title: Weights of nn.ModuleList() are adjusted even if no forward propagation was applied

body:
I'm trying to use nn.ModuleList() to conduct some multitask learning, but the weights of all list elements (i.e. tasks) are adjusted when they were trained before. The following code (based on this notebook) creates a neural network object called MTL.

import torch
import torch.nn as nn
import numpy as np
import os
from torch.autograd import Variable
import math
import sklearn.preprocessing as sk
from sklearn.model_selection import KFold
from sklearn import metrics
from sklearn.feature_selection import VarianceThreshold
from sklearn.linear_model import Ridge
from sklearn.linear_model import RidgeCV
from sklearn.model_selection import train_test_split
import random

#creating training data
seed = 42
random.seed(seed)
torch.cuda.manual_seed_all(seed)

N = 10000
M = 100
c = 0.5
p = 0.9
k = np.random.randn(M)
u1 = np.random.randn(M)
u1 -= u1.dot(k) * k / np.linalg.norm(k)**2
u1 /= np.linalg.norm(u1)
k /= np.linalg.norm(k)
u2 = k
w1 = c*u1
w2 = c*(p*u1+np.sqrt((1-p**2))*u2)
X = np.random.normal(0, 1, (N, M))
eps1 = np.random.normal(0, 0.01)
eps2 = np.random.normal(0, 0.01)
Y1 = np.matmul(X, w1) + np.sin(np.matmul(X, w1))+eps1
Y2 = np.matmul(X, w2) + np.sin(np.matmul(X, w2))+eps2
split = list(np.random.permutation(N))

X_train = X[split[0:8000],:]
Y1_train = Y1[split[0:8000]]
Y2_train = Y2[split[0:8000]]
X_valid = X[8000:9000,:]
Y1_valid = Y1[8000:9000]
Y2_valid = Y2[8000:9000]
X_test = X[9000:10000,:]
Y1_test = Y1[9000:10000]
Y2_test = Y2[9000:10000]

X_train = torch.from_numpy(X_train)
X_train = X_train.float()
Y1_train = torch.tensor(Y1_train)
Y1_train = Y1_train.float()
Y2_train = torch.tensor(Y2_train)
Y2_train = Y2_train.float()

X_valid = torch.from_numpy(X_valid)
X_valid = X_valid.float()
Y1_valid = torch.tensor(Y1_valid)
Y1_valid = Y1_valid.float()
Y2_valid = torch.tensor(Y2_valid)
Y2_valid = Y2_valid.float()

X_test = torch.from_numpy(X_test)
X_test = X_test.float()
Y1_test = torch.tensor(Y1_test)
Y1_test = Y1_test.float()
Y2_test = torch.tensor(Y2_test)
Y2_test = Y2_test.float()

input_size, feature_size = X.shape

LR = 0.001
epoch = 50
mb_size = 100

#the network
class MTLnet(nn.Module):
    def __init__(self):
        super(MTLnet, self).__init__()

        self.sharedlayer = nn.Sequential(
            nn.Linear(feature_size, 64),
            nn.ReLU(),
            nn.Dropout()
        )

        output = ['tower1', 'tower2']
        self.scoring_list = nn.ModuleList()

        for task, lab in enumerate(output):
            tower = nn.Sequential(
                nn.Linear(64, 32),
                nn.ReLU(),
                nn.Dropout(),
                nn.Linear(32, 16),
                nn.ReLU(),
                nn.Dropout(),
                nn.Linear(16, 1)
            )
            self.scoring_list.append(tower)

    def forward(self, x, task_id):
        h_shared = self.sharedlayer(x)
        logits = self.scoring_list[task_id](h_shared)

        return logits

def random_mini_batches(XE, R1E, R2E, mini_batch_size = 3, seed = 42):
    # Creating the mini-batches
    np.random.seed(seed)
    m = XE.shape[0]
    mini_batches = []
    permutation = list(np.random.permutation(m))
    shuffled_XE = XE[permutation,:]
    shuffled_X1R = R1E[permutation]
    shuffled_X2R = R2E[permutation]
    num_complete_minibatches = math.floor(m/mini_batch_size)
    for k in range(0, int(num_complete_minibatches)):
        mini_batch_XE = shuffled_XE[k * mini_batch_size : (k+1) * mini_batch_size, :]
        mini_batch_X1R = shuffled_X1R[k * mini_batch_size : (k+1) * mini_batch_size]
        mini_batch_X2R = shuffled_X2R[k * mini_batch_size : (k+1) * mini_batch_size]
        mini_batch = (mini_batch_XE, mini_batch_X1R, mini_batch_X2R)
        mini_batches.append(mini_batch)
    Lower = int(num_complete_minibatches * mini_batch_size)
    Upper = int(m - (mini_batch_size * math.floor(m/mini_batch_size)))
    if m % mini_batch_size != 0:
        mini_batch_XE = shuffled_XE[Lower : Lower + Upper, :]
        mini_batch_X1R = shuffled_X1R[Lower : Lower + Upper]
        mini_batch_X2R = shuffled_X2R[Lower : Lower + Upper]
        mini_batch = (mini_batch_XE, mini_batch_X1R, mini_batch_X2R)
        mini_batches.append(mini_batch)

    return mini_batches

MTL = MTLnet()
optimizer = torch.optim.Adam(MTL.parameters(), lr=LR)
loss_func = nn.MSELoss()

The neural network has the following structure:

<bound method Module.parameters of MTLnet(
  (sharedlayer): Sequential(
    (0): Linear(in_features=100, out_features=64, bias=True)
    (1): ReLU()
    (2): Dropout(p=0.5)
  )
  (scoring_list): ModuleList(
    (0): Sequential(
      (0): Linear(in_features=64, out_features=32, bias=True)
      (1): ReLU()
      (2): Dropout(p=0.5)
      (3): Linear(in_features=32, out_features=16, bias=True)
      (4): ReLU()
      (5): Dropout(p=0.5)
      (6): Linear(in_features=16, out_features=1, bias=True)
    )
    (1): Sequential(
      (0): Linear(in_features=64, out_features=32, bias=True)
      (1): ReLU()
      (2): Dropout(p=0.5)
      (3): Linear(in_features=32, out_features=16, bias=True)
      (4): ReLU()
      (5): Dropout(p=0.5)
      (6): Linear(in_features=16, out_features=1, bias=True)
    )
  )
)>

The initial values of the weights are (yours will differ):

print(MTL.state_dict()['scoring_list.0.6.weight'])
print(MTL.state_dict()['scoring_list.1.6.weight'])

Output:

tensor([[-0.0240, -0.1798, -0.2393, -0.2149, -0.1393,  0.1718, -0.1476,  0.0346,
          0.2485, -0.0305, -0.1574,  0.1500, -0.2356, -0.0597,  0.0291,  0.0521]])
tensor([[ 0.2046, -0.1277, -0.2103, -0.1006, -0.1311,  0.1902, -0.0969, -0.0953,
          0.1340,  0.1506, -0.1222, -0.0638, -0.0661,  0.1118, -0.1009, -0.1438]])

The following code trains the neural network and prints the weights after every epoch. The first epoch will train the shared layer and the first element of nn.ModuleList() (i.e. task1). The second epoch will train the shared layer and the second element of the nn.ModuleList() (i.e. task2).

trainTask1 = True
epoch = 2

for it in range(epoch):
    minibatches = random_mini_batches(X_train, Y1_train, Y2_train, mb_size)
    for minibatch in minibatches:
        XE, YE1, YE2  = minibatch

        if trainTask1:
            Yhat = MTL(XE, 0)
            loss = loss_func(Yhat, YE1.view(-1,1))
        else:
            Yhat = MTL(XE, 1)
            loss = loss_func(Yhat, YE2.view(-1,1))

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        #Shows you the weights of task 1 and they get adjusted during the 2 epoch even if no training has happened
        #print("Task 1 weights {}".format(MTL.state_dict()['scoring_list.0.6.weight']))

    #@prosti suggested to freeze the layers of the task which aren't trained
    if trainTask1:
        trainTask1 = False
        for param in MTL.scoring_list[0].parameters():
            param.requires_grad = False
    else:
        trainTask1 = True
        for param in MTL.scoring_list[1].parameters():
            param.requires_grad = False

    print(it)
    print(MTL.state_dict()['scoring_list.0.6.weight'])
    print(MTL.state_dict()['scoring_list.1.6.weight'])

Output:

0
tensor([[-0.0283, -0.2025, -0.2181, -0.2183, -0.1438,  0.1605, -0.1715,  0.0863,
          0.2765, -0.0153, -0.1519,  0.1704, -0.2453, -0.0539,  0.0220,  0.0739]])
tensor([[ 0.2046, -0.1277, -0.2103, -0.1006, -0.1311,  0.1902, -0.0969, -0.0953,
          0.1340,  0.1506, -0.1222, -0.0638, -0.0661,  0.1118, -0.1009, -0.1438]])
1
tensor([[-0.0311, -0.2114, -0.2162, -0.2214, -0.1463,  0.1614, -0.1800,  0.1003,
          0.2850, -0.0148, -0.1576,  0.1809, -0.2511, -0.0575,  0.0221,  0.0844]])
tensor([[ 0.2693, -0.0921, -0.2313, -0.1483, -0.0995,  0.2497, -0.1028, -0.1108,
          0.1405,  0.1997, -0.1266, -0.0725, -0.0871,  0.1472, -0.0924, -0.0994]])

After the first epoch the weights of task1 were adjusted but not the weights of task2 (as expected), but after the second epoch the weights of both tasks were adjusted. That shouldn't happen.

You can also see that when you print the weights during the over minibatches (just uncomment the print), they are always get adjusted for the first task even if all layers were freezed and no calculation has happened.

Is there another cache I have to clear besides optimizer.zero_grad()?


title: DCGAN debugging. Getting just garbage

body:
Introduction:
I am trying to get a CDCGAN (Conditional Deep Convolutional Generative Adversarial Network) to work on the MNIST dataset which should be fairly easy considering that the library (PyTorch) I am using has a tutorial on its website.
But I can't seem to get It working it just produces garbage or the model collapses or both.
What I tried:

making the model Conditional semi-supervised learning
using batch norm
using dropout on each layer besides the input/output layer on the generator and discriminator
label smoothing to combat overconfidence
adding noise to the images (I guess you call this instance noise) to get a better data distribution
use leaky relu to avoid vanishing gradients
using a replay buffer to combat forgetting of learned stuff and overfitting
playing with hyperparameters
comparing it to the model from PyTorch tutorial
basically what I did besides some things like Embedding layer ect.

Images my Model generated:
Hyperparameters:
batch_size=50, learning_rate_discrimiantor=0.0001, learning_rate_generator=0.0003, shuffle=True, ndf=64, ngf=64, droupout=0.5

batch_size=50, learning_rate_discriminator=0.0003, learning_rate_generator=0.0003, shuffle=True, ndf=64, ngf=64, dropout=0

Images Pytorch tutorial Model generated:
Code for the pytorch tutorial dcgan model
As comparison here are the images from the DCGAN from the pytorch turoial:

My Code:
import torch
import torch.nn as nn
import torchvision
from torchvision import transforms, datasets
import torch.nn.functional as F
from torch import optim as optim
from torch.utils.tensorboard import SummaryWriter

import numpy as np

import os
import time

class Discriminator(torch.nn.Module):
    def __init__(self, ndf=16, dropout_value=0.5):  # ndf feature map discriminator
        super().__init__()
        self.ndf = ndf
        self.droupout_value = dropout_value

        self.condi = nn.Sequential(
            nn.Linear(in_features=10, out_features=64 * 64)
        )

        self.hidden0 = nn.Sequential(
            nn.Conv2d(in_channels=2, out_channels=self.ndf, kernel_size=4, stride=2, padding=1, bias=False),
            nn.LeakyReLU(0.2),
        )
        self.hidden1 = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf, out_channels=self.ndf * 2, kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ndf * 2),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )
        self.hidden2 = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf * 2, out_channels=self.ndf * 4, kernel_size=4, stride=2, padding=1, bias=False),
            #nn.BatchNorm2d(self.ndf * 4),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )
        self.hidden3 = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf * 4, out_channels=self.ndf * 8, kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ndf * 8),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )
        self.out = nn.Sequential(
            nn.Conv2d(in_channels=self.ndf * 8, out_channels=1, kernel_size=4, stride=1, padding=0, bias=False),
            torch.nn.Sigmoid()
        )

    def forward(self, x, y):
        y = self.condi(y.view(-1, 10))
        y = y.view(-1, 1, 64, 64)

        x = torch.cat((x, y), dim=1)

        x = self.hidden0(x)
        x = self.hidden1(x)
        x = self.hidden2(x)
        x = self.hidden3(x)
        x = self.out(x)

        return x

class Generator(torch.nn.Module):
    def __init__(self, n_features=100, ngf=16, c_channels=1, dropout_value=0.5):  # ngf feature map of generator
        super().__init__()
        self.ngf = ngf
        self.n_features = n_features
        self.c_channels = c_channels
        self.droupout_value = dropout_value

        self.hidden0 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.n_features + 10, out_channels=self.ngf * 8,
                               kernel_size=4, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(self.ngf * 8),
            nn.LeakyReLU(0.2)
        )

        self.hidden1 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.ngf * 8, out_channels=self.ngf * 4,
                               kernel_size=4, stride=2, padding=1, bias=False),
            #nn.BatchNorm2d(self.ngf * 4),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )

        self.hidden2 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.ngf * 4, out_channels=self.ngf * 2,
                               kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ngf * 2),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )

        self.hidden3 = nn.Sequential(
            nn.ConvTranspose2d(in_channels=self.ngf * 2, out_channels=self.ngf,
                               kernel_size=4, stride=2, padding=1, bias=False),
            nn.BatchNorm2d(self.ngf),
            nn.LeakyReLU(0.2),
            nn.Dropout(self.droupout_value)
        )

        self.out = nn.Sequential(
            # "out_channels=1" because gray scale
            nn.ConvTranspose2d(in_channels=self.ngf, out_channels=1, kernel_size=4,
                               stride=2, padding=1, bias=False),
            nn.Tanh()
        )

    def forward(self, x, y):
        x_cond = torch.cat((x, y), dim=1)  # Combine flatten image with conditional input (class labels)

        x = self.hidden0(x_cond)           # Image goes into a "ConvTranspose2d" layer
        x = self.hidden1(x)
        x = self.hidden2(x)
        x = self.hidden3(x)
        x = self.out(x)

        return x

class Logger:
    def __init__(self, model_name, model1, model2, m1_optimizer, m2_optimizer, model_parameter, train_loader):
        self.out_dir = "data"
        self.model_name = model_name
        self.train_loader = train_loader
        self.model1 = model1
        self.model2 = model2
        self.model_parameter = model_parameter
        self.m1_optimizer = m1_optimizer
        self.m2_optimizer = m2_optimizer

        # Exclude Epochs of the model name. This make sense e.g. when we stop a training progress and continue later on.
        self.experiment_name = '_'.join("{!s}={!r}".format(k, v) for (k, v) in model_parameter.items())\
            .replace("Epochs" + "=" + str(model_parameter["Epochs"]), "")

        self.d_error = 0
        self.g_error = 0

        self.tb = SummaryWriter(log_dir=str(self.out_dir + "/log/" + self.model_name + "/runs/" + self.experiment_name))

        self.path_image = os.path.join(os.getcwd(), f'{self.out_dir}/log/{self.model_name}/images/{self.experiment_name}')
        self.path_model = os.path.join(os.getcwd(), f'{self.out_dir}/log/{self.model_name}/model/{self.experiment_name}')

        try:
            os.makedirs(self.path_image)
        except Exception as e:
            print("WARNING: ", str(e))

        try:
            os.makedirs(self.path_model)
        except Exception as e:
            print("WARNING: ", str(e))

    def log_graph(self, model1_input, model2_input, model1_label, model2_label):
        self.tb.add_graph(self.model1, input_to_model=(model1_input, model1_label))
        self.tb.add_graph(self.model2, input_to_model=(model2_input, model2_label))

    def log(self, num_epoch, d_error, g_error):
        self.d_error = d_error
        self.g_error = g_error

        self.tb.add_scalar("Discriminator Train Error", self.d_error, num_epoch)
        self.tb.add_scalar("Generator Train Error", self.g_error, num_epoch)

    def log_image(self, images, epoch, batch_num):
        grid = torchvision.utils.make_grid(images)
        torchvision.utils.save_image(grid, f'{self.path_image}\\Epoch_{epoch}_batch_{batch_num}.png')

        self.tb.add_image("Generator Image", grid)

    def log_histogramm(self):
        for name, param in self.model2.named_parameters():
            self.tb.add_histogram(name, param, self.model_parameter["Epochs"])
            self.tb.add_histogram(f'gen_{name}.grad', param.grad, self.model_parameter["Epochs"])

        for name, param in self.model1.named_parameters():
            self.tb.add_histogram(name, param, self.model_parameter["Epochs"])
            self.tb.add_histogram(f'dis_{name}.grad', param.grad, self.model_parameter["Epochs"])

    def log_model(self, num_epoch):
        torch.save({
            "epoch": num_epoch,
            "model_generator_state_dict": self.model1.state_dict(),
            "model_discriminator_state_dict": self.model2.state_dict(),
            "optimizer_generator_state_dict":  self.m1_optimizer.state_dict(),
            "optimizer_discriminator_state_dict":  self.m2_optimizer.state_dict(),
        }, str(self.path_model + f'\\{time.time()}_epoch{num_epoch}.pth'))

    def close(self, logger, images, num_epoch,  d_error, g_error):
        logger.log_model(num_epoch)
        logger.log_histogramm()
        logger.log(num_epoch, d_error, g_error)
        self.tb.close()

    def display_stats(self, epoch, batch_num, dis_error, gen_error):
        print(f'Epoch: [{epoch}/{self.model_parameter["Epochs"]}] '
              f'Batch: [{batch_num}/{len(self.train_loader)}] '
              f'Loss_D: {dis_error.data.cpu()}, '
              f'Loss_G: {gen_error.data.cpu()}')

def get_MNIST_dataset(num_workers_loader, model_parameter, out_dir="data"):
    compose = transforms.Compose([
        transforms.Resize((64, 64)),
        transforms.CenterCrop((64, 64)),
        transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.5], std=[0.5])
    ])

    dataset = datasets.MNIST(
        root=out_dir,
        train=True,
        download=True,
        transform=compose
    )

    train_loader = torch.utils.data.DataLoader(dataset,
                                               batch_size=model_parameter["batch_size"],
                                               num_workers=num_workers_loader,
                                               shuffle=model_parameter["shuffle"])

    return dataset, train_loader

def train_discriminator(p_optimizer, p_noise, p_images, p_fake_target, p_real_target, p_images_labels, p_fake_labels, device):
    p_optimizer.zero_grad()

    # 1.1 Train on real data
    pred_dis_real = discriminator(p_images, p_images_labels)
    error_real = loss(pred_dis_real, p_real_target)

    error_real.backward()

    # 1.2 Train on fake data
    fake_data = generator(p_noise, p_fake_labels).detach()
    fake_data = add_noise_to_image(fake_data, device)
    pred_dis_fake = discriminator(fake_data, p_fake_labels)
    error_fake = loss(pred_dis_fake, p_fake_target)

    error_fake.backward()

    p_optimizer.step()

    return error_fake + error_real

def train_generator(p_optimizer, p_noise, p_real_target, p_fake_labels, device):
    p_optimizer.zero_grad()

    fake_images = generator(p_noise, p_fake_labels)
    fake_images = add_noise_to_image(fake_images, device)
    pred_dis_fake = discriminator(fake_images, p_fake_labels)
    error_fake = loss(pred_dis_fake, p_real_target)  # because
    """
    We use "p_real_target" instead of "p_fake_target" because we want to
    maximize that the discriminator is wrong.
    """

    error_fake.backward()

    p_optimizer.step()

    return fake_images, pred_dis_fake, error_fake

# TODO change to a Truncated normal distribution
def get_noise(batch_size, n_features=100):
    return torch.FloatTensor(batch_size, n_features, 1, 1).uniform_(-1, 1)

# We flip label of real and fate data. Better gradient flow I have told
def get_real_data_target(batch_size):
    return torch.FloatTensor(batch_size, 1, 1, 1).uniform_(0.0, 0.2)

def get_fake_data_target(batch_size):
    return torch.FloatTensor(batch_size, 1, 1, 1).uniform_(0.8, 1.1)

def image_to_vector(images):
    return torch.flatten(images, start_dim=1, end_dim=-1)

def vector_to_image(images):
    return images.view(images.size(0), 1, 28, 28)

def get_rand_labels(batch_size):
    return torch.randint(low=0, high=9, size=(batch_size,))

def load_model(model_load_path):
    if model_load_path:
        checkpoint = torch.load(model_load_path)

        discriminator.load_state_dict(checkpoint["model_discriminator_state_dict"])
        generator.load_state_dict(checkpoint["model_generator_state_dict"])

        dis_opti.load_state_dict(checkpoint["optimizer_discriminator_state_dict"])
        gen_opti.load_state_dict(checkpoint["optimizer_generator_state_dict"])

        return checkpoint["epoch"]

    else:
        return 0

def init_model_optimizer(model_parameter, device):
    # Initialize the Models
    discriminator = Discriminator(ndf=model_parameter["ndf"], dropout_value=model_parameter["dropout"]).to(device)
    generator = Generator(ngf=model_parameter["ngf"], dropout_value=model_parameter["dropout"]).to(device)

    # train
    dis_opti = optim.Adam(discriminator.parameters(), lr=model_parameter["learning_rate_dis"], betas=(0.5, 0.999))
    gen_opti = optim.Adam(generator.parameters(), lr=model_parameter["learning_rate_gen"], betas=(0.5, 0.999))

    return discriminator, generator, dis_opti, gen_opti

def get_hot_vector_encode(labels, device):
    return torch.eye(10)[labels].view(-1, 10, 1, 1).to(device)

def add_noise_to_image(images, device, level_of_noise=0.1):
    return images[0].to(device) + (level_of_noise) * torch.randn(images.shape).to(device)

if __name__ == "__main__":
    # Hyperparameter
    model_parameter = {
        "batch_size": 500,
        "learning_rate_dis": 0.0002,
        "learning_rate_gen": 0.0002,
        "shuffle": False,
        "Epochs": 10,
        "ndf": 64,
        "ngf": 64,
        "dropout": 0.5
    }

    # Parameter
    r_frequent = 10        # How many samples we save for replay per batch (batch_size / r_frequent).
    model_name = "CDCGAN"   # The name of you model e.g. "Gan"
    num_workers_loader = 1  # How many workers should load the data
    sample_save_size = 16   # How many numbers your saved imaged should show
    device = "cuda"         # Which device should be used to train the neural network
    model_load_path = ""    # If set load model instead of training from new
    num_epoch_log = 1       # How frequent you want to log/
    torch.manual_seed(43)   # Sets a seed for torch for reproducibility

    dataset_train, train_loader = get_MNIST_dataset(num_workers_loader, model_parameter)  # Get dataset

    # Initialize the Models and optimizer
    discriminator, generator, dis_opti, gen_opti = init_model_optimizer(model_parameter, device)  # Init model/Optimizer

    start_epoch = load_model(model_load_path)  # when we want to load a model

    # Init Logger
    logger = Logger(model_name, generator, discriminator, gen_opti, dis_opti, model_parameter, train_loader)

    loss = nn.BCELoss()

    images, labels = next(iter(train_loader))  # For logging

    # For testing
    # pred = generator(get_noise(model_parameter["batch_size"]).to(device), get_hot_vector_encode(get_rand_labels(model_parameter["batch_size"]), device))
    # dis = discriminator(images.to(device), get_hot_vector_encode(labels, device))

    logger.log_graph(get_noise(model_parameter["batch_size"]).to(device), images.to(device),
                     get_hot_vector_encode(get_rand_labels(model_parameter["batch_size"]), device),
                     get_hot_vector_encode(labels, device))

    # Array to store
    exp_replay = torch.tensor([]).to(device)

    for num_epoch in range(start_epoch, model_parameter["Epochs"]):
        for batch_num, data_loader in enumerate(train_loader):
            images, labels = data_loader
            images = add_noise_to_image(images, device)  # Add noise to the images

            # 1. Train Discriminator
            dis_error = train_discriminator(
                                            dis_opti,
                                            get_noise(model_parameter["batch_size"]).to(device),
                                            images.to(device),
                                            get_fake_data_target(model_parameter["batch_size"]).to(device),
                                            get_real_data_target(model_parameter["batch_size"]).to(device),
                                            get_hot_vector_encode(labels, device),
                                            get_hot_vector_encode(
                                                get_rand_labels(model_parameter["batch_size"]), device),
                                            device
                                            )

            # 2. Train Generator
            fake_image, pred_dis_fake, gen_error = train_generator(
                                                                  gen_opti,
                                                                  get_noise(model_parameter["batch_size"]).to(device),
                                                                  get_real_data_target(model_parameter["batch_size"]).to(device),
                                                                  get_hot_vector_encode(
                                                                      get_rand_labels(model_parameter["batch_size"]),
                                                                      device),
                                                                  device
                                                                  )

            # Store a random point for experience replay
            perm = torch.randperm(fake_image.size(0))
            r_idx = perm[:max(1, int(model_parameter["batch_size"] / r_frequent))]
            r_samples = add_noise_to_image(fake_image[r_idx], device)
            exp_replay = torch.cat((exp_replay, r_samples), 0).detach()

            if exp_replay.size(0) >= model_parameter["batch_size"]:
                # Train on experienced data
                dis_opti.zero_grad()

                r_label = get_hot_vector_encode(torch.zeros(exp_replay.size(0)).numpy(), device)
                pred_dis_real = discriminator(exp_replay, r_label)
                error_real = loss(pred_dis_real,  get_fake_data_target(exp_replay.size(0)).to(device))

                error_real.backward()

                dis_opti.step()

                print(f'Epoch: [{num_epoch}/{model_parameter["Epochs"]}] '
                      f'Batch: Replay/Experience batch '
                      f'Loss_D: {error_real.data.cpu()}, '
                      )

                exp_replay = torch.tensor([]).to(device)

            logger.display_stats(epoch=num_epoch, batch_num=batch_num, dis_error=dis_error, gen_error=gen_error)

            if batch_num % 100 == 0:
                logger.log_image(fake_image[:sample_save_size], num_epoch, batch_num)

        logger.log(num_epoch, dis_error, gen_error)
        if num_epoch % num_epoch_log == 0:
            logger.log_model(num_epoch)
            logger.log_histogramm()
    logger.close(logger, fake_image[:sample_save_size], num_epoch, dis_error, gen_error)

First link to my Code (Pastebin)
Second link to my Code (0bin)
Conclusion:
Since I implemented all these things (e.g. label smoothing) which are considered beneficial to a GAN/DCGAN.
And my Model still performs worse than the Tutorial DCGAN from PyTorch I think I might have a bug in my code but I can't seem to find it.
Reproducibility:
You should be able to just copy the code and run it if you have the libraries that I imported installed to look for yourself if you can find anything.
I appreciate any feedback.


title: When to use padding in Conv2d() and when to do ReflectionPad2d() Pytorch

body:
I have two PyTorch models that are equivalent (I think), the only difference between them is the padding:

import torch
import torch.nn as nn

i = torch.arange(9, dtype=torch.float).reshape(1,1,3,3)
# First model:
model1 = nn.Conv2d(1, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), padding_mode='reflection')
# tensor([[[[-0.6095, -0.0321,  2.2022],
#           [ 0.1018,  1.7650,  5.5392],
#           [ 1.7988,  3.9165,  5.6506]]]], grad_fn=<MkldnnConvolutionBackward>)

# Second model:
model2 = nn.Sequential(nn.ReflectionPad2d((1, 1, 1, 1)),
             nn.Conv2d(1, 1, kernel_size=3))
# tensor([[[[1.4751, 1.5513, 2.6566],
#           [4.0281, 4.1043, 5.2096],
#           [2.6149, 2.6911, 3.7964]]]], grad_fn=<MkldnnConvolutionBackward>)

I was wondering why and when you use both approaches, the output of both is different but as I see it they should be the same, because the padding is of type reflection.

Would appreciate some help in understanding it.

EDIT

After what @Ash said, I wanted to check wheter or not the weights had influence so I pinned all of them to the same value and still there is a difference between the 2 methods:

import torch
import torch.nn as nn

i = torch.arange(9, dtype=torch.float).reshape(1,1,3,3)
# First model:
model1 = nn.Conv2d(1, 1, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), padding_mode='reflection')
model1.weight.data = torch.full(model1.weight.data.shape, 0.4)
print(model1(i))
print(model1.weight)
# tensor([[[[ 3.4411,  6.2411,  5.0412],
#           [ 8.6411, 14.6411, 11.0412],
#           [ 8.2411, 13.4411,  9.8412]]]], grad_fn=<MkldnnConvolutionBackward>)
# Parameter containing:
# tensor([[[[0.4000, 0.4000, 0.4000],
#           [0.4000, 0.4000, 0.4000],
#           [0.4000, 0.4000, 0.4000]]]], requires_grad=True)

# Second model:
model2 = [nn.ReflectionPad2d((1, 1, 1, 1)),
             nn.Conv2d(1, 1, kernel_size=3)]
model2[1].weight.data = torch.full(model2[1].weight.data.shape, 0.4)
model2 = nn.Sequential(*model2)
print(model2(i))
print(model2[1].weight)
# tensor([[[[ 9.8926, 11.0926, 12.2926],
#           [13.4926, 14.6926, 15.8926],
#           [17.0926, 18.2926, 19.4926]]]], grad_fn=<MkldnnConvolutionBackward>)
# Parameter containing:
# tensor([[[[0.4000, 0.4000, 0.4000],
#           [0.4000, 0.4000, 0.4000],
#           [0.4000, 0.4000, 0.4000]]]], requires_grad=True)


title: How to calculate the 3x3 covariance matrix for RGB values across an image dataset?

body:
I need to calculate the covariance matrix for RGB values across an image dataset, and then apply Cholesky decomposition to the final result.
The covariance matrix for RGB values is a 3x3 matrix M, where M_(i, i) is the variance of channel i and M_(i, j) is the covariance between channels i and j.
The end result should be something like this:
([[0.26, 0.09, 0.02],
[0.27, 0.00, -0.05],
[0.27, -0.09, 0.03]])

I'd prefer to stick to PyTorch functions even though Numpy has a Cov function.
I attempted to recreate the numpy Cov function in PyTorch here based on other cov implementations and clones:
def pytorch_cov(tensor, tensor2=None, rowvar=True):
    if tensor2 is not None:
        tensor = torch.cat((tensor, tensor2), dim=0)
    tensor = tensor.view(1, -1) if tensor.dim() < 2 else tensor
    tensor = tensor.t() if not rowvar and tensor.size(0) != 1 else tensor
    tensor = tensor - torch.mean(tensor, dim=1, keepdim=True)
    return 1 / (tensor.size(1) - 1) * tensor.mm(tensor.t())

def cov_vec(x):
    c = x.size(0)
    m1 = x - torch.sum(x, dim=[1],keepdims=True)/ c
    out = torch.einsum('ijk,ilk->ijl',m1,m1)  / (c - 1)
    return out

The dataset loading would be like this:
dataset = torchvision.datasets.ImageFolder(data_path)
loader = torch.utils.data.DataLoader(dataset)

for images, _ in loader:
    batch_size = images.size(0)
    ...

For the moment I'm just experimenting with images created with torch.randn(batch_size, 3, height, width).
Edit:
I'm attempting to replicate the matrix from Tensorflow's Lucid here, and somewhat explained on distill.pub here.
Second Edit:
In order to make the output resemble the example one, you have to do this instead of using Cholesky:
rgb_cov_tensor = rgb_cov_tensor / len(loader.dataset)
U,S,V = torch.svd(rgb_cov_tensor)
epsilon = 1e-10
svd_sqrt = U @ torch.diag(torch.sqrt(S + epsilon))

The resulting matrix can then be used to perform color decorrelation, which is useful for visualizing features (DeepDream). I've implemented it in my project here.


title: Windows keeps crashing when trying to install PyTorch via pip

body:
I am currently trying to install PyTorch (using the installation commands posted on PyTorch.org for pip) and when I run the command, my computer completely freezes.
I tried this multiple times with the same result. I had to restart the computer a few times as well. On my current try, I put "-v" when trying to install and the pip seems to be stuck on
"Looking up  in the cache".
I do not know how to proceed. As I mentioned, I've already tried this method multiple times. It worked the first time but did not install PyTorch as it gave me an error for not using "--user".
Are there any solutions to this?
EDIT: I did want to add that I have Python 3.8.6 (64bit)


title: Pytorch 3-GPUs, just can only use 2 of them to train

body:
I have three 1080TI, but when train I can only use 2 of them..

Code:

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.cuda()

criterion = nn.CrossEntropyLoss().cuda()
optimizer_conv = optim.SGD(model.classifier.parameters(), lr=0.0001, momentum=0.9)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_conv, step_size=7, gamma=0.1)

train part:

outputs = nn.parallel.data_parallel(model,inputs,device_ids=[0,1,2])

With "CUDA_VISIBLE_DEVICES="1,2,3" python train.py"
Got this:

| 22%   35C    P8    10W / 250W |     12MiB / 11178MiB |      0%
| 43%   59C    P2    92W / 250W |   1169MiB / 11178MiB |     49%
| 44%   60C    P2    91W / 250W |   1045MiB / 11175MiB |     54%

With "CUDA_VISIBLE_DEVICES="0,1,2" python train.py"
Got this:

| 21%   38C    P2    95W / 250W |   1169MiB / 11178MiB |     78%      Default |
| 42%   63C    P2    93W / 250W |    777MiB / 11178MiB |     76%      Default |
| 43%   64C    P0    85W / 250W |    282MiB / 11175MiB |      0%      Default |


title: How to define a custom gradient in pytorch with an optimizer?

body:
I am trying to learn an embedding with a custom gradient function. Basically, I have an embedding

my_embedding = torch.nn.Embedding(
                  num_embeddings=n,
                  embedding_dim=d)

and an optimizer, for instance

my_optimizer = torch.optim.SGD(
                  my_embedding.parameters(),
                  lr=0.01)

I am computing some gradient manually (denoted as my_gradient below) and I am doing my_embedding.weight.grad = my_gradient.
However, when I'm doing my_optimizer.step() it does not update anything.
Am I missing something ? (in this case, I could directly skip the optimizer but I would like to understand why it's not working)


title: how see inside object tensor in spyder?

body:
I am struggling to see inside an object in a spyder. when I run this snippet, in variable explorer I can see only the attached fig, however, I need to see the probs and other object tensor and also I need to access to the value of probs? any comment would appreciate.

import torch
from U_Net_demo import device
from dataset import test_loader
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
import torch.nn as nn
import numpy

criterion = nn.NLLLoss()

def test():

model_load = torch.load('model.pth')

#test model
model_load.eval()
total = 0
test_loss = 0
correct = 0
count = 0
#iterate through test dataset
for ii, data in enumerate(test_loader):

            t_image, mask = data
            t_image, mask = t_image.to(device), mask.to(device)
            with torch.no_grad():

                outputs = model_load(t_image)
                #print(outputs.shape) # torch.Size([1, 2, 240, 320])
                test_loss += criterion(outputs, mask).item() / len(test_loader)

                probs = torch.exp(outputs)

                _, predicted = torch.max(outputs.data, 1)

                total += mask.nelement()
                correct += predicted.eq(mask.data).sum().item()
                accuracy = 100 * correct / total

                count +=1
                print(count, "Test Loss: {:.3f}".format(test_loss), "Test Accuracy: %d %%" % (accuracy))

if __name__=='__main__':
    test = test()


title: Adding class objects to Pytorch Dataloader: batch must contain tensors

body:
I have a custom Pytorch dataset that returns a dictionary containing a class object "queries".
class QueryDataset(torch.utils.data.Dataset):

    def __init__(self, queries, values, targets):
        super(QueryDataset).__init__()
        self.queries = queries
        self.values = values
        self.targets = targets

    def __len__(self):
        return self.values.shape[0]

    def __getitem__(self, idx):
        sample = DeviceDict({'query': self.queries[idx],
                             "values": self.values[idx],
                             "targets": self.targets[idx]})
        return sample

The problem is that when I put the queries in a data loader I get default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'query.Query'>. Is there a way to have a class object in my data loader? It blows up at next(iterator) in the code below.
train_queries = QueryDataset(train_queries)
train_loader = torch.utils.data.DataLoader(train_queries,
                                           batch_size=10],
                                           shuffle=True,
                                           drop_last=False)
for i in range(epochs):
    iterator = iter(train_loader)
    for i in range(len(train_loader)):
        batch = next(iterator)
        out = model(batch)
        loss = criterion(out["pred"], batch["targets"])
        self.optimizer.zero_grad()
        loss.sum().backward()
        self.optimizer.step()


title: clang in neovim giving pp_file_not_found error for c++/pytorch basic example

body:
I'm following this very basic c++/pytorch example:
pytorch_installing
And I can walk through this example with no errors. However, when creating the example-app.cpp file (or editing it at any point in time) with neovim, clang throws an error 'torch/torch.h' file not found [clang: pp_file_not_found]. My CMakeLists.txt file looks like this:
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(example-app)

find_package(Torch REQUIRED)
include_directories(SYSTEM /home/username/Downloads/libtorch/)
set(CMAKE_PREFIX_PATH "/home/username/Downloads/libtorch/share/cmake/Torch")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")

add_executable(example-app example-app.cpp)
target_link_libraries(example-app "${TORCH_LIBRARIES}")
set_property(TARGET example-app PROPERTY CXX_STANDARD 14)

This isn't a big issue for me with this small of a project. But I will be working on a larger project with libtorch and would like clang to recognize <torch/torch.h>. There were a couple similar Stackoverflow questions, but no answers.
Update:
I believe this is happening because clang is not seeing torch/torch.h because it is not part of the include paths. I printed the include paths for clang and it was not list. So I tried adding the include paths of libtorch to /usr/include/ but then it has issues seeing other header files referenced by header files I added. So I added the libtorch/include/torch/csrc/api/include/torch/ directory to /usr/include so that it can read #include <torch/torch.h. But then other header files are referenced in those header files with specified directory paths outside of the libtorch/include/torch/csrc/api/include/torch/ path.
For example, libtorch/include/torch/csrc/ contains WindowsTorchApiMacro.h. There are other paths within the libtorch directory that contain more header files. I tried to add all header files to \usr\include but still received an error. I'm sure the package as a whole needs to be used. The original cmake file looks like this:
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
project(example-app)

find_package(Torch REQUIRED)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TORCH_CXX_FLAGS}")

add_executable(example-app example-app.cpp)
target_link_libraries(example-app "${TORCH_LIBRARIES}")
set_property(TARGET example-app PROPERTY CXX_STANDARD 14)

So cmake is able to find and reference all of those header files properly, and I know you can compile it at the command line with clang, but how can clang within neovim (during editing) see this package without error? I thought about using a package manager like vcpkg, but they do not have libtorch available, unfortunately. I can include output from a verbose and successful cmake run if needed.


title: pytorch gesv gives different result than scipy sparse solve

body:
I'm trying to implement baseline als subtraction in pytorch so that I can run it on my GPU but I am running into problems because pytorch.gesv gives a different result than scipy.linalg.spsolve. Here is my code for scipy:
def baseline_als(y, lam, p, niter=10):
  L = len(y)
  D = sparse.diags([1,-2,1],[0,-1,-2], shape=(L,L-2))
  w = np.ones(L)
  for i in range(niter):
    W = sparse.spdiags(w, 0, L, L)
    Z = W + lam * D.dot(D.transpose())
    z = spsolve(Z, w*y)
    w = p * (y > z) + (1-p) * (y < z)
  return z

and here is my code for pytorch
def baseline_als_pytorch(y, lam, p, niter=10):
    diag = torch.tensor(np.repeat(1, L))
    diag = torch.diag(diag, 0)
    diag_minus_one = torch.tensor(np.repeat(-2, L - 1))
    diag_minus_one = torch.diag(diag_minus_one, -1)
    diag_minus_two = torch.tensor(np.repeat(1, L - 2))
    diag_minus_two = torch.diag(diag_minus_two, -2)
    D = diag + diag_minus_one + diag_minus_two
    D = D[:, :L - 2].double()
    w = torch.tensor(np.repeat(1, L)).double()
    for i in range(10):
        W = diag.double()
        Z = W + lam * torch.mm(D, D.permute(1, 0))
        z = torch.gesv(w * y, Z)
        z = z[0].squeeze()
        w = p * (y > z).double() + (1 - p) * (y < z).double()
    return z

Sorry that the pytorch code looks so bad I'm just starting out in it.
I've confirmed that Z, w, and y are all the same going into both scipy and pytorch and that z is different between them right after I try to solve the system of equations.
Thanks for the comment, here is an example:
I use 100000 for lam and 0.001 for p.
Using the dummy input: y =  (5,5,5,5,5,10,10,5,5,5,10,10,10,5,5,5,5,5,5,5),
I get (3.68010263,  4.90344214,  6.12679489,  7.35022406,  8.57384278, 9.79774074, 11.02197199, 12.2465927 , 13.47164891, 14.69711435,15.92287813, 17.14873257, 18.37456982, 19.60038184, 20.82626043,22.05215157, 23.27805103, 24.50400438, 25.73010693, 26.95625922) from scipy and
(6.4938312 , 6.46912395, 6.44440175, 6.41963499, 6.39477958,6.36977727, 6.34455582, 6.31907933, 6.29334844, 6.26735058, 6.24106029, 6.21443939, 6.18748732, 6.16024137, 6.13277694,6.10515785, 6.07743658, 6.04965455, 6.02184242, 5.99402035) from pytorch.
This is with just one iteration of the loop. Scipy is correct, pytorch is not.
Interestingly, if I use a shorter dummy input (5,5,5,5,5,10,10,5,5,5), I get the same answer from both. My real input is 1011 dimensional.


title: How to make sure PyTorch has deallocated GPU memory?

body:
Say we have a function like this:
        def trn_l(totall_lc, totall_lw, totall_li, totall_lr):
            self.model_large.cuda()
            self.model_large.train()
            self.optimizer_large.zero_grad()

            for fb in range(self.fake_batch):
                val_x, val_y = next(self.valid_loader)
                val_x, val_y = val_x.cuda(), val_y.cuda()

                logits_main, emsemble_logits_main = self.model_large(val_x)
                cel = self.criterion(logits_main, val_y)
                loss_weight = cel / (self.fake_batch)
                loss_weight.backward(retain_graph=False)
                cel = cel.cpu().detach()
                emsemble_logits_main = emsemble_logits_main.cpu().detach()

                totall_lw += float(loss_weight.item())
                val_x = val_x.cpu().detach()
                val_y = val_y.cpu().detach()

            loss_weight = loss_weight.cpu().detach()
            self._clip_grad_norm(self.model_large)
            self.optimizer_large.step()
            self.model_large.train(mode=False)
            self.model_large = self.model_large.cpu()
            return totall_lc, totall_lw, totall_li, totall_lr

On the first call, it allocates 8GB of GPU memory. On the next call, no new memory gets allocated, yet 8GBs are still occupied. I want to have after it is called and the produced first result to have 0 allocated GPU memory or as low as possible.
What I have tried: do  retain_graph=False and .cpu().detach() everywhere - no positive effects.
Memory snapshot before
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |   33100 KB |   33219 KB |   40555 KB |    7455 KB |
|       from large pool |    3072 KB |    3072 KB |    3072 KB |       0 KB |
|       from small pool |   30028 KB |   30147 KB |   37483 KB |    7455 KB |
|---------------------------------------------------------------------------|
| Active memory         |   33100 KB |   33219 KB |   40555 KB |    7455 KB |
|       from large pool |    3072 KB |    3072 KB |    3072 KB |       0 KB |
|       from small pool |   30028 KB |   30147 KB |   37483 KB |    7455 KB |
|---------------------------------------------------------------------------|
| GPU reserved memory   |   51200 KB |   51200 KB |   51200 KB |       0 B  |
|       from large pool |   20480 KB |   20480 KB |   20480 KB |       0 B  |
|       from small pool |   30720 KB |   30720 KB |   30720 KB |       0 B  |
|---------------------------------------------------------------------------|
| Non-releasable memory |   18100 KB |   20926 KB |   56892 KB |   38792 KB |
|       from large pool |   17408 KB |   18944 KB |   18944 KB |    1536 KB |
|       from small pool |     692 KB |    2047 KB |   37948 KB |   37256 KB |
|---------------------------------------------------------------------------|
| Allocations           |   12281    |   12414    |   12912    |     631    |
|       from large pool |       2    |       2    |       2    |       0    |
|       from small pool |   12279    |   12412    |   12910    |     631    |
|---------------------------------------------------------------------------|
| Active allocs         |   12281    |   12414    |   12912    |     631    |
|       from large pool |       2    |       2    |       2    |       0    |
|       from small pool |   12279    |   12412    |   12910    |     631    |
|---------------------------------------------------------------------------|
| GPU reserved segments |      16    |      16    |      16    |       0    |
|       from large pool |       1    |       1    |       1    |       0    |
|       from small pool |      15    |      15    |      15    |       0    |
|---------------------------------------------------------------------------|
| Non-releasable allocs |       3    |      30    |     262    |     259    |
|       from large pool |       1    |       1    |       1    |       0    |
|       from small pool |       2    |      29    |     261    |     259    |
|===========================================================================|

And after calliing function and
torch.cuda.empty_cache()
torch.cuda.synchronize()

We get:
|===========================================================================|
|                  PyTorch CUDA memory summary, device ID 0                 |
|---------------------------------------------------------------------------|
|            CUDA OOMs: 0            |        cudaMalloc retries: 0         |
|===========================================================================|
|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |
|---------------------------------------------------------------------------|
| Allocated memory      |   10957 KB |    8626 MB |  272815 MB |  272804 MB |
|       from large pool |       0 KB |    8596 MB |  272477 MB |  272477 MB |
|       from small pool |   10957 KB |      33 MB |     337 MB |     327 MB |
|---------------------------------------------------------------------------|
| Active memory         |   10957 KB |    8626 MB |  272815 MB |  272804 MB |
|       from large pool |       0 KB |    8596 MB |  272477 MB |  272477 MB |
|       from small pool |   10957 KB |      33 MB |     337 MB |     327 MB |
|---------------------------------------------------------------------------|
| GPU reserved memory   |    8818 MB |    9906 MB |   19618 MB |   10800 MB |
|       from large pool |    8784 MB |    9874 MB |   19584 MB |   10800 MB |
|       from small pool |      34 MB |      34 MB |      34 MB |       0 MB |
|---------------------------------------------------------------------------|
| Non-releasable memory |    5427 KB |    3850 MB |  207855 MB |  207850 MB |
|       from large pool |       0 KB |    3850 MB |  207494 MB |  207494 MB |
|       from small pool |    5427 KB |       5 MB |     360 MB |     355 MB |
|---------------------------------------------------------------------------|
| Allocations           |    3853    |   13391    |   34339    |   30486    |
|       from large pool |       0    |     557    |   12392    |   12392    |
|       from small pool |    3853    |   12838    |   21947    |   18094    |
|---------------------------------------------------------------------------|
| Active allocs         |    3853    |   13391    |   34339    |   30486    |
|       from large pool |       0    |     557    |   12392    |   12392    |
|       from small pool |    3853    |   12838    |   21947    |   18094    |
|---------------------------------------------------------------------------|
| GPU reserved segments |     226    |     226    |     410    |     184    |
|       from large pool |     209    |     209    |     393    |     184    |
|       from small pool |      17    |      17    |      17    |       0    |
|---------------------------------------------------------------------------|
| Non-releasable allocs |      46    |     358    |   12284    |   12238    |
|       from large pool |       0    |     212    |    7845    |    7845    |
|       from small pool |      46    |     279    |    4439    |    4393    |
|===========================================================================|


title: saving and loading RNN hidden states in PyTorch

body:
I am trying to use an RNN network in PyTorch for regression task. In the training phase the model is learned. I want to use the trained model in testing phase. For this purpose I have saved the learned model by:

torch.save(learned_model, "model_path")

Then I can load the model again by:

loaded_model = torch.load("model_path")

For testing phase I must use this loaded model but I want to know what is the value of the first hidden state of the model? I can initialize the first hidden state by zero but I think maybe this is not correct. Is there any function other than torch.save which can return the last hidden state in the learned mode? Then I can restore that hidden state and use it as the first hidden state in the loaded model for testing phase.

Thanks in advance.


title: Understanding nn.Sequential in convolutional layers

body:
I am new to PyTorch/Deep learning and I am trying to understand the use of the following line to define a convolutional layer:

self.layer1 = nn.Sequential(nn.Conv1d(input_dim, n_conv_filters, kernel_size=7, padding=0), nn.ReLU(), nn.MaxPool1d(3))

I understand that that it is creating a 1d convolutional layer to the network with max pooling 3 wide. However, I don't understand the function of the sequential module or RelU. How do these function in creating a layer?

For reference, the rest of the code can be found here: https://github.com/ArdalanM/nlp-benchmarks/blob/master/src/cnn/net.py


title: minimum the cosine similarity of two tensors and output one scalar. Pytorch

body:
I use Pytorch cosine similarity function as follows. I have two feature vectors and my goal is to make them dissimilar to each other. So, I thought I could minimum their cosine similarity. I have some doubts about the way I have coded. I appreciate your suggestions about the following questions.

I don't know why here are some negative values in val1?

I have done three steps to convert val1 to a scalar. Am I doing it in the right way? Is there any other way?

To minimum the similarity, I have used 1/var1. Is it a standard way to do this? Is it correct if I use 1-var1?
def loss_func(feat1, feat2):
     cosine_loss = torch.nn.CosineSimilarity(dim=1, eps=1e-6)
     val1 = cosine_loss(feat1, feat2).tolist()
     # 1. calculate the absolute values of each element,
     # 2. sum all values together,
     # 3. divide it by the number of values
     val1 = 1/(sum(list(map(abs, val1)))/int(len(val1)))
     val1 = torch.tensor(val1, device='cuda', requires_grad=True)
     return val1


title: Installing pytorch fails - fatal error: libavutil/motion_vector.h: No such file or directory

body:
I'm trying to install pytorch

python setup.py install

but it fails as it can't locate libavutil/motion_vector.h

[ 66%] Building CXX object caffe2/CMakeFiles/torch.dir/video/video_input_op.cc.o
In file included from /root/pytorch/caffe2/video/video_input_op.h:15:0,
                 from /root/pytorch/caffe2/video/video_input_op.cc:1:
/root/pytorch/caffe2/video/video_decoder.h:15:37: fatal error: libavutil/motion_vector.h: No such file or directory
 #include <libavutil/motion_vector.h>
                                     ^
compilation terminated.
[ 66%] Building CXX object caffe2/CMakeFiles/torch.dir/video/video_io.cc.o
make[2]: *** [caffe2/CMakeFiles/torch.dir/video/video_input_op.cc.o] Error 1
make[2]: *** Waiting for unfinished jobs....
In file included from /root/pytorch/caffe2/video/video_io.h:6:0,
                 from /root/pytorch/caffe2/video/video_io.cc:1:
/root/pytorch/caffe2/video/video_decoder.h:15:37: fatal error: libavutil/motion_vector.h: No such file or directory
 #include <libavutil/motion_vector.h>
                                     ^
compilation terminated.
make[2]: *** [caffe2/CMakeFiles/torch.dir/video/video_io.cc.o] Error 1
make[1]: *** [caffe2/CMakeFiles/torch.dir/all] Error 2
make: *** [all] Error 2
Traceback (most recent call last):
  File "setup.py", line 759, in <module>
    build_deps()
  File "setup.py", line 321, in build_deps
    cmake=cmake)
  File "/root/pytorch/tools/build_pytorch_libs.py", line 63, in build_caffe2
    cmake.build(my_env)
  File "/root/pytorch/tools/setup_helpers/cmake.py", line 330, in build
    self.run(build_args, my_env)
  File "/root/pytorch/tools/setup_helpers/cmake.py", line 143, in run
    check_call(command, cwd=self.build_dir, env=env)
  File "/usr/lib/python2.7/subprocess.py", line 540, in check_call
    raise CalledProcessError(retcode, cmd)
subprocess.CalledProcessError: Command '['cmake', '--build', '.', '--target', 'install', '--config', 'Release', '--', '-j', '12']' returned non-zero exit status 2

What I've tried so far:

installed libavutil-dev
ensured ffmpeg is installed
followed https://github.com/facebookresearch/VMZ/blob/master/tutorials/Installation_guide.md

Any other ideas?


title: Different results from PyTorch's conv1d and SciPy's convolve

body:
I'm building a PyTorch model to estimate Impuse Responses.
Currently I am calculating the loss from the real and estimated impulse response.
I would like to convolve both the estimated and real impulse response with a signal and then calculate the loss from those.

The pyroomaccoustics package uses SciPy's fftconvolve to convolve the impulse response with a given signal. I cannot use this since it would break PyTorch's computation graph. PyTorch's conv1d uses cross-correlation. From this answer it seems that by flipping the filter conv1d can be used for convolution.

I am confused as to why the following code gives a different result for conv1d and convolve and what must be changed to get the outputs to be equal.

import torch
from scipy.signal import convolve

a = torch.tensor([.1, .2, .3, .4, .5])
b = torch.tensor([.0, .1, .0])

a1 = a.view(1, 1, -1)
b1 = torch.flip(b, (0,)).view(1, 1, -1)

print(torch.nn.functional.conv1d(a1, b1).view(-1))
# >>> tensor([0.0200, 0.0300, 0.0400])

print(convolve(a, b))
# >>> [0.   0.01 0.02 0.03 0.04 0.05 0.  ]


title: torch.save(model.state_dict()) line comes error while model complated train and trying to save it how can i solve?

body:
I'm working on some code about nlp. I want to train and save model but here comes this error. I searched some documentation but i didn't find right solution. How can i solve this problem?

import torch,time
import torch.nn as nn

input_dim = 5
hidden_dim = 10
n_layers = 1

lstm_layer = nn.LSTM(input_dim, hidden_dim, n_layers, batch_first=True)

batch_size = 1
seq_len = 1

inp = torch.randn(batch_size, seq_len, input_dim)
hidden_state = torch.randn(n_layers, batch_size, hidden_dim)
cell_state = torch.randn(n_layers, batch_size, hidden_dim)
hidden = (hidden_state, cell_state)

out, hidden = lstm_layer(inp, hidden)
print("Output shape: ", out.shape)
print("Hidden: ", hidden)

seq_len = 3
inp = torch.randn(batch_size, seq_len, input_dim)
out, hidden = lstm_layer(inp, hidden)
print(out.shape)

# Obtaining the last output
out = out.squeeze()[-1, :]
print(out.shape)

import bz2
from collections import Counter
import re
import nltk
import numpy as np
#nltk.download('punkt')

train_file = bz2.BZ2File('C:/Users/DELL/Dogal-Dil-Isleme/Xml-Files/trwiktionary-20200301-pages-articles-multistream.xml.bz2')
test_file = bz2.BZ2File('C:/Users/DELL/Dogal-Dil-Isleme/Xml-Files/trwikisource-20200601-pages-articles.xml.bz2')

train_file = train_file.readlines()
test_file = test_file.readlines()

num_train = 200
num_test = 50

train_file = [x.decode('utf-8') for x in train_file[:num_train]]
test_file = [x.decode('utf-8') for x in test_file[:num_test]]

train_labels = [0 if x.split(' ')[0] == '__label__1' else 1 for x in train_file]
train_sentences = [x.split(' ', 1)[1][:-1].lower() for x in train_file]

test_labels = [0 if x.split(' ')[0] == '__label__1' else 1 for x in test_file]
test_sentences = [x.split(' ', 1)[1][:-1].lower() for x in test_file]

for i in range(len(train_sentences)):
    train_sentences[i] = re.sub('\d','0',train_sentences[i])

for i in range(len(test_sentences)):
    test_sentences[i] = re.sub('\d','0',test_sentences[i])

for i in range(len(train_sentences)):
    if 'www.' in train_sentences[i] or 'http:' in train_sentences[i] or 'https:' in train_sentences[i] or '.com' in train_sentences[i]:
        train_sentences[i] = re.sub(r"([^ ]+(?<=\.[a-z]{3}))", "<url>", train_sentences[i])

for i in range(len(test_sentences)):
    if 'www.' in test_sentences[i] or 'http:' in test_sentences[i] or 'https:' in test_sentences[i] or '.com' in test_sentences[i]:
        test_sentences[i] = re.sub(r"([^ ]+(?<=\.[a-z]{3}))", "<url>", test_sentences[i])

words = Counter()  # Dictionary that will map a word to the number of times it appeared in all the training sentences
for i, sentence in enumerate(train_sentences):
    train_sentences[i] = []
    for word in nltk.word_tokenize(sentence):
        words.update([word.lower()])
        train_sentences[i].append(word)
    if i%20000 == 0:
        print(str((i*100)/num_train) + "% done")
print("100% done")

words = {k:v for k,v in words.items() if v>1}
words = sorted(words, key=words.get, reverse=True)
words = ['_PAD','_UNK'] + words
word2idx = {o:i for i,o in enumerate(words)}
idx2word = {i:o for i,o in enumerate(words)}

for i, sentence in enumerate(train_sentences):
    train_sentences[i] = [word2idx[word] if word in word2idx else 0 for word in sentence]

for i, sentence in enumerate(test_sentences):
    # For test sentences, we have to tokenize the sentences as well
    test_sentences[i] = [word2idx[word.lower()] if word.lower() in word2idx else 0 for word in nltk.word_tokenize(sentence)]

def pad_input(sentences, seq_len):
    features = np.zeros((len(sentences), seq_len),dtype=int)
    for ii, review in enumerate(sentences):
        if len(review) != 0:
            features[ii, -len(review):] = np.array(review)[:seq_len]
    return features

seq_len = 200  # The length that the sentences will be padded/shortened to

train_sentences = pad_input(train_sentences, seq_len)
test_sentences = pad_input(test_sentences, seq_len)

train_labels = np.array(train_labels)
test_labels = np.array(test_labels)

split_frac = 0.5 # 50% validation, 50% test
split_id = int(split_frac * len(test_sentences))
val_sentences, test_sentences = test_sentences[:split_id], test_sentences[split_id:]
val_labels, test_labels = test_labels[:split_id], test_labels[split_id:]

import torch
from torch.utils.data import TensorDataset, DataLoader
import torch.nn as nn

train_data = TensorDataset(torch.from_numpy(train_sentences), torch.from_numpy(train_labels))
val_data = TensorDataset(torch.from_numpy(val_sentences), torch.from_numpy(val_labels))
test_data = TensorDataset(torch.from_numpy(test_sentences), torch.from_numpy(test_labels))

batch_size = 200

train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
val_loader = DataLoader(val_data, shuffle=True, batch_size=batch_size)
test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)

# torch.cuda.is_available() checks and returns a Boolean True if a GPU is available, else it'll return False
is_cuda = torch.cuda.is_available()

# If we have a GPU available, we'll set our device to GPU. We'll use this device variable later in our code.
if is_cuda:
    device = torch.device("cuda")
else:
    device = torch.device("cpu")

class SentimentNet(nn.Module):
    def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):
        super(SentimentNet, self).__init__()
        self.output_size = output_size
        self.n_layers = n_layers
        self.hidden_dim = hidden_dim

        self.embedding = nn.Embedding(vocab_size, embedding_dim)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, dropout=drop_prob, batch_first=True)
        self.dropout = nn.Dropout(drop_prob)
        self.fc = nn.Linear(hidden_dim, output_size)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x, hidden):
        batch_size = x.size(0)
        x = x.long()
        embeds = self.embedding(x)
        lstm_out, hidden = self.lstm(embeds, hidden)
        lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)

        out = self.dropout(lstm_out)
        out = self.fc(out)
        out = self.sigmoid(out)

        out = out.view(batch_size, -1)
        out = out[:,-1]
        return out, hidden

    def init_hidden(self, batch_size):
        weight = next(self.parameters()).data
        hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().to(device),
                      weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().to(device))
        return hidden

vocab_size = len(word2idx) + 1
output_size = 1
embedding_dim = 400
hidden_dim = 512
n_layers = 2

model = SentimentNet(vocab_size, output_size, embedding_dim, hidden_dim, n_layers)
model.to(device)

lr=0.005
criterion = nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=lr)

epochs = 2
counter = 0
print_every = 1000
clip = 5
valid_loss_min = np.Inf

model.train()
for i in range(epochs):
    h = model.init_hidden(batch_size)

    for inputs, labels in train_loader:
        counter += 1
        h = tuple([e.data for e in h])
        inputs, labels = inputs.to(device), labels.to(device)
        model.zero_grad()
        output, h = model(inputs, h)
        loss = criterion(output.squeeze(), labels.float())
        loss.backward()
        nn.utils.clip_grad_norm_(model.parameters(), clip)
        optimizer.step()

        if counter%print_every == 0:
            val_h = model.init_hidden(batch_size)
            val_losses = []
            model.eval()
            for inp, lab in val_loader:
                val_h = tuple([each.data for each in val_h])
                inp, lab = inp.to(device), lab.to(device)
                out, val_h = model(inp, val_h)
                val_loss = criterion(out.squeeze(), lab.float())
                val_losses.append(val_loss.item())

            model.train()
            print("Epoch: {}/{}...".format(i+1, epochs),
                  "Step: {}...".format(counter),
                  "Loss: {:.6f}...".format(loss.item()),
                  "Val Loss: {:.6f}".format(np.mean(val_losses)))
            if np.mean(val_losses) <= valid_loss_min:
                torch.save(model.state_dict(), 'C:/Users/DELL/Dogal-Dil-Isleme/Models/state_dict.pt')
                print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,np.mean(val_losses)))
                valid_loss_min = np.mean(val_losses)

time.sleep(1)
# Loading the best model
model.load_state_dict(torch.load('C:/Users/DELL/Dogal-Dil-Isleme/Models/state_dict.pt'))

test_losses = []
num_correct = 0
h = model.init_hidden(batch_size)

model.eval()
for inputs, labels in test_loader:
    h = tuple([each.data for each in h])
    inputs, labels = inputs.to(device), labels.to(device)
    output, h = model(inputs, h)
    test_loss = criterion(output.squeeze(), labels.float())
    test_losses.append(test_loss.item())
    pred = torch.round(output.squeeze())  # Rounds the output to 0/1
    correct_tensor = pred.eq(labels.float().view_as(pred))
    correct = np.squeeze(correct_tensor.cpu().numpy())
    num_correct += np.sum(correct)

print("Test loss: {:.3f}".format(np.mean(test_losses)))
test_acc = num_correct/len(test_loader.dataset)
print("Test accuracy: {:.3f}%".format(test_acc*100))

i tried a create new folder and change path but all the ways comes error :)
i read pytorch documentation and change recommended code but error still came.
i will share some link for your reading about this issue.

same issue
pytorch documentation

how can i fix or is there any alternative way to save model?


title: How to get the file name of image that I put into Dataloader in Pytorch

body:
I use pytorch to load images like this:
inf_data = InfDataloader(img_folder=args.imgs_folder, target_size=args.img_size)
inf_dataloader = DataLoader(inf_data, batch_size=1, shuffle=True, num_workers=2)

And then:
    with torch.no_grad():
        for batch_idx, (img_np, img_tor) in enumerate(inf_dataloader, start=1):

            img_tor = img_tor.to(device)
            pred_masks, _ = model(img_tor)

But I want to get the file name of the image. Can anyone help me with this?
Thanks a lot!


title: 'BertEmbeddings' object has no attribute 'bias' while converting tf checkpoint

body:
When trying to convert the checkpoint of a self pre-trained tensorflow BERT model (using the create-pretraining.py script from google) into a pytorch model using the convert_bert_original_tf_checkpoint_to_pytorch.py from Huggingface.
I always end up with the following error:
AttributeError: 'BertEmbeddings' object has no attribute 'bias'
The init_vars names (just the first ones) look like this:
['bert/embeddings/layer_normalization/beta', 'bert/embeddings/layer_normalization/beta/adam_m', 'bert/embeddings/layer_normalization/beta/adam_v', 'bert/embeddings/layer_normalization/gamma', 'bert/embeddings/layer_normalization/gamma/adam_m', 'bert/embeddings/layer_normalization/gamma/adam_v']

Code that produces the error looks like this:
for m_name in name:
            if re.fullmatch(r"[A-Za-z]+_\d+", m_name):
                scope_names = re.split(r"_(\d+)", m_name)
            else:
                scope_names = [m_name]
            if scope_names[0] == "kernel" or scope_names[0] == "gamma":
                pointer = getattr(pointer, "weight")
            elif scope_names[0] == "output_bias" or scope_names[0] == "beta":
                print(scope_names)
                pointer = getattr(pointer, "bias")
            elif scope_names[0] == "output_weights":
                pointer = getattr(pointer, "weight")
            elif scope_names[0] == "squad":
                pointer = getattr(pointer, "classifier")
            else:
                try:
                    pointer = getattr(pointer, scope_names[0])
                except AttributeError:
                    logger.info("Skipping {}".format("/".join(name)))

Going through all the names and getting the right attributes from the model. When it comes to the Layer Normalization in the BertEmbeddings the script produces an error. Did anyone else encouter that error before? How did you fix this?
Here again the whole stacktrace:
Traceback (most recent call last):
  File "convert_bert_original_tf_checkpoint_to_pytorch.py", line 62, in <module>
    convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, args.bert_config_file, args.pytorch_dump_path)
  File "convert_bert_original_tf_checkpoint_to_pytorch.py", line 37, in convert_tf_checkpoint_to_pytorch
    load_tf_weights_in_bert(model, config, tf_checkpoint_path)
  File "/modeling_bert.py", line 136, in load_tf_weights_in_bert
    pointer = getattr(pointer, "bias")
  File "module.py", line 594, in __getattr__
    type(self).__name__, name))
AttributeError: 'BertEmbeddings' object has no attribute 'bias'

Bert Config is the following:
Building PyTorch model from configuration: BertConfig {
  "attention_probs_dropout_prob": 0.1,
  "gradient_checkpointing": false,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 512,
  "initializer_range": 0.02,
  "intermediate_size": 2048,
  "layer_norm_eps": 1e-12,
  "max_position_embeddings": 512,
  "model_type": "bert",
  "num_attention_heads": 8,
  "num_hidden_layers": 8,
  "pad_token_id": 0,
  "type_vocab_size": 2,
  "vocab_size": 30522
}


title: How to improve cats and dogs classification using CNN with pytorch

body:
I tried to follow the architecture of CNN in this paper, ImageNet Classification with Deep Convolutional Neural Networks (https://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks.pdf). In this paper, they tried to classify 1000 classes, whereas I am just trying to classify 2 classes.

But, my test accuracy got stuck at 50%, and the model is not learning.

I am training with 23K images of cats and dogs, and test with 2500 images.

This is URL to my notebook https://github.com/jinglescode/workspace/blob/master/my-journey-computer-vision/codes/Cats_and_Dogs.ipynb

Could anyone advise what's wrong? What have I missed out? Willing to learn.


title: Installing a precompiled library that depends on another in a bitbake recipe

body:
I am trying to include pytorch/libtorch in our Yocto-based distribution and running into troubles when it comes to installing precompiled libraries with dependencies among them onto the target.

After trying to compile pytorch from source to no avail I thought I could use the pre-compiled shared libraries and move onto the next problem. This however proves so far more difficult than initially anticipated.

Specifically, one of the shared libraries I'd like to install, libtorch.so depends on another shared library distributed along it libgomp-753e6e92.so.1. When running ldd it looks something like this:

$ ldd libtorch.so
    linux-vdso.so.1 (0x00007ffc27bcb000)
    libgomp-753e6e92.so.1 => /home/user/Desktop/libtorch-1.2.0/libtorch/lib/./libgomp-753e6e92.so.1 (0x00007f3052954000)
    libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f3052735000)
    librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f305252d000)
    libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007f3052315000)
    libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f3052111000)
    libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f3051d73000)
    libc10.so => /home/user/Desktop/libtorch-1.2.0/libtorch/lib/./libc10.so (0x00007f3051b2c000)
    libstdc++.so.6 => /usr/lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007f30517a3000)
    libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f30513b2000)
    /lib64/ld-linux-x86-64.so.2 (0x00007f305ef4a000)

The bitbake recipe that tries to put everything into place looks like this:

SUMMARY = "Facebook PyTorch AI"
DESCRIPTION = "Facebook PyTorch AI"
HOMEPAGE = "https://pytorch.org/"
LICENSE = "MIT"
LIC_FILES_CHKSUM = "file://LICENSE;md5=acf4d595f99e159bf31797aa872aef57"

S = "${WORKDIR}/libtorch"
LOCAL_INCLUDE = "${S}/include"
LOCAL_LIB = "${S}/lib"
TARGET_INCLUDE = "${D}${includedir}"
TARGET_LIB = "${D}${libdir}"

SRC_URI = "\
    https://download.pytorch.org/libtorch/cpu/libtorch-cxx11-abi-shared-with-deps-${PV}.zip \
    file://LICENSE \
"
SRC_URI[md5sum] = "f34c5b6e46331f79100cd93522bad3ae"

INSANE_SKIP_${PN} = "ldflags"
INSANE_SKIP_${PN} += "already-stripped"
INHIBIT_PACKAGE_STRIP = "1"
INHIBIT_SYSROOT_STRIP = "1"
SOLIBS = ".so"
FILES_SOLIBSDEV = ""

# RDEPENDS_${PN} += "libgomp"

do_install() {
    # Create include directory
    install -d ${TARGET_INCLUDE}
    # Copy all header files
    cp -R --no-dereference --preserve=mode,links -v ${LOCAL_INCLUDE}/* ${TARGET_INCLUDE}

    # Create library directory
    install -d ${TARGET_LIB}
    # Install shared libraries
    install -m 0755 ${LOCAL_LIB}/*.so* ${TARGET_LIB}
}

FILES_${PN} += "\
    ${includedir}/* \
    ${libdir}/* \
"

Unfortunately, I keep getting an error telling me that nothing ir rproviding the libgomp library which is required by libtorch.
I have tried multiple things including putting the library in question in its own recipe, explicitly installing it or using the already provided libgomp runtime dependency but without any luck so far. The error is always something in the likes of:

ERROR: pytorch-1.2.0-r0 do_package_qa: QA Issue: /usr/lib/libtorch.so contained in package pytorch requires libgomp-753e6e92.so.1(GOMP_4.0)(64bit), but no providers found in RDEPENDS_pytorch? [file-rdeps]
ERROR: pytorch-1.2.0-r0 do_package_qa: QA Issue: /usr/lib/libtorch.so contained in package pytorch requires libgomp-753e6e92.so.1(OMP_1.0)(64bit), but no providers found in RDEPENDS_pytorch? [file-rdeps]
ERROR: pytorch-1.2.0-r0 do_package_qa: QA Issue: /usr/lib/libtorch.so contained in package pytorch requires libgomp-753e6e92.so.1()(64bit), but no providers found in RDEPENDS_pytorch? [file-rdeps]
ERROR: pytorch-1.2.0-r0 do_package_qa: QA Issue: /usr/lib/libtorch.so contained in package pytorch requires libgomp-753e6e92.so.1(GOMP_1.0)(64bit), but no providers found in RDEPENDS_pytorch? [file-rdeps]
ERROR: pytorch-1.2.0-r0 do_package_qa: QA run found fatal errors. Please consider fixing them.
ERROR: pytorch-1.2.0-r0 do_package_qa: Function failed: do_package_qa

Any clues on what I am missing?


title: How can I convert this keras cnn model to pytorch version

body:
This is the example keras code that I want to convert to pytorch. My input dataset is 10000*1*102 (two dimensions for labels). The dataset includes 10000 samples. Each sample contains one row with 102 features. I am thinking to use 1dcnn for regression.

PS: hyper-parameter (e.g. filters, kernel_size, stride, padding) could be adjusted based on my 10000*1*102 dataset.

model = Sequential()
model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(n_timesteps,n_features)))
model.add(Conv1D(filters=64, kernel_size=3, activation='relu'))
model.add(Dropout(0.5))
model.add(MaxPooling1D(pool_size=2))
model.add(Flatten())
model.add(Dense(100, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))


title: Pytorch: RAM explodes when using multiprocessing SharedMemory and CUDA

body:
I would like to use multiprocessing to launch multiple training instances on CUDA device. Since the data is common between the processes, I want to avoid data copy for every process. I'm using python 3.8's SharedMemory from multiprocessing module to achieve this following this SO example.
I can allocate a memory block using SharedMemory and create as many processes as I'd like with constant memory (RAM) usage. However, when I try to send tensors to CUDA, the memory scales linearly with the number of processes. It appears as if when c.to(device) is called, the base data is copied for every process.
Does any one know why this is happening? Any ideas to mitigate this issue?
Here is the sample code I'm using:
import numpy as np
from multiprocessing import shared_memory, get_context
import time
import torch
import copy

dim = 10000
batch_size = 10
sleep_time = 2
npe = 1  # number of parallel executions

# cuda
if torch.cuda.is_available():
    dev = 'cuda:0'
else:
    dev = "cpu"
device = torch.device(dev)

def step(i, shr_name):
    existing_shm = shared_memory.SharedMemory(name=shr_name)
    np_arr = np.ndarray((dim, dim), dtype=np.float32, buffer=existing_shm.buf)
    b = np_arr[i * batch_size: (i + 1) * batch_size, :]
    b = torch.Tensor(b)

    # This is just to explicitly copy the tensor so that it has nothing to do
    # with the shared memory block
    c = copy.deepcopy(b)

    # If tensor c is sent to the cuda device, then RAM scales linearly
    # with the number of parallel executions.
    # If c is not sent to cuda device, memory consumption is constant.
    c = c.to(device)

    time.sleep(sleep_time)
    existing_shm.close()

def create_shared_block():
    a = np.random.random((dim, dim)).astype(np.float32)
    shm = shared_memory.SharedMemory(create=True, size=a.nbytes, name='sha')
    np_arr = np.ndarray(a.shape, dtype=np.float32, buffer=shm.buf)
    np_arr[:] = a[:]
    return shm, np_arr

if __name__ == '__main__':
    # create shared memory block
    shm, np_arr = create_shared_block()
    # create list of inputs to be executed in parallel
    inp = [[x, 'sha'] for x in range(npe)]
    print(inp)

    # sleep added before and after launching multiprocessing to monitor the memory consumption
    print('before pool')  # to check memory with top or htop
    time.sleep(sleep_time)

    context = get_context('spawn')
    with context.Pool(npe) as pool:
        print('after pool')  # to check memory with top or htop
        time.sleep(sleep_time)

        pool.starmap(step, inp)

    time.sleep(sleep_time)

    shm.close()
    shm.unlink()


title: Pytorch - AttributeError: 'tuple' object has no attribute 'dim'

body:
I am trying to use this architecture:
class Net(BaseFeaturesExtractor):

    def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):
        super(Net, self).__init__(observation_space, features_dim)

        self.conv1 = nn.Conv2d(1, 64, 3, stride=1, padding=1)
        self.conv2 = nn.Conv2d(64, 64, 3, stride=1, padding=1)
        self.conv3 = nn.Conv2d(64, 64, 3, stride=1)
        self.conv4 = nn.Conv2d(64, 64, 3, stride=1)

        self.bn1 = nn.BatchNorm2d(64)
        self.bn2 = nn.BatchNorm2d(64)
        self.bn3 = nn.BatchNorm2d(64)
        self.bn4 = nn.BatchNorm2d(64)

        self.fc1 = nn.Linear(64 * (7 - 4) * (6 - 4), 128)
        self.fc_bn1 = nn.BatchNorm1d(128)

        self.fc2 = nn.Linear(128, 64)
        self.fc_bn2 = nn.BatchNorm1d(64)

        self.fc3 = nn.Linear(64, 7)

        self.fc4 = nn.Linear(64, 1)

    def forward(self, s):
        #                                                            s: batch_size x board_x x board_y
        s = s.view(-1, 1, 7, 6)  # batch_size x 1 x board_x x board_y
        s = F.relu(self.bn1(self.conv1(s)))  # batch_size x num_channels x board_x x board_y
        s = F.relu(self.bn2(self.conv2(s)))  # batch_size x num_channels x board_x x board_y
        s = F.relu(self.bn3(self.conv3(s)))  # batch_size x num_channels x (board_x-2) x (board_y-2)
        s = F.relu(self.bn4(self.conv4(s)))  # batch_size x num_channels x (board_x-4) x (board_y-4)
        s = s.view(-1,64 * (7 - 4) * (6 - 4))

        s = F.dropout(
            F.relu(self.fc_bn1(self.fc1(s))),
            p=0.3,
            training=self.training)  # batch_size x 128
        s = F.dropout(
            F.relu(self.fc_bn2(self.fc2(s))),
            p=0.3,
            training=self.training)  # batch_size x 64

        pi = self.fc3(s)  # batch_size x action_size
        v = self.fc4(s)  # batch_size x 1

        return F.log_softmax(pi, dim=1), th.tanh(v)

When I am trying to use this architecture, I am getting following error:
Traceback (most recent call last):
  File "/Users/joe/Documents/JUPYTER/ConnectX/training3.py", line 130, in <module>
    learner.learn(total_timesteps=iterations, callback=eval_callback)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/stable_baselines3/ppo/ppo.py", line 264, in learn
    reset_num_timesteps=reset_num_timesteps,
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/stable_baselines3/common/on_policy_algorithm.py", line 222, in learn
    continue_training = self.collect_rollouts(self.env, callback, self.rollout_buffer, n_rollout_steps=self.n_steps)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/stable_baselines3/common/on_policy_algorithm.py", line 154, in collect_rollouts
    actions, values, log_probs = self.policy.forward(obs_tensor)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/stable_baselines3/common/policies.py", line 545, in forward
    latent_pi, latent_vf, latent_sde = self._get_latent(obs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/stable_baselines3/common/policies.py", line 564, in _get_latent
    latent_pi, latent_vf = self.mlp_extractor(features)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/stable_baselines3/common/torch_layers.py", line 220, in forward
    shared_latent = self.shared_net(features)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/modules/container.py", line 117, in forward
    input = module(input)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/modules/linear.py", line 93, in forward
    return F.linear(input, self.weight, self.bias)
  File "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/torch/nn/functional.py", line 1688, in linear
    if input.dim() == 2 and bias is not None:
AttributeError: 'tuple' object has no attribute 'dim'

How this problem can be fixed?


title: Out of memory error exception while running a Deep Learning python code

body:
I am trying to run the Deep Image Prior's Super Resolution. While running it, I am getting this error. Can anyone tell me where I am going wrong ? I am using CUDA-9.2, GPU 12GB which are sufficient to run this code. Someone told me to reduce the batch size and I don't know how to do it. I am new to this.

from __future__ import print_function
import matplotlib.pyplot as plt
%matplotlib inline

import argparse
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'

import numpy as np
from models import *

import torch
import torch.optim

from skimage.measure import compare_psnr
from models.downsampler import Downsampler

from utils.sr_utils import *

torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark =True
dtype = torch.cuda.FloatTensor

imsize = -1
factor = 4 # 8
enforse_div32 = 'CROP' # we usually need the dimensions to be divisible by a power of two (32 in this case)
PLOT = True

# To produce images from the paper we took *_GT.png images from LapSRN viewer for corresponding factor,
# e.g. x4/zebra_GT.png for factor=4, and x8/zebra_GT.png for factor=8
path_to_image = '/home/smitha/Documents/Falcon.png'
imgs = load_LR_HR_imgs_sr(path_to_image , imsize, factor, enforse_div32)

imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np'] = get_baselines(imgs['LR_pil'], imgs['HR_pil'])

if PLOT:
    plot_image_grid([imgs['HR_np'], imgs['bicubic_np'], imgs['sharp_np'], imgs['nearest_np']], 4,12);
    print ('PSNR bicubic: %.4f   PSNR nearest: %.4f' %  (
                                        compare_psnr(imgs['HR_np'], imgs['bicubic_np']),
                                        compare_psnr(imgs['HR_np'], imgs['nearest_np'])))
input_depth = 32

INPUT =     'noise'
pad   =     'reflection'
OPT_OVER =  'net'
KERNEL_TYPE='lanczos2'

LR = 0.01
tv_weight = 0.0

OPTIMIZER = 'adam'

if factor == 4:
    num_iter = 2000
    reg_noise_std = 0.03
elif factor == 8:
    num_iter = 4000
    reg_noise_std = 0.05
else:
    assert False, 'We did not experiment with other factors'
net_input = get_noise(input_depth, INPUT, (imgs['HR_pil'].size[1], imgs['HR_pil'].size[0])).type(dtype).detach()

NET_TYPE = 'skip' # UNet, ResNet
net = get_net(input_depth, 'skip', pad,
              skip_n33d=128,
              skip_n33u=128,
              skip_n11=4,
              num_scales=5,
              upsample_mode='bilinear').type(dtype)

# Losses
mse = torch.nn.MSELoss().type(dtype)

img_LR_var = np_to_torch(imgs['LR_np']).type(dtype)

downsampler = Downsampler(n_planes=3, factor=factor, kernel_type=KERNEL_TYPE, phase=0.5, preserve_size=True).type(dtype)
def closure():
    global i, net_input

    if reg_noise_std > 0:
        net_input = net_input_saved + (noise.normal_() * reg_noise_std)

    out_HR = net(net_input)
    out_LR = downsampler(out_HR)

    total_loss = mse(out_LR, img_LR_var)

    if tv_weight > 0:
        total_loss += tv_weight * tv_loss(out_HR)

    total_loss.backward()

    # Log
    psnr_LR = compare_psnr(imgs['LR_np'], torch_to_np(out_LR))
    psnr_HR = compare_psnr(imgs['HR_np'], torch_to_np(out_HR))
    print ('Iteration %05d    PSNR_LR %.3f   PSNR_HR %.3f' % (i, psnr_LR, psnr_HR), '\r', end='')

    # History
    psnr_history.append([psnr_LR, psnr_HR])

    if PLOT and i % 100 == 0:
        out_HR_np = torch_to_np(out_HR)
        plot_image_grid([imgs['HR_np'], imgs['bicubic_np'], np.clip(out_HR_np, 0, 1)], factor=13, nrow=3)

    i += 1

    return total_loss
psnr_history = []
net_input_saved = net_input.detach().clone()
noise = net_input.detach().clone()

i = 0
p = get_params(OPT_OVER, net, net_input)
optimize(OPTIMIZER, p, closure, LR, num_iter)
out_HR_np = np.clip(torch_to_np(net(net_input)), 0, 1)
result_deep_prior = put_in_center(out_HR_np, imgs['orig_np'].shape[1:])

# For the paper we acually took `_bicubic.png` files from LapSRN viewer and used `result_deep_prior` as our result
plot_image_grid([imgs['HR_np'],
                 imgs['bicubic_np'],
                 out_HR_np], factor=4, nrow=1);

The Error is :

Starting optimization with ADAM

/home/smitha/anaconda3/lib/python3.6/site-packages/torch/nn/modules/upsampling.py:122: UserWarning: nn.Upsampling is deprecated. Use nn.functional.interpolate instead.
  warnings.warn("nn.Upsampling is deprecated. Use nn.functional.interpolate instead.")
/home/smitha/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py:1961: UserWarning: Default upsampling behavior when mode=bilinear is changed to align_corners=False since 0.4.0. Please specify align_corners=True if the old behavior is desired. See the documentation of nn.Upsample for details.
  "See the documentation of nn.Upsample for details.".format(mode))

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-6-0fe781a02812> in <module>()
      5 i = 0
      6 p = get_params(OPT_OVER, net, net_input)
----> 7 optimize(OPTIMIZER, p, closure, LR, num_iter)

~/Documents/deep-image-prior/utils/common_utils.py in optimize(optimizer_type, parameters, closure, LR, num_iter)
    227         for j in range(num_iter):
    228             optimizer.zero_grad()
--> 229             closure()
    230             optimizer.step()
    231     else:

<ipython-input-5-887ba7755977> in closure()
      5         net_input = net_input_saved + (noise.normal_() * reg_noise_std)
      6
----> 7     out_HR = net(net_input)
      8     out_LR = downsampler(out_HR)
      9

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     89     def forward(self, input):
     90         for module in self._modules.values():
---> 91             input = module(input)
     92         return input
     93

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     89     def forward(self, input):
     90         for module in self._modules.values():
---> 91             input = module(input)
     92         return input
     93

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    299     def forward(self, input):
    300         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 301                         self.padding, self.dilation, self.groups)
    302
    303

RuntimeError: CUDA error: out of memory

If I reduce the size of the input image I get this error.

RuntimeError                              Traceback (most recent call last)
<ipython-input-6-0fe781a02812> in <module>()
      5 i = 0
      6 p = get_params(OPT_OVER, net, net_input)
----> 7 optimize(OPTIMIZER, p, closure, LR, num_iter)

~/Documents/deep-image-prior/utils/common_utils.py in optimize(optimizer_type, parameters, closure, LR, num_iter)
    227         for j in range(num_iter):
    228             optimizer.zero_grad()
--> 229             closure()
    230             optimizer.step()
    231     else:

<ipython-input-5-887ba7755977> in closure()
      8     out_LR = downsampler(out_HR)
      9
---> 10     total_loss = mse(out_LR, img_LR_var)
     11
     12     if tv_weight > 0:

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    475             result = self._slow_forward(*input, **kwargs)
    476         else:
--> 477             result = self.forward(*input, **kwargs)
    478         for hook in self._forward_hooks.values():
    479             hook_result = hook(self, input, result)

~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    419
    420     def forward(self, input, target):
--> 421         return F.mse_loss(input, target, reduction=self.reduction)
    422
    423

~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py in mse_loss(input, target, size_average, reduce, reduction)
   1714     else:
   1715         reduction = _Reduction.get_enum(reduction)
-> 1716     return _pointwise_loss(lambda a, b: (a - b) ** 2, torch._C._nn.mse_loss, input, target, reduction)
   1717
   1718

~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py in _pointwise_loss(lambd, lambd_optimized, input, target, reduction)
   1672         return torch.mean(d) if reduction == 'elementwise_mean' else torch.sum(d)
   1673     else:
-> 1674         return lambd_optimized(input, target, reduction)
   1675
   1676

RuntimeError: input and target shapes do not match: input [1 x 3 x 64 x 64], target [1 x 1 x 64 x 64] at /opt/conda/conda-bld/pytorch_1532502421238/work/aten/src/THCUNN/generic/MSECriterion.cu:12


title: Character level CNN - 1D or 2D

body:
I want to implement a character-level CNN in Pytorch.

My input has 4 dimensions:

  (batch_size, seq_length, padded_character_length, embedding_dim)

I'm wondering whether I should merge two dimensions and use a Conv1D-layer or instead use a Conv2D-layer on the existing dimensions.

Given the dimensions of the input both would technically work fine, I also have seen implementations for both versions. So I'm wondering which method to prefer.

Does one of the two methods have particular advantages over the other?


title: I used pack_padded_sequence() and put in lstm layer, but I got start () + length () exceeds dimension size (). error

body:
In my model, there are a embedding layer, a conv1d layer, a lstm layer. And I'm using Pytorch.
My question is:
When I put pack = pack_padded_sequence(conv) in the lstm layer, I got RuntimeError: start (pack[0].size(0)) + length (1) exceeds dimension size (pack[0].size(0)). error. Why I got this error? And What does mean?? Please help me..

class CNNModel(nn.Module):
    def __init__(self, vocab_size: int, embed_size: int, out_size: int, filter_size: List[int], hidden_size: int,
                 layer_num: int, pad: int = 1):
        super().__init__()
        self.vocab_size = vocab_size
        self.embed_size = embed_size
        self.pad = pad
        self.out_size = out_size
        self.filter_size = filter_size
        self.hidden_size = hidden_size
        self.layer_num = layer_num

        self.embed = nn.Embedding(
            num_embeddings=self.vocab_size,
            embedding_dim=self.embed_size,
            padding_idx=self.pad,
        )

        self.cnn = nn.Conv1d(
            in_channels=self.embed_size,
            out_channels=self.out_size,
            kernel_size=self.filter_size[0],
            bias=True,
        )

        self.lstm = nn.LSTM(
            input_size=self.out_size,
            hidden_size=self.hidden_size,
            num_layers=self.layer_num,
            bidirectional=True,
            batch_first=True,
        )

    def forward(self, batch):
        embed = self.embed(batch.word[0])  # [B(64), L(465), F(256)]
        embed = embed.transpose(1, 2)  # [B(64), L(256), F(465)]

        conv = self.cnn(embed)  # [B(64), F(64), L(465)]
        conv = conv.transpose(1, 2)  # [B(64), L(465), F(64)]

        encoding = pack_padded_sequence(conv,
                                        [tensor.item() for tensor in batch.word[1]],
                                        batch_first=True)
        print(f'encoding => {encoding[0].size()}')  # [8093, 64]
                print(f'encoding => {encoding[1].size()}')  # [465]
        _, (h, _) = self.lstm(encoding)  # [bsz, sln, dim3]
        print(f'h => {h}')

And result like below.

 File "/Users/user_name/anaconda3/lib/python3.7/site-packages/torch/nn/modules/rnn.py", line 182, in forward
    self.num_layers, self.dropout, self.training, self.bidirectional)
RuntimeError: start (8093) + length (1) exceeds dimension size (8093).