title: How can I fix this expected CUDA got CPU error in PyTorch?

body:
I've been struggling to find what's wrong in my code. I'm trying to implement DCGAN paper and from the past 2 days, I'm going through these errors. Could anyone please help me fix this?
I'm training this on Google colab with GPU runtime but I'm getting this error. Yesterday, I implemented the first GAN paper by Ian Goodfellow and I did not got this error. I don't know what's happening any help would be appreciated. Also, please check whether the gen_input is correct or not.
I already asked this question and no one replied to the old post. one person replied to it but all he said was to change the lines up and down which also gives the same error. please please help me
Here is the code:
import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import torchvision.transforms as transforms
from torchvision.utils import save_image
import torch.optim as optim

lr = 0.00002 #learning rate
nc = 3 #color channels
nz = 100 #size of latent vector or size of generator input
ngf = 64 #size of feature maps in generator
ndf = 64 #size of feature maps in discriminator
height = 128 #height of the image
width = 128 #width of the image
num_epochs = 5 #the variable name tells everything
workers = 2 #number of workers to load the data in batches
batch_size = 64 #batch size
image_size = 128 #resizing parameter
root = './simpsons/' #path to the training directory
beta1 = 0.5

img_shape = (nc, height, width)

class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        self.convt1 = nn.ConvTranspose2d(in_channels = nz, out_channels = ngf*8, kernel_size = 4, stride = 1, padding = 0, bias = False)
        self.convt2 = nn.ConvTranspose2d(in_channels = ngf*8, out_channels = ngf*4, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.convt3 = nn.ConvTranspose2d(in_channels = ngf*4, out_channels = ngf*2, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.convt4 = nn.ConvTranspose2d(in_channels = ngf*2, out_channels = ngf, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.convt5 = nn.ConvTranspose2d(in_channels = ngf, out_channels = 3, kernel_size=4, stride = 2, padding = 1, bias = False)

    def forward(self, t):
        t = self.convt1(t)
        t = nn.BatchNorm2d(t)
        t = F.relu(t)

        t = self.convt2(t)
        t = nn.BatchNorm2d(t)
        t = F.relu(t)

        t = self.convt3(t)
        t = nn.BatchNorm2d(t)
        t = F.relu(t)

        t = self.convt4(t)
        t = nn.BatchNorm2d(t)
        t = F.relu(t)

        t = self.convt5(t)
        t = F.tanh(t)

        return t

class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()
        self.conv1 = nn.Conv2d(in_channels = 3, out_channels = ndf, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.conv2 = nn.Conv2d(in_channels = ndf, out_channels = ndf*2, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.conv3 = nn.Conv2d(in_channels = ndf*2, out_channels = ndf*4, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.conv4 = nn.Conv2d(in_channels = ndf*4, out_channels = ndf*8, kernel_size = 4, stride = 2, padding = 1, bias = False)
        self.conv5 = nn.Conv2d(in_channels = ndf*8, out_channels = 1, kernel_size = 4, stride = 1, padding = 0, bias = False)

    def forward(self, t):
        t = self.conv1(t)
        t = F.leaky_relu(t, 0.2)

        t = self.conv2(t)
        t = nn.BatchNorm2d(t)
        t = F.leaky_relu(t, 0.2)

        t = self.conv3(t)
        t = nn.BatchNorm2d(t)
        t = F.leaky_relu(t, 0.2)

        t = self.conv4(t)
        t = nn.BatchNorm2d(t)
        t = F.leaky_relu(t, 0.2)

        t = self.conv5(t)
        t = F.sigmoid(t)

        return t

def weights_init(m):
    classname = m.__class__.__name__ #returns the class name(eg: Conv2d or ConvTranspose2d)
    if classname.find('Conv') != -1:
        nn.init.normal_(m.weight.data, 0.0, 0.02) #0.0 is mean and 0.02 is standard deviation
    elif classname.find('BatchNorm') != -1:
        nn.init.normal_(m.weight.data, 1, 0.02) #1 is mean and 0.02 is standard deviation
        nn.init.constant_(m.bias.data, 0.0)

def load_data(image_size, root):
    transform = transforms.Compose([
        transforms.Resize(image_size),
        transforms.ToTensor(),
        transforms.Normalize((0.486, 0.486, 0.486), (0.486, 0.486, 0.486))
        ])

    train_set = torchvision.datasets.ImageFolder(root = root, transform = transform)

    return train_set

#getting the batches of data
train_set = load_data(image_size, root)
dataloader = torch.utils.data.DataLoader(train_set, batch_size = batch_size, shuffle = True, num_workers = workers)

generator = Generator()
discriminator = Discriminator()

generator.apply(weights_init)
discriminator.apply(weights_init)

print(generator)
print(discriminator)

criterion = nn.BCELoss()

noise = torch.randn(64, nz, 1, 1)

optimizer_G = optim.Adam(generator.parameters(), lr = lr, betas=(beta1, 0.999))
optimizer_D = optim.Adam(discriminator.parameters(), lr = lr, betas=(beta1, 0.999))

if torch.cuda.is_available():
    print("CUDA available")
    generator = generator.to('cuda')
    discriminator = discriminator.to('cuda')
    criterion = criterion.cuda('cuda')
    Tensor = torch.cuda.FloatTensor
    print("Networks moved on to cuda")

for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(dataloader):

        val = Tensor(images.size(0), 1).fill_(1.0)
        fake = Tensor(images.size(0),1).fill_(0.0)

        real_images = images

        optimizer_G.zero_grad()

        gen_input = Tensor(np.random.normal(0,1,(512,100,4,4)))
        gen = generator(gen_input)

        g_loss = loss_func(discriminator(gen), val)

        g_loss.backward()
        optimizer_G.step()

        optimizer_D.zero_grad()

        real_loss = loss_func(discriminator(real_images), val)
        fake_loss = loss_func(discriminator(gen.detach()),fake)
        d_loss = (real_loss + fake_loss)/2

        d_loss.backward()
        optimizer_D.step()

        if i%900 == 0:
            print("[EPOCH %d/%d] [Batch %d/%d] [D loss: %f] [G loss: %f]"%(epoch, num_epochs, i, len(dataset), d_loss.item(), g_loss.item()))

        total_batch = epoch * len(dataset) + i
        if total_batch%400 == 0:
            save_image(gen.data[:25], 'output/%d.png' % total_batch, nrow=5)

And here's the error:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-36-0af32f223344> in <module>()
     18         gen_input = gen_input.cuda()
     19         #we then pass it to generator()
---> 20         gen = generator(gen_input) #this returns a image
     21
     22         #now calculate the loss wrt to discriminator output

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/batchnorm.py in __init__(self, num_features, eps, momentum, affine, track_running_stats)
     40         self.track_running_stats = track_running_stats
     41         if self.affine:
---> 42             self.weight = Parameter(torch.Tensor(num_features))
     43             self.bias = Parameter(torch.Tensor(num_features))
     44         else:

TypeError: expected CPU (got CUDA)

Any help would be appreciated. Thank you!


title: ResNet object has no attribute 'predict'

body:
I have trained a CNN model in PyTorch to detect skin diseases in 6 different classes. My model came out with an accuracy of 92% and I saved it in a .pth file. I wish to use this model for predictions but I don't know how to do so. If anyone can aid me in the necessary steps, I will be grateful.
I have tried just taking the image input straight from the folder, resizing it, and then running it through the model for predictions. The error I face is a ModuleAttributeAError which says there is no attribute named predict. Now I do not understand where I went wrong and I know this is a simple task for most but I was hoping for some guidance in this regard. The dataset I used is the Skin Cancer MNIST: HAM10000 dataset from Kaggle and trained it on ResNet18. If anyone has any pointers on fine-tuning the model, I would greatly appreciate it.
TLDR: I get an error called ModuleAttributeError that says the 'ResNet' module has no attribute 'predict'.
The image is preprocessed here as follows:
import os, cv2,itertools
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import pickle
from tqdm import tqdm
from glob import glob
from PIL import Image

# pytorch libraries
import torch
from torch import optim,nn
from torch.autograd import Variable
from torch.utils.data import DataLoader,Dataset
from torchvision import models,transforms

# sklearn libraries
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

np.random.seed(10)
torch.manual_seed(10)
torch.cuda.manual_seed(10)

print(os.listdir("/content/drive/My Drive/input"))

from google.colab import drive
drive.mount('/content/drive')

"""**Data analysis and preprocessing**"""

data_dir = '/content/drive/My Drive/input'
all_image_path = glob(os.path.join(data_dir, '*', '*.jpg'))
imageid_path_dict = {os.path.splitext(os.path.basename(x))[0]: x for x in all_image_path}
lesion_type_dict = {
    'nv': 'Melanocytic nevi',
    'mel': 'Melanoma',
    'bkl': 'Benign keratosis-like lesions ',
    'bcc': 'Basal cell carcinoma',
    'akiec': 'Actinic keratoses',
    'vasc': 'Vascular lesions',
    'df': 'Dermatofibroma'
}

def compute_img_mean_std(image_paths):
    """
        computing the mean and std of three channel on the whole dataset,
        first we should normalize the image from 0-255 to 0-1
    """

    img_h, img_w = 224, 224
    imgs = []
    means, stdevs = [], []

    for i in tqdm(range(len(image_paths))):
        img = cv2.imread(image_paths[i])
        img = cv2.resize(img, (img_h, img_w))
        imgs.append(img)

    imgs = np.stack(imgs, axis=3)
    print(imgs.shape)

    imgs = imgs.astype(np.float32) / 255.

    for i in range(3):
        pixels = imgs[:, :, i, :].ravel()  # resize to one row
        means.append(np.mean(pixels))
        stdevs.append(np.std(pixels))

    means.reverse()  # BGR --> RGB
    stdevs.reverse()

    print("normMean = {}".format(means))
    print("normStd = {}".format(stdevs))
    return means,stdevs

# norm_mean,norm_std = compute_img_mean_std(all_image_path)

norm_mean = (0.763035, 0.54564625, 0.5700399)
norm_std = (0.1409281, 0.15261264, 0.16997051)

df_original = pd.read_csv(os.path.join(data_dir, 'HAM10000_metadata.csv'))
df_original['path'] = df_original['image_id'].map(imageid_path_dict.get)
df_original['cell_type'] = df_original['dx'].map(lesion_type_dict.get)
df_original['cell_type_idx'] = pd.Categorical(df_original['cell_type']).codes
df_original.head()

# this will tell us how many images are associated with each lesion_id
df_undup = df_original.groupby('lesion_id').count()
# now we filter out lesion_id's that have only one image associated with it
df_undup = df_undup[df_undup['image_id'] == 1]
df_undup.reset_index(inplace=True)
df_undup.head()

# here we identify lesion_id's that have duplicate images and those that have only one image.
def get_duplicates(x):
    unique_list = list(df_undup['lesion_id'])
    if x in unique_list:
        return 'unduplicated'
    else:
        return 'duplicated'

# create a new colum that is a copy of the lesion_id column
df_original['duplicates'] = df_original['lesion_id']
# apply the function to this new column
df_original['duplicates'] = df_original['duplicates'].apply(get_duplicates)
df_original.head()

df_original['duplicates'].value_counts()

# now we filter out images that don't have duplicates
df_undup = df_original[df_original['duplicates'] == 'unduplicated']
df_undup.shape

# now we create a val set using df because we are sure that none of these images have augmented duplicates in the train set
y = df_undup['cell_type_idx']
_, df_val = train_test_split(df_undup, test_size=0.2, random_state=101, stratify=y)
df_val.shape

df_val['cell_type_idx'].value_counts()

# This set will be df_original excluding all rows that are in the val set
# This function identifies if an image is part of the train or val set.
def get_val_rows(x):
    # create a list of all the lesion_id's in the val set
    val_list = list(df_val['image_id'])
    if str(x) in val_list:
        return 'val'
    else:
        return 'train'

# identify train and val rows
# create a new colum that is a copy of the image_id column
df_original['train_or_val'] = df_original['image_id']
# apply the function to this new column
df_original['train_or_val'] = df_original['train_or_val'].apply(get_val_rows)
# filter out train rows
df_train = df_original[df_original['train_or_val'] == 'train']
print(len(df_train))
print(len(df_val))

df_train['cell_type_idx'].value_counts()

df_val['cell_type'].value_counts()

# Copy fewer class to balance the number of 7 classes
data_aug_rate = [15,10,5,50,0,40,5]
for i in range(7):
    if data_aug_rate[i]:
        df_train=df_train.append([df_train.loc[df_train['cell_type_idx'] == i,:]]*(data_aug_rate[i]-1), ignore_index=True)
df_train['cell_type'].value_counts()

# # We can split the test set again in a validation set and a true test set:
# df_val, df_test = train_test_split(df_val, test_size=0.5)
df_train = df_train.reset_index()
df_val = df_val.reset_index()
# df_test = df_test.reset_index()

Here is where I build the model:
# feature_extract is a boolean that defines if we are finetuning or feature extracting.
# If feature_extract = False, the model is finetuned and all model parameters are updated.
# If feature_extract = True, only the last layer parameters are updated, the others remain fixed.
def set_parameter_requires_grad(model, feature_extracting):
    if feature_extracting:
        for param in model.parameters():
            param.requires_grad = False

def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):
    # Initialize these variables which will be set in this if statement. Each of these
    #   variables is model specific.
    model_ft = None
    input_size = 0

    if model_name == "resnet":
        """ Resnet18, resnet34, resnet50, resnet101
        """
        model_ft = models.resnet18(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs, num_classes)
        input_size = 224

    elif model_name == "vgg":
        """ VGG11_bn
        """
        model_ft = models.vgg11_bn(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier[6].in_features
        model_ft.classifier[6] = nn.Linear(num_ftrs,num_classes)
        input_size = 224

    elif model_name == "densenet":
        """ Densenet121
        """
        model_ft = models.densenet121(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        num_ftrs = model_ft.classifier.in_features
        model_ft.classifier = nn.Linear(num_ftrs, num_classes)
        input_size = 224

    elif model_name == "inception":
        """ Inception v3
        """
        model_ft = models.inception_v3(pretrained=use_pretrained)
        set_parameter_requires_grad(model_ft, feature_extract)
        # Handle the auxilary net
        num_ftrs = model_ft.AuxLogits.fc.in_features
        model_ft.AuxLogits.fc = nn.Linear(num_ftrs, num_classes)
        # Handle the primary net
        num_ftrs = model_ft.fc.in_features
        model_ft.fc = nn.Linear(num_ftrs,num_classes)
        input_size = 299

    else:
        print("Invalid model name, exiting...")
        exit()
    return model_ft, input_size

# resnet,vgg,densenet,inception
model_name = 'resnet'
num_classes = 7
feature_extract = False
# Initialize the model for this run
model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True)
# Define the device:
device = torch.device('cuda:0')
# Put the model on the device:
model = model_ft.to(device)

# norm_mean = (0.49139968, 0.48215827, 0.44653124)
# norm_std = (0.24703233, 0.24348505, 0.26158768)
# define the transformation of the train images.
train_transform = transforms.Compose([transforms.Resize((input_size,input_size)),transforms.RandomHorizontalFlip(),
                                      transforms.RandomVerticalFlip(),transforms.RandomRotation(20),
                                      transforms.ColorJitter(brightness=0.1, contrast=0.1, hue=0.1),
                                        transforms.ToTensor(), transforms.Normalize(norm_mean, norm_std)])
# define the transformation of the val images.
val_transform = transforms.Compose([transforms.Resize((input_size,input_size)), transforms.ToTensor(),
                                    transforms.Normalize(norm_mean, norm_std)])

# Define a pytorch dataloader for this dataset
class HAM10000(Dataset):
    def __init__(self, df, transform=None):
        self.df = df
        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, index):
        # Load data and get label
        X = Image.open(self.df['path'][index])
        y = torch.tensor(int(self.df['cell_type_idx'][index]))

        if self.transform:
            X = self.transform(X)

        return X, y

# Define the training set using the table train_df and using our defined transitions (train_transform)
training_set = HAM10000(df_train, transform=train_transform)
train_loader = DataLoader(training_set, batch_size=64, shuffle=True, num_workers=4)
# Same for the validation set:
validation_set = HAM10000(df_val, transform=train_transform)
val_loader = DataLoader(validation_set, batch_size=64, shuffle=False, num_workers=4)

# we use Adam optimizer, use cross entropy loss as our loss function
optimizer = optim.Adam(model.parameters(), lr=1e-5)
criterion = nn.CrossEntropyLoss().to(device)

Lastly, is the training process with a prediction function:
# this function is used during training process, to calculation the loss and accuracy
class AverageMeter(object):
    def __init__(self):
        self.reset()

    def reset(self):
        self.val = 0
        self.avg = 0
        self.sum = 0
        self.count = 0

    def update(self, val, n=1):
        self.val = val
        self.sum += val * n
        self.count += n
        self.avg = self.sum / self.count

total_loss_train, total_acc_train = [],[]
def train(train_loader, model, criterion, optimizer, epoch):
    model.train()
    train_loss = AverageMeter()
    train_acc = AverageMeter()
    curr_iter = (epoch - 1) * len(train_loader)
    for i, data in enumerate(train_loader):
        images, labels = data
        N = images.size(0)
        # print('image shape:',images.size(0), 'label shape',labels.size(0))
        images = Variable(images).to(device)
        labels = Variable(labels).to(device)

        optimizer.zero_grad()
        outputs = model(images)

        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()
        prediction = outputs.max(1, keepdim=True)[1]
        train_acc.update(prediction.eq(labels.view_as(prediction)).sum().item()/N)
        train_loss.update(loss.item())
        curr_iter += 1
        if (i + 1) % 100 == 0:
            print('[epoch %d], [iter %d / %d], [train loss %.5f], [train acc %.5f]' % (
                epoch, i + 1, len(train_loader), train_loss.avg, train_acc.avg))
            total_loss_train.append(train_loss.avg)
            total_acc_train.append(train_acc.avg)
    return train_loss.avg, train_acc.avg

def validate(val_loader, model, criterion, optimizer, epoch):
    model.eval()
    val_loss = AverageMeter()
    val_acc = AverageMeter()
    with torch.no_grad():
        for i, data in enumerate(val_loader):
            images, labels = data
            N = images.size(0)
            images = Variable(images).to(device)
            labels = Variable(labels).to(device)

            outputs = model(images)
            prediction = outputs.max(1, keepdim=True)[1]

            val_acc.update(prediction.eq(labels.view_as(prediction)).sum().item()/N)

            val_loss.update(criterion(outputs, labels).item())

    print('------------------------------------------------------------')
    print('[epoch %d], [val loss %.5f], [val acc %.5f]' % (epoch, val_loss.avg, val_acc.avg))
    print('------------------------------------------------------------')
    return val_loss.avg, val_acc.avg

import cv2
from PIL import Image, ImageOps
import numpy as np

model = model_ft
model.load_state_dict(torch.load("/content/drive/MyDrive/input/trainbest.pth"))
model.eval()

def import_and_predict(image_data, model):
  size = (224, 224)
  image = ImageOps.fit(image_data, size, Image.ANTIALIAS)
  img = np.asarray(image)
  image_reshape = img[np.newaxis,...]
  prediction = model.predict(img_reshape)
  return prediction

image = Image.open('/content/0365-0596-abd-88-05-0712-gf03.jpg')
# st.image(image, use_column_width = True)
predictions = import_and_predict(image, model)
class_names = ["Melanocytic nevi", "dermatofibroma", "Benign keratosis-like lesions", "Basal cell carcinoma", "Actinic keratoses", "Vascular lesions", "Dermatofibroma"]
string = "It is: " + class_names[np.argmax(predictions)]
print(string)

Here is the error that comes immediately after this is executed.
---------------------------------------------------------------------------
ModuleAttributeError                      Traceback (most recent call last)
<ipython-input-219-d563271b78c6> in <module>()
     32 image = Image.open('/content/0365-0596-abd-88-05-0712-gf03.jpg')
     33 # st.image(image, use_column_width = True)
---> 34 predictions = import_and_predict(image, model)
     35 class_names = ["Melanocytic nevi", "dermatofibroma", "Benign keratosis-like lesions", "Basal cell carcinoma", "Actinic keratoses", "Vascular lesions", "Dermatofibroma"]
     36 string = "It is: " + class_names[np.argmax(predictions)]

1 frames
<ipython-input-219-d563271b78c6> in import_and_predict(image_data, model)
     27   img = np.asarray(image)
     28   image_reshape = img[np.newaxis,...]
---> 29   prediction = model.predict(img_reshape)
     30   return prediction
     31

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __getattr__(self, name)
    777                 return modules[name]
    778         raise ModuleAttributeError("'{}' object has no attribute '{}'".format(
--> 779             type(self).__name__, name))
    780
    781     def __setattr__(self, name: str, value: Union[Tensor, 'Module']) -> None:

ModuleAttributeError: 'ResNet' object has no attribute 'predict'

If anyone can help me fix the issue and get this to work as a classifier for skin diseases, I would be ever so thankful.


title: XOR neural network does not learn

body:
I am trying to solve the very simple non-linear problem. It is XOR gate.
I my school knowledge. XOR can be solve by using 2 input nodes, 2 hidden layer nodes. And 1 output. It is binary classification problem.

I generate the 1000 of random integer number it is 0 or 1 and then do backpropagation. But for some unknown reason my network has not learned anything. The training accuracy is constant at 50.

# coding: utf-8
import matplotlib
import torch
import torch.nn as nn
from torch.autograd import Variable

matplotlib.use('TkAgg')  # My buggy OSX 10.13.6 requires this
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from tqdm import tqdm
import random

N = 1000
batch_size = 10
epochs = 40
hidden_size = 2
output_size = 1
lr = 0.1

def return_xor(N):
    tmp_x = []
    tmp_y = []
    for i in range(N):
        a = (random.randint(0, 1) == 1)
        b = (random.randint(0, 1) == 1)
        if (a and not b) or (not a and b):
            q = True
        else:
            q = False
        input_features = (a, b)
        output_class = q
        tmp_x.append(input_features)
        tmp_y.append(output_class)
    return tmp_x, tmp_y

# In[495]:

# Training set
x, y = return_xor(N)
x = torch.tensor(x, dtype=torch.float, requires_grad=True)
y = torch.tensor(y, dtype=torch.float, requires_grad=True)
# Test dataset
x_test, y_test = return_xor(100)
x_test = torch.tensor(x_test)
y_test = torch.tensor(y_test)

class MyDataset(Dataset):
    """Define my own `Dataset` in order to use `Variable` with `autograd`"""

    def __init__(self, x, y):
        self.x = x
        self.y = y

    def __getitem__(self, index):
        return self.x[index], self.y[index]

    def __len__(self):
        return len(self.x)

dataset = MyDataset(x, y)
test_dataset = MyDataset(x_test, y_test)

print(dataset.x.shape)
print(dataset.y.shape)

# Make data iterable by loading to a loader. Shuffle, batch_size kwargs put them here in order to remind I myself
train_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)

print(f"They are {len(train_loader)} batches in the dataset")
shown = 0
for (x, y) in train_loader:
    if shown == 1:
        break
    print(f"{x.shape} {x.dtype}")
    print(f"{y.shape} {y.dtype}")
    shown += 1

class MyModel(nn.Module):
    """
    Binary classification
    2 input nodes
    2 hidden nodes
    1 output node
    """

    def __init__(self, input_size, hidden_size, output_size):
        super().__init__()
        self.fc1 = torch.nn.Linear(input_size, hidden_size)
        self.fc2 = torch.nn.Linear(hidden_size, output_size)
        self.sigmoid = torch.nn.Sigmoid()

    def forward(self, out):
        out = self.fc1(out)
        out = self.fc2(out)
        out = self.sigmoid(out)
        return out

# Create my network
net = MyModel(dataset.x.shape[1], hidden_size, output_size)
CUDA = torch.cuda.is_available()
if CUDA:
    net = net.cuda()
criterion = torch.nn.BCELoss(reduction='elementwise_mean')
optimizer = torch.optim.SGD(net.parameters(), lr=lr)

# Train the network
correct_train = 0
total_train = 0
for epoch in range(epochs):
    for i, (batches, labels) in enumerate(train_loader):
        batcesh = Variable(batches.float())
        labels = Variable(labels.float())
        output = net(batches)  # Forward pass
        optimizer.zero_grad()

        loss = criterion(output, labels.view(10, 1))
        loss.backward()
        optimizer.step()
        total_train += labels.size(0)
        correct_train += (predicted == labels.long()).sum()
        if (i + 1) % 10 == 0:
            print(f"""
                Epoch {epoch+1}/{epochs},
                Iteration {i+1}/{len(dataset)//batch_size},
                Training Loss: {loss.item()},
                Training Accuracy: {100*correct_train/total_train}
              """)

Solution:
I did initialized weight, Adaptive learning rate
https://github.com/elcolie/nnbootcamp/blob/master/Study-XOR.ipynb


title: How can I solve this elusive error in my multi-GPU Pytorch setup?

body:
I have spent the past day trying to figure out how to use multiple GPUs. In theory, parallelizing models across multiple GPUs is supposed to be as as easy as simply wrapping models with nn.DataParallel. However, I have found that this does not work for me. To use the most simple and canonical thing I could find for proof of this, I ran the code in the Data Parallelism tutorial, line for line.

I have tried everything from only having a specific permutation of my GPUs be visible to CUDA to reinstalling everything related to CUDA but can't figure out why I cannot run with multiple GPUs. Some information about my machine:
Operating System: Ubuntu 16.04
GPUS: 4 1080tis
Pytorch version: 1.01
CUDA version: 10.0

The error code is the following:

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-3-0f0d83e9ef13> in <module>
      1 for data in rand_loader:
      2     input = data.to(device)
----> 3     output = model(input)
      4     print("Outside: input size", input.size(),
      5           "output_size", output.size())

/usr/local/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

/usr/local/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py in forward(self, *inputs, **kwargs)
    141             return self.module(*inputs[0], **kwargs[0])
    142         replicas = self.replicate(self.module, self.device_ids[:len(inputs)])
--> 143         outputs = self.parallel_apply(replicas, inputs, kwargs)
    144         return self.gather(outputs, self.output_device)
    145

/usr/local/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py in parallel_apply(self, replicas, inputs, kwargs)
    151
    152     def parallel_apply(self, replicas, inputs, kwargs):
--> 153         return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
    154
    155     def gather(self, outputs, output_device):

/usr/local/lib/python3.6/site-packages/torch/nn/parallel/parallel_apply.py in parallel_apply(modules, inputs, kwargs_tup, devices)
     73             thread.start()
     74         for thread in threads:
---> 75             thread.join()
     76     else:
     77         _worker(0, modules[0], inputs[0], kwargs_tup[0], devices[0])

/usr/local/lib/python3.6/threading.py in join(self, timeout)
   1054
   1055         if timeout is None:
-> 1056             self._wait_for_tstate_lock()
   1057         else:
   1058             # the behavior of a negative timeout isn't documented, but

/usr/local/lib/python3.6/threading.py in _wait_for_tstate_lock(self, block, timeout)
   1070         if lock is None:  # already determined that the C code is done
   1071             assert self._is_stopped
-> 1072         elif lock.acquire(block, timeout):
   1073             lock.release()
   1074             self._stop()

KeyboardInterrupt:

Any insight into this error would be very much appreciated. From my relatively limited systems and CUDA knowledge, it has to do with some sort of locking, but I can't for the life of me figure out how to fix this.


title: In PyTorch, what is the difference between forward() and an ordinary method?

body:
How is implementing the forward() method of a custom nn.Module class different from adding an ordinary method to that class?

I heard that the forward() method should only accept and return tensors, because PyTorch has implemented special processing on the input and output of the forward() method. But I tried inputting/outputting non-tensor objects on a forward() method, and implementing a module that doesn't have a forward() method (instead, there are multiple custom-named methods which act like forward() methods). Both ways worked well.


title: pytorch load _IncompatibleKeys

body:
I trained a model of Efficentnet-b6(Architechture is as follows):

https://github.com/lukemelas/EfficientNet-PyTorch

Now, I tried to load a model I trained with it:

checkpoint  = torch.load('model.pth', map_location=torch.device('cpu'))
model.load_state_dict(checkpoint, strict=False)

but then I got the following error:

_IncompatibleKeys

missing_keys=['_conv_stem.weight', '_bn0.weight', '_bn0.bias', ...]
unexpected_keys=['module._conv_stem.weight', 'module._bn0.weight', 'module._bn0.bias', ...]

Please let me know how can I fix that, what am I missing?
Thank you!


title: how to calcullate the reconstruction probability for VAE anomaly detection

body:
I'm implementing the reconstruction probability of VAE in paper "Variational Autoencoder based Anomaly Detection using Reconstruction Probability". But I got a problem with the shape of mean_x' and sigma_x' for multivariate normal distribution.
The logic of reconstruction probability is:

input x to Encoder, get the mean(mu_z) and variance(sigma_z) of latent vector;
take L sample from the mu_z and sigma_z;
input these L samples into Decoder, get the mean(mu_x') and variance(sigma_x') of generated x';
calculate the log probability of x in the multivariate normal distribution with mu_z' and sigma_x';

Reconstruction probability
As I know, the shape of mu_x' and sigma_x' is the same when output from the decoder and should be (sequence_length,batch_size,num_of_features), like (240,32,1). To calculate the multivariate normal distribution log_probability, we should input these two parameters into distribution, but as the distribution parameter, the covariance matrix should not be as the same shape as the mean. So how should I convert sigma_x' and input it into MultivariateNormal function? Here's my code. The "#Problem line" is the line with the problem.

reconstructed_prob = np.zeros((len(dataset),), dtype='float32')

test_loader = DataLoader(dataset=dataset,
                         batch_size=self.batch_size,
                         shuffle=False,
                         drop_last=True)
if self.is_fitted:
    with torch.no_grad():
        for t, x in enumerate(test_loader):  #t=0, x.shape(32,240,1)
            x = x[0]
            x = x.permute(1, 0, 2) #(240,32,1)
            x = Variable(x[:, :, :].type(self.dtype), requires_grad=True)
            _, _, cell_output = self.encoder(x)  # h_hidden, (batch_size,units) (32,90)
            latent_mean, latent_logvar = self.lmbd(cell_output)  # input: last hidden state of encoder return: latent vector(batch_size32, z_dims5)

            score = 0
            for _ in range(L):

                # Sample a random c and z vector and reparametrize
                # epsilon_c = torch.randn(net.batch_size, net.T, net.z_dim)
                epsilon_z = torch.randn_like(latent_mean)

                if torch.cuda.is_available():
                    # epsilon_c = epsilon_c.to(torch.device(0))
                    epsilon_z = epsilon_z.to(torch.device(0))

                # c = mu_c + epsilon_c * sigma_c
                z = epsilon_z.mul(latent_logvar).add_(latent_mean)  #(32,5)

                # Pass sample through decoder and calculate reconstruction prob
                mu_x, sigma_x = self.decoder(z) # (240,32,1)
                pdf = torch.distributions.multivariate_normal.MultivariateNormal(mu_x,sigma_x)  #problem line
                score += pdf.log_prob(x)
                p_l = multivariate_normal.pdf(x, mu_x, sigma_x)
            reconstructed_prob[t] = score/L


title: PyTorch equivalent to tf.nn.softmax_cross_entropy_with_logits and tf.nn.sigmoid_cross_entropy_with_logits

body:
I found the post here. Here, we try to find an equivalence of tf.nn.softmax_cross_entropy_with_logits in PyTorch. The answer is still confusing to me.
Here is the Tensorflow 2 code
import tensorflow as tf
import numpy as np

# here we assume 2 batch size with 5 classes

preds = np.array([[.4, 0, 0, 0.6, 0], [.8, 0, 0, 0.2, 0]])
labels = np.array([[0, 0, 0, 1.0, 0], [1.0, 0, 0, 0, 0]])

tf_preds = tf.convert_to_tensor(preds, dtype=tf.float32)
tf_labels = tf.convert_to_tensor(labels, dtype=tf.float32)

loss = tf.nn.softmax_cross_entropy_with_logits(logits=tf_preds, labels=tf_labels)

It give me the loss as
<tf.Tensor: shape=(2,), dtype=float32, numpy=array([1.2427604, 1.0636061], dtype=float32)>

Here is the PyTorch code
import torch
import numpy as np

preds = np.array([[.4, 0, 0, 0.6, 0], [.8, 0, 0, 0.2, 0]])
labels = np.array([[0, 0, 0, 1.0, 0], [1.0, 0, 0, 0, 0]])

torch_preds = torch.tensor(preds).float()
torch_labels = torch.tensor(labels).float()

loss = torch.nn.functional.cross_entropy(torch_preds, torch_labels)

However, it raises:

RuntimeError: 1D target tensor expected, multi-target not supported

It seems that the problem is still unsolved. How to implement tf.nn.softmax_cross_entropy_with_logits in PyTorch?
What about tf.nn.sigmoid_cross_entropy_with_logits?


title: Image size in DefaultPredictor of Detectron2

body:
For object detection, I'm using detectron2.
I want to fix the input image size so I made my customized dataloader:
def build_train_loader(cls, cfg):
    dataloader = build_detection_train_loader(cfg,
        mapper=DatasetMapper(cfg, is_train=True, augmentations=[
            T.Resize((1200, 1200))
        ]))

What I wonder is for the prediction, I can use the DefaultPredictor of detectron2 and resize my images to (1200, 1200) as prepossessing before sending to the predictor?
Or the DefaultPredictor is resizing the image before the prediction and I have to override a function to resize to (1200, 1200)?


title: Most efficient way to prep data for pytorch lstm from pandas dataframe

body:
I am trying to feed data into my lstm. I have the data across multiple csv's, so I created a generator to load it in. However, I am having some issues on prepping the data for my lstm.

I understand this code (I got it from the pytorch docs)

seq_len = 5
batch_size= 3
cols_num = 10
hidden_size=20
num_layers = 2

rnn = nn.LSTM(input_size=cols_num, hidden_size=hidden_size, num_layers=num_layers)

data = torch.randn(seq_len, batch_size, cols_num)
h0 = torch.randn(batch_size, seq_len, hidden_size)
c0 = torch.randn(batch_size, seq_len, hidden_size)
output, (hn, cn) = rnn(data)

However, I think my disconnect is with using actual data not the torch.randn().

This is my current generator:

def data_loader(batch_size, fp, dropcol, seq_len):
    while True:
        for f in fp:
            gc.collect()
            df=pd.read_csv(f)
            df=df.replace(np.nan, 0)
            df=df.drop(dropcol,1)
            df['minute'] = df['minute'].apply(lambda x: min_idx(x))
            row_count, col_count = df.shape
            encoder_input = []
            prev = 0
            for idx, b in enumerate(range(1, row_count)):
                end = prev + batch_size
                window = df.iloc[prev:end]
                prev = end - 1
                w = np.array(window, dtype='float64')
                if w.shape[0] != batch_size:  break
                encoder_input.append(w)
                if idx == seq_len:
                    w0 = encoder_input
                    encoder_input = []
                    yield w0

but I get errors when I run this:

loader = data_loader(batch_size=batch_size, fp=<list of csvs>, dropcol=idcol, seq_len=2)
lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, batch_first=batch_first)

for batch in loader:
    b = torch.tensor(batch)
    output, hidden = lstm(b)

The error:
RuntimeError: Expected object of scalar type Float but got scalar type Double for argument #4 'mat1'

What is the error in how I am thinking? Also, how should I format an h0 or c0 from the data?


title: TypeError: 'tuple' object cannot be interpreted as an integer while creating data generators PyTorch

body:
I am new to PyTorch and I am learning to create batches of data for segmentation. The code is shown below:

class NumbersDataset(Dataset):
    def __init__(self):
        self.X = list(df['input_img'])
        self.y = list(df['mask_img'])

    def __len__(self):
        return len(self.X), len(self.y)

    def __getitem__(self, idx):
        return self.X[idx], self.y[idx]

if __name__ == '__main__':
    dataset = NumbersDataset()
    dataloader = DataLoader(dataset, batch_size=50, shuffle=True, num_workers=2)
    # print(len(dataset))
    # plt.imshow(dataset[100])
    # plt.show()
    print(next(iter(dataloader)))

where df['input_img'] column contains the location of the image ('/path/to/pic/480p/boxing-fisheye/00010.jpg') and df['mask_img'] contains the location of all the mask images. I am trying to load the images but I get the error:

  TypeError: 'tuple' object cannot be interpreted as an integer

However, if I don't use DataLoader and just do the following:

     dataset = NumbersDataset()
     print(len(dataset))
     print(dataset[10:20])

then I get what I expect. Can someone tell me what am I doing wrong?


title: Convert weight and bias to sparse tensor pytorch

body:
I'm trying to convert torch.nn.Parameters to sparse tensor. Pytorch documents say that Parameters is a Tensor's subclass. Tensor support to_sparse method but if I convert a Parameters to sparse, it will give me:
TypeError: cannot assign 'torch.cuda.sparse.FloatTensor' as parameter 'weight' (torch.nn.Parameter or None expected)
Is there a way to bypass this and use sparse tensor for Parameters?
Here is example code to produce the problem:

for name, module in net.named_modules():
    if isinstance(module, torch.nn.Conv2d):
        module.weight = module.weight.data.to_sparse()
        module.bias = module.bias.data.to_sparse()


title: IndexError: index_select(): Index is supposed to be a vector

body:
for batch_id, (data, target) in enumerate(tqdm(train_loader)):
        print(target)
        print('Entered for loop')
        target = torch.sparse.torch.eye(10).index_select(dim=0, index=target)
        data, target = Variable(data), Variable(target)

The line which contains the index_select function gives this error and I am not able to find a solution to it anywhere. The target variable on printing looks like this:
tensor([[4],
[1],
[8],
[5],
[9],
[5],
[5],
[8],
[4],
[6]])

How do I convert the target variable into a vector? Isn’t it already a vector?


title: Why the same configuration network in caffe and pytorch behaves so differently?

body:
the code in pytorch is here, I only used the vgg19 arch.

In order to make the cifar10 dataset preprocessed same in caffe and pytorch, I remove all the transforms in main.py but toTensor(). I found the cifar10 dataset's range is [0,1] in pytorch, but caffe is [0,255], so I scale 1/255 below. Any extra preprocessing is canceled to make things easier.

here is my caffe net definition prototxt:

layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TRAIN
  }
  transform_param {
    scale: 0.00392156862745
  }
  data_param {
    source: "/home/snk/Documents/digits-jobs/20180106-135745-4cac/train_db"
    batch_size: 128
    backend: LMDB
  }
}
layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TEST
  }
  transform_param {
    scale: 0.00392156862745
  }
  data_param {
    source: "/home/snk/Documents/digits-jobs/20180106-135745-4cac/val_db"
    batch_size: 128
    backend: LMDB
  }
}
layer {
  name: "conv1_1"
  type: "Convolution"
  bottom: "data"
  top: "conv1_1"
  convolution_param {
    num_output: 64
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu1_1"
  type: "ReLU"
  bottom: "conv1_1"
  top: "conv1_1"
}
layer {
  name: "conv1_2"
  type: "Convolution"
  bottom: "conv1_1"
  top: "conv1_2"
  convolution_param {
    num_output: 64
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu1_2"
  type: "ReLU"
  bottom: "conv1_2"
  top: "conv1_2"
}
layer {
  name: "pool1"
  type: "Pooling"
  bottom: "conv1_2"
  top: "pool1"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "conv2_1"
  type: "Convolution"
  bottom: "pool1"
  top: "conv2_1"
  convolution_param {
    num_output: 128
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu2_1"
  type: "ReLU"
  bottom: "conv2_1"
  top: "conv2_1"
}
layer {
  name: "conv2_2"
  type: "Convolution"
  bottom: "conv2_1"
  top: "conv2_2"
  convolution_param {
    num_output: 128
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu2_2"
  type: "ReLU"
  bottom: "conv2_2"
  top: "conv2_2"
}
layer {
  name: "pool2"
  type: "Pooling"
  bottom: "conv2_2"
  top: "pool2"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "conv3_1"
  type: "Convolution"
  bottom: "pool2"
  top: "conv3_1"
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu3_1"
  type: "ReLU"
  bottom: "conv3_1"
  top: "conv3_1"
}
layer {
  name: "conv3_2"
  type: "Convolution"
  bottom: "conv3_1"
  top: "conv3_2"
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu3_2"
  type: "ReLU"
  bottom: "conv3_2"
  top: "conv3_2"
}
layer {
  name: "conv3_3"
  type: "Convolution"
  bottom: "conv3_2"
  top: "conv3_3"
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu3_3"
  type: "ReLU"
  bottom: "conv3_3"
  top: "conv3_3"
}
layer {
  name: "conv3_4"
  type: "Convolution"
  bottom: "conv3_3"
  top: "conv3_4"
  convolution_param {
    num_output: 256
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu3_4"
  type: "ReLU"
  bottom: "conv3_4"
  top: "conv3_4"
}
layer {
  name: "pool3"
  type: "Pooling"
  bottom: "conv3_4"
  top: "pool3"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "conv4_1"
  type: "Convolution"
  bottom: "pool3"
  top: "conv4_1"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu4_1"
  type: "ReLU"
  bottom: "conv4_1"
  top: "conv4_1"
}
layer {
  name: "conv4_2"
  type: "Convolution"
  bottom: "conv4_1"
  top: "conv4_2"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu4_2"
  type: "ReLU"
  bottom: "conv4_2"
  top: "conv4_2"
}
layer {
  name: "conv4_3"
  type: "Convolution"
  bottom: "conv4_2"
  top: "conv4_3"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu4_3"
  type: "ReLU"
  bottom: "conv4_3"
  top: "conv4_3"
}
layer {
  name: "conv4_4"
  type: "Convolution"
  bottom: "conv4_3"
  top: "conv4_4"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu4_4"
  type: "ReLU"
  bottom: "conv4_4"
  top: "conv4_4"
}
layer {
  name: "pool4"
  type: "Pooling"
  bottom: "conv4_4"
  top: "pool4"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "conv5_1"
  type: "Convolution"
  bottom: "pool4"
  top: "conv5_1"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu5_1"
  type: "ReLU"
  bottom: "conv5_1"
  top: "conv5_1"
}
layer {
  name: "conv5_2"
  type: "Convolution"
  bottom: "conv5_1"
  top: "conv5_2"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu5_2"
  type: "ReLU"
  bottom: "conv5_2"
  top: "conv5_2"
}
layer {
  name: "conv5_3"
  type: "Convolution"
  bottom: "conv5_2"
  top: "conv5_3"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu5_3"
  type: "ReLU"
  bottom: "conv5_3"
  top: "conv5_3"
}
layer {
  name: "conv5_4"
  type: "Convolution"
  bottom: "conv5_3"
  top: "conv5_4"
  convolution_param {
    num_output: 512
    pad: 1
    kernel_size: 3
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu5_4"
  type: "ReLU"
  bottom: "conv5_4"
  top: "conv5_4"
}
layer {
  name: "pool5"
  type: "Pooling"
  bottom: "conv5_4"
  top: "pool5"
  pooling_param {
    pool: MAX
    kernel_size: 2
    stride: 2
  }
}
layer {
  name: "drop6"
  type: "Dropout"
  bottom: "pool5"
  top: "drop_pool5"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc6"
  type: "InnerProduct"
  bottom: "drop_pool5"
  top: "fc6"
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu6"
  type: "ReLU"
  bottom: "fc6"
  top: "fc6"
}
layer {
  name: "drop7"
  type: "Dropout"
  bottom: "fc6"
  top: "fc6"
  dropout_param {
    dropout_ratio: 0.5
  }
}
layer {
  name: "fc7"
  type: "InnerProduct"
  bottom: "fc6"
  top: "fc7"
  inner_product_param {
    num_output: 512
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "relu7"
  type: "ReLU"
  bottom: "fc7"
  top: "fc7"
}
layer {
  name: "fc8"
  type: "InnerProduct"
  bottom: "fc7"
  top: "fc8"
  inner_product_param {
    num_output: 10
    weight_filler {
      type: "xavier"
    }
    bias_filler {
      type: "xavier"
    }
  }
}
layer {
  name: "acc"
  type: "Accuracy"
  bottom: "fc8"
  bottom: "label"
  top: "accuracy"
  include {
    phase: TEST
  }
}
layer {
  name: "loss"
  type: "SoftmaxWithLoss"
  bottom: "fc8"
  bottom: "label"
  top: "loss"
}

Here is my solver.prototxt

test_iter: 79
test_interval: 391
base_lr: 0.05
display: 40
max_iter: 117300
lr_policy: "step"
gamma: 0.5
momentum: 0.9
weight_decay: 0.0005
stepsize: 11730
snapshot: 3910
snapshot_prefix: "snapshot"
solver_mode: GPU
net: "train_val.prototxt"
solver_type: SGD

The fact is pytorch can train the model at a very quick speed, the acc after one epoch is about 20%, but in caffe the loss is always around 2.3XXX (about -log(0.1), random guess loss), I suspect it was because the different weight initialization, so I changed caffe's filler.hpp's xavier to 'efficient backprop'(i.e U(-std, std), std=1/(sqrt(fan_in))), but it didn't work.

Now, the only difference is the bias initialization method, in pytorch it uses the weight's fan_in, but in caffe I think it uses the output_num as fan_in (because its shape is [1,N], N is the number of output neurons, and in filler.hpp it use blob.count() / blob.num() as fan_in for xavier ).

Who can help me ? I think if all the configuration is the same, the training process is almost the same ,but it breaked my opinion.


title: Accuracy not increasing with BERT Large model

body:
I used both BERT_base_cased and BERT_large_Cased model for multi class text classification. With BERT_base_cased, I got satisfactory results. When I tried with BERT_large_cased model, the accuracy is same for all the epochs

With BERT_base_cased, there is no such problem. But with BERT_large_cased, why accuracy is same in all the epochs? Any help is really appreciated.............


title: Inverted colors in Tensorboard SummaryWriter add_image() function

body:
There is an image stored in image_tensor (image_tensor of size (3,256,512), storing values in the interval 0,255)  which I would like to display in Tensorboard (TensorboardX for PyTorch, more specifically) via the add_image() function for SummaryWriter. When I add the image to the Tensorboard via writer.add_image("color_image",image_tensor,self.step),  the colors are inverted.

When I write the image to a file via scipy.misc.imsave("/write/to/path/image.png",np.transpose(image_tensor.data.cpu().numpy(),(1,2,0))), the image is perfectly fine.

Only thing I change for the second line is changing CxHxW to HxWxC, but I don't think that this is the root of this color inversion issue. What might be the problem?


title: Is there any way to convert weight.pt model to weight.h5 or weight.weights to weight.h5 format

body: Is there any way to convert weight.pt model(weight files in pytorch format) to weight.h5(weight files in keras format) or weight.weights(weight files in darknet format) to weight.h5 format


title: How do I install the d2l library on Windows without conda?

body: The d2l.ai textbook includes its own Python library, d2l.  Their instructions recommend installing with conda, a Python package manager that also functions as a general library manager.  But I've been burned by conda a couple times, and prefer pure pip.  How can I install d2l on Windows with pure pip?  I only need PyTorch, not MxNet or Tensorflow.


title: Normalisation in recurrent network PyTorch LSTM

body:
I am trying to normalise between layers of my stacked LSTM network in PyTorch. The network looks something like this:
class LSTMClassifier(nn.Module):
    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):
        super().__init__()
        self.hidden_dim = hidden_dim
        self.layer_dim = layer_dim
        self.lstm1 = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)
        self.lstm2 = nn.LSTM(input_dim, hidden_dim, hidden_dim, batch_first=True)
        self.fc1 = nn.Linear(hidden_dim, 32)
        self.fc2 = nn.Linear(32, 1)
        self.dropout = nn.Dropout(p=0.2)
        self.batch_normalisation1 = nn.BatchNorm1d(hidden_dim)
        self.batch_normalisation2 = nn.BatchNorm1d(hidden_dim)

    def forward(self, x):
        h0, c0 = self.init_hidden(x)
        out, (hn1, cn1) = self.lstm1(x, (h0, c0))
        out = self.dropout(out)                     # error line
        out = self.batch_normalisation1(out)

        h1, c1 = self.init_hidden(out)
        out, (hn2, cn2) = self.lstm2(out, (h1, c1))
        out = self.dropout(out)
        out = self.batch_normalisation1(out)

        h2, c2 = self.init_hidden(out)
        out, (hn3, cn3) = self.lstm2(out, (h2, c2))
        out = self.dropout(out)
        out = self.batch_normalisation1(out)

        out = self.fc1(out[:, -1, :])
        out = self.dropout(out)
        out = self.fc2(out)
        return out

    def init_hidden(self, x):
        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)
        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)
        return [t for t in (h0, c0)]

There is an error being raise where I have commented in the above, which is happening due to the BatchNorm1d expecting a 2 dimensional input.
In particular, I am initialising the model and passing batched data to the network as:
model = LSTMClassifier(5, 128, 3, 1)
model(X)

error:
RuntimeError: running_mean should contain 3 elements not 128
The X input tensor has shape  torch.Size([10, 3, 5]), i.e. batch size is 10 and each input has dimensions 3 x 5 i.e. 5 features and 3 time steps.
The error is arising due to the BatchNorm1d trying to normalise across the wrong dimension - in the network the variable out has shape torch.Size([1, 3, 128]), i.e. the 5 input features are mapped to 128 hyper variables.
I could reshape the variable put inside the forward function, but this seems unnecessary. I have also tried using BatchNorm2d, but it requires a 4d Tensor, which my variable out is not. Is there some way to over come this?
Additionally, I am trying to add normalisation in my network to speed the training - I am not entirely sure how the PyTorch BatchNorm functions work, so would appreciate an explanation of this v. much. Specifically, why should we normalise across the time dimension and not the feature dimensions?


title: Loading n-dimensional tensor for pytorch from text file

body:
I have a 3-dimensional tensor that I create outside of my python code and am able to encode in any human-readable format. I need to load the values of this tensor as a frozen layer to my pytorch NN. I've tried to encode the tensor as a text file in the form [[[a,b],[c,d]], [[e,f], [g,h]], [[k,l],[m,n]]] which seemed to be the most logical way for that. Then I tried to read its value via
tensor = torch.from_numpy(np.loadtxt("./arrays/tensor.txt"))

but got the exception in npyio.py
ValueError: could not convert string to float: '[[[-2.888356,'

Apparently, that's not how it works and the values are to be written as plain numbers separated by spaces and \n, but then I don't see how to easily read the data of dimension >= 2 with numpy.
What could be other simple methods to write down and read the tensor value into a pytorch tensor?


title: Can't install pytorch: no matching distribution found for torch==1.7.0+cpu

body:
When trying to install PyTorch, I can't progress past these errors:
ERROR: Could not find a version that satisfies the requirement torch==1.7.0+cpu (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2)
ERROR: No matching distribution found for torch==1.7.0+cpu


title: Pytorch: why the orders of weights in ConvTranspose2d and Conv2d are different?

body:
Pytorch code:

up = nn.ConvTranspose2d(3, 128, 2, stride=2)
conv = nn.Conv2d(3, 128, 2)

inputs = Variable(torch.rand(1, 3, 64, 64))
print('up conv output size:', up(inputs).size())

inputs = Variable(torch.rand(1, 3, 64, 64))
print('conv output size:', conv(inputs).size())

print('up conv weight size:', up.weight.data.shape)
print('conv weight size:', conv.weight.data.shape)

Result:

up conv output size: torch.Size([1, 128, 128, 128])
conv output size: torch.Size([1, 128, 63, 63])
up conv weight size: torch.Size([3, 128, 2, 2])
conv weight size: torch.Size([128, 3, 2, 2])

Why the orders are different between ConvTranspose2d (3,128) and Conv2d (128, 3)?

Is it supposed to behave like this?


title: Pytorch autograd: compute batch gradient w.r.t to input

body:
I need to compute the gradient of the network output (not the loss) with respect to the input but I'm getting the error grad can be implicitly created only for scalar outputs. I'm trying to reproduce what's described in this paper and I need the gradient I mentioned to perform step 7

This is what I'd like to do:

out = netD(input).view(-1)
out.backward(retain_graph=True)
grad = input.grad


title: How to run perticular code in gpu using PyTorch?

body:
I am using an image processing code in python opencv. Since that process is taking a lot of time to process say 30 images. I tried to process these image parallel using Multiprocessing. The multiprocessing part is working good in CPU but I want to use that multiprocessing thing in GPU(cuda).

I use torch.multiprocessing for running task in parallel. So I am using torch.device('cuda') for our class to run whole thing in to this perticular device. When I run the code it's showing device using "cuda" but not using any GPU processing.

import cv2
import numpy as np
import torch
import torch.nn as nn
from torch.multiprocessing import Process, Pool, Manager, set_start_method
import sys
import os

class RoadShoulderWidth(nn.Module):
    def __init__(self):
        super(RoadShoulderWidth, self).__init__()
        pass

    // Want to run below method in parallel for 30 images.
    @staticmethod
    def get_dim(image, road_shoulder_width_list):
        ..... code

    def get_road_shoulder_width(self, _root_dir, _img_path_list):

    manager = Manager()
    road_shoulder_width_list = manager.list()
    processes = []
    for img_path in img_path_list[:30]:
        img = cv2.imread(_root_dir + '/' + img_path)
        img = img[72 * 5:72 * 6, 0:1280]
        # Do work
        p = Process(target=self.get_dim,args=(img,road_shoulder_width_list))
        p.start()
        processes.append(p)

    for p in processes:
        p.join()
    return road_shoulder_width_list

Use below set of code to run your class

if __name__ == '__main__':
    root_dir = '/home/nikhil_m/r'
    img_path_list = os.listdir(root_dir)

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    print('Using device:', device)
    dataloader_kwargs = {'pin_memory': True}
    set_start_method('fork')
    obj = RoadShoulderWidth().to(device)
    val = obj.get_road_shoulder_width(str(root_dir), img_path_list)
    print(val)
    print(torch.cuda.is_available())

Can anybody suggest me how to fix this?


title: Is there a Pytorch equivalent of Tensorflow class "tf.nn.rnn_cell.MultiRNNCell"?

body:
guys.
Is there pytorch equivalent of “tf.nn.rnn_cell.MultiRNNCell” that stacks multiple cells?
Could it be torch.nn.LSTM()?
Then how to use it to create same network as MultiRNNCell?


title: Error getting state_dict from a serialized model in Pytorch

body:
Already trained my encoder-decoder model and saved using :

model_state = {
             'encoder': encoder,
             'encoder_optimizer': encoder_optimizer,
             'decoder': decoder,
             'decoder_optimizer': decoder_optimizer
             }
torch.save(model_state, "best_model.pth.tar")

This works fine when I use the model stand-alone, but it gives me errors when I try to use my model within another app. Thus, I'm trying to load the model and save the encoder and decoder as state_dicts instead. This works for my encoder, but when I try:

checkpoint = torch.load(path_to_model, map_location=torch.device("cpu"))
decoder = checkpoint['decoder']
decoder = decoder.to(device)
encoder = checkpoint['encoder']
encoder = encoder.to(device)
torch.save(encoder.state_dict(), 'encoder.dict')
torch.save(decoder.state_dict(), 'decoder.dict')

It fails on torch.save(decoder.state_dict(), 'decoder.dict') and I get the error :

 File "<stdin>", line 1, in <module>
  File "caption.py", line 31, in load_maps
    torch.save(decoder.state_dict(), 'decoder.dict')
  File "/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 695, in state_dict
    module.state_dict(destination, prefix + name + '.', keep_vars=keep_vars)
  File "/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 695, in state_dict
    module.state_dict(destination, prefix + name + '.', keep_vars=keep_vars)
  File "/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 696, in state_dict
    for hook in self._state_dict_hooks.values():
  File "/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py", line 585, in __getattr__
    type(self).__name__, name))
AttributeError: 'Softmax' object has no attribute '_state_dict_hooks'

Is there a way to bypass this error or recreate the state_dict without having to retrain my model? I don't get why if I saved the whole model that I can't get the state_dict out of it, which is presumably a part of the model.

This is the output from calling for m in decoder.modules(): print(m):

DecoderWithAttention(
  (attention): Attention(
    (encoder_att): Linear(in_features=2048, out_features=512, bias=True)
    (decoder_att): Linear(in_features=512, out_features=512, bias=True)
    (full_att): Linear(in_features=512, out_features=1, bias=True)
    (relu): ReLU()
    (softmax): Softmax(dim=1)
  )
  (embedding): Embedding(9490, 512)
  (dropout): Dropout(p=0.5, inplace=False)
  (decode_step): LSTMCell(2560, 512, bias=1)
  (init_h): Linear(in_features=2048, out_features=512, bias=True)
  (init_c): Linear(in_features=2048, out_features=512, bias=True)
  (f_beta): Linear(in_features=512, out_features=2048, bias=True)
  (sigmoid): Sigmoid()
  (fc): Linear(in_features=512, out_features=9490, bias=True)
)
Attention(
  (encoder_att): Linear(in_features=2048, out_features=512, bias=True)
  (decoder_att): Linear(in_features=512, out_features=512, bias=True)
  (full_att): Linear(in_features=512, out_features=1, bias=True)
  (relu): ReLU()
  (softmax): Softmax(dim=1)
)
Linear(in_features=2048, out_features=512, bias=True)
Linear(in_features=512, out_features=512, bias=True)
Linear(in_features=512, out_features=1, bias=True)
ReLU()
Softmax(dim=1)
Embedding(9490, 512)
Dropout(p=0.5, inplace=False)
LSTMCell(2560, 512, bias=1)
Linear(in_features=2048, out_features=512, bias=True)
Linear(in_features=2048, out_features=512, bias=True)
Linear(in_features=512, out_features=2048, bias=True)
Sigmoid()
Linear(in_features=512, out_features=9490, bias=True)


title: How can I convert Tensor into Bitmap on PyTorch Mobile?

body:
I found that solution (https://itnext.io/converting-pytorch-float-tensor-to-android-rgba-bitmap-with-kotlin-ffd4602a16b6) but when I tried to convert that way I found that the size of inputTensor.dataAsFloatArray is more than bitmap.width*bitmap.height. How works converting tensor to float array or is there any other possible method to convert pytorch tensor to bitmap?

val inputTensor = TensorImageUtils.bitmapToFloat32Tensor(
    bitmap,
    TensorImageUtils.TORCHVISION_NORM_MEAN_RGB, TensorImageUtils.TORCHVISION_NORM_STD_RGB
)

// Float array size is 196608 when width and height are 256x256 = 65536

val res = floatArrayToGrayscaleBitmap(inputTensor.dataAsFloatArray, bitmap.width, bitmap.height)

fun floatArrayToGrayscaleBitmap (
    floatArray: FloatArray,
    width: Int,
    height: Int,
    alpha :Byte = (255).toByte(),
    reverseScale :Boolean = false
) : Bitmap {

    // Create empty bitmap in RGBA format (even though it says ARGB but channels are RGBA)
    val bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
    val byteBuffer = ByteBuffer.allocate(width*height*4)
    Log.d("App", floatArray.size.toString() + " " + (width * height * 4).toString())

    // mapping smallest value to 0 and largest value to 255
    val maxValue = floatArray.max() ?: 1.0f
    val minValue = floatArray.min() ?: 0.0f
    val delta = maxValue-minValue
    var tempValue :Byte

    // Define if float min..max will be mapped to 0..255 or 255..0
    val conversion = when(reverseScale) {
        false -> { v: Float -> ((v-minValue)/delta*255).toByte() }
        true -> { v: Float -> (255-(v-minValue)/delta*255).toByte() }
    }

    // copy each value from float array to RGB channels and set alpha channel
    floatArray.forEachIndexed { i, value ->
        tempValue = conversion(value)
        byteBuffer.put(4*i, tempValue)
        byteBuffer.put(4*i+1, tempValue)
        byteBuffer.put(4*i+2, tempValue)
        byteBuffer.put(4*i+3, alpha)
    }

    bmp.copyPixelsFromBuffer(byteBuffer)

    return bmp
}


title: Confusion in understanding the output of BERTforTokenClassification class from Transformers library

body:
It is the example given in the documentation of transformers pytorch library

from transformers import BertTokenizer, BertForTokenClassification
import torch

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertForTokenClassification.from_pretrained('bert-base-uncased',
                      output_hidden_states=True, output_attentions=True)

input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute",
                         add_special_tokens=True)).unsqueeze(0)  # Batch size 1
labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0)  # Batch size 1
outputs = model(input_ids, labels=labels)

loss, scores, hidden_states,attentions = outputs

Here hidden_states is a tuple of length 13 and contains hidden-states of the model at the output of each layer plus the initial embedding outputs. I would like to know, whether hidden_states[0] or hidden_states[12] represent the final hidden state vectors?


I use this python script from AllenAI to integrate CRF module (Conditional random fields) layer in my ML-system ...
and execute the following example to test the CRF module.
import torch
from allennlp.modules import ConditionalRandomField

num_tags = 2

model = ConditionalRandomField(num_tags)

seq_length = 3  # maximum sequence length in a batch
batch_size = 2  # number of samples in the batch
emissions = torch.randn(seq_length, batch_size, num_tags)
tags = torch.tensor([[0.0, 1.0], [1.0, 1.0], [0.0, 1.0]], dtype=torch.long)  # (seq_length, batch_size)
model(emissions, tags)

Upon execution, I get the following error.
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-3-415939853a93> in <module>
  7 emissions = torch.randn(seq_length, batch_size, num_tags)
  8 tags = torch.tensor([[0.0, 1.0], [1.0, 1.0], [0.0, 1.0]], dtype=torch.long)  #
(seq_length, batch_size)
----> 9 model(emissions, tags)

~/anaconda3/envs/torch/lib/python3.6/site-packages/torch/nn/modules/module.py in
__call__(self, *input, **kwargs)
545                     result = (result,)
546                 input = result
--> 547         if torch._C._get_tracing_state():
548             result = self._slow_forward(*input, **kwargs)
549         else:

<ipython-input-2-2d984bd97cf1> in forward(self, inputs, tags, mask)
329             mask = mask.to(torch.bool)
330
--> 331         log_denominator = self._input_likelihood(inputs, mask)
332         log_numerator = self._joint_likelihood(inputs, tags, mask)
333

<ipython-input-2-2d984bd97cf1> in _input_likelihood(self, logits, mask)
249             # In valid positions (mask == True) we want to take the logsumexp over the
current_tag dimension
250             # of `inner`. Otherwise (mask == False) we want to retain the previous
alpha.
--> 251             alpha = util.logsumexp(inner, 1) * mask[i].view(batch_size, 1) + alpha *
(
252                 ~mask[i]
253             ).view(batch_size, 1)

RuntimeError: expected device cpu and dtype Float but got device cpu and dtype Bool

I do understand that this is a problem of PyTorch version incompatibility with the functions used in allennlp package, but I am not sure what versions of torch and torchvision it expects.
These are the versions of PyTorch installations on my machine.
torch                            1.5.1
torchvision                      0.4.2
allennlp                         1.0.0

Any help or hint will be appreciated.


title: Feature extraction in loop seems to cause memory leak in pytorch

body:
I have spent considerable time trying to debug some pytorch code which I have created a minimal example of for the purpose of helping to better understand what the issue might be.
I have removed all necessary portions of the code which are unrelated to the issue so the remaining piece of code won't make much sense from a functional standpoint but it still displays the error I'm facing.
The overall task I'm working on is in a loop and every pass of the loop is computing the embedding of the image and adding it to a variable storing it. It's effectively aggregating it (not concatenating, so the size remains the same). I don't expect the number of iterations to force the datatype to overflow, I don't see this happening here nor in my code.

I have added multiple metrics to evaluate the size of the tensors I'm working with to make sure they're not growing in memory footprint
I'm checking the overall GPU memory usage to verify the issue leading to the final RuntimeError: CUDA out of memory..

My environment is as follows:
 - python 3.6.2
 - Pytorch 1.4.0
 - Cudatoolkit 10.0
 - Driver version 410.78
 - GPU: Nvidia GeForce GT 1030  (2GB VRAM)
(though I've replicated this experiment with the same result on a Titan RTX with 24GB,
same pytorch version and cuda toolkit and driver, it only goes out of memory further in the loop).

Complete code below. I have marked 2 lines as culprits, as deleting them removes the issue, though obviously I need to find a way to execute them without having memory issues. Any help would be much appreciated! You may try with any image named "source_image.bmp" to replicate the issue.
import torch
from PIL import Image
import torchvision
from torchvision import transforms
from pynvml import nvmlDeviceGetHandleByIndex, nvmlDeviceGetMemoryInfo, nvmlInit
import sys
import os
os.environ["CUDA_VISIBLE_DEVICES"]='0'      # this is necessary on my system to allow the environment to recognize my nvidia GPU for some reason
os.environ['CUDA_LAUNCH_BLOCKING'] = '1'    # to debug by having all CUDA functions executed in place
torch.set_default_tensor_type('torch.cuda.FloatTensor')

# Preprocess image
tfms = transforms.Compose([
    transforms.Resize(256),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),])
img = tfms(Image.open('source_image.bmp')).unsqueeze(0).cuda()

model = torchvision.models.resnet50(pretrained=True).cuda()
model.eval()    # we put the model in evaluation mode, to prevent storage of gradient which might accumulate

nvmlInit()
h = nvmlDeviceGetHandleByIndex(0)
info = nvmlDeviceGetMemoryInfo(h)
print(f'Total available memory   : {info.total / 1000000000}')

feature_extractor = torch.nn.Sequential(*list(model.children())[:-1])
orig_embedding = feature_extractor(img)

embedding_depth = 2048

mem0 = 0

embedding = torch.zeros(2048, img.shape[2], img.shape[3]) #, dtype=torch.float)

patch_size=[4,4]
patch_stride=[2,2]
patch_value=0.0

# Here, we iterate over the patch placement, defined at the top left location
for row in range(img.shape[2]-1):
    for col in range(img.shape[3]-1):
        print("######################################################")

        ######################################################
        # Isolated line, culprit 1 of the GPU memory leak
        ######################################################
        patched_embedding = feature_extractor(img)

        delta_embedding = (patched_embedding - orig_embedding).view(-1, 1, 1)

        ######################################################
        # Isolated line, culprit 2 of the GPU memory leak
        ######################################################
        embedding[:,row:row+1,col:col+1] = torch.add(embedding[:,row:row+1,col:col+1], delta_embedding)

        print("img size:\t\t", img.element_size() * img.nelement())
        print("patched_embedding size:\t", patched_embedding.element_size() * patched_embedding.nelement())
        print("delta_embedding size:\t", delta_embedding.element_size() * delta_embedding.nelement())
        print("Embedding size:\t\t", embedding.element_size() * embedding.nelement())

        del patched_embedding, delta_embedding
        torch.cuda.empty_cache()

        info = nvmlDeviceGetMemoryInfo(h)
        print("\nMem usage increase:\t", info.used / 1000000000 - mem0)
        mem0 = info.used / 1000000000
        print(f'Free:\t\t\t {(info.total - info.used) / 1000000000}')

print("Done.")


title: Backpropagation without updating weights (DCGAN)

body:
The basic idea is, that I load a trained model (DCGAN) and make a anomaly detection with it on images. For the anomaly detection I have to do some iterations on the test phase to evaluate it, if it is a anomaly or not.

For that I have two Loss-Functions in the test setup, which should be calculating a backpropagation to the generator input and update the latent vector. But it only should update the latent vector, not the weights on the graph.

Is this possible?

Probably, if I use only a pytorch-variable of my latent vector and set the variable output of the generator to
"requires_grad=False"
Like in the docs --> Pytorch


title: Pytorch FloatClassNLLCriterion_updateOutput error

body:
I am getting the following error message while computing the loss of my neural net: TypeError: FloatClassNLLCriterion_updateOutput received an invalid combination of arguments - got (int, torch.FloatTensor, !torch.FloatTensor!, torch.FloatTensor, bool, NoneType, torch.FloatTensor, int), but expected (int state, torch.FloatTensor input, torch.LongTensor target, torch.FloatTensor output, bool sizeAverage, [torch.FloatTensor weights or None], torch.FloatTensor total_weight, int ignore_index) on this line: loss = criterion(outputs,one_hot_target).

I have tried several things and searched the web, but I can't seem to find my mistake. Does someone have an idea?

The code used:

class Net(nn.Module):
def _init_(self,input_size,hidden_size, num_classes):
    super(Net, self)._init_()
    self.l1 = nn.Linear(input_size,hidden_size)
    self.l2 = nn.Linear(hidden_size,num_classes)

def forward(self,x):
    x = self.l1(x)
    x = F.tanh(x)
    x = self.l2(x)
    x = F.softmax(x)
    return x

mlp = Net(input_size,hidden_size,num_classes)

criterion = nn.NLLLoss()
optimizer = torch.optim.SGD(mlp.parameters(), lr=learning_rate)

for i in range(N-1,len(word_list)):

  # Define input vector x
  x = None

  for j in range(0,N-1):
      try:
          x = np.r_[x,w2v[word_list[j]]]
      except:
          x = w2v[word_list[j-N+1]]

  # Done with defining  x

  np.reshape(x,(len(x),1))
  x = autograd.Variable(torch.FloatTensor(x))
  optimizer.zero_grad()

  outputs = mlp(x)
  outputs = outputs.unsqueeze(0)
  outputs = outputs.transpose(0,1)

  index = w2i[word_list[i]]
  one_hot_target = np.zeros([num_classes,1],dtype=float)

  one_hot_target[index] = float(1)

  one_hot_target = autograd.Variable(torch.Tensor(one_hot_target))
  print (one_hot_target)
  loss = criterion(outputs,one_hot_target)
  #loss.backward()
  #optimizer.step()


title: How to use PNASNet5 as encoder in Unet in pytorch

body:
I want use PNASNet5Large as encoder for my Unet here is my wrong aproach for the PNASNet5Large but working for resnet:

class UNetResNet(nn.Module):
def __init__(self, encoder_depth, num_classes, num_filters=32, dropout_2d=0.2,
                 pretrained=False, is_deconv=False):
        super().__init__()
        self.num_classes = num_classes
        self.dropout_2d = dropout_2d

        if encoder_depth == 34:
            self.encoder = torchvision.models.resnet34(pretrained=pretrained)
            bottom_channel_nr = 512
        elif encoder_depth == 101:
            self.encoder = torchvision.models.resnet101(pretrained=pretrained)
            bottom_channel_nr = 2048
        elif encoder_depth == 152: #this works
            self.encoder = torchvision.models.resnet152(pretrained=pretrained)
            bottom_channel_nr = 2048
        elif encoder_depth == 777: #coded version for the pnasnet
            self.encoder = PNASNet5Large()
            bottom_channel_nr = 4320 #this unknown for me as well

        self.pool = nn.MaxPool2d(2, 2)
        self.relu = nn.ReLU(inplace=True)
        self.conv1 = nn.Sequential(self.encoder.conv1,
                                   self.encoder.bn1,
                                   self.encoder.relu,
                                   self.pool)

        self.conv2 = self.encoder.layer1 #PNASNet5Large doesn't have such layers
        self.conv3 = self.encoder.layer2
        self.conv4 = self.encoder.layer3
        self.conv5 = self.encoder.layer4
        self.center = DecoderCenter(bottom_channel_nr, num_filters * 8 *2, num_filters * 8, False)

        self.dec5 =  DecoderBlock(bottom_channel_nr + num_filters * 8, num_filters * 8 * 2, num_filters * 8,   is_deconv)
        self.dec4 = DecoderBlock(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8, is_deconv)
        self.dec3 = DecoderBlock(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2, is_deconv)
        self.dec2 = DecoderBlock(bottom_channel_nr // 8 + num_filters * 2, num_filters * 2 * 2, num_filters * 2 * 2,
                                   is_deconv)
        self.dec1 = DecoderBlock(num_filters * 2 * 2, num_filters * 2 * 2, num_filters, is_deconv)
        self.dec0 = ConvRelu(num_filters, num_filters)
        self.final = nn.Conv2d(num_filters, num_classes, kernel_size=1)

    def forward(self, x):
        conv1 = self.conv1(x)
        conv2 = self.conv2(conv1)
        conv3 = self.conv3(conv2)
        conv4 = self.conv4(conv3)
        conv5 = self.conv5(conv4)
        center = self.center(conv5)
        dec5 = self.dec5(torch.cat([center, conv5], 1))
        dec4 = self.dec4(torch.cat([dec5, conv4], 1))
        dec3 = self.dec3(torch.cat([dec4, conv3], 1))
        dec2 = self.dec2(torch.cat([dec3, conv2], 1))
        dec1 = self.dec1(dec2)
        dec0 = self.dec0(dec1)
        return self.final(F.dropout2d(dec0, p=self.dropout_2d))

1) How to get how many bottom channels pnasnet has. It ends up following way:

...
 self.cell_11 = Cell(in_channels_left=4320, out_channels_left=864,
                            in_channels_right=4320, out_channels_right=864)
        self.relu = nn.ReLU()
        self.avg_pool = nn.AvgPool2d(11, stride=1, padding=0)
        self.dropout = nn.Dropout(0.5)
        self.last_linear = nn.Linear(4320, num_classes)

Is 4320 the answer or not, in_channels_left and out_channels_left - something new for me

2) Resnet has somekind of 4 big layers which I use and encoders in my Unet arch, how get similar layer from pnasnet

I'm using pytorch 3.1 and this is the link to the Pnasnet directory

3) AttributeError: 'PNASNet5Large' object has no attribute 'conv1' - so doesn't have conv1 as well

UPD: tried smth like this but failed

class UNetPNASNet(nn.Module):
    def init(self, encoder_depth,  num_classes, num_filters=32, dropout_2d=0.2,
                     pretrained=False, is_deconv=False):
            super().init()
            self.num_classes = num_classes
            self.dropout_2d = dropout_2d
            self.encoder = PNASNet5Large()
            bottom_channel_nr = 4320
            self.center = DecoderCenter(bottom_channel_nr, num_filters * 8 *2, num_filters * 8, False)

        self.dec5  =  DecoderBlockV2(bottom_channel_nr + num_filters * 8, num_filters * 8 * 2, num_filters * 8,   is_deconv)
        self.dec4  = DecoderBlockV2(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8, is_deconv)
        self.dec3  = DecoderBlockV2(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2, is_deconv)
        self.dec2  = DecoderBlockV2(num_filters * 4 * 4, num_filters * 4 * 4, num_filters, is_deconv)
        self.dec1  = DecoderBlockV2(num_filters * 2 * 2, num_filters * 2 * 2, num_filters, is_deconv)
        self.dec0  = ConvRelu(num_filters, num_filters)
        self.final = nn.Conv2d(num_filters, num_classes, kernel_size=1)

def forward(self, x):
        features = self.encoder.features(x)
        relued_features = self.encoder.relu(features)
        avg_pooled_features = self.encoder.avg_pool(relued_features)
        center = self.center(avg_pooled_features)
        dec5 = self.dec5(torch.cat([center, avg_pooled_features], 1))
        dec4 = self.dec4(torch.cat([dec5, relued_features], 1))
        dec3 = self.dec3(torch.cat([dec4, features], 1))
        dec2 = self.dec2(dec3)
        dec1 = self.dec1(dec2)
        dec0 = self.dec0(dec1)
        return self.final(F.dropout2d(dec0, p=self.dropout_2d))

RuntimeError: Given input size: (4320x4x4). Calculated output size: (4320x-6x-6). Output size is too small at /opt/conda/conda-bld/pytorch_1525796793591/work/torch/lib/THCUNN/generic/SpatialAveragePooling.cu:63


title: usage about rnn.pack_padded_sequence

body:
When I use torch.nn.utils.rnn.pack_padded_sequence(), an error occurred.
Here is my code:

import torch
import numpy as np
x = torch.from_numpy(np.array([[1,2,3,4,5,6,0,0],[6,7,8,9,0,0,0,0],[12,83,84,0,0,0,0,0]]))
length =[6,4,3]
print(torch.nn.utils.rnn.pack_padded_sequence(input=x, lengths=length, batch_first=True))

The error is as follows：

Traceback (most recent call last):
  File "/home/pc/PycharmProjects/padded/padded.py", line 112, in <module>
    print(torch.nn.utils.rnn.pack_padded_sequence(input=x, lengths=length, batch_first=True))
  File "/home/pc/anaconda3/lib/python3.6/site-packages/torch/onnx/__init__.py", line 67, in wrapper
    if not might_trace(args):
  File "/home/pc/anaconda3/lib/python3.6/site-packages/torch/onnx/__init__.py", line 141, in might_trace
    first_arg = args[0]
IndexError: tuple index out of range

But when I enter the following code:

import torch
import numpy as np
x = torch.from_numpy(np.array([[1,2,3,4,5,6,0,0],[6,7,8,9,0,0,0,0],[12,83,84,0,0,0,0,0]]))
length =[6,4,3]
print(torch.nn.utils.rnn.pack_padded_sequence(x, lengths=length, batch_first=True))

The result is normal. And I don't know why.

Can you help me solve the problem?


title: pytorch CUDA out of memory when training on big image

body:
When I train a model with 512*512 images with batch_size=4, it works,
but when I train it with 1024*1024 images with batch_size=1, the cuda out of memory error happens.

I am confused, the total number of pixels are equal. Why one 1024*1024 image uses more memory than four 512*512 images?


title: No performance improvement using quantization model in pytorch

body:
I have trained a model in pytorch with float data type. I want to improve my inference time by converting this model to quantized model. I have used torch.quantization.convert api to convert my model's weight to uint8 data type. However, when I use this model for inference, I do not get any performance improvement. Am I doing something wrong here ?

The Unet Model code:

def gen_initialization(m):
    if type(m) == nn.Conv2d:
        sh = m.weight.shape
        nn.init.normal_(m.weight, std=math.sqrt(2.0 / (sh[0]*sh[2]*sh[3])))
        nn.init.constant_(m.bias, 0)
    elif type(m) == nn.BatchNorm2d:
        nn.init.constant_(m.weight, 1)
        nn.init.constant_(m.bias, 0)

class TripleConv(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(TripleConv, self).__init__()
        mid_ch = (in_ch + out_ch) // 2
        self.conv = nn.Sequential(
            nn.Conv2d(in_ch, mid_ch, kernel_size=3, stride=1, padding=1, bias=True),
            nn.BatchNorm2d(num_features=mid_ch),
            nn.LeakyReLU(negative_slope=0.1),
            nn.Conv2d(mid_ch, mid_ch, kernel_size=3, stride=1, padding=1, bias=True),
            nn.BatchNorm2d(num_features=mid_ch),
            nn.LeakyReLU(negative_slope=0.1),
            nn.Conv2d(mid_ch, out_ch, kernel_size=3, stride=1, padding=1, bias=True),
            nn.BatchNorm2d(num_features=out_ch),
            nn.LeakyReLU(negative_slope=0.1)
        )
        self.conv.apply(gen_initialization)

    def forward(self, x):
        return self.conv(x)

class Down(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(Down, self).__init__()
        self.triple_conv = TripleConv(in_ch, out_ch)
        self.avg_pool_conv = nn.AvgPool2d(2, 2)
        self.in_ch = in_ch
        self.out_ch = out_ch

    def forward(self, x):
        self.cache = self.triple_conv(x)
        pad = torch.zeros(x.shape[0], self.out_ch - self.in_ch, x.shape[2], x.shape[3], device=x.device)
        x = torch.cat((x, pad), dim=1)
        self.cache += x
        return self.avg_pool_conv(self.cache)

class Center(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(Center, self).__init__()

        self.conv = nn.Sequential(
            nn.Conv2d(in_ch, out_ch, kernel_size=3, stride=1, padding=1, bias=True),
            nn.BatchNorm2d(num_features=out_ch),
            nn.LeakyReLU(negative_slope=0.1, inplace=True)
        )
        self.conv.apply(gen_initialization)

    def forward(self, x):
        return self.conv(x)

class Up(nn.Module):
    def __init__(self, in_ch, out_ch):
        super(Up, self).__init__()
        self.upsample = nn.Upsample(scale_factor=2, mode='bilinear',
                                    align_corners=True)
        self.triple_conv = TripleConv(in_ch, out_ch)

    def forward(self, x, cache):
        x = self.upsample(x)
        x = torch.cat((x, cache), dim=1)
        x = self.triple_conv(x)
        return x

class UNet(nn.Module):
    def __init__(self, in_ch, first_ch=None):
        super(UNet, self).__init__()

        if not first_ch:
            first_ch = 32

        self.down1 = Down(in_ch, first_ch)
        self.down2 = Down(first_ch, first_ch*2)
        self.down3 = Down(first_ch*2, first_ch*4)
        self.down4 = Down(first_ch*4, first_ch*8)
        self.center = Center(first_ch*8, first_ch*8)
        self.up4 = Up(first_ch*8*2, first_ch*4)
        self.up3 = Up(first_ch*4*2, first_ch*2)
        self.up2 = Up(first_ch*2*2, first_ch)
        self.up1 = Up(first_ch*2, first_ch)
        self.output = nn.Conv2d(first_ch, in_ch, kernel_size=3, stride=1,
                                padding=1, bias=True)
        self.output.apply(gen_initialization)

    def forward(self, x):
        x = self.down1(x)
        x = self.down2(x)
        x = self.down3(x)
        x = self.down4(x)
        x = self.center(x)
        x = self.up4(x, self.down4.cache)
        x = self.up3(x, self.down3.cache)
        x = self.up2(x, self.down2.cache)
        x = self.up1(x, self.down1.cache)
        return self.output(x)

The inference code:

from tqdm import tqdm
import os
import numpy as np
import torch
import gan_network
import torch.nn.parallel
from torch.utils.data import DataLoader
import torch.utils.data as data
import random
import glob
import scipy.io
import time
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"   # see issue #152
os.environ["CUDA_VISIBLE_DEVICES"]="0"

class DataFolder(data.Dataset):
    def __init__(self, file):
        super(DataFolder, self).__init__()
        self.image_names = []
        fid = file
        for line in fid:
            # line = line[:-1]
            if line == '':
                continue
            # print(line)
            self.image_names.append(line)
        random.shuffle(self.image_names)
        self.image_names = self.image_names[0:]

    def __len__(self):
        return len(self.image_names)

    def __getitem__(self, index):
        path = self.image_names[index]
        img = np.load(path)
        img = np.rollaxis(img, 2, 0)
        img = torch.from_numpy(img[:, :, :])
        return img, path

if __name__ == '__main__':
    batch_size = 1
    image_size = 2048
    channels = 6
    model_path = 'D:/WorkProjects/Network_Training_Aqusens/FullFovReconst/network/network_epoch9.pth'
    test_data = glob.glob('D:/save/temp/*.npy')
    dest_dir = 'D:/save/temp/results/'

    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    net = gan_network.UNet(6, 32)
    if torch.cuda.device_count() > 1:
        net = torch.nn.DataParallel(net)
    net.to(device)
    net.load_state_dict(torch.load(model_path))
    quantized_model = torch.quantization.convert(net, {torch.nn.Conv2d, torch.nn.BatchNorm2d}, inplace=False)

    dataset = DataFolder(file=test_data)
    print(f'{len(dataset)}')
    data_loader = DataLoader(dataset=dataset, num_workers=4,
                             batch_size=batch_size, shuffle=False,
                             drop_last=False, pin_memory=True)
    input = torch.Tensor(batch_size, channels, image_size, image_size).to(device)

    t0 = time.time()
    with torch.no_grad():
        for i, batch in enumerate(tqdm(data_loader)):
            input.copy_(batch[0])
            output = net(input).cpu().clone().numpy()
            np.array(output)
            output = np.rollaxis(output, 1, 4)
            for num in range(batch_size):
                arr = output[num, :, :, :]
                file_name = os.path.basename(batch[1][num])
                save_name = os.path.join(dest_dir, file_name)
                save_name = save_name.replace(".npy", "")
                scipy.io.savemat(save_name+'.mat', {'output': arr})
    t1 = time.time()
    print(f'Elapsed time = {t1-t0}')

For models net and quantized model, i get the elapsed time around 30 seconds for 12 images passed through them.


title: Why TFBertForTokenClassification.from_pretrained("allenai/scibert_scivocab_uncased",from_pt=True,config=config) gives weights warning?

body:
I want to do Named Entity Recognition on scientific articles.
tokenizer = AutoTokenizer.from_pretrained('allenai/scibert_scivocab_cased')
model=TFBertForTokenClassification.from_pretrained("allenai/scibert_scivocab_uncased",from_pt=True,config=config)

But it gives following warning.

Some weights of the PyTorch model were not used when initializing the
TF 2.0 model TFBertForTokenClassification: ['classifier.weight',
'classifier.bias'] - This IS expected if you are initializing
TFBertForTokenClassification from a TF 2.0 model trained on another
task or with another architecture (e.g. initializing a
BertForSequenceClassification model from a TFBertForPretraining
model). - This IS NOT expected if you are initializing
TFBertForTokenClassification from a TF 2.0 model that you expect to be
exactly identical (initializing a BertForSequenceClassification model
from a TFBertForSequenceClassification model). Some weights or buffers
of the PyTorch model TFBertForTokenClassification were not initialized
from the TF 2.0 model and are newly initialized:
['cls.predictions.transform.dense.weight',
'cls.predictions.decoder.bias',
'cls.predictions.transform.LayerNorm.weight',
'cls.predictions.transform.LayerNorm.bias',
'cls.predictions.decoder.weight', 'cls.seq_relationship.weight',
'cls.predictions.bias', 'cls.predictions.transform.dense.bias',
'cls.seq_relationship.bias'] You should probably TRAIN this model on a
down-stream task to be able to use it for predictions and inference.

And when I try to fit the model, it gives follwing warning.

WARNING:tensorflow:Gradients do not exist for variables
['tf_bert_for_token_classification/bert/pooler/dense/kernel:0',
'tf_bert_for_token_classification/bert/pooler/dense/bias:0'] when
minimizing the loss.

Does anyone know why it is giving these warnings?