title: Pytorch: Variable data has to be a tensor, but got Variable

body:
i am stuck and confused... I am using the same code at 2 places one is my personal computer using Pycharm and other on my university provided server (GPU is bigger). And code is working fine and training data and producing result on my computer which has (8 gigs of GPU) while the same code is not working on the server (i am using "apollo") and giving the following error:

yeosiz@apollo:~/YAN/color$ python3 main.py
<IPython.core.display.Image object>
Traceback (most recent call last):
  File "main.py", line 133, in <module>
    model = ColorizationNet()
  File "main.py", line 55, in __init__
    resnet.conv1.weight = nn.Parameter(resnet.conv1.weight.sum(dim=1).unsqueeze(1))
  File "/data/yeosiz/anaconda3/lib/python3.6/site-packages/torch/nn/parameter.py", line 25, in __new__
    return super(Parameter, cls).__new__(cls, data, requires_grad=requires_grad)
RuntimeError: Variable data has to be a tensor, but got Variable

Error block of code 1:

class ColorizationNet(nn.Module):
  def __init__(self, input_size=128):
    super(ColorizationNet, self).__init__()
    MIDLEVEL_FEATURE_SIZE = 128

    ## First half: ResNet
    resnet = models.resnet18(num_classes=365)
    # Change first conv layer to accept single-channel (grayscale) input
    resnet.conv1.weight = nn.Parameter(resnet.conv1.weight.sum(dim=1).unsqueeze(1))
    # Extract midlevel features from ResNet-gray
    self.midlevel_resnet = nn.Sequential(*list(resnet.children())[0:6])

    ## Second half: Upsampling
    self.upsample = nn.Sequential(
      nn.Conv2d(MIDLEVEL_FEATURE_SIZE, 128, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(128),
      nn.ReLU(),
      nn.Upsample(scale_factor=2),
      nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(64),
      nn.ReLU(),
      nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(64),
      nn.ReLU(),
      nn.Upsample(scale_factor=2),
      nn.Conv2d(64, 32, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(32),
      nn.ReLU(),
      nn.Conv2d(32, 2, kernel_size=3, stride=1, padding=1),
      nn.Upsample(scale_factor=2)
    )

  def forward(self, input):

    # Pass input through ResNet-gray to extract features
    midlevel_features = self.midlevel_resnet(input)

    # Upsample to get colors
    output = self.upsample(midlevel_features)
    return output

and

model = ColorizationNet()

I tried to use out.data.numpy() but did not work !!

This is the code:

  # Move data into training and validation directories
import os

import matplotlib
from IPython.display import Image, display

# For plotting
import numpy as np
import matplotlib.pyplot as plt
#%matplotlib inline

# For conversion
from skimage.color import lab2rgb, rgb2lab, rgb2gray
from skimage import io

# For everything
import torch
import torch.nn as nn
import torch.nn.functional as F

# For our model
import torchvision.models as models
from torchvision import datasets, transforms
# For utilities

import os, shutil, time

os.makedirs('images/train/class/', exist_ok=True)  # 40,000 images
os.makedirs('images/val/class/', exist_ok=True)  # 1,000 images
for i, file in enumerate(os.listdir('testSet_resize')):
    if i < 1000:  # first 1000 will be val
        os.rename('testSet_resize/' + file, 'images/val/class/' + file)
    else:  # others will be val
        os.rename('testSet_resize/' + file, 'images/train/class/' + file)

        # Make sure the images are there

display(Image(filename='images/val/class/0b4803802caabfdc98dfe1ba22298848.jpg'))

# Check if GPU is available
use_gpu = torch.cuda.is_available()

class ColorizationNet(nn.Module):
  def __init__(self, input_size=128):
    super(ColorizationNet, self).__init__()
    MIDLEVEL_FEATURE_SIZE = 128

    ## First half: ResNet
    resnet = models.resnet18(num_classes=365)
    # Change first conv layer to accept single-channel (grayscale) input
    resnet.conv1.weight = nn.Parameter(resnet.conv1.weight.sum(dim=1).unsqueeze(1))
    # Extract midlevel features from ResNet-gray
    self.midlevel_resnet = nn.Sequential(*list(resnet.children())[0:6])

    ## Second half: Upsampling
    self.upsample = nn.Sequential(
      nn.Conv2d(MIDLEVEL_FEATURE_SIZE, 128, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(128),
      nn.ReLU(),
      nn.Upsample(scale_factor=2),
      nn.Conv2d(128, 64, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(64),
      nn.ReLU(),
      nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(64),
      nn.ReLU(),
      nn.Upsample(scale_factor=2),
      nn.Conv2d(64, 32, kernel_size=3, stride=1, padding=1),
      nn.BatchNorm2d(32),
      nn.ReLU(),
      nn.Conv2d(32, 2, kernel_size=3, stride=1, padding=1),
      nn.Upsample(scale_factor=2)
    )

  def forward(self, input):

    # Pass input through ResNet-gray to extract features
    midlevel_features = self.midlevel_resnet(input)

    # Upsample to get colors
    output = self.upsample(midlevel_features)
    return output
class GrayscaleImageFolder(datasets.ImageFolder):
  '''Custom images folder, which converts images to grayscale before loading'''
  def __getitem__(self, index):
    path, target = self.imgs[index]
    img = self.loader(path)
    if self.transform is not None:
      img_original = self.transform(img)
      img_original = np.asarray(img_original)
      img_lab = rgb2lab(img_original)
      img_lab = (img_lab + 128) / 255
      img_ab = img_lab[:, :, 1:3]
      img_ab = torch.from_numpy(img_ab.transpose((2, 0, 1))).float()
      img_original = rgb2gray(img_original)
      img_original = torch.from_numpy(img_original).unsqueeze(0).float()
    if self.target_transform is not None:
      target = self.target_transform(target)
    return img_original, img_ab, target

class AverageMeter(object):
  '''A handy class from the PyTorch ImageNet tutorial'''
  def __init__(self):
    self.reset()
  def reset(self):
    self.val, self.avg, self.sum, self.count = 0, 0, 0, 0
  def update(self, val, n=1):
    self.val = val
    self.sum += val * n
    self.count += n
    self.avg = self.sum / self.count

def to_rgb(grayscale_input, ab_input, save_path=None, save_name=None):
  '''Show/save rgb image from grayscale and ab channels
     Input save_path in the form {'grayscale': '/path/', 'colorized': '/path/'}'''
  plt.clf() # clear matplotlib
  color_image = torch.cat((grayscale_input, ab_input), 0).numpy() # combine channels
  color_image = color_image.transpose((1, 2, 0))  # rescale for matplotlib
  color_image[:, :, 0:1] = color_image[:, :, 0:1] * 100
  color_image[:, :, 1:3] = color_image[:, :, 1:3] * 255 - 128
  color_image = lab2rgb(color_image.astype(np.float64))
  grayscale_input = grayscale_input.squeeze().numpy()
  if save_path is not None and save_name is not None:
    plt.imsave(arr=grayscale_input, fname='{}{}'.format(save_path['grayscale'], save_name), cmap='gray')
    plt.imsave(arr=color_image, fname='{}{}'.format(save_path['colorized'], save_name))

model = ColorizationNet()
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-2, weight_decay=0.0)

# Training
train_transforms = transforms.Compose([transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip()])
train_imagefolder = GrayscaleImageFolder('images/train', train_transforms)
train_loader = torch.utils.data.DataLoader(train_imagefolder, batch_size=64, shuffle=True)

# Validation
val_transforms = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224)])
val_imagefolder = GrayscaleImageFolder('images/val' , val_transforms)
val_loader = torch.utils.data.DataLoader(val_imagefolder, batch_size=64, shuffle=False)

def validate(val_loader, model, criterion, save_images, epoch):
  model.eval()

  # Prepare value counters and timers
  batch_time, data_time, losses = AverageMeter(), AverageMeter(), AverageMeter()

  end = time.time()
  already_saved_images = False
  for i, (input_gray, input_ab, target) in enumerate(val_loader):
    data_time.update(time.time() - end)

    # Use GPU
    if use_gpu: input_gray, input_ab, target = input_gray.cuda(), input_ab.cuda(), target.cuda()

    # Run model and record loss
    output_ab = model(input_gray) # throw away class predictions
    loss = criterion(output_ab, input_ab)
    losses.update(loss.item(), input_gray.size(0))

    # Save images to file
    if save_images and not already_saved_images:
      already_saved_images = True
      for j in range(min(len(output_ab), 10)): # save at most 5 images
        save_path = {'grayscale': 'outputs/gray/', 'colorized': 'outputs/color/'}
        save_name = 'img-{}-epoch-{}.jpg'.format(i * val_loader.batch_size + j, epoch)
        to_rgb(input_gray[j].cpu(), ab_input=output_ab[j].detach().cpu(), save_path=save_path, save_name=save_name)

    # Record time to do forward passes and save images
    batch_time.update(time.time() - end)
    end = time.time()

    # Print model accuracy -- in the code below, val refers to both value and validation
    if i % 25 == 0:
      print('Validate: [{0}/{1}]\t'
            'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
            'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
             i, len(val_loader), batch_time=batch_time, loss=losses))

  print('Finished validation.')
  return losses.avg

def train(train_loader, model, criterion, optimizer, epoch):
    print('Starting training epoch {}'.format(epoch))
    model.train()

    # Prepare value counters and timers
    batch_time, data_time, losses = AverageMeter(), AverageMeter(), AverageMeter()

    end = time.time()
    for i, (input_gray, input_ab, target) in enumerate(train_loader):

        # Use GPU if available
        if use_gpu: input_gray, input_ab, target = input_gray.cuda(), input_ab.cuda(), target.cuda()

        # Record time to load data (above)
        data_time.update(time.time() - end)

        # Run forward pass
        output_ab = model(input_gray)
        loss = criterion(output_ab, input_ab)
        losses.update(loss.item(), input_gray.size(0))

        # Compute gradient and optimize
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # Record time to do forward and backward passes
        batch_time.update(time.time() - end)
        end = time.time()

        # Print model accuracy -- in the code below, val refers to value, not validation
        if i % 25 == 0:
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'.format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses))

    print('Finished training epoch {}'.format(epoch))

# Move model and loss function to GPU
if use_gpu:
  criterion = criterion.cuda()
  model = model.cuda()

# Make folders and set parameters
os.makedirs('outputs/color', exist_ok=True)
os.makedirs('outputs/gray', exist_ok=True)
os.makedirs('checkpoints', exist_ok=True)
save_images = True
best_losses = 1e10
epochs = 100

# Train model
for epoch in range(epochs):
  # Train for one epoch, then validate
  train(train_loader, model, criterion, optimizer, epoch)
  with torch.no_grad():
    losses = validate(val_loader, model, criterion, save_images, epoch)
  # Save checkpoint and replace old best model if current model is better
  if losses < best_losses:
    best_losses = losses
    torch.save(model.state_dict(), 'checkpoints/model-epoch-{}-losses-{:.3f}.pth'.format(epoch+1,losses))

# Show images
import matplotlib.image as mpimg
image_pairs = [('outputs/color/img-2-epoch-0.jpg', 'outputs/gray/img-2-epoch-0.jpg'),
               ('outputs/color/img-7-epoch-0.jpg', 'outputs/gray/img-7-epoch-0.jpg')]
for c, g in image_pairs:
  color = mpimg.imread(c)
  gray  = mpimg.imread(g)
  f, axarr = plt.subplots(1, 2)
  f.set_size_inches(15, 15)
  axarr[0].imshow(gray, cmap='gray')
  axarr[1].imshow(color)
  axarr[0].axis('off'), axarr[1].axis('off')
  plt.show()

So I do not know how to solve this or how to convert that variable to tensor !! and why it is working on my pc?  Thanks alot everyone!!


title: pytorch DataLoader very slow when dataset is big and shuffle is True

body:
I have a big dataset with lots of images and I found that the speed of dataloader is very slow.
I did many tests and found when the number of images is big:

Direct read the dataset is fast
Set shuffle = False with num_workers=0, also fast (1.1 times slower than 1st one)
Set shuffle = False with num_workers=8, becomes slow (2.8 times slower than 1st one)
Set shuffle = True with num_workers=0, slower (7.2 times slower than 1st one)
Set shuffle = True with num_workers=8,(9.2 times slower than 1st one)

Here is the sample code:
class MyTestDataset(torchvision.datasets.vision.VisionDataset):
    def __init__(self,
                 transform=None,
                 target_transform=None):
        super(MyTestDataset, self).__init__(None, transform=transform,
                                          target_transform=target_transform)

    def __getitem__(self, index):
        time.sleep(0.001)
        return np.random.rand(256, 256),random.randint(0, 300)

    def __len__(self):
        return 17250000

dataset = MyTestDataset()

since = time.time()
imgs = []
for idx in range(128):
    img,label = dataset[10000+idx*1000]
    imgs.append(img)
base = time.time() - since
print("next dataset",base)

loader = torch.utils.data.DataLoader(dataset, batch_size=128, shuffle=False, num_workers=0)
since = time.time()
imgs = next(iter(loader))
cost = time.time() - since
print("loader no shuffle num_workers-0",cost,cost/base)

loader = torch.utils.data.DataLoader(dataset, batch_size=128, shuffle=False, num_workers=8)
since = time.time()
imgs = next(iter(loader))
cost = time.time() - since
print("loader no shuffle num_workers-8",cost,cost/base)

loader = torch.utils.data.DataLoader(dataset, batch_size=128, shuffle=True, num_workers=0)
since = time.time()
imgs = next(iter(loader))
cost = time.time() - since
print("loader shuffle num_workers-0",cost,cost/base)

dataset = MyTestDataset()
loader = torch.utils.data.DataLoader(dataset, batch_size=128, shuffle=True, num_workers=8)
since = time.time()
imgs = next(iter(loader))
cost = time.time() - since
print("loader shuffle num_workers-8",cost,cost/base)

and below is the output
next dataset 0.2839939594268799
loader no shuffle num_workers-0 0.3133578300476074 1.103396109832709
loader no shuffle num_workers-8 0.811976432800293 2.859132759157693
loader shuffle num_workers-0 2.041795253753662 7.189572827091643
loader shuffle num_workers-8 2.617912769317627 9.218198776483705

Seems shuffle make it 9 times slow?


title: 4d Input Tensor vs 1d Input Tensor (aka vector) to a neural network

body:
Reading about machine learning, I keep seeing references to the "input vector" or "feature vector", a 1d tensor that holds the input to the neural network. So for example a 28x28 grayscale image would be a 784 dimensional vector.

Then I also keep seeing references to images being a 4 dimensional tensor with the dimensions being number in batch, color channel, height, and width. For example, this is how it's described in "Deep Learning with Python, by Francois Chollet".

I'm wondering, why is it described in the different ways? When would one be used vs the other?


title: Pass pretrained weights in CNN Pytorch to a CNN in Tensorflow

body:
I have trained this network in Pytorch for 224x224 size images and 4 classes.

class CustomConvNet(nn.Module):
    def __init__(self, num_classes):
        super(CustomConvNet, self).__init__()

        self.layer1 = self.conv_module(3, 64)
        self.layer2 = self.conv_module(64, 128)
        self.layer3 = self.conv_module(128, 256)
        self.layer4 = self.conv_module(256, 256)
        self.layer5 = self.conv_module(256, 512)
        self.gap = self.global_avg_pool(512, num_classes)
        #self.linear = nn.Linear(512, num_classes)
        #self.relu = nn.ReLU()
        #self.softmax = nn.Softmax()

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = self.layer5(out)
        out = self.gap(out)
        out = out.view(-1, 4)
        #out = self.linear(out)

        return out

    def conv_module(self, in_num, out_num):
        return nn.Sequential(
            nn.Conv2d(in_num, out_num, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2), stride=None))

    def global_avg_pool(self, in_num, out_num):
        return nn.Sequential(
            nn.Conv2d(in_num, out_num, kernel_size=3, stride=1, padding=1),
            #nn.BatchNorm2d(out_num),
            #nn.LeakyReLU(),

            nn.ReLU(),
            nn.Softmax(),
            nn.AdaptiveAvgPool2d((1, 1)))

I got the weights from the first Conv2D and it's size torch.Size([64, 3, 3, 3])

I have saved it as:

weightsCNN = net.layer1[0].weight.data
np.save('CNNweights.npy', weightsCNN)

This is my model I built in Tensorflow. I would like to pass those weights I saved from the Pytorch model into this Tensorflow CNN.

    model = models.Sequential()
    model.add(layers.Conv2D(64, (3, 3), activation='relu', input_shape=(224, 224, 3)))
    model.add(layers.MaxPooling2D((2, 2)))

    model.add(layers.Conv2D(128, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))

    model.add(layers.Conv2D(256, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))

    model.add(layers.Conv2D(256, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))

    model.add(layers.Conv2D(512, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D((2, 2)))

    model.add(layers.Conv2D(512, (3, 3), activation='relu'))

    model.add(layers.GlobalAveragePooling2D())
    model.add(layers.Dense(4, activation='softmax'))
    print(model.summary())

    adam = optimizers.Adam(learning_rate=0.0001, amsgrad=False)
    model.compile(loss='categorical_crossentropy',
                  optimizer=adam,
                  metrics=['accuracy'])

    nb_train_samples = 6596
    nb_validation_samples = 1290
    epochs = 10
    batch_size = 256

    history = model.fit_generator(
        train_generator,
        steps_per_epoch=np.ceil(nb_train_samples/batch_size),
        epochs=epochs,
        validation_data=validation_generator,
        validation_steps=np.ceil(nb_validation_samples / batch_size)
        )

How should I actually do that? What shape of weights does Tensorflow require? Thanks!


title: Preferred way to decrease learning rate for Adam optimiser in PyTorch

body:
I have been seeing code that uses an Adam optimizer . And the way they decrease the learning rate is as follows:

    optimizer = torch.optim.Adam(net.parameters(),lr=0.01)
    (training...
    optimizer.step()...)

    if iteration >= some_threshold:
        for param_group in optimizer.param_groups:
            param_group['lr'] = 0.001

I thought we have the same learning rate for all parameters. So why then iterate over the param_groups and individually set learning rate for each parameter?

Wouldn't the following be faster and have an identical effect?

    optimizer = torch.optim.Adam(net.parameters(),lr=0.01)
    scheduler = MultiStepLR(optimizer, milestones=[some_threshold], gamma=0.1)
    (training...
    optimizer.step()
    scheduler.step())

Thank you


title: Pytorch: randomly subsample a vector

body:
I need to randomly subsample a vector in pytorch.
The equivalent in Matlab would be something like

sample_size = 5
a = rand(10,1)
idx = randperm(10)
b = a(idx(1:sample_size))

Are there similar functions for pytorch?

I'm trying to randomly subsample my prediction and target vector for computing the loss.


title: Why Skorch show NAN in the every epoch?

body:
I want to create my own dataset class based on Dataset class of Skorch because I want to differentiate categorical columns and continuous columns. These categorical columns will be passed through the embedding layers in the model. The result is weird because it show NAN
like this:

  epoch    train_loss    valid_loss     dur
-------  ------------  ------------  ------
      1           nan           nan  0.2187
      2           nan           nan  0.1719
      3           nan           nan  0.1719
      4           nan           nan  0.1562
      5           nan           nan  0.1406

Can you help me fix it ? I am using data from this kaggle:
Here

from skorch import NeuralNetRegressor
from skorch.dataset import Dataset
import torch
import torch.nn as nn
import torch.nn.functional as F
import pandas as pd
import numpy as np
from sklearn.preprocessing import LabelEncoder

class TabularDataset(Dataset):
    def __init__(self, data, cat_cols=None, output_col=None):
        self.n = data.shape[0]

        if output_col:
            self.y = data[output_col].astype(np.float32).values.reshape(-1, 1)
        else:
            self.y = np.zeros((self.n, 1))

        self.cat_cols = cat_cols if cat_cols else []
        self.cont_cols = [col for col in data.columns
                          if col not in self.cat_cols + [output_col]]

        if self.cont_cols:
            self.cont_X = data[self.cont_cols].astype(np.float32).values
        else:
            self.cont_X = np.zeros((self.n, 1))

        if self.cat_cols:
            self.cat_X = data[self.cat_cols].astype(np.int64).values
        else:
            self.cat_X = np.zeros((self.n, 1))

    def __len__(self):
        # Denotes the total number of sampoes
        return self.n

    def __getitem__(self, idx):
        # generates one sample of data
        return [self.cont_X[idx], self.cat_X[idx]], self.y[idx]

class FeedForwardNN(nn.Module):

    def __init__(self, emb_dims, no_of_cont, lin_layer_sizes,
                 output_size, emb_dropout, lin_layer_dropouts):

        """
        Parameters
        ----------
        emb_dims: List of two element tuples
          This list will contain a two element tuple for each
          categorical feature. The first element of a tuple will
          denote the number of unique values of the categorical
          feature. The second element will denote the embedding
          dimension to be used for that feature.
        no_of_cont: Integer
          The number of continuous features in the data.
        lin_layer_sizes: List of integers.
          The size of each linear layer. The length will be equal
          to the total number
          of linear layers in the network.
        output_size: Integer
          The size of the final output.
        emb_dropout: Float
          The dropout to be used after the embedding layers.
        lin_layer_dropouts: List of floats
          The dropouts to be used after each linear layer.
        """

        super().__init__()

        # Embedding layers
        self.emb_layers = nn.ModuleList([nn.Embedding(x, y)
                                         for x, y in emb_dims])

        no_of_embs = sum([y for x, y in emb_dims])
        self.no_of_embs = no_of_embs
        self.no_of_cont = no_of_cont

        # Linear Layers
        first_lin_layer = nn.Linear(self.no_of_embs + self.no_of_cont,
                                    lin_layer_sizes[0])

        self.lin_layers = \
            nn.ModuleList([first_lin_layer] + \
                          [nn.Linear(lin_layer_sizes[i], lin_layer_sizes[i + 1])
                           for i in range(len(lin_layer_sizes) - 1)])

        for lin_layer in self.lin_layers:
            nn.init.kaiming_normal_(lin_layer.weight.data)

        # Output Layer
        self.output_layer = nn.Linear(lin_layer_sizes[-1],
                                      output_size)
        nn.init.kaiming_normal_(self.output_layer.weight.data)

        # Batch Norm Layers
        self.first_bn_layer = nn.BatchNorm1d(self.no_of_cont)
        self.bn_layers = nn.ModuleList([nn.BatchNorm1d(size)
                                        for size in lin_layer_sizes])

        # Dropout Layers
        self.emb_dropout_layer = nn.Dropout(emb_dropout)
        self.droput_layers = nn.ModuleList([nn.Dropout(size)
                                            for size in lin_layer_dropouts])

    def forward(self, X):
        cont_data = X[0]
        cat_data = X[1]
        if self.no_of_embs != 0:
            x = [emb_layer(cat_data[:, i])
                 for i, emb_layer in enumerate(self.emb_layers)]
            x = torch.cat(x, 1)
            x = self.emb_dropout_layer(x)

        if self.no_of_cont != 0:
            normalized_cont_data = self.first_bn_layer(cont_data)

            if self.no_of_embs != 0:
                x = torch.cat([x, normalized_cont_data], 1)
            else:
                x = normalized_cont_data

        for lin_layer, dropout_layer, bn_layer in \
                zip(self.lin_layers, self.droput_layers, self.bn_layers):
            x = F.relu(lin_layer(x))
            x = bn_layer(x)
            x = dropout_layer(x)

        x = self.output_layer(x)

        return x

# Read data
data = pd.read_csv("data/train.csv", usecols=["SalePrice", "MSSubClass", "MSZoning", "LotFrontage", "LotArea",
                                              "Street", "YearBuilt", "LotShape", "1stFlrSF", "2ndFlrSF"]).dropna()

categorical_features = ["MSSubClass", "MSZoning", "Street", "LotShape", "YearBuilt"]
output_feature = "SalePrice"

# Label Encode Categorial Features
label_encoders = {}
for cat_col in categorical_features:
    label_encoders[cat_col] = LabelEncoder()
    data[cat_col] = label_encoders[cat_col].fit_transform(data[cat_col])

# feed Forward NN
cat_dims = [int(data[col].nunique()) for col in categorical_features]

emb_dims = [(x, min(50, (x + 1) // 2)) for x in cat_dims]

net = FeedForwardNN(emb_dims, no_of_cont=4, lin_layer_sizes=[50, 100],
                    output_size=1, emb_dropout=0.04,
                    lin_layer_dropouts=[0.001, 0.01])

# Fit
ds = TabularDataset(data=data, cat_cols=categorical_features,
                    output_col=output_feature)
X = data.drop(['SalePrice'], axis=1)
y = data['SalePrice'].values.reshape(-1, 1)
net = NeuralNetRegressor(
    net,
    max_epochs=5,
    lr=0.1,
    dataset=ds
)
net.fit(X, y)


title: EnvironmentNotWritableError in Conda

body:
I am trying to run this command conda install pytorch torchvision cuda80 -c soumith
 from this link https://github.com/kenshohara/video-classification-3d-cnn-pytorch and I am getting error as below. I don't have root privileges. Can anyone tell me what I should do to fix this issue?

    EnvironmentNotWritableError: The current user does not have write permissions to the target environment.
  environment location: /usr/local/anaconda-2019.07
  uid: 3792
  gid: 1556


title: Plot the Decision Boundary of a Neural Network in PyTorch

body:
I've been trying to plot the decision boundary of my neural network which I used for binary classification with the sigmoid function in the output layer but with no success, I found many posts discussing the plotting of the decision boundary of a scikit-learn classifier but not a neural network built in PyTorch.
Below is my neural network:

class NeuralNetwork(torch.nn.Module):
  def __init__(self):
    super(NeuralNetwork, self).__init__()
    self.fc1 = torch.nn.Linear(23, 16)
    self.fc2 = torch.nn.Linear(16, 14)
    self.fc3 = torch.nn.Linear(14, 10)
    self.fc4 = torch.nn.Linear(10, 5)
    self.fc5 = torch.nn.Linear(5, 1)

  def forward(self, x):
    x = torch.relu(self.fc1(x))
    x = torch.relu(self.fc2(x))
    x = torch.relu(self.fc3(x))
    x = torch.relu(self.fc4(x))
    x = torch.sigmoid(self.fc5(x))
    return x

model = NeuralNetwork().double()

CUDA = torch.cuda.is_available()
if CUDA:
  model.cuda()

criterion = torch.nn.BCELoss(reduction='mean')
optimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.9)

model_1.train()

Precision = []
Cost = []

for epoch in range(10001):

  if CUDA:
    inputs = X_train.cuda()
    label = Y_train.cuda()
  else:
    inputs = X_train
    label = Y_train

  prediction = model_1(inputs)
  loss = criterion(prediction, label)
  accuracy = ((prediction > 0.5) == label).float().mean().item()

  Cost.append(loss.item())
  Precision.append(accuracy)

  if epoch % 1000 == 0 or epoch == 30000:
    print("Epoch:", epoch, ",", "Loss:", loss.item(), ",", "Accuracy:", accuracy)

  # Backpropagation process
  optimizer.zero_grad()
  loss.backward()
  optimizer.step()

model_1.eval()

X_test = torch.from_numpy(X[27000:,:])
Y_test = torch.from_numpy(y[27000:,:]).double()

with torch.no_grad():

  y_pred = model_1(X_test)
  print("Accuracy: ", ((y_pred > 0.5) == Y_test).float().mean().item())

Here is my attempt trying to generate a similar plot here:

# I've chosen two features because the data is multi-dimensional
X0 = X_test[:,[0,]].reshape(3000)
X5 = X_test[:,[5,]].reshape(3000)
X0, X5 = np.meshgrid(X0, X5)

xx, yy = np.meshgrid(X0, X5)
grid = np.c_[xx.ravel(), yy.ravel()]
probs = y_pred

f, ax = plt.subplots(figsize=(8, 6))
contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
                  vmin=0, vmax=1)
ax_c = f.colorbar(contour)
ax_c.set_label("$P(y = 1)$")
ax_c.set_ticks([0, .25, .5, .75, 1])

ax.scatter(X0, X5, c=Y_test, s=50,cmap="RdBu", vmin=-.2, vmax=1.2,edgecolor="white", linewidth=1)

ax.set(aspect="equal",
   xlim=(-5, 5), ylim=(-5, 5),
   xlabel="$X0$", ylabel="$X5$")

but unfortunately I get the following error:

TypeError                                 Traceback (most recent call last)

<ipython-input-52-fb941749621a> in <module>()
  1 f, ax = plt.subplots(figsize=(8, 6))
  2 contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
----> 3                       vmin=0, vmax=1)
  4 ax_c = f.colorbar(contour)
  5 ax_c.set_label("$P(y = 1)$")

5 frames

/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py in _check_xyz(self, args, kwargs)
1549             raise TypeError("Input z must be a 2D array.")
1550         elif z.shape[0] < 2 or z.shape[1] < 2:
-> 1551             raise TypeError("Input z must be at least a 2x2 array.")
1552         else:
1553             Ny, Nx = z.shape

TypeError: Input z must be at least a 2x2 array.

I would greatly appreciate your help, thanks in advance.


title: Running transformers predictions on CPU with multiprocessing

body:
I am trying to use pre-trained model from transformers to predict on CPU with multiprocessing. Below code code gets stuck at step-4. Any thoughts on how to resolve this?
I referred to below but no luck:
https://pytorch.org/docs/stable/notes/multiprocessing.html

import os
import torch
from transformers import BertForQuestionAnswering
from transformers import BertTokenizer
from torch.multiprocessing import Pool, Process, set_start_method

def answer_question(question, answer_text, model, tokenizer):
    input_ids = tokenizer.encode(question, answer_text)
    print('Query has {:,} tokens.\n'.format(len(input_ids)))
    sep_index = input_ids.index(tokenizer.sep_token_id)
    print(".. step -- 1")
    num_seg_a = sep_index + 1
    print(".. step -- 2")
    num_seg_b = len(input_ids) - num_seg_a
    print(".. step -- 3")
    segment_ids = [0]*num_seg_a + [1]*num_seg_b
    print(".. step -- 4")
    assert len(segment_ids) == len(input_ids)
    # ======== Evaluate ========
    # Run our example question through the model.
    start_scores, end_scores = model(torch.tensor([input_ids]), # The tokens representing our input text.
                                    token_type_ids=torch.tensor([segment_ids])) # The segment IDs to differentiate question from answer_text
    print(".. step -- 5")
    # ======== Reconstruct Answer ========
    # Find the tokens with the highest `start` and `end` scores.
    answer_start = torch.argmax(start_scores)
    answer_end = torch.argmax(end_scores)
    print(".. step -- 6")
    tokens = tokenizer.convert_ids_to_tokens(input_ids)
    print(".. step -- 7")
    answer = tokens[answer_start]
    print(".. step -- 8")
    for i in range(answer_start + 1, answer_end + 1):
        if tokens[i][0:2] == '##':
            answer += tokens[i][2:]
        else:
            answer += ' ' + tokens[i]
    print('Answer: "' + answer + '"')

if __name__ == "__main__":
    size = 2
    processes = []
    print("reading bert model")
    bert_model = BertForQuestionAnswering.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
    print("reading bert tokenizer")
    bert_tokenizer = BertTokenizer.from_pretrained('bert-large-uncased-whole-word-masking-finetuned-squad')
    q_list = ["How many parameters does BERT-large have?","How many parameters does BERT-large have?"]
    a_list = ["BERT-large is really big... it has 24-layers and an embedding size of 1,024, for a total of 340M parameters! Altogether it is 1.34GB, so expect it to take a couple minutes to download to your Colab instance.", "BERT-large is really big... it has 24-layers and an embedding size of 1,024, for a total of 340M parameters! Altogether it is 1.34GB, so expect it to take a couple minutes to download to your Colab instance."]

    for a in zip(q_list, a_list):
        print(a)
        q = Process(target = answer_question, args= (a[0], a[1], bert_model, bert_tokenizer))
        q.start()
        processes.append(q)

    for q in processes:
        q.join()


title: PyTorch and Convolutional Neural Networks

body:
I have a image input 340px*340px and I want to classify it to 2 classes.
I want to create convolution neural network (PyTorch framework). I have a problem with input and output of layer.

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        # 3 channels (RGB), kernel=5, but i don't understand why 6.
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        #why 16?
        self.conv2 = nn.Conv2d(6, 16, 5)
        #why 107584 = 328*328
        self.fc1 = nn.Linear(107584, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 2)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        # i dont understand this line
        x = x.view(x.size(0),  -1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

Is it network is correct?


title: Why will GPU usage run low in NN training?

body:
I'm running a NN training on my GPU with pytorch.
But the GPU usage is strangely "limited" at about 50-60%.
That's a waste of computing resources but I can't make it a bit higher.

I'm sure that the hardware is fine because running 2 of my process at the same time,or training a simple NN (DCGAN,for instance) can both occupy 95% or more GPU.(which is how it supposed to be)
My NN contains several convolution layers and it should use more GPU resources.

Besides, I guess that the data from dataset has been feeding fast enough,because I used workers=64 in my dataloader instance and my disk works just fine.

I just confused about what is happening.

Dev details:

GPU : Nvidia GTX 1080 Ti

os:Ubuntu 64-bit


title: Why is simple seq2seq in pytorch always returning NaN?

body:
I have the following trimmed down model:

import torch.nn as nn
import torch
import argparse
import torch
import torch.utils.data
from torch import nn, optim
from torch.autograd import Variable
from torch.nn import functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
import json
import numpy as np
import datetime
import os

class EncoderRNN(nn.Module):
    def __init__(self, input_size=8, hidden_size=10, num_layers=2):
        super(EncoderRNN, self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_layers = num_layers

        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
        self.relu = nn.ReLU()
        self.sigmoid = nn.Sigmoid()

        #initialize weights
        nn.init.xavier_uniform(self.lstm.weight_ih_l0, gain=np.sqrt(2))
        nn.init.xavier_uniform(self.lstm.weight_hh_l0, gain=np.sqrt(2))

    def forward(self, input):
        tt = torch
        print input.shape
        h0 = Variable(tt.FloatTensor(self.num_layers, input.size(0), self.hidden_size))
        c0 = Variable(tt.FloatTensor(self.num_layers, input.size(0), self.hidden_size))
        encoded_input, hidden = self.lstm(input, (h0, c0))
        encoded_input = self.sigmoid(encoded_input)
        return encoded_input

train_x = torch.from_numpy(np.random.random((2000,19,8))).float()

train_loader = torch.utils.data.DataLoader(train_x,
    batch_size=64, shuffle=True)

model = EncoderRNN()

optimizer = optim.Adam(model.parameters(), lr=1e-6)

optimizer.zero_grad()

loss_function = torch.nn.BCELoss(reduce=True)

def train(epoch):
    model.train()
    train_loss = 0
    for batch_idx, (data_x) in enumerate(train_loader):
        x = model(Variable(data_x))
        print("output has nan: " + str(np.isnan(x.detach().numpy()).any()))

train(0)

To summarize, I think I'm basically just feeding an input into an LSTM with random initialized hidden values, and then taking the sigmoid of the output of that LSTM.  Then I'm feeding that output to a decoder LSTM, and taking the sigmoid of the output of the decoder output and using that as my final value.

Unfortunately, even in the first iteration, the model often outputs a vector of the correct shape (batch_size, seq_length, seq_dim), but contains at least one and sometimes all NaN values.  What am I doing wrong?

Thanks!

What I've tried so far:

changing LSTM to GRU
changing sigmoid to relu
changing the dimension of the hidden representation
getting the failing input down to the encoder

Edit: Apologies to everyone who tried to help when I had broken code - I really do value your time, and thanks so much for helping!


title: What do * and mean stand for in this PyTorch expression?

body:
I do not understand how to evaluate this expression:
x.view(*(x.shape[:-2]),-1).mean(-1)`,

if x.shape == (N, C, H, W).
What does the asterisk * stand for? And what is mean(-1) ?


title: 2-step 3D rotation of x,y,z tensors

body:
In Python, using PyTorch, I'm trying to rotate 3 cubes x, y, and z, first about the x-axis and then about the z-axis. However, the rotation about the z-axis seems to be behaving in a way I would not expect.
I am creating 3 cubes to rotate below and then rotating them 90 degrees about the x-axis and then 90 degrees about the z-axis.
import torch
from numpy import pi
import matplotlib.pyplot as plt

def rotation(x,y,z,theta,phi):
    ### A function for rotating about the x and then z axes ###
    xx = (x*torch.cos(theta)) - (((y*torch.cos(phi)) - (z*torch.sin(phi)))*torch.sin(theta))
    yy = (x*torch.sin(theta)) + (((y*torch.cos(phi)) - (z*torch.sin(phi)))*torch.cos(theta))
    zz = (y*torch.sin(phi)) + (z*torch.cos(phi))
    return xx,yy,zz

### Creating the 3 cubes: x, y, z ###
l = torch.arange(-2,3,1)
x,y,z=torch.meshgrid(l,l,l)

###  Scaling the cubes so they can be differentiated from one another ###
x = x.clone().T
y = y.clone().T*2
z = z.clone().T*3

### Defining the amount of rotation about the x and z axes
phi = torch.tensor([pi/2]).to(torch.float) # about the x axis
theta = torch.tensor([pi/2]).to(torch.float) # about the z axis

### Performing the rotation
x_r,y_r,z_r = rotation(x, y, z, theta, phi)

By visualising the first slice of each cube I can see that the rotation has not been successful as at first glance it looks like the cubes have actually been rotated about the x-axis followed by the y-axis instead.

Is there a specific way that Python handles rotations like this that I'm missing, such as the axes changing along with rotations, meaning the initial rotation matrix operation no longer applies?

Extra information
If one were to replace theta as 0 instead of pi/2, it can be seen that the first rotation behaves as expected by looking at the first slice of each rotated cube:
Code for visualising:
plt.figure()
plt.subplot(231)
x_before = plt.imshow(x[0,:,:])
plt.xlabel('x-before'); plt.colorbar(x_before,fraction=0.046, pad=0.04)
plt.subplot(232)
y_before = plt.imshow(y[0,:,:])
plt.xlabel('y-before'); plt.colorbar(y_before,fraction=0.046, pad=0.04)
plt.subplot(233)
z_before = plt.imshow(z[0,:,:])
plt.xlabel('z-before'); plt.colorbar(z_before,fraction=0.046, pad=0.04)
plt.subplot(234)
x_after = plt.imshow(x_r[0,:,:])
plt.xlabel('x-after'); plt.colorbar(x_after,fraction=0.046, pad=0.04)
plt.subplot(235)
y_after = plt.imshow(y_r[0,:,:])
plt.xlabel('y-after'); plt.colorbar(y_after,fraction=0.046, pad=0.04)
plt.subplot(236)
z_after = plt.imshow(z_r[0,:,:])
plt.xlabel('z-after'); plt.colorbar(z_after,fraction=0.046, pad=0.04)
plt.tight_layout()


title: How to build pytorch from source using numpy

body:
I am trying to build pytorch v1.4.0 from source because I need it for another module.
I have succeeded in building pytorch from source but when I try to run the intended python script I get this error:
RuntimeError: PyTorch was compiled without NumPy support

So I looked up what I did wrong and it turns out I needed to install numpy before i build pytorch from source, so thats what I did.
I used the command:
pip3 install numpy==1.19.4

When installing pytorch the console gives me a lot of information, including the build settings. The build settings looked like this:
--   Compile definitions   : ONNX_ML=1;ONNX_NAMESPACE=onnx_torch;HAVE_MMAP=1;_FILE_OFFSET_BITS=64;HAVE_SHM_OPEN=1;HAVE_SHM_UNLINK=1;HAVE_MALLOC_USABLE_SIZE=1
--   CMAKE_PREFIX_PATH     : /home/elvygcp/venv/lib/python3.6/site-packages;/usr/local/cuda
--   CMAKE_INSTALL_PREFIX  : /home/elvygcp/venv/pytorch-1.4.0/torch
--
--   TORCH_VERSION         : 1.4.0
--   CAFFE2_VERSION        : 1.4.0
--   BUILD_CAFFE2_MOBILE   : ON
--   USE_STATIC_DISPATCH   : OFF
--   BUILD_BINARY          : OFF
--   BUILD_CUSTOM_PROTOBUF : ON
--     Link local protobuf : ON
--   BUILD_DOCS            : OFF
--   BUILD_PYTHON          : True
--     Python version      : 3.6.9
--     Python executable   : /home/elvygcp/venv/bin/python3
--     Pythonlibs version  : 3.6.9
--     Python library      : /usr/lib/libpython3.6m.so.1.0
--     Python includes     : /usr/include/python3.6m
--     Python site-packages: lib/python3.6/site-packages
--   BUILD_CAFFE2_OPS      : ON
--   BUILD_SHARED_LIBS     : ON
--   BUILD_TEST            : True
--   BUILD_JNI             : OFF
--   INTERN_BUILD_MOBILE   :
--   USE_ASAN              : OFF
--   USE_CUDA              : ON
--     CUDA static link    : OFF
--     USE_CUDNN           : OFF
--     CUDA version        : 10.2
--     CUDA root directory : /usr/local/cuda
--     CUDA library        : /usr/local/cuda/lib64/stubs/libcuda.so
--     cudart library      : /usr/local/cuda/lib64/libcudart.so
--     cublas library      : /usr/lib/x86_64-linux-gnu/libcublas.so
--     cufft library       : /usr/local/cuda/lib64/libcufft.so
--     curand library      : /usr/local/cuda/lib64/libcurand.so
--     nvrtc               : /usr/local/cuda/lib64/libnvrtc.so
--     CUDA include path   : /usr/local/cuda/include
--     NVCC executable     : /usr/local/cuda/bin/nvcc
--     CUDA host compiler  : /usr/bin/cc
--     USE_TENSORRT        : OFF
--   USE_ROCM              : OFF
--   USE_EIGEN_FOR_BLAS    : ON
--   USE_FBGEMM            : ON
--   USE_FFMPEG            : OFF
--   USE_GFLAGS            : OFF
--   USE_GLOG              : OFF
--   USE_LEVELDB           : OFF
--   USE_LITE_PROTO        : OFF
--   USE_LMDB              : OFF
--   USE_METAL             : OFF
--   USE_MKL               : OFF
--   USE_MKLDNN            : ON
--   USE_MKLDNN_CBLAS      : OFF
--   USE_NCCL              : ON
--     USE_SYSTEM_NCCL     : OFF
--   USE_NNPACK            : ON
--   USE_NUMPY             : OFF
--   USE_OBSERVERS         : ON
--   USE_OPENCL            : OFF
--   USE_OPENCV            : OFF
--   USE_OPENMP            : ON
--   USE_TBB               : OFF
--   USE_PROF              : OFF
--   USE_QNNPACK           : ON
--   USE_REDIS             : OFF
--   USE_ROCKSDB           : OFF
--   USE_ZMQ               : OFF
--   USE_DISTRIBUTED       : ON
--     USE_MPI             : OFF
--     USE_GLOO            : ON
--   BUILD_NAMEDTENSOR   : OFF

There are 2 things I do not understand:

The line: USE_CUDNN           : OFF, which I think is strange since i followed the pytorch build instructions from their github page which mentions I need CuDNN to build pytorch from source, and CuDNN is installed on my system.

The line: USE_NUMPY             : OFF, I dont know why but pytorch does not recognize numpy.

My system:
OS: Ubuntu 18.04 LTS
Cuda version: 10.2
CuDNN version: 8
python venv in Google Cloud Compute Engine VM

How I build pytorch 1.4.0 from source:
git clone --branch v1.4.0 https://github.com/pytorch/pytorch.git pytorch-1.4.0
cd pytorch-1.4.0/
git submodule update --init --recursive
sudo apt install cmake -y
sudo apt-get update
cd ../
sudo apt install python3-venv -y
python3 -m venv venv/
cd venv
source bin/activate
cd pytorch-1.4.0/
pip install pyyaml
python3 setup.py install
cd ../
git clone --branch v0.5.0 https://github.com/pytorch/vision.git torchvision-0.5.0
cd torchvision-0.5.0/
python3 setup.py install
cd ../

If anyone could tell how I make pytorch recognize and build with numpy I'd be very grateful.
If I need to supply more information I'd be happy to.


title: How to use spatial transformer to crop the image in pytorch?

body:
the paper of the spatial transformer network claims that it can be used to crop the image.

Given the crop region (top_left, bottom_right)=(x1,y1,x2,y2), how to interpret the region as a transformation matrix and crop the image in pytorch?

Here is a introduction about the spatial transformer network in torch (http://torch.ch/blog/2015/09/07/spatial_transformers.html), in the introduction, it visualize the bounding box where the transformer look at, How can we determine the bounding box given the transformation matrix?

[Edit]

I just found out the answer to the first question [given the crop region, find out a transformation matrix]


title: Debug Pytorch Optimizer

body:
When I run optimizer.step on my code, I get this error

RuntimeError: sqrt not implemented for 'torch.LongTensor'

C:\Program Files\Anaconda3\lib\site-packages\IPython\core\magic.py in <lambda>(f, *a, **k)
    186     # but it's overkill for just that one bit of state.
    187     def magic_deco(arg):
--> 188         call = lambda f, *a, **k: f(*a, **k)
    189
    190         if callable(arg):

C:\Program Files\Anaconda3\lib\site-packages\IPython\core\magics\execution.py in time(self, line, cell, local_ns)
   1178         else:
   1179             st = clock2()
-> 1180             exec(code, glob, local_ns)
   1181             end = clock2()
   1182             out = None

<timed exec> in <module>()

C:\Program Files\Anaconda3\lib\site-packages\torch\optim\adam.py in step(self, closure)
     98                     denom = max_exp_avg_sq.sqrt().add_(group['eps'])
     99                 else:
--> 100                     denom = exp_avg_sq.sqrt().add_(group['eps'])
    101
    102                 bias_correction1 = 1 - beta1 ** state['step']

RuntimeError: sqrt not implemented for 'torch.LongTensor'

I am using my own loss function. My question is how will I debug this error? Is there a quick way to see the type of all my variables? I am manually doing it and all of them are type float (including the output of my custom loss). I can't figure out why we are even getting an error related to a LongTensor. How does the optimizer.step function work in PyTorch?

Just in case, below is most of the code.
This is the model:

class LSTM(nn.Module):

    def __init__(self, mel_channels=40, frames=81, hidden_dim=768, proj_dim=256):
        super(LSTM, self).__init__()
        self.hidden_dim = hidden_dim
        self.mel_channels = mel_channels
        self.frames = frames
        self.proj_dims = proj_dim

        weight = torch.tensor([10])
        bias = torch.tensor([-5])
        self.w = nn.Parameter(weight)
        self.b = nn.Parameter(bias)

        # The LSTM takes word embeddings as inputs, and outputs hidden states
        # with dimensionality hidden_dim.
        self.lstm1 = nn.LSTM(mel_channels, hidden_dim, batch_first=False)
        print("here1")
        self.lstm2 = nn.LSTM(proj_dim, hidden_dim, batch_first=False)
        self.lstm3 = nn.LSTM(proj_dim, hidden_dim, batch_first=False)
        self.lstms = [self.lstm1, self.lstm2, self.lstm3]

        self.proj1 = nn.Linear(hidden_dim, proj_dim)
        self.proj2 = nn.Linear(hidden_dim, proj_dim)
        self.proj3 = nn.Linear(hidden_dim, proj_dim)
        self.projs = [self.proj1, self.proj2, self.proj3]

    def init_states(self, batchsize):
        # Before we've done anything, we dont have any hidden state.
        # Refer to the Pytorch documentation to see exactly
        # why they have this dimensionality.
        # The axes semantics are (num_layers, minibatch_size, hidden_dim)
        return [(torch.zeros(1, batchsize, self.hidden_dim),
                 torch.zeros(1, batchsize, self.hidden_dim)),
                (torch.zeros(1, batchsize, self.hidden_dim),
                 torch.zeros(1, batchsize, self.hidden_dim)),
                (torch.zeros(1, batchsize, self.hidden_dim),
                 torch.zeros(1, batchsize, self.hidden_dim)),
                ]

    def forward(self, inputs, states=None):
        time, batchsize, inputdim = list(inputs.shape)

        if states is None:
            states = self.init_states(batchsize)
        output = inputs

        print(output.type())

        for i in range(3):
            print(output.type())
            output, state = self.lstms[i](output, states[i])
            output = self.projs[i](output)
        # perform normalization on this output here
        output = output[-1]
        print(output.type())
        output = F.normalize(output, p=2, dim=-1)
        print(output.type())

        self.state = state

        print(output.type())

        return output

    def get_w(self):
        print(get_w.type())
        return(self.w)

    def get_b(self):
        print(get_b.type())
        return(self.b)

    def get_state(self):
        print(get_state())
        return(self.state)

This is the custom loss:

class CustomLoss(_Loss):

    def __init__(self,  size_average=True, reduce=True):

        super(CustomLoss, self).__init__(size_average, reduce)

    def forward(self, S, N, M, type='softmax',):

        return self.loss_cal(S, N, M, type)

    def loss_cal(self, S, N, M, type="softmax",):

        self.A = torch.cat([S[i * M:(i + 1) * M, i:(i + 1)]
                           for i in range(N)], dim=0)

        if type == "softmax":
            self.B = torch.log(torch.sum(torch.exp(S.float()), dim=1, keepdim=True) + 1e-8)
            total = torch.abs(torch.sum(self.A - self.B))
        else:
            raise AssertionError("loss type should be softmax or contrast !")
        return total

Finally, this is the main file

model=LSTM()
optimizer = optim.Adam(list(model.parameters()), lr=LEARNING_RATE)
model = model.to(device)

best_loss = 100.
generator = SpeakerVerificationDataset()
dataloader = DataLoader(generator, batch_size=4,
                        shuffle=True, num_workers=0)

loss_history = []
update_counter = 1

for epoch in range(NUM_EPOCHS):
    print("Epoch # : ", epoch + 1)

    for step in range(STEPS_PER_EPOCH):
        # get batch dataset
        for i_batch, sample_batched in enumerate(dataloader):
            print(sample_batched['MelData'].size())
            inputs = sample_batched['MelData'].float()
            inputs=sample_batched['MelData'].view(180, M*N, 40).float()
            print((inputs.size()))
            inputs = inputs
            #print(here)
            # remove previous gradients
            optimizer.zero_grad()

            # get gradients and loss at this iteration
            #predictions,state,w,b = model(inputs)
            predictions = model(inputs)
            w = model.w
            b = model.b
            predictions = similarity(output=predictions,w=w,b=b)
            #loss = CustomLoss()
            S = predictions

            loss_func = CustomLoss()
            loss = loss_func.loss_cal(S=S,N=N,M=M)
            loss.backward()

            # update the weights
            print("start optimizing")
            optimizer.step()

            loss_history.append(loss.item())
            print(update_counter, ":", loss_history[-1])

            update_counter += 1
    print()

    # save the weights
    torch.save(model.state_dict(), CHECKPOINT_PATH)

    print("Saving weights")
    print()
print()


title: Why did I get 2 different results from two models with same parameters and inputs?

body:
I loaded resnet18 into my two models (model1 and model2), with pretrained weights.

I want to use them as feature extractors
For model1: I freezed the parameters except the last linear layer model1.fc, then train it. After training, I set model1.fc into torch.nn.Identity()
For model2: I directly set model2.fc into torch.nn.Identity()

Then these 2 models should be the same, but I get different forward result from the same inputs.

If the training of model1 is not done, they will have the same result, maybe something wrong with the freezing of  parameter.
However, I checked their parameters after the training of model1 and setting the last layer of both models to identity layer, and they seems to be the same.

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms, models

# Load weights pretrained on ImageNet
def load_weights(model):
    model_dir = "....."
    model.load_state_dict(tor`enter code here`ch.utils.model_zoo.load_url("https://download.pytorch.org/models/resnet18-5c106cde.pth", model_dir=model_dir))
    return model

model1=models.resnet18()
model1=load_weights(model1)
for param in model1.parameters():
    param.requires_grad = False
model1.fc=nn.Linear(512, 2)
model1.cuda()
optimizer = optim.SGD(model1.fc.parameters(), lr=1e-2, momentum=0.9)
result_freeze = \
    run_training(model1, optimizer, device, train_loader, val_loader,num_epochs=10)

model2=models.resnet18()
model2=load_weights(model2)
model2.fc=nn.Identity()
model2.cuda()

model1.fc=nn.Identity()
model1.cuda()

# checking forward results(extracting features)

# The batch size is one here
for batch_idx, (data, target) in enumerate(X_train):
        data, target = data.to(device), target.to(device)
        d=data
        X_train_feature[batch_idx]=model1(data).cpu().detach().numpy()
        y_train[batch_idx]=target.cpu().detach().numpy()
        X_train2_feature[batch_idx]=model2(d).cpu().detach().numpy()
        y_train2[batch_idx]=target.cpu().detach().numpy()
        print(sum(X_train_feature[batch_idx]==X_train2_feature[batch_idx]))
        print(sum(y_train[batch_idx]==y_train2[batch_idx]))
        print(torch.sum(d==data))
for batch_idx, (data, target) in enumerate(X_test):
        data, target = data.to(device), target.to(device)
        d=data
        X_test_feature[batch_idx]=model1(data).cpu().detach().numpy()
        y_test[batch_idx]=target.cpu().detach().numpy()
        X_test2_feature[batch_idx]=model2(d).cpu().detach().numpy()
        y_test2[batch_idx]=target.cpu().detach().numpy()
        print(sum(X_test_feature[batch_idx]==X_test2_feature[batch_idx]))
        print(sum(y_test[batch_idx]==y_test2[batch_idx]))
        print(torch.sum(d==data))

# checking parameters
for a,b in zip(model1.parameters(),model2.parameters()):
    print(torch.sum(a!=b))

Expect to get the same forward results from model1 and model2, but they are different. And If they produce different forward results, why do they have exactly the same parameters?


title: I've installed PyTorch, but CUDA is not recognized

body:
I installed PyTorch using this command:

pip3 install torch===1.2.0 torchvision===0.4.0 -f https://download.pytorch.org/whl/torch_stable.html

And it seemed to work. When importing torch, there isn't any error.
My laptop has a GeForce 1060 TI, which I assumed would work with CUDA.

Here is the error in the IDE (Eclipse):


title: How Can I Install pytorch on Intel & python2.7?

body:
I often saw how to install pytorch on intel & python3
but no python2...!

How can I install pytorch on intel & python2


title: what is the pytorch's view equivalence with tensorflow 2.0?

body:
l_conv7 = self.loc_conv7(conv7_feats)  # (N, 24, 19, 19)
l_conv7 = l_conv7.permute(0, 2, 3, 1).contiguous()  # (N, 19, 19, 24)
l_conv7 = l_conv7.view(batch_size, -1, 4)  # (N, 2166, 4), there are a total 2116 boxes on this feature map

what is the equivalence with torch's view in TensorFlow?
how to change the l_conv7.view in TensorFlow 2.0?


title: Process got killed when building docker image on GCP

body:
I am a newbie of cloud computing and struggling to deploy my web app on google cloud platform for the first time. When I was building docker image, which means I ran the following code
docker build -t gcr.io/${PROJECT_ID}/insurance-streamlit:v1 .

the process got killed, showing the following error:
Killed
The command '/bin/sh -c pip install -r requirements.txt' returned a non-zero code: 137

I guessed my app might be too large because the weight file has been more than 100MB. So is there a way to fix it? Please tell me the details, thanks in advance!
PS: my Dockerfile is as below:
FROM python:3.7
RUN pip install virtualenv
ENV VIRTUAL_ENV=/venv
RUN virtualenv venv -p python3
ENV PATH="VIRTUAL_ENV/bin:$PATH"

WORKDIR /app
ADD . /app

# Install dependencies
RUN pip install -r requirements.txt

# copying all files over
COPY . /app

# Expose port
ENV PORT 8501

# cmd to launch app when container is run
CMD streamlit run app.py

# streamlit-specific commands for config
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
RUN mkdir -p /root/.streamlit
RUN bash -c 'echo -e "\
[general]\n\
email = \"\"\n\
" > /root/.streamlit/credentials.toml'

RUN bash -c 'echo -e "\
[server]\n\
enableCORS = false\n\
" > /root/.streamlit/config.toml

And my requirements.txt is like:
albumentations==0.4.5
numpy==1.19.0
opencv-python==4.1.0.25
opencv-python-headless==4.2.0.34
pandas==1.0.5
Pillow==7.1.2
streamlit==0.62.0
torch==1.4.0
torchvision==0.5.0
matplotlib


title: Pytorch crossentropy loss with 3d input

body:
I have a network which outputs a 3D tensor of size (batch_size, max_len, num_classes). My groud truth is in the shape (batch_size, max_len). If I do perform one-hot encoding on the labels, it'll be of shape (batch_size, max_len, num_classes) i.e the values in max_len are integers in the range [0, num_classes]. Since the original code is too long, I have written a simpler version that reproduces the original error.
criterion = nn.CrossEntropyLoss()
batch_size = 32
max_len = 350
num_classes = 1000
pred = torch.randn([batch_size, max_len, num_classes])
label = torch.randint(0, num_classes,[batch_size, max_len])
pred = nn.Softmax(dim = 2)(pred)
criterion(pred, label)

the shape of pred and label are respectively,torch.Size([32, 350, 1000]) and torch.Size([32, 350])
The error encountered is

ValueError: Expected target size (32, 1000), got torch.Size([32, 350, 1000])

If I one-hot encode labels for computing the loss
x = nn.functional.one_hot(label)
criterion(pred, x)

it'll throw the following error

ValueError: Expected target size (32, 1000), got torch.Size([32, 350, 1000])


title: Error when compiling pytorch: 'cstdint' file not found

body:
I was attempting to compile pytorch using NO_CUDA=1 python setup.py install on Mac OS X, but I got these errors:

In file included from /Users/ezyang/Dev/pytorch-tmp/torch/lib/tmp_install/include/THPP/Tensor.hpp:3:
/Users/ezyang/Dev/pytorch-tmp/torch/lib/tmp_install/include/THPP/Storage.hpp:6:10: fatal error:
      'cstdint' file not found
#include <cstdint>
         ^
1 error generated.
In file included from torch/csrc/autograd/functions/init.cpp:2:
In file included from torch/csrc/autograd/functions/batch_normalization.h:4:
In file included from /Users/ezyang/Dev/pytorch-tmp/torch/lib/tmp_install/include/THPP/THPP.h:4:
/Users/ezyang/Dev/pytorch-tmp/torch/lib/tmp_install/include/THPP/Storage.hpp:6:10: fatal error:
      'cstdint' file not found
#include <cstdint>
         ^
1 error generated.
1 error generated.
1 error generated.
1 error generated.
In file included from torch/csrc/autograd/python_hook.cpp:5:
In file included from /Users/ezyang/Dev/pytorch-tmp/torch/csrc/THP.h:32:
/Users/ezyang/Dev/pytorch-tmp/torch/csrc/utils.h:6:10: fatal error:
      'type_traits' file not found
#include <type_traits>
         ^
13 errors generated.
1 error generated.
1 error generated.
error: command 'gcc' failed with exit status 1


title: How to progressively grow a neural network in pytorch?

body:
I am trying to make a progressive autoencoder and I have thought a couple ways of growing my network during training. However, I am always stuck on this one part where I don't know if changing the input(encoder) and output(decoder) channel would affect my network. See the example below.

X = torch.randn( 8, 1, 4, 4,) # A batch of 8 grayscale images of 4x4 pixels in size

Encoder = nn.Sequential( Conv2D( 1, 16, 3, 1, 1 ), nn.ReLU() ) # starting setup 16 3x3 kernels

if I print the above weights from the network I would get a size of [ 1, 16, 3, 3 ], 16 kernels each of size 3x3
if I want to grow the network I would need to do save those weight because hopefully its already well trained on those 4x4 image inputs.

X = torch.randn( 8, 1, 8, 8) # increase the image size from 4x4 to 8x8

...
new_model = nn.Sequential()

then do...
# copy the previous layer and its weights from the original encoder

# BTW My issue starts here.

# Add/grow the new_model with new layers concat with the old layer, also modify the input channela so they can link correctly

# Final result would be like something below.

new_model = nn.Sequential( Conv2D( **1**, 8, 3, 1, 1 ), nn.ReLU(), Conv2D( **8**, 16,  3, 1, 1 ), nn.ReLU()   )

Encoder = new_model

# Repeat process

Everything looks good however because I change the input channel the size of the weights changed as well, and this is the issue that I have been stuck on for a while now. You can simply check this by running,

foo_1 = nn.Conv2d(1, 1, 3, 1, 1) # You can think this as the starting Conv2D from the starting encoder

foo_2 = nn.Conv2d(3, 1, 3, 1, 1) # You can think this as the modfiied starting Conv2D with an outer layer outputting 3 channels connecting to it

print(foo_1.weight.size()) # torch.Size([1, 1, 3, 3])

print(foo_2.weight.size()) # torch.Size([1, 3, 3, 3])

Initially, I thought foo_1 and foo_2 would both have the same weight size, as in both would only use one 3x3 kernel but it doesn't see to be the case. I hope you can see my dilemma now, after x amount of epochs I need to grow another convolution and I have to mess with the input size to make new layers chain properly but if I change the input size the shape of the weight is different and I don't know how pasting the old state would work.

I have been looking at pro gan implementations in pytorch and IMO they are not easy to read. How can I build more institutions on how to properly progressively grow your network?


title: Understanding behavior of index_put in PyTorch

body:
I am trying to understand the behavior of index_put in PyTorch, but the document is not clear to me.
Given
a = torch.zeros(2, 3)
a.index_put([torch.tensor(1, 0), torch.tensor([1, 1])], torch.tensor(1.))

it returns
tensor([[1., 1., 0.],
       [0., 0., 0.])

While given
a = torch.zeros(2, 3)
a.index_put([torch.tensor(0, 0), torch.tensor([1, 1])], torch.tensor(1.))

it returns
tensor([[0., 1., 0.],
       [0., 0., 0.])

I am wondering what the rule of index_put on earth? What if I want to put three values to a, such that it returns
tensor([0., 1., 1.,],
       [0., 1., 0.])

Any help is appreciated!


title: Sentence embedding using T5

body:
I would like to use state-of-the-art LM T5 to get sentence embedding vector.
I found this repository https://github.com/UKPLab/sentence-transformers
As I know, in BERT I should take the first token as [CLS] token, and it will be the sentence embedding.
In this repository I see the same behaviour on T5 model:
cls_tokens = output_tokens[:, 0, :]  # CLS token is first token

Does this behaviour correct? I have taken encoder from T5 and encoded two phrases with it:
"I live in the kindergarden"
"Yes, I live in the kindergarden"

The cosine similarity between them was only "0.2420".
I just need to understand how sentence embedding works - should I train network to find similarity to reach correct results? Or I it is enough of base pretrained language model?


title: TypeError: forward() takes 2 positional arguments but 3 were given in pytorch

body:
I have the following error in my training loop and I don't really understand what the issue is. I am currently in the process of writing this code so stuff isn't final but I cannot figure out what this problem is.
I have tried googling the error and read some of the answers but still couldn't seem to understand the crux of the issue.
Dataset and Dataloader
(X and Y are already given to me, they are both [2000, 40, 1] tensors)
class TrainingDataset(data.Dataset):
  def __init__(self, X, y):
    self.X = X
    self.y = y

  def __len__(self):
    return Nf

  # returns corresponding input/output pairs
  def __getitem__(self, t):
    X = self.X[t]
    y = self.y[t]

    #print(X.shape, y.shape)

    return X, y

# prints torch.Size([2000, 40, 1]) torch.Size([2000, 40, 1])
print(x.size(), y.size())

dataset = TrainingDataset(x,y)
batchSize = 20
dataIter = data.DataLoader(dataset, batchSize)

Model:
class Encoder(nn.Module):
  def __init__(self, num_inputs = 40, num_outputs = 40):
    super(Encoder, self).__init__()

    self.num_inputs = num_inputs
    self.num_hidden = num_hidden
    self.num_outputs = num_outputs

    self.layers = nn.Sequential(
        nn.Linear(num_inputs, num_outputs),
        nn.ReLU(),
        nn.Linear(num_outputs, num_outputs),
        nn.ReLU(),
        nn.Linear(num_outputs, num_outputs)
    )

  def forward(self, x_c, y_c):
    return self.layers(x_c, y_c)

Training Loop:
for epoch in range(epochs):
  for batch in dataIter:
    optimiser.zero_grad()
    l = loss(encoder(x_c=batch[0], y_c=batch[1]), batch[1])
    l.backward()
    optimiser.step()

Error:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-15-aa1c60616d82> in <module>()
      6   for batch in dataIter:
      7     optimiser.zero_grad()
----> 8     l = loss(encoder(x_c=batch[0], y_c=batch[1]), batch[1])
      9     l.backward()
     10     optimiser.step()

2 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

TypeError: forward() takes 2 positional arguments but 3 were given

Can anyone point me in the right direction? I have just started to learn and do pytorch so I am not good at any of this yet.


title: PyTorch: efficiently interleave two tensors in a custom order

body:
I want to create a new tensor z from two tensors, say x and y with dimensions [N_samples, S, N_feats] and [N_samples, T, N_feats] respectively. The aim is to combine both tensors on the 2nd dim by mixing the elements of the 2nd dim in a specific ordering, which is stored in a variable order with dim [N_samples, U].

The ordering is different for every sample and is basically which index to extract from which tensor. It looks like this for a given sample order[0] - [x_0, x_1, y_0, x_2, y_1, ... ], where the letter indicates the tensor and the number indicates the index of the 2nd dim. So z[0] would be

z[0] = [x[0, 0, :], x[0, 1, :], y[0, 0, :], x[0, 2, :], y[0, 1, :] ... ]

How would I achieve this? I've written something that uses torch.gather that tries to do this.

x = torch.rand((2, 4, 5))
y = torch.rand((2, 3, 5))

# new ordering of second dim
# positive means take (n-1)th element from x
# negative means take (n-1)th element from y
order = [[1, 2, -1, 3, -2, 4, 3],
         [1, -1, -2, 2, 3, 4, -3]]

# simple concat for gather
combined = torch.cat([x, y], dim=1)

# add a zero padding on top of combined tensor to ease gather
zero = torch.zeros_like(x)[:, 1:2]
combined = torch.cat([zero, combined], dim=1)

def _create_index_for_gather(index, offset, n_feats):
    new_index = [abs(i) + offset if i < 0 else i for i in index]

    # need to repeat index for each dim for torch.gather
    new_index = [[x] * n_feats for x in new_index]
    return new_index

_, offset, n_feats = x.shape
index_for_gather = [_create_index_for_gather(i, offset, n_feats) for i in order]

z = combined.gather(dim=1, index=torch.tensor(index_for_gather))

Is there a more efficient way of doing this?


title: PyTorch sigmoid gives exit code 132 SIGILL

body:
I've been getting the following error when I try to run most PyTorch examples:

Process finished with exit code 132 (interrupted by signal 4: SIGILL)

I've been able to find a minimal example that generates this error:

import torch

tensor = torch.zeros(1)
torch.sigmoid(tensor)

I'm using PyCharm and a virtual environment, on Ubuntu 18.04.2. I can provide any more information if needed. Thank you for your help!


title: TypeError: Cannot handle this data type

body:
Trying to put the saliency map to the image and make a new data set

trainloader = utilsxai.load_data_cifar10(batch_size=1,test=False)
testloader =  utilsxai.load_data_cifar10(batch_size=128, test=True)

this load_cifar10 is torchvision

data = trainloader.dataset.data

trainloader.dataset.data = (data * sal_maps_hf).reshape(data.shape)

  sal_maps_hf shape with (50000,32,32,3)  and trainloader shape
  with (50000,32,32,3)

but when I run this

for idx,img in enumerate(trainloader):

  --------------------------------------------------------------------------- KeyError                                  Traceback (most recent call
  last) ~/venv/lib/python3.7/site-packages/PIL/Image.py in
  fromarray(obj, mode)    2644             typekey = (1, 1) + shape[2:],
  arr["typestr"]
  -> 2645             mode, rawmode = _fromarray_typemap[typekey]    2646         except KeyError:

  KeyError: ((1, 1, 3), '

  During handling of the above exception, another exception occurred:

  TypeError                                 Traceback (most recent call
  last)  in
  ----> 1 show_images(trainloader)

   in show_images(trainloader)
        1 def show_images(trainloader):
  ----> 2     for idx,(img,target) in enumerate(trainloader):
        3         img = img.squeeze()
        4         #pritn(img)
        5         img = torch.tensor(img)

  ~/venv/lib/python3.7/site-packages/torch/utils/data/dataloader.py in
  next(self)
      344     def next(self):
      345         index = self._next_index()  # may raise StopIteration
  --> 346         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
      347         if self._pin_memory:
      348             data = _utils.pin_memory.pin_memory(data)

  ~/venv/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py in
  fetch(self, possibly_batched_index)
       42     def fetch(self, possibly_batched_index):
       43         if self.auto_collation:
  ---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
       45         else:
       46             data = self.dataset[possibly_batched_index]

  ~/venv/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py in
  (.0)
       42     def fetch(self, possibly_batched_index):
       43         if self.auto_collation:
  ---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
       45         else:
       46             data = self.dataset[possibly_batched_index]

  ~/venv/lib/python3.7/site-packages/torchvision/datasets/cifar.py in
  getitem(self, index)
      120         # doing this so that it is consistent with all other datasets
      121         # to return a PIL Image
  --> 122         img = Image.fromarray(img)
      123
      124         if self.transform is not None:

  ~/venv/lib/python3.7/site-packages/PIL/Image.py in fromarray(obj,
  mode)    2645             mode, rawmode = _fromarray_typemap[typekey]
  2646         except KeyError:
  -> 2647             raise TypeError("Cannot handle this data type")    2648     else:    2649         rawmode = mode

  TypeError: Cannot handle this data type

trainloader.dataset.__getitem__

  getitem of Dataset CIFAR10
      Number of datapoints: 50000
      Root location: /mnt/3CE35B99003D727B/input/pytorch/data
      Split: Train
      StandardTransform Transform: Compose(
                 Resize(size=32, interpolation=PIL.Image.BILINEAR)
                 ToTensor()
             )


title: How to convert a Label matrix to colour matrix for image segmentation?

body:
I have a label matrix of 256*256 for example. And the classes are 0-11 so 12 classes. I want to convert the label matrix to colour matrix. I tried do it in a code like this

`for i in range(256):
    for j in range(256):
        if x[i][j] == 11:
            dummy[i][j] = [255,255,255]
        if x[i][j] == 1:
            dummy[i][j] = [144,0,0]
        if x[i][j] == 2:
            dummy[i][j] = [0,255,0]
        if x[i][j] == 3:
            dummy[i][j] = [0,0,255]
        if x[i][j] == 4:
            dummy[i][j] = [144,255,0]
        if x[i][j] == 5:
            dummy[i][j] = [144,0,255]
        if x[i][j] == 6:
            dummy[i][j] = [0,255,255]
        if x[i][j] == 7:
            dummy[i][j] = [122,0,0]
        if x[i][j] == 8:
            dummy[i][j] = [0,122,0]
        if x[i][j] == 9:
            dummy[i][j] = [0,0,122]
        if x[i][j] == 10:
            dummy[i][j] = [122,0,122]
        if x[i][j] == 11:
            dummy[i][j] = [122,122,0]
            `

It is highly inefficient. PS: the shape of x is [256 256] and dummy is [256 256 3]. Is there any better way to do it ?


title: Keras learning rate decay in pytorch

body:
I have a question concerning learning rate decay in Keras. I need to understand how the option decay works inside optimizers in order to translate it to an equivalent PyTorch formulation.

From the source code of SGD I see that the update is done this way after every batch update:

lr = self.lr * (1. / (1. + self.decay * self.iterations))

Does this mean that after every batch update the lr is updated starting from its value from its previous update or from its initial value? I mean, which of the two following interpretation is the correct one?

lr = lr_0 * (1. / (1. + self.decay * self.iterations))

or

lr = lr * (1. / (1. + self.decay * self.iterations)),

where lr is the lr updated after previous iteration and lr_0 is always the initial learning rate.

If the correct answer is the first one, this would mean that, in my case, the learning rate would decay from 0.001 to just 0.0002 after 100 epochs, whereas in the second case it would decay from 0.001 at around 1e-230 after 70 epochs.

Just to give you some context, I'm working with a CNN for a regression problem from images and I just have to translate Keras code into Pytorch code. So far, with the second of the afore-mentioned interpretations I manage to only always predict the same value, disregarding of batch size and input at test time.

Thanks in advance for your help!


title: Pytorch's model can't feed forward a DataLoader dataset, NotImplementedError

body:
so I am experimenting with PyTorch library to train a CNN. There is nothing wrong with the model (I can feed forward a data w/ no error) and I prepare a custom dataset with DataLoader function.

This is my code for data prep (I've omitted some irrelevant variable declaration, etc.):

# Initiliaze model
class neural_net_model(nn.Module):
      # omitted
      ...

# Prep the dataset
train_data = torchvision.datasets.ImageFolder(root = TRAIN_DATA_PATH, transform = TRANSFORM_IMG)
train_data_loader = data_utils.DataLoader(train_data, batch_size = BATCH_SIZE, shuffle = True)

test_data = torchvision.datasets.ImageFolder(root = TEST_DATA_PATH, transform = TRANSFORM_IMG)
test_data_loader = data_utils.DataLoader(test_data, batch_size = BATCH_SIZE, shuffle = True)

But, in the training code (which I follow based on various online references), there is an error when I feed forward the model with this instruction:

...

for step, (data, label) in enumerate(train_data_loader):
    outputs = neural_net_model(data)
    ...

Which raise an error:

NotImplementedError                       Traceback (most recent call last)
<ipython-input-12-690cfa6916ec> in <module>
      6
      7         # Forward pass
----> 8         outputs = neural_net_model(images)
      9         loss = criterion(outputs, labels)
     10

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in forward(self, *input)
     83             registered hooks while the latter silently ignores them.
     84         """
---> 85         raise NotImplementedError
     86
     87     def register_buffer(self, name, tensor):

NotImplementedError:

I can't find a similar problems on the internet and it seems strange because I've followed the code much exactly as the references and the error is not really well defined in the docs (NotImplementedError:)

Do you guys know the cause and solution to this problem?

This is the code for the network

from torch import nn, from_numpy
import torch
import torch.nn.functional as F

class DeXpression(nn.Module):
    def __init__(self, ):
        super(DeXpression, self).__init__()

        # Layer 1
        self.convolution1 = nn.Conv2d(in_channels = 1, out_channels = 64, kernel_size = 7, stride = 2, padding = 3)
        self.pooling1 = nn.MaxPool2d(kernel_size = 3, stride = 2, padding = 0)

        # Layer FeatEx1
        self.convolution2a = nn.Conv2d(in_channels = 64, out_channels = 96, kernel_size = 1, stride = 1, padding = 0)
        self.convolution2b = nn.Conv2d(in_channels = 96, out_channels = 208, kernel_size = 3, stride = 1, padding = 1)

        self.pooling2a = nn.MaxPool2d(kernel_size = 3, stride = 1, padding = 1)
        self.convolution2c = nn.Conv2d(in_channels = 64, out_channels = 64, kernel_size = 1, stride = 1, padding = 0)

        self.pooling2b = nn.MaxPool2d(kernel_size = 3, stride = 2, padding = 0)

        # Layer FeatEx2
        self.convolution3a = nn.Conv2d(in_channels = 272, out_channels = 96, kernel_size = 1, stride = 1, padding = 0)
        self.convolution3b = nn.Conv2d(in_channels = 96, out_channels = 208, kernel_size = 3, stride = 1, padding = 1)

        self.pooling3a = nn.MaxPool2d(kernel_size = 3, stride = 1, padding = 1)
        self.convolution3c = nn.Conv2d(in_channels = 272, out_channels = 64, kernel_size = 1, stride = 1, padding = 0)

        self.pooling3b = nn.MaxPool2d(kernel_size = 3, stride = 2, padding = 0)

        # Fully-connected Layer
        self.fc1 = nn.Linear(45968, 1024)
        self.fc2 = nn.Linear(1024, 64)
        self.fc3 = nn.Linear(64, 8)

    def net_forward(self, x):
        # Layer 1
        x = F.relu(self.convolution1(x))
        x = F.local_response_norm(self.pooling1(x), size = 2)
        y1 = x
        y2 = x
        # Layer FeatEx1
        y1 = F.relu(self.convolution2a(y1))
        y1 = F.relu(self.convolution2b(y1))

        y2 = self.pooling2a(y2)
        y2 = F.relu(self.convolution2c(y2))

        x = torch.zeros([y1.shape[0], y1.shape[1] + y2.shape[1], y1.shape[2], y1.shape[3]])
        x[:, 0:y1.shape[1], :, :] = y1
        x[:,  y1.shape[1]:, :, :] = y2

        x = self.pooling2b(x)
        y1 = x
        y2 = x
        # Layer FeatEx2
        y1 = F.relu(self.convolution3a(y1))
        y1 = F.relu(self.convolution3b(y1))

        y2 = self.pooling3a(y2)
        y2 = F.relu(self.convolution3c(y2))

        x = torch.zeros([y1.shape[0], y1.shape[1] + y2.shape[1], y1.shape[2], y1.shape[3]])
        x[:, 0:y1.shape[1], :, :] = y1
        x[:,  y1.shape[1]:, :, :] = y2

        x = self.pooling3b(x)
        # Fully-connected layer
        x = x.view(-1, x.shape[0] * x.shape[1] * x.shape[2] * x.shape[3])
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.log_softmax(self.fc3(x), dim = None)

        return x


title: Difficulty in training with callback 'register_backward_hook' in PyTorch

body:
I have a simple Hook class which works pretty fine with forward pass but has difficulty during the backward pass.

The hooks are successfully registered on each layer of my deep model but are not recording the gradients though the forward is working fine.

The model was run for 5 iterations and problem with backward pass can be seen below. Backward hooks for layer 0,1,2 are empty which should not have been the case. For each iteration, my code return the mean the variance rather than returning the full tensor.

Forward Hook of layer 0
Mean: [-1.511833667755127, -1.6455317735671997, -2.1622400283813477, -2.1323182582855225, -1.8739168643951416], Std: [2.0011091232299805, 1.855371117591858, 2.315523624420166, 1.3520175218582153, 2.242307424545288]

 Forward Hook of layer 1
Mean: [0.11147132515907288, -0.08157677203416824, -0.010751131922006607, -0.0921892449259758, -0.04136423021554947], Std: [0.20785003900527954, 0.24797917902469635, 0.08388562500476837, 0.0, 0.17534032464027405]

 Forward Hook of layer 2
Mean: [-0.17542554438114166, 0.06612542271614075, -0.0477512925863266, 0.03804510459303856, 0.0076016997918486595], Std: [0.13048705458641052, 0.09858430176973343, 0.0582481250166893, 0.02330612950026989, 0.07083183526992798]

 Forward Hook of layer 3
Mean: [0.41979557275772095, -0.5407723188400269, -3.3170995712280273, -9.282533645629883, -19.208200454711914], Std: [0.6125074028968811, 0.375322163105011, 1.5524002313613892, 3.9379470348358154, 4.83458948135376]

 Backward Hook of layer 0
Mean: [], Std: [] # Why is this empty??

 Backward Hook of layer 1
Mean: [], Std: [] # Why is this empty??

 Backward Hook of layer 2
Mean: [], Std: [] # Why is this empty??

 Backward Hook of layer 3
Mean: [5.980815887451172, 5.21722936630249, 6.572475910186768, 5.469051837921143, 4.7423529624938965], Std: [3.251680850982666, 1.677825927734375, 2.518986225128174, 2.810399055480957, 2.7709243297576904]

Can someone please let me know why the gradients are empty for layer 0,1 and 2?

-----------My full code is below

import torch
import torch.nn as nn

# creat a dummy deep net
class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()

        self.conv1 = nn.Conv2d(1,1, kernel_size=3, stride=1, padding=1, bias=True)
        self.conv2 = nn.Conv2d(1,1, kernel_size=3, stride=1, padding=1, bias=True)
        self.conv3 = nn.Conv2d(1,1, kernel_size=3, stride=1, padding=1, bias=True)
        self.seq = nn.Sequential(
                    nn.Conv2d(1,2, kernel_size=3, stride=1, padding=1, bias=True),
                    nn.LeakyReLU(negative_slope=0, inplace=True),
                    nn.Conv2d(2,1, kernel_size=3, stride=1, padding=1, bias=True),
                    )
        self.relu = nn.LeakyReLU(negative_slope=0, inplace=True)

    def forward(self, x):
        print('conv1')
        out = self.conv1(x)
        print('relu')
        out = self.relu(out)
        print('conv2')
        out =self.conv2(out)
        print('conv3')
        out = self.conv3(out)
        print('ResBlock Identity addition')
        out = out + x
        print('sequential')
        out = self.seq(x)

        return out

net = Net()
print(net)
criterion = nn.MSELoss()

# ------------------The Hook class begins to calculate each layer stats
class Hook():
    def __init__(self, module, backward=False):
        if backward==False:
            self.hook = module.register_forward_hook(self.hook_fn)
        else:
            self.hook = module.register_backward_hook(self.hook_fn)

        self.Mean = []
        self.Std = []
        self.backward = backward

    def hook_fn(self, module, input, output):
        #self.inputMean.append(input[0][0,...].mean().item())
        print('Hook is called on {}'.format(module))
        #calculate only for 1st image in the batch
        if self.backward:
            self.Mean.append(output[0][0,...].mean().item())
            self.Std.append(output[0][0,...].std().item())

        else:
            self.Mean.append(output[0,...].mean().item())
            self.Std.append(output[0,...].std().item())

# create hooks on each layer
hookF = []
hookB = []
for i,layer in enumerate(list(net.children())):
    if not isinstance(layer,nn.ReLU) and not isinstance(layer,nn.LeakyReLU):
        print('Hooked to {}'.format(layer))
        hookF.append(Hook(layer))
        hookB.append(Hook(layer,backward=True))

optimizer = torch.optim.Adam(net.parameters(),lr=0.1)

for _ in range(5):
    print('Iteration --------')
    data = torch.rand(2,1,3,3)*10
    print('Input mean is {}'.format(data[0,...].mean()))
    target = data.clone()
    print('Forward xxxxxx')
    out = net(data)
    print('backward xxxxxx')
    out.backward(target,retain_graph=True)

    #loss = criterion(out, target)
    #loss.backward()

    optimizer.step()
    optimizer.zero_grad()

for i,h in enumerate(hookF):

    print('\n Forward Hook of layer {}'.format(i))
    print('Mean: {}, Std: {}'.format(h.Mean, h.Std))
    h.hook.remove()

for i,h in enumerate(hookB):
    print('\n Backward Hook of layer {}'.format(i))
    print('Mean: {}, Std: {}'.format(h.Mean, h.Std))
    h.hook.remove()

Thankyou


title: huggingface transformers bert model without classification layer

body:
I want to do a joint-embedding from vgg16 and bert for classification.

The thing with huggingface transformers bert is that it has the classification layer which has num_labels dimension.

But, I want the output from BertPooler (768 dimensions) which I will use as a text-embedding for an extended model.

from transformers import BertForSequenceClassification

model = BertForSequenceClassification.from_pretrained('bert-base-uncased')

This gives the following model:

BertForSequenceClassification(
...
...
        (11): BertLayer(
          (attention): BertAttention(
            (self): BertSelfAttention(
              (query): Linear(in_features=768, out_features=768, bias=True)
              (key): Linear(in_features=768, out_features=768, bias=True)
              (value): Linear(in_features=768, out_features=768, bias=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
            (output): BertSelfOutput(
              (dense): Linear(in_features=768, out_features=768, bias=True)
              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
              (dropout): Dropout(p=0.1, inplace=False)
            )
          )
          (intermediate): BertIntermediate(
            (dense): Linear(in_features=768, out_features=3072, bias=True)
          )
          (output): BertOutput(
            (dense): Linear(in_features=3072, out_features=768, bias=True)
            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)
            (dropout): Dropout(p=0.1, inplace=False)
          )
        )
      )
    )
    (pooler): BertPooler(
      (dense): Linear(in_features=768, out_features=768, bias=True)
      (activation): Tanh()
    )
  )
  (dropout): Dropout(p=0.1, inplace=False)
  (classifier): Linear(in_features=768, out_features=2, bias=True)
)

How can I get rid of the classifier layer?


title: Result of slicing in an empty tensor

body:
Can anyone help me to understand why this works:

lens = list(range(170,1,-1))
xs = Variable(torch.randn(169, 200, 1))
packed = torch.nn.utils.rnn.pack_padded_sequence(xs, lens, batch_first=True)

and this does not:

lens = [294, 289, 288, 282, 273, 270, 261, 260, 240, 235, 231, 228, 228, 227, 226, 226, 199, 195, 194, 192, 190, 189, 177, 176, 165, 165, 161, 156, 153, 149, 149, 142, 142, 137, 136, 136, 135, 134, 134, 132, 131, 129, 122, 121, 121, 114, 113, 113, 112, 110, 109, 108, 107, 107, 106, 105, 105, 103, 102, 100, 99, 99, 98, 96, 95, 93, 92, 91, 91, 90, 88, 88, 87, 79, 78, 78, 77, 76, 75, 74, 73, 72, 72, 71, 71, 71, 71, 69, 69, 69, 68, 68, 68, 68, 68, 68, 67, 67, 66, 66, 65, 65, 64, 64, 64, 63, 63, 61, 61, 61, 61, 60, 60, 59, 59, 59, 59, 57, 57, 57, 57, 57, 56, 56, 55, 55, 54, 54, 54, 54, 54, 53, 53, 52, 52, 52, 51, 51, 51, 51, 51, 50, 50, 50, 49, 49, 49, 48, 47, 47, 47, 47, 46, 46, 46, 45, 44, 44, 44, 44, 43, 42, 39, 38, 36, 30, 30, 25, 23]
xs = Variable(torch.randn(169, 200, 1))
packed = torch.nn.utils.rnn.pack_padded_sequence(xs, lens, batch_first=True)

I am getting a "ValueError: result of slicing is an empty tensor"


title: Minimize angle in pytorch

body:
I'm trying to figure out how to minimize a scalar in pytorch that represents the angle of an axis/angle rotation. My target is a sample set of 3D vectors, my input is the target rotated by a particular axis/angle rotation (plus some gaussian noise). The axis is known and fixed. I want to find the angle using pytorch.

What I have so far is this nn.Module:

class AngleModel(torch.nn.Module):
    def __init__(self):
        super(AngleModel, self).__init__()
        self.angle = nn.Parameter(Variable(torch.Tensor([0.0]), requires_grad=True))
        self.qw = torch.cos(self.angle / 2.)
        self.qx = 0.0
        self.qy = 0.0
        self.qz = torch.sin(self.angle / 2.)

    def forward(self, input):
        matrix_np = transform_from_pq([0, 0, 0, self.qw, self.qx, self.qy, self.qz])
        matrix = torch.from_numpy(matrix_np)
        input_ext = torch.cat((input, torch.ones(input.size(0)).reshape(-1, 1)), 1)
        output = torch.matmul(input_ext.float(), matrix.float())
        return output[:, :3]

However using this in an optimizer fails because model.parameters() returns an empty list (i.e. "empty" generator).

I'm completely new to pytorch. What am I doing wrong?

Relevant code

Initialization:

def _init_model(self):
    self.model = AngleModel()
    self.crit = torch.nn.MSELoss()
    l_rate = 0.01
    print(list(self.model.parameters()))
    self.optim = torch.optim.SGD(self.model.parameters(), lr = l_rate)
    self.epochs = 2000

Learning function:

for epoch in range(self.epochs):
    inputs = torch.from_numpy(normalize(input))
    labels = torch.from_numpy(normalize(target))
    _x = Variable(inputs, requires_grad = True) # without it complains about no grad_fn
    _y = Variable(labels)
    self.optim.zero_grad()

    outputs = self.model.forward(_x)
    loss = self.crit(outputs, _y)
    loss.backward()
    self.optim.step()
    print("Epoch %6d loss %05.3f; %s" % (epoch, loss.data[0], self.model.angle))

Update 1:

Code of the Module updated
Surrounding code added
This at least does not throw any errors anymore but also does not optimize angle


title: how sort randomly images and its mask?

body:
before splitting the dataset I need to randomly load the data and then do splitting . This is the snippet for splitting the dataset which is not randomly. I am wondering how can I do this for images and corresponding mask in folder_mask?

folder_data = glob.glob("D:\\Neda\\Pytorch\\U-net\\my_data\\imagesResized\\*.png")
folder_mask = glob.glob("D:\\Neda\\Pytorch\\U-net\\my_data\\labelsResized\\*.png")

 # split these path using a certain percentage
  len_data = len(folder_data)
  print("count of dataset: ", len_data)
  # count of dataset:  992

  split_1 = int(0.6 * len(folder_data))
  split_2 = int(0.8 * len(folder_data))

  #folder_data.sort()

  train_image_paths = folder_data[:split_1]
  print("count of train images is: ", len(train_image_paths))

  valid_image_paths = folder_data[split_1:split_2]
  print("count of validation image is: ", len(valid_image_paths))

  test_image_paths = folder_data[split_2:]
  print("count of test images is: ", len(test_image_paths))

   train_mask_paths = folder_mask[:split_1]
   valid_mask_paths = folder_mask[split_1:split_2]
   test_mask_paths = folder_mask[split_2:]

   train_dataset = CustomDataset(train_image_paths, train_mask_paths)
   train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1,
   shuffle=True, num_workers=2)

   valid_dataset = CustomDataset(valid_image_paths, valid_mask_paths)
   valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=1,
   shuffle=True, num_workers=2)

    test_dataset = CustomDataset(test_image_paths, test_mask_paths)
    test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1,
     shuffle=False, num_workers=2)

     dataLoaders = {
          'train': train_loader,
          'valid': valid_loader,
           'test': test_loader,
                 }


title: Perform max pooling on Integer tensor in Pytorch

body:
I am defining a max pool layer and passing in an integer tensor as follows.

max_pool = nn.MaxPool2d(3, stride=2)
max_pool(torch.IntTensor(3,5,5).random_(0, 10))

It throws the following error:

RuntimeError: _thnn_max_pool2d_with_indices_forward not supported on CPUType for Int


title: Why the backpropagation process can still work when I included 'loss.backward()' in 'with torch.no_grad():'?

body:
I'm working with a linear regression example in PyTorch. I know I did wrong including 'loss.backward()' in 'with torch.no_grad():', but why it worked well with my code?

According to pytorch docs, torch.autograd.no_grad is a context-manager that disabled gradient calculation. So I'm really confused.

Code here:

import torch
import torch.nn as nn
import numpy as np
import matplotlib.pyplot as plt
# Toy dataset
x_train = np.array([[3.3], [4.4], [5.5], [6.71], [6.93], [4.168],
                    [9.779], [6.182], [7.59], [2.167], [7.042],
                    [10.791], [5.313], [7.997], [3.1]], dtype=np.float32)

y_train = np.array([[1.7], [2.76], [2.09], [3.19], [1.694], [1.573],
                    [3.366], [2.596], [2.53], [1.221], [2.827],
                    [3.465], [1.65], [2.904], [1.3]], dtype=np.float32)

input_size = 1
output_size = 1
epochs = 100
learning_rate = 0.05

model = nn.Linear(input_size, output_size)

criterion = nn.MSELoss(reduction='sum')
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

# training
for epoch in range(epochs):
    # convert numpy to tensor
    inputs = torch.from_numpy(x_train)
    targets = torch.from_numpy(y_train)

    # forward
    out = model(inputs)
    loss = criterion(out, targets)

    # backward
    with torch.no_grad():
        model.zero_grad()
        loss.backward()
        optimizer.step()
        print('inputs grad : ', inputs.requires_grad)
    if epoch % 5 == 0:
        print ('Epoch [{}/{}], Loss: {:.4f}'.format(epoch+1, epochs, loss.item()))

predicted = model(torch.from_numpy(x_train)).detach().numpy()
plt.plot(x_train, y_train, 'ro', label='Original data')
plt.plot(x_train, predicted, label='Fitted line')
plt.legend()
plt.show()

# Save the model checkpoint
torch.save(model.state_dict(), 'model\linear_model.ckpt')

Thanks in advance for answering my question.