title: Pytorch model (transfer learning) not learning

body:
I am going through the transfer learning tutorial in Pytorch:
https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html

I have taken most of the code from the transfer learning tutorial,
   and made some changes to print out the val_acc more often to fit my dataset.

Despite the original transfer learning tutorial code working (Densenet model was training), my code with the changes doesn't seem to be working anymore.
  I can't figure out why.  I have played around with the learning rates and yet no change.

 from __future__ import print_function, division

import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy

def prepare_dataset_from_folder(data_dir, size, batch_size):
    """prepare datasets into dataloaders
    args
    ------
    data_dir = root_dir holding 'train_dir' and 'val_dir'

    size = size of image

    batch_size = batch_size
    """

    data_transforms = {
        'training': transforms.Compose([
           transforms.Resize((size,size)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]),
        'validation': transforms.Compose([
            transforms.Resize((size,size)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]),
    }

    image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
                                              data_transforms[x])
                      for x in ['training', 'validation']}

    dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size= batch_size,
                                                 shuffle=True, num_workers=1) #num_workers leads to errors
                  for x in ['training', 'validation']}
    dataset_sizes = {x: len(image_datasets[x]) for x in ['training', 'validation']}
    class_names = image_datasets['training'].classes

    return dataloaders, dataset_sizes, class_names

def train_model(model, criterion, optimizer, scheduler, num_epochs=3, best_acc = 80.0, batch_size = 5):
    since = time.time()
    dataloaders, dataset_sizes, class_names =  prepare_dataset_from_folder(data_dir, size, batch_size)

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        running_loss = 0.0
        running_corrects = 0

        for num, data in enumerate(dataloaders["training"]):

            if num % 100 == 0:
                   val_running_loss = 0.0
                   val_running_corrects = 0
                   model.eval()

                   for val_num, val_data in enumerate(dataloaders["validation"]):
                       inputs, labels = val_data
                       inputs = inputs.to(device)
                       labels = labels.to(device)
                       with torch.set_grad_enabled(False):
                           outputs = model(inputs)
                           _, preds = torch.max(outputs, 1)
                           loss = criterion(outputs, labels)

                       val_running_loss += loss.item() * inputs.size(0)
                       val_running_corrects += torch.sum(preds == labels.data)

                   print(dataset_sizes['validation'])
                   val_loss = ( val_running_loss/dataset_sizes['validation'])
                   val_acc = (val_running_corrects.double() / dataset_sizes['validation'])#.cpu().numpy()
                   print('val_loss  {:.4f} val_acc: {:.4f}'.format(val_loss, val_acc))
                   if val_acc > best_acc:
                      print("Saving due to high val accuracy")
                      x = datetime.datetime.now()
                      time_stamp_str = str(x).split(' ')[0] + "_"
                      save_path = os.path.join(SAVE_DIR, f"val_acc_{val_acc}.pt")
                      torch.save(model, save_path)

            scheduler.step()
            model.train()
            inputs, labels = data
            inputs = inputs.to(device)
            labels = labels.to(device)
             #what is this

            optimizer.zero_grad()

            with torch.set_grad_enabled(True):
                    #print(torch.is_grad_enabled())
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)
                    loss.backward()
                    optimizer.step()

            running_loss += loss.item() * inputs.size(0)
            running_corrects += torch.sum(preds == labels.data)

            if num % 100 == 0:
                    print("100 batches")
                    temp_running_corrects = (running_corrects.double() / ( len(inputs) *  num + 1)).cpu().numpy()
                    print(temp_running_corrects)

        epoch_loss = running_loss / dataset_sizes['training']
        epoch_acc = running_corrects.double() / dataset_sizes['training']

        print('Training Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
    time_elapsed // 60, time_elapsed % 60))

    return model

if __name__=="__main__":

    data_dir = r'TEST 10'
    device = torch.device("cuda:0")
    PATH =" "
    size = 224
    batch_size = 2

    # pick your model
    #model_ft = torch.load(PATH)
    #model_ft = models.resnet18(pretrained=True)
    model_ft = models.densenet201(pretrained=True)

    #num_ftrs = model_ft.fc.in_features #in features
    num_ftrs = 1920 #densenet 201
    #https://discuss.pytorch.org/t/what-does-the-fc-in-feature-mean/4889

    model_ft.fc = nn.Linear(num_ftrs, 2)

    model_ft = model_ft.to(device)

    criterion = nn.CrossEntropyLoss()

    # Observe that all parameters are being optimized
    optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.0001, momentum=0.9)

    # Decay LR by a factor of 0.1 every 7 epochs
    exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=2, gamma=0.1)

    model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,num_epochs=3, batch_size = batch_size)

Here is the result:

Training Loss: 8.5898 Acc: 0.0102
Epoch 1/2
val_loss 8.5163 val_acc: 0.0191
val_loss 8.9667 val_acc: 0.0255
val_loss 9.3078 val_acc: 0.0127
val_loss 8.7855 val_acc: 0.0318
val_loss 8.6217 val_acc: 0.0255
val_loss 9.0631 val_acc: 0.0191
val_loss 8.5167 val_acc: 0.0255
val_loss 9.0499 val_acc: 0.0191
val_loss 9.0549 val_acc: 0.0255
val_loss 8.8373 val_acc: 0.0191
val_loss 8.9288 val_acc: 0.0191
val_loss 8.9968 val_acc: 0.0127
val_loss 9.2790 val_acc: 0.0127
val_loss 9.4389 val_acc: 0.0191
val_loss 8.6907 val_acc: 0.0318
val_loss 9.0903 val_acc: 0.0191
val_loss 9.0093 val_acc: 0.0191
val_loss 9.4387 val_acc: 0.0127
val_loss 9.1059 val_acc: 0.0191
val_loss 9.3480 val_acc: 0.0127
val_loss 8.9435 val_acc: 0.0191
val_loss 8.4412 val_acc: 0.0318
val_loss 8.8712 val_acc: 0.0382
val_loss 8.9125 val_acc: 0.0191
val_loss 9.3815 val_acc: 0.0127
val_loss 9.0214 val_acc: 0.0191
val_loss 9.4234 val_acc: 0.0127
val_loss 9.1625 val_acc: 0.0191

However if I use the original tutorial,
my model seems to be learning.

from __future__ import print_function, division
import pretrainedmodels as ptmodels

import torch
import torch.nn as nn
import torch.optim as optim
from torch.optim import lr_scheduler
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time
import os
import copy

def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['training', 'validation']:
            if phase == 'training':
                scheduler.step()  #what is this
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for num, data in enumerate(dataloaders[phase]):
                inputs, labels = data
                inputs = inputs.to(device)
                labels = labels.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'training'):
                    outputs = model(inputs)
                    _, preds = torch.max(outputs, 1)
                    loss = criterion(outputs, labels)

                    # backward + optimize only if in training phase
                    if phase == 'training':
                        loss.backward()
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
                running_corrects += torch.sum(preds == labels.data)

                if num % 100 == 0:
                    print("500 batches")
                    temp_running_corrects = (running_corrects.double() / ( len(inputs) *  num + 1)).cpu().numpy()
                    print(temp_running_corrects)
                    #print('running_corrects', running_corrects/( len(inputs) *  num + 1).numpy())

            epoch_loss = running_loss / dataset_sizes[phase]
            epoch_acc = running_corrects.double() / dataset_sizes[phase]

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(
                phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'validation' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    save_path = os.path.join(SAVE_DIR, f"val_acc_{val_acc}.pt")
    torch.save(model, save_path)
    return model

if __name__=="__main__":

    img_size = 224
    data_transforms = {
        'training': transforms.Compose([
           transforms.Resize((img_size,img_size)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]),
        'validation': transforms.Compose([
            transforms.Resize((img_size,img_size)),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ]),
    }
    data_dir = r'TEST 10'

    image_datasets = {x: datasets.ImageFolder(os.path.join(data_dir, x),
                                              data_transforms[x])
                      for x in ['training', 'validation']}
    dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=2,
                                                 shuffle=True, num_workers=4) #num_workers leads to errors
                  for x in ['training', 'validation']}
    dataset_sizes = {x: len(image_datasets[x]) for x in ['training', 'validation']}
    class_names = image_datasets['training'].classes

    device = torch.device("cuda:0")

    #model_ft = models.resnet18(pretrained=True)
    #model_ft = models.resnet152(pretrained = True)
    #model_ft = ptmodels.__dict__['polynet'](num_classes=1000, pretrained='imagenet')
    model_ft = models.densenet201(pretrained=True)
    #model_ft = models.resnext101_32x8d(pretrained=True).fc.in_features
    print(model_ft.features)

    #num_ftrs = model_ft.fc.in_features #in features
    num_ftrs = 1920 #densenet 201
    #num_ftrs = 2048
    #https://discuss.pytorch.org/t/what-does-the-fc-in-feature-mean/4889

    model_ft.fc = nn.Linear(num_ftrs, 2)
    #model_ft.last_linear = nn.Linear(num_ftrs, 2)

    model_ft = model_ft.to(device)

    criterion = nn.CrossEntropyLoss()

    # Observe that all parameters are being optimized
    optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)

    # Decay LR by a factor of 0.1 every 7 epochs
    exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=2, gamma=0.1)

    model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                           num_epochs=3)

https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html

Training Loss:
Epoch 0/2

500 batches
0.0
500 batches
0.681592039800995
500 batches
0.6359102244389028
500 batches
0.6422628951747088
500 batches
0.6541822721598003
500 batches
0.6763236763236763
500 batches
0.694421315570358
500 batches
0.7059243397573162
500 batches
0.7108057464084947
500 batches
0.7096057745696835
500 batches
0.7126436781609196
500 batches
0.7142208087233075
500 batches
0.7251145356101625
500 batches
0.7293348712033834
500 batches
0.729382363441628
500 batches
0.7340886371209597
500 batches
0.7363323961262105

I have been stuck at it for a couple of days, and can't figure it out.
I am just attempting to print out the validation accuracy more frequently.

Thank you.


title: Serving pytorch model with django and uwsgi, how can the system support multi requests concurrency?

body:
If I street-test the API with 10 concurrency requests:

using django built-in server, python3 manage.py runserver, the average response time is about 10 times longer than 1 concurrency request
using uwsgi to start 10 progress, the average response time is about 5 times longer than 1 concurrency request

How can I handle multi requests in instant response time?


title: Some problems when loading weights in Pytorch

body:
I searched lots of resources to solve this issue, but still stuck here.

I followed the pytorch tutorial and saved parameters with

torch.save(the_model.state_dict(), PATH)

then loaded parameters with

the_model = TheModelClass(*args, **kwargs)
the_model.load_state_dict(torch.load(PATH))

After I loaded parameters, I printed my model. It came out an error.

IncompatibleKeys(missing_keys=[], unexpected_keys=[])

I found some people facing the same problem, but it seems that it can be ignored?! Then I try to input data into model with ```model.forward()''', it came out another error.

AttributeError: 'IncompatibleKeys' object has no attribute 'forward'

I know this kind of saving method(the_model.state_dict()) is just save "weights". It should be used only .eval() due to some important information not be saved (dropout, batchnorm, etc.). So I try model.eval(), it still has the same error.

AttributeError: 'IncompatibleKeys' object has no attribute 'eval'

Here's are some relative code:

Initialize model:

model = VAE(some constructor parameters)
After training:

checkpoint_path = os.path.join(save_path, "E%02d.pkl" % ep)
torch.save(model.state_dict(), checkpoint_path)

Initialize a same model and load parameters into model:

model = VAE(some constructor parameters)
checkpoint = torch.load("E24.pkl", map_location='cuda:0')
model = model.load_state_dict(checkpoint)

I won't train this model anymore. I just want to load parameters then check the performance. Thanks for reading. :)


title: Passing weights to cross entropy loss

body:
I am trying to assign different weights to different classes, so I have modified my loss criterion as such:

I had to convert the weight tensor to double torch.DoubleTensor(weight) since my model is already moved to double(). Am I doing it correctly ?

weights = [0.4,0.8,1.0]

class_weights = torch.DoubleTensor(weights).cuda()

criterion = nn.CrossEntropyLoss(weight=class_weights)


title: How to extract output of torch model in c++?

body:
I have got trained keras model and converted it using mmdnn. Then I try use it in c++ code:

#include <iostream>

#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>

#include <torch.h>

int main()
{
    cv::Mat image;
    image= cv::imread("test_img.png", cv::IMREAD_GRAYSCALE);   // Read the file

try
{
    torch::jit::script::Module module;
    module = torch::jit::load("my_model.pth");

    torch::IntArrayRef input_dim = std::vector<int64_t>({ 1, 2, 256, 256});

    cv::Mat input_img;
    image.convertTo(input_img, CV_32FC3, 1 / 255.0);
    torch::Tensor x = torch::from_blob(input_img.data, { 1, 2, 256, 256 }, torch::kFloat);
    torch::NoGradGuard no_grad;

    auto output = module.forward({ x });

    float* data = static_cast<float*>(output.toTensor().data_ptr());

    cv::Mat output_img = cv::Mat(256, 256, CV_32FC3, data);
    cv::imwrite("output_img.png", output_img);
}
catch (std::exception &ex)
{
    std::cout << "exception! " << ex.what() << std::endl;
}

    return 0;
}

This code throws an exception:

  exception! isTensor() INTERNAL ASSERT FAILED at
  E:\20B\pytorch\pytorch\aten\src\ATen/core/ivalue_inl.h:112, please
  report a bug to PyTorch. Expected Tensor but got Tuple (toTensor at
  E:\20B\pytorch\pytorch\aten\src\ATen/core/ivalue_inl.h:112) (no
  backtrace available)

This was thrown in line float* data = static_cast<float*>(output.toTensor().data_ptr()); when the function toTensor() was called. If I use toTuple() instead of toTensor() then the result doesn't have the function data_ptr(), but I need this for extracting data (and putting it into opencv image).

How to extract image from the model output?


title: How to split test and train data in a dataset based on number of targets in each category

body:
I have an imageFolder in PyTorch which holds my categorized data images. Each folder is the name of the category and in the folder are images of that category.
I've loaded data and split train and test data via a sampler with random train_test_split. But the problem is my data distribution isn't good and some classes have lots of images and some classes have fewer.
So for solving this problem I want to choose 20% of each class as my test data and the rest would be the train data
ds = ImageFolder(filePath, transform=transform)
batch_size = 64
validation_split = 0.2

indices = list(range(len(ds))) # indices of the dataset

# TODO: fix spliting
train_indices,test_indices = train_test_split(indices,test_size=0.2)

# Creating PT data samplers and loaders:
train_sampler = SubsetRandomSampler(train_indices)
test_sampler = SubsetRandomSampler(test_indices)

train_loader = torch.utils.data.DataLoader(ds, batch_size=batch_size, sampler=train_sampler, num_workers=16)
test_loader = torch.utils.data.DataLoader(ds, batch_size=batch_size, sampler=test_sampler, num_workers=16)

Any idea of how I should fix it?


I am trying to visualize cnn network features map for conv1 layer based on the code and architecture below. It’s working properly without DataParallel, but when I am activating model = nn.DataParallel(model) it raised with error: ‘DataParallel’ object has no attribute ‘conv1’. Any suggestion appreciated.
class Model(nn.Module):
    def __init__(self, kernel, num_filters, res = ResidualBlock):
        super(Model, self).__init__()

        self.conv0 = nn.Sequential(
            nn.Conv2d(4, num_filters, kernel_size = kernel*3,
                       padding = 4),
            nn.BatchNorm2d(num_filters),
            nn.ReLU(inplace=True))

        self.conv1 = nn.Sequential(
            nn.Conv2d(num_filters, num_filters*2, kernel_size = kernel,
                      stride=2, padding = 1),
            nn.BatchNorm2d(num_filters*2),
            nn.ReLU(inplace=True))

        self.conv2 = nn.Sequential(
            nn.Conv2d(num_filters*2, num_filters*4, kernel_size = kernel, stride=2, padding = 1),
            nn.BatchNorm2d(num_filters*4),
            nn.ReLU(inplace=True))

        self.tsconv0 = nn.Sequential(
            nn.ConvTranspose2d(num_filters*4, num_filters*2, kernel_size = kernel, padding = 1),
            nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(num_filters*2))

        self.tsconv1 = nn.Sequential(
            nn.ConvTranspose2d(num_filters*2, num_filters, kernel_size = kernel, padding = 1),
            nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True),
            nn.ReLU(inplace=True),
            nn.BatchNorm2d(num_filters))

        self.tsconv2 = nn.Sequential(
            nn.Conv2d(num_filters, 1, kernel_size = kernel*3, padding = 4, bias=False),
            nn.ReLU(inplace=True))

model = Model(kernel, num_filters)
model = nn.DataParallel(model)

The code for feature map visualization:
def get_activation(name):
    def hook(model, x_train_batch, y_train_pred):
        activation[name] = y_train_pred.detach()
    return hook

model.conv3.register_forward_hook(get_activation('conv3'))
x_train_batch[0,0,:,:]
y_train_pred = model(x_train_batch)

act = activation['conv3'].squeeze()
act1 = act.cpu().detach().numpy()
act=act[0,:,:,:]

fig, axarr = plt.subplots(6,16)
k = 0
for idx in range(act.size(0)//16):
    for idy in range(act.size(0)//6):
        axarr[idx, idy].imshow(act[k])
        k += 1


I'm trying to load an existing pytorch model with -

from mymodels import model
m = torch.load("model.pth", map_location=torch.device("cpu"))

I'm getting the error -

  line 613, in _load

  result = unpickler.load()

  Attribute error: 'ReduceLROnPlateau' object has no attribute _cmp

my pytorch version is 1.3.1. How do I fix this?


title: Low accuracy when fine-tuning BERT for question answering

body:
I'm trying to fine-tune CamemBERT (french version of Roberta) for question answering.

At the first I'm using CamemBERT model to generate the input embedding of question and text and a output linear layer to output the start and end logits that corresponds to the start and the end of the answer.

In the official results in the paper the performance for question answering is (88 %, 77%) of (F1 score, EM) but the results I get are (71%, 46%).

My question is why the results are not close enough ?

This is a part of the script that I'm using to train and evaluate the model on FQuAD dataset with the same hyper-parameters as the official model:

MAX_SEQ_LENGTH = 384
TRAIN_BATCH_SIZE = 12
n_epochs = 3
learning_rate = 3e-5
EVAL_BATCH_SIZE = 12
dropout = 0
BERT_TYPE =  "fmikaelian/camembert-base-fquad"

class CamemBERTQA(nn.Module):
   def __init__(self,bert_type, hidden_size, num_labels):
       super(CamemBERTQA, self).__init__()
       self.bert_type = bert_type
       self.hidden_size = hidden_size
       self.num_labels = num_labels
       self.camembert = AutoModel.from_pretrained(self.bert_type)
       self.qa_outputs = nn.Linear(self.hidden_size, self.num_labels)

   def forward(self, input_ids):
       output = self.camembert(input_ids = input_ids)[0]
       logits = self.qa_outputs(output)
       start_logits, end_logits = logits.split(1, dim=-1)
       start_logits = start_logits.squeeze(-1)
       end_logits = end_logits.squeeze(-1)
       outputs = (start_logits, end_logits,)
       return outputs

def train_eval_model(model, n_epochs, scheduler=None):
   train_lossess = []
   valid_lossess = []
   avg_train_losses = []
   avg_valid_losses = []
   res = []
   for epoch in trange(n_epochs):
       #######################################################################################
       ################################### train the model ###################################
       #######################################################################################
       model.train()
       for batch, d in enumerate(tqdm_notebook(train_dataloader, desc="Iteration")):
           ids = d['ids']
           start_pos = d['start_pos']
           end_pos = d['end_pos']
           ids = ids.to(device, dtype = torch.long)
           start_pos = start_pos.to(device, dtype = torch.long)
           end_pos = end_pos.to(device, dtype = torch.long)
           optimizer.zero_grad()
           start_and_end_scores = model(ids) # Forward pass return start and end positions
           loss = loss_func(start_and_end_scores, start_pos, end_pos)
           loss.backward()
           optimizer.step()
           if scheduler is not None:
               scheduler.step()
           train_lossess.append(loss.item())
       ##########################################################################################
       ################################### validate the model ###################################
       ##########################################################################################
       model.eval()
       pred_s = None
       pred_e = None
       # eval_loss = 0.0
       # eval_steps = 0

       for batch, d in enumerate(eval_dataloader):
           ids = d['ids']
           start_pos = d['start_pos']
           end_pos = d['end_pos']

           ids = ids.to(device, dtype = torch.long)
           start_pos = start_pos.to(device, dtype = torch.long)
           end_pos = end_pos.to(device, dtype = torch.long)

           with torch.no_grad():
               start_and_end_scores = model(ids)
               loss = loss_func(start_and_end_scores, start_pos, end_pos)
               valid_lossess.append(loss.item())
           # eval_steps += 1
           if pred_s is None:
               pred_s = start_and_end_scores[0].detach().cpu().numpy()
               pred_e = start_and_end_scores[1].detach().cpu().numpy()
           else:
               pred_s = np.append(pred_s, start_and_end_scores[0].detach().cpu().numpy(), axis=0)
               pred_e = np.append(pred_e, start_and_end_scores[1].detach().cpu().numpy(), axis=0)
       pred_start = np.argmax(pred_s, axis=1)
       pred_end = np.argmax(pred_e, axis=1)
       res.append([pred_start,pred_end])
       train_loss = np.average(train_lossess)
       valid_loss = np.average(valid_lossess)
       avg_train_losses.append(train_loss)
       avg_valid_losses.append(valid_loss)
       epoch_len = len(str(n_epochs))
       print_msg = (f'[{epoch:>{epoch_len}}/{n_epochs:>{epoch_len}}] ' +
                    f'train_loss: {train_loss:.5f} ' +
                    f'valid_loss: {valid_loss:.5f}')
       print(print_msg)
       train_lossess = []
       valid_lossess = []
   return  model, avg_train_losses, avg_valid_losses, res


title: Efficient way of calculating sum of unequal sized chunks of tensor in Pytorch

body:
I am wondering if there is a way of calculating the following sum of unequal sized chunks of a tensor in pytorch

import torch
import numpy as np

x = torch.rand(1000,100)
y = np.unique(np.random.choice(1000,10)

here I have a tensor x of size (1000,10), I want to calculated the sum of chucks along the first axis. These chunks are split along the first axis and y indicate the end line of each chunk. They are in general of unequal size. For example, I can do this with the following for loop

cum_pos_lik = torch.FloatTensor(y.size, 100)
y = np.append(0, y)
for i in range(1, y.size):
    cum_pos_lik[i-1, :] = x[y[i-1]:y[i], :].sum(0)

But I need this to be faster for my application. Clearly the sum of each chunk can be parallelized. I am wondering if there is a simple way in pytorch of doing it. It would also be nice if a way of vectorizing it in numpy is provided.

Thank you!


title: Assertion error (assert cur_offset == offset) in loss.backward() pytorch

body:
I am trying to train a GRU model, but running into this assertion error at the point of loss.backward(). Here is my code with the error. Any help is appreciated.

class AttendResistance(nn.Module):
       def __init__(self, nb_classes, nb_tokens, embedding_matrix,
   embed_dropout_rate=0, final_dropout_rate=0, return_attention=False):
        super(AttendResistance, self).__init__()
    embedding_dim = 20
    hidden_size = 32

    self.embed_dropout_rate = embed_dropout_rate
    self.final_dropout_rate = final_dropout_rate
    self.return_attention = return_attention
    self.hidden_size = hidden_size
    self.nb_classes = nb_classes

    self.embed = nn.Embedding(nb_tokens, embedding_dim)
    self.embed.weight = nn.Parameter(embedding_matrix)
    self.embed_dropout = nn.Dropout2d(embed_dropout_rate)
    self.gru = nn.GRU(embedding_dim, hidden_size, num_layers = 1, batch_first=True, dropout = 0.5,
                    bidirectional=False)
    self.final_drop = nn.Dropout(final_dropout_rate)
    self.linear = nn.Linear(hidden_size, nb_classes)
    self.softmax = nn.Softmax(dim = 1)

def forward(self, input_seqs):
    print (input_seqs.size())
    x = self.embed(input_seqs)
    print (x.size())
    x = nn.Tanh()(x)
    print (x.size())
    x = self.embed_dropout(x)
    print (x.size())
    x, _ = self.gru(x)
    print (x.size())
    x = self.final_drop(x)
    print (x.size())
    x = self.linear(x[:, -1, :].float())
    print (x.size())
    outputs = self.softmax(x)
    print (outputs.size())

    if self.return_attention:
        return outputs, att_weights
    else:
        return outputs

attn_res = AttendResistance(268, 20, embedding_matrix, 0.5, 0.3, True)
attn_res = attn_res.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(attn_res.parameters())

num_epochs = 10
for epoch in range(num_epochs):
    for i, (prot_seqs, labels) in enumerate(train_loader):
        prot_seqs = Variable(prot_seqs.long()).cuda()
        labels = Variable(labels.long()).cuda()

        #print (prot_seqs)
        #print (labels)
        # Forward + Backward + Optimize
        optimizer.zero_grad()
        outputs, att_weights = attn_res(prot_seqs)
        print (outputs)
        loss = criterion(outputs, torch.max(labels, 1)[1])
        print (loss)
        loss.backward()
        optimizer.step()

        if (i+1) % 100 == 0:
            print ('Epoch [%d/%d], Step [%d/%d], Loss: %.4f'
                   %(epoch+1, num_epochs, i+1, len(X_train)//batch_size, loss.data[0]))

And here is the error with the print output:

torch.Size([64, 1602])
torch.Size([64, 1602, 20])
torch.Size([64, 1602, 20])
torch.Size([64, 1602, 20])
torch.Size([64, 1602, 32])
torch.Size([64, 1602, 32])
torch.Size([64, 268])
torch.Size([64, 268])
Variable containing:
1.00000e-03 *
 3.5743  3.7436  4.2370  ...   3.9607  4.2058  4.2674
 3.5743  3.7436  4.2370  ...   3.9607  4.2058  4.2674
 3.5743  3.7436  4.2370  ...   3.9607  4.2058  4.2674
          ...             ⋱             ...
 3.5743  3.7436  4.2370  ...   3.9607  4.2058  4.2674
 3.5743  3.7436  4.2370  ...   3.9607  4.2058  4.2674
 3.5743  3.7436  4.2370  ...   3.9607  4.2058  4.2674
[torch.cuda.FloatTensor of size 64x268 (GPU 0)]

Variable containing:
 5.5909
[torch.cuda.FloatTensor of size 1 (GPU 0)]

---------------------------------------------------------------------------
AssertionError                            Traceback (most recent call last)
<ipython-input-89-a32cf2edb4cc> in <module>()
     17         print (torch.sum(att_weights))
     18         print (loss)
---> 19         loss.backward()
     20         optimizer.step()
     21

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/autograd/variable.py in backward(self, gradient, retain_graph, create_graph, retain_variables)
    165                 Variable.
    166         """
--> 167         torch.autograd.backward(self, gradient, retain_graph, create_graph, retain_variables)
    168
    169     def register_hook(self, hook):

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/autograd/__init__.py in backward(variables, grad_variables, retain_graph, create_graph, retain_variables)
     97
     98     Variable._execution_engine.run_backward(
---> 99         variables, grad_variables, retain_graph)
    100
    101

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/autograd/function.py in _do_backward(self, gradients, retain_variables)
    333     def _do_backward(self, gradients, retain_variables):
    334         self.retain_variables = retain_variables
--> 335         result = super(NestedIOFunction, self)._do_backward(gradients, retain_variables)
    336         if not retain_variables:
    337             del self._nested_output

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/autograd/function.py in backward(self, *gradients)
    341     def backward(self, *gradients):
    342         nested_gradients = _unflatten(gradients, self._nested_output)
--> 343         result = self.backward_extended(*nested_gradients)
    344         return tuple(_iter_None_tensors(result))
    345

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in backward_extended(self, grad_output, grad_hy)
    333                 output,
    334                 weight,
--> 335                 grad_weight)
    336         else:
    337             grad_weight = [(None,) * len(layer_weight) for layer_weight in weight]

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/backends/cudnn/rnn.py in backward_weight(fn, input, hx, output, weight, grad_weight)
    466
    467         # copy the weights from the weight_buf into grad_weight
--> 468         grad_params = get_parameters(fn, handle, dw)
    469         _copyParams(grad_params, grad_weight)
    470         return grad_weight

/home/nafizh/anaconda3/lib/python3.6/site-packages/torch/backends/cudnn/rnn.py in get_parameters(fn, handle, weight_buf)
    169                     layer_params.append(param)
    170                 else:
--> 171                     assert cur_offset == offset
    172
    173                 cur_offset = offset + filter_dim_a[0]

AssertionError:

I am a newbie with pytorch. Because the error is not giving me any explicit message, I don’t know what am I doing wrong here. I am aware it is an assertion error that is happening. But I don’t know what are these cur_offset and offset variables are. Running on,

0.3.0.post4

Cuda compilation tools, release 8.0, V8.0.61


title: TypeError: argument 0 is not a Variable

body:
I am trying to learn CNN with my own data. The shape of the data is (1224, 15, 23). 1224 is the number of data, and each data is (15, 23). CNN is built with PyTorch.

I think there is no logical error because conv2D needs 4-D tensor, and I feed (batch, channel, x, y).

when I build an instance of the Net class I got this error.

TypeError: argument 0 is not a Variable

I have been using PyTroch for half of a year but this error is the first time and I am still confused.

Here is my code.

class Net(nn.Module):
    def __init__(self, n):
        super(Net,self).__init__()

        self.conv = nn.Sequential(nn.Conv2d(1, 32, kernel_size=3, stride=1),
                                 nn.ReLU(),
                                 nn.Conv2d(32, 64, kernel_size=3, stride=1),
                                 nn.ReLU(),
                                 nn.Conv2d(64, 64, kernel_size=3, stride=1),  # 64 x 9 x 17
                                 nn.ReLU()
                                )

        conv_out_size = self._get_conv_out(input_shape)
        self.fc = nn.Sequential(nn.Linear(64 * 9 * 17, 128),
                               nn.ReLU(),
                               nn.Linear(128, n)
                               )

    def _get_conv_out(self, shape):
        o = self.conv(torch.zeros(1, *shape))
        return int(np.prod(o.size()))

    def forward(self, x):
        conv_out = self.conv(x).view(x.size()[0], -1)
        return sefl.fc(conv_out)

if __name__=='__main__':
    num_epochs = 1
    num_classes = 2
    input_shape = train_img[0].shape  # 1, 15, 23

    net = Net(num_classes)
    iteration = 51
    BATCH_SIZE = 24
    LEARNING_RATE = 0.0001

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(net.parameters(), lr=LEARNING_RATE)

    loss_list= []
    batch_index = 0

    # train
    for epoch in range(num_epochs):
        for i in range(iteration):
            input_img = torch.FloatTensor(train_img[batch_index: batch_index + BATCH_SIZE])
            print(input_img.size())  # 24, 1, 15, 23

            outputs = net(input_img)
            loss = criterion(outputs, labels)
            loss_list.append(loss.item())

            # Backprop
            opimizer.zero_grad()
            loss.backward()
            optimizer.step()

And the error message:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-179-0f6bc7588c29> in <module>
      4     input_shape = train_img[0].shape  # 1, 15, 23
      5
----> 6     net = Net(num_classes)
      7     iteration = 51
      8     BATCH_SIZE = 24

<ipython-input-178-8a68d4a0dc4a> in __init__(self, n)
     11                                 )
     12
---> 13         conv_out_size = self._get_conv_out(input_shape)
     14         self.fc = nn.Sequential(nn.Linear(64 * 9 * 17, 128),
     15                                nn.ReLU(),

<ipython-input-178-8a68d4a0dc4a> in _get_conv_out(self, shape)
     18
     19     def _get_conv_out(self, shape):
---> 20         o = self.conv(torch.zeros(1, *shape))
     21         return int(np.prod(o.size()))
     22

C:\DTools\Anaconda3\envs\tensorflow\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    355             result = self._slow_forward(*input, **kwargs)
    356         else:
--> 357             result = self.forward(*input, **kwargs)
    358         for hook in self._forward_hooks.values():
    359             hook_result = hook(self, input, result)

C:\DTools\Anaconda3\envs\tensorflow\lib\site-packages\torch\nn\modules\container.py in forward(self, input)
     65     def forward(self, input):
     66         for module in self._modules.values():
---> 67             input = module(input)
     68         return input
     69

C:\DTools\Anaconda3\envs\tensorflow\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    355             result = self._slow_forward(*input, **kwargs)
    356         else:
--> 357             result = self.forward(*input, **kwargs)
    358         for hook in self._forward_hooks.values():
    359             hook_result = hook(self, input, result)

C:\DTools\Anaconda3\envs\tensorflow\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    280     def forward(self, input):
    281         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 282                         self.padding, self.dilation, self.groups)
    283
    284

C:\DTools\Anaconda3\envs\tensorflow\lib\site-packages\torch\nn\functional.py in conv2d(input, weight, bias, stride, padding, dilation, groups)
     88                 _pair(0), groups, torch.backends.cudnn.benchmark,
     89                 torch.backends.cudnn.deterministic, torch.backends.cudnn.enabled)
---> 90     return f(input, weight, bias)
     91
     92

TypeError: argument 0 is not a Variable


title: Use SageMaker Pytorch image for training

body:
I am trying to containerize the training process for a fine tuned BERT model and run it on SageMaker. I was planning to use the pre-built SageMaker Pytorch GPU containers (https://aws.amazon.com/releasenotes/available-deep-learning-containers-images/) as my starting point but I am having issues pulling the images during my build process.
My Dockerfile looks like this:
# SageMaker PyTorch image
FROM 763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.5.0-gpu-py36-cu101-ubuntu16.04

ENV PATH="/opt/ml/code:${PATH}"

# /opt/ml and all subdirectories are utilized by SageMaker, we use the /code subdirectory to store our user code.
COPY /bert /opt/ml/code

# this environment variable is used by the SageMaker PyTorch container to determine our user code directory.
ENV SAGEMAKER_SUBMIT_DIRECTORY /opt/ml/code

# this environment variable is used by the SageMaker PyTorch container to determine our program entry point
# for training and serving.
# For more information: https://github.com/aws/sagemaker-pytorch-container
ENV SAGEMAKER_PROGRAM bert/train

My build_and_push script:
#!/usr/bin/env bash

# This script shows how to build the Docker image and push it to ECR to be ready for use
# by SageMaker.

# The argument to this script is the image name. This will be used as the image on the local
# machine and combined with the account and region to form the repository name for ECR.
IMAGE="my-bert"

# parameters
PY_VERSION="py36"

# Get the account number associated with the current IAM credentials
account=$(aws sts get-caller-identity --query Account --output text)

if [ $? -ne 0 ]
then
    exit 255
fi

chmod +x bert/train

# Get the region defined in the current configuration (default to us-west-2 if none defined)
region=$(aws configure get region)
region=${region:-us-east-2}

# If the repository doesn't exist in ECR, create it.
aws ecr describe-repositories --repository-names ${IMAGE} || aws ecr create-repository --repository-name ${IMAGE}

echo "---> repository done.."
# Get the login command from ECR and execute it directly
aws ecr get-login-password --region $region | docker login --username AWS --password-stdin $account.dkr.ecr.$region.amazonaws.com
echo "---> logged in to account ecr.."

# Get the login command from ECR in order to pull down the SageMaker PyTorch image
# aws ecr get-login-password --region $region | docker login --username AWS --password-stdin 763104351884.dkr.ecr.us-east-1.amazonaws.com
# echo "---> logged in to pytorch ecr.."

echo "Building image with arch=gpu, region=${region}"
TAG="gpu-${PY_VERSION}"
FULLNAME="${account}.dkr.ecr.${region}.amazonaws.com/${IMAGE}:${TAG}"
docker build -t ${IMAGE}:${TAG} --build-arg ARCH="$arch" -f "Dockerfile" .
docker tag ${IMAGE}:${TAG} ${FULLNAME}
docker push ${FULLNAME}

I get the following message during the push and the sagemaker pytorch image is not pulled:
Get https://763104351884.dkr.ecr.us-east-1.amazonaws.com/v2/pytorch-training/manifests/1.5.0-gpu-py36-cu101-ubuntu16.04: no basic auth credentials

Please let me know if this is the correct way to use a pre-built SageMaker image and what I could do to fix this error.


title: pytorch backports.functools_lru_cache conflict

body:
I'm using Windows 10, and my the installation dir is:
anacoda2/python2.7/python3.6/opencv/cdua10/cudnn ....

Now I want to install pytorch, with this command: conda install pytorch -c pytorch

But as result I'm getting this error:

C:\Users\MM>conda install pytorch -c pytorch
Solving environment: failed

UnsatisfiableError: The following specifications were found to be in conflict:
  - backports.functools_lru_cache
  - pytorch
Use "conda info <package>" to see the dependencies for each package.

The version of backports.functools_lru_cache is 1.4.

Does anyone know how to solve this?


title: Conditional GAN - shuffle/split two datasets identically

body:
I'm trying to use train a DCGAN to colorize some images. When doing so, I condition my GAN on grayscale versions of the images. I then want to train my GAN/discriminator first with a batch of real images, and then with a batch of fake images. Every so often, I want to compare the colorized, grayscale and ground truth version of the images. I therefore need the batches of the real/gray images to be split the same way. I use pytorch. Looking at the code I've included, these should give the same batches. However, they don't.

I have tried without worker_init_fn. I have also tried different random function calls and passed them to worker_init_fn with no avail.

dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size,
                                          shuffle=True, num_workers=workers, worker_init_fn = random.seed(seed))

dataloader_gray = torch.utils.data.DataLoader(dataset_gray, batch_size=batch_size,
                                          shuffle=True, num_workers=workers, worker_init_fn = random.seed(seed))

for i, (data, data_gray) in enumerate(zip(dataloader, dataloader_gray)):
    doStuff()


title: why does the Accuracy decrease when using a ReLu activation after Linear layers

body:
so I started using Pytorch and I'm building a very basic CNN on the FashionMNIST Dataset. I noticed some weird Behaviour while using the NN and I don't know why this is happening, in the Forward Function the Accuracy of the NN decrease when I use a Relu function after every Linear Layer.

here is the Code for my custom NN:

# custom class neural network
class FashionMnistClassifier(nn.Module):
  def __init__(self, n_inputs, n_out):
    super().__init__()
    self.cnn1 = nn.Conv2d(n_inputs, out_channels=32, kernel_size=5).cuda(device)
    self.cnn2 = nn.Conv2d(32, out_channels=64, kernel_size=5).cuda(device)
    #self.cnn3 = nn.Conv2d(n_inputs, out_channels=32, kernel_size=5)
    self.fc1 = nn.Linear(64*4*4, out_features=100).cuda(device)
    self.fc2 = nn.Linear(100, out_features=n_out).cuda(device)
    self.relu = nn.ReLU().cuda(device)
    self.pool = nn.MaxPool2d(kernel_size=2).cuda(device)
    self.soft_max = nn.Softmax().cuda(device)

  def forward(self, x):
    x.cuda(device)
    out = self.relu(self.cnn1(x))
    out = self.pool(out)
    out = self.relu(self.cnn2(out))
    out = self.pool(out)
    #print("out shape in classifier forward func: ", out.shape)
    out = self.fc1(out.view(out.size(0), -1))
    #out = self.relu(out) # if I uncomment these then the Accuracy decrease from 90 to 50!!!
    out = self.fc2(out)
    #out = self.relu(out) # this too
    return out

n_batch = 100
n_outputs = 10
LR = 0.001

model = FashionMnistClassifier(1, 10).cuda(device)
optimizer = optim.Adam(model.parameters(), lr=LR)
criterion = nn.CrossEntropyLoss()

so if I use the ReLu only after the CNN layers I get an Accuracy of 90% but when I uncomment that section and use the Relu activation after the Linear Layers the Accuracy decrease to 50% I have no idea why is this happening since I thought it is always better to use activation after every linear layer to get a better accuracy for Classification. I thought always that we should always use the activation function if we have a Classification Problem and for Linear Regression we don't have to do this, but here in my case although it is a Classification Problem, I get a better performance if I don't use the Activation Function after the Linear Layers. Can someone maybe clarify this to me?


title: How are token vectors calculated in spacy-pytorch-transformers

body:
I am currently working with the spacy-pytorch-transformer package to experiment with the respective embeddings.
When reading the introductionary article (essentially the GitHub README), my understanding was that the token-level embeddings are the mean over the embeddings of all corresponding word pieces, i.e. embed(complex) would be the same as 1/2 * embed(comp#) * embed(#lex).

According to the BERT paper, this should simply utilize the last_hidden_state property of the network, but my MCVE below shows that this is not the same for Spacy 2.1.8 and spacy-pytorch-transformers 0.4.0, for at least BERT and RoBERTa (have not verified it for more models):

import spacy
import numpy as np
nlp = spacy.load("en_pytt_robertabase_lg")  # either this or the BERT model
test = "This is a test"  # Note that all tokens are directly aligned, so no mean has to be calculated.
doc = nlp(test)
# doc[0].vector and doc.tensor[0] are equal, so the results are equivalent.
print(np.allclose(doc[0].vector, doc._.pytt_last_hidden_state[1, :]))
# returns False

The offset of 1 for the hidden states is due to the <CLS> token as the first input, which corresponds to the sentence classification task; I even checked with any available other token for my sentence (which has no token alignment problems according to doc._.pytt_alignment), so there is no way I missed something here.

According to the source code, the corresponding hook overwrites simply to return the corresponding row in the tensor, so I do not see any transformation here. Is there something obvious that I am missing here, or is this deviating from the expected behavior?


title: PyTorch: ImportError: libcurand.so.9.1: cannot open shared object file: No such file or directory

body:
I tried to install PyTorch==0.4.1 with Anaconda.

I tried using the command "conda install pytorch=0.4.1 torchvision==0.2.1 cuda91 -c pytorch" and "conda install pytorch=0.4.1 torchvision==0.2.1 cuda90 -c pytorch", but both installation will result in the error

ImportError: libcurand.so.9.1: cannot open shared object file: No such file or directory

when importing torch.

Also, running "locate libcurand.so.9.1" gives me

/usr/local/cuda-9.1/lib64/libcurand.so.9.1
/usr/local/cuda-9.1/lib64/libcurand.so.9.1.85

I just couldn't figure out what caused the problem and how to fix it..

(CUDA in the system is version 9.1, although I think Anaconda install its own CUDA toolkit)


title: why does batch normalization make my batches so abnormal?

body:
I'm playing with pytorch for the first time, and I've noticed that when training my neural net, about one time in four or so the loss takes a left turn towards infinity, then nan shortly after that. I've seen a few other questions about nan-ing, but the recommendations there seem to be essentially to do normalization; but the first layer in my net below is such a normalization, and I still see this problem! The full net is a bit convoluted, but I've done some debugging to try to produce a very small, understandable net that still displays the same issue.
The code is below; it consists of sixteen inputs, 0-1, which are passed through a batch normalization and then a fully-connected layer to a single output. I'd like it to learn the function that always outputs 1, so I take the squared error from 1 for the loss.
import torch as t
import torch.nn as tn
import torch.optim as to

if __name__ == '__main__':
    board = t.rand([1,1,1,16])
    net = tn.Sequential \
        ( tn.BatchNorm2d(1)
        , tn.Conv2d(1, 1, [1,16])
        )
    optimizer = to.SGD(net.parameters(), lr=0.1)
    for i in range(10):
        net.zero_grad()
        nn_outputs = net.forward(board)
        loss = t.sum((nn_outputs - 1)**2)
        print(i, nn_outputs, loss)
        loss.backward()
        optimizer.step()

If you run it a few times, eventually you'll see a run that looks like this:
0 tensor([[[[-0.7594]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(3.0953, grad_fn=<SumBackward0>)
1 tensor([[[[4.0954]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(9.5812, grad_fn=<SumBackward0>)
2 tensor([[[[5.5210]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(20.4391, grad_fn=<SumBackward0>)
3 tensor([[[[-3.4042]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(19.3966, grad_fn=<SumBackward0>)
4 tensor([[[[823.6523]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(676756.7500, grad_fn=<SumBackward0>)
5 tensor([[[[3.5471e+08]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(1.2582e+17, grad_fn=<SumBackward0>)
6 tensor([[[[2.8560e+25]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(inf, grad_fn=<SumBackward0>)
7 tensor([[[[inf]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(inf, grad_fn=<SumBackward0>)
8 tensor([[[[nan]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(nan, grad_fn=<SumBackward0>)
9 tensor([[[[nan]]]], grad_fn=<MkldnnConvolutionBackward>) tensor(nan, grad_fn=<SumBackward0>)

Why does my loss go to nan, and what can I do about it?


title: How does bias work in pytorch quantized convolution?

body:
I'm trying to perform static post-train quantization in PyTorch. For this example, I tried quantizing a Conv2d layer with a bias:
def quantize(model, input_shape):
    with torch.no_grad():
        # model = tq.QuantWrapper(model)
        observer = tq.PerChannelMinMaxObserver()
        model.qconfig = torch.quantization.QConfig(activation=tq.MinMaxObserver,
                                                   weight=observer.with_args(dtype=torch.qint8,
                                                                             qscheme=torch.per_channel_affine))
        #model.qconfig = torch.quantization.get_default_qconfig('qnnpack')
        model = tq.QuantWrapper(model)
        tq.prepare(model, inplace=True)

        for i in range(1000):
            x = torch.ones(2, *input_shape)
            #x = torch.randn(2, *input_shape)
            tmp = model(x)
        tq.convert(model, inplace=True)
    return model

input_shape = (5, 7, 7)
model_b = nn.Conv2d(input_shape[0], 2, 3, bias=True)
for p in model_b.parameters():
    torch.nn.init.zeros_(p)
model_b.bias.data.fill_(.5)
model_b = quantize(model_b, input_shape)
model_b.eval()

The PyTorch documentation explicitly states that bias is not quantized and is kept as a float tensor.
The integer representation of the output yields:
tensor([[[[255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255]],

         [[255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255],
          [255, 255, 255, 255, 255]]]], dtype=torch.uint8)

However, the float representation yields:
tensor([[[[0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000]],

         [[0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000],
          [0.5000, 0.5000, 0.5000, 0.5000, 0.5000]]]], size=(1, 2, 5, 5),
       dtype=torch.quint8, quantization_scheme=torch.per_tensor_affine,
       scale=0.0019607844296842813, zero_point=0)

I have searched for information on the issue and came to the conclusion that the scales and zero points used for requantizing convolution output take the bias into account, and also that during GEMM operations the bias is quantized to int32_t before being added to int32_t result of the GEMM. From the example above, if it were simply cast to int32_t, the integer and float outputs would be 0.
My question is: how is the bias quantized to int32_t, if it isn't converted to a quantized tensor?


title: Size of input tensor and input format are different

body:
I am new to PyTorch and working with GANs for a project.
I am trying to visualize real and fake images using tensorboard.
writer_real = SummaryWriter(f"path")
writer_fake = SummaryWriter(f"path")
writer_real.add_image("Temp_real", x_b)
writer_fake.add_image("Temp_fake", fake_b)

In the above code, both x_b and fake_b are tensors
But this code is giving an error:
AssertionError: size of input tensor and input format are different.         tensor shape: (4, 3, 128, 128), input_format: CHW
My original image is of (128x128x3) (HWC).
Please tell me if any other information is needed.
Please Help!


title: Whats the equivalent of tf.keras.Input() in pytorch?

body:
can someone tell me what the equivalent of tf.keras.Input() in pytorch is?
At the documentation it says, "Initiates a Keras Tensor", so does it just creates a new empty tensor?
Thanks


title: How can I build an RNN without using nn.RNN

body:
I need to build an RNN (without using nn.RNN) with following specifications :

It should have set of weights [

It is a chanracter RNN.
It should have 1 hidden layer
Wxh (from input layer to hidden layer )
Whh (from the recurrent connection in the hidden layer)
W ho  (from hidden layer to output layer)
I need to use Tanh for hidden layer
I need to use softmax for output layer.

I have implemented the code . I am using CrossEntropyLoss() as loss function .
Which is giving me error as

RuntimeError                              Traceback (most recent call last)
<ipython-input-33-94b42540bc4f> in <module>()
     25         print("target ",target_tensor[timestep])
     26
---> 27         loss += criterion(output,target_tensor[timestep].view(1,n_vocab))
     28
     29     loss.backward()

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

/opt/anaconda/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    145         _assert_no_grad(target)
    146         return F.nll_loss(input, target, self.weight, self.size_average,
--> 147                           self.ignore_index, self.reduce)
    148
    149

/opt/anaconda/lib/python3.6/site-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce)
   1047         weight = Variable(weight)
   1048     if dim == 2:
-> 1049         return torch._C._nn.nll_loss(input, target, weight, size_average, ignore_index, reduce)
   1050     elif dim == 4:
   1051         return torch._C._nn.nll_loss2d(input, target, weight, size_average, ignore_index, reduce)

RuntimeError: multi-target not supported at /opt/conda/conda-bld/pytorch_1513368888240/work/torch/lib/THNN/generic/ClassNLLCriterion.c:22

Here is my code for model :

class CharRNN(torch.nn.Module):

    def __init__(self,input_size,hidden_size,output_size, n_layers = 1):

        super(CharRNN, self).__init__()
        self.input_size  = input_size
        self.hidden_size = hidden_size
        self.n_layers    = 1

        self.x2h_i = torch.nn.Linear(input_size + hidden_size, hidden_size)
        self.x2h_f = torch.nn.Linear(input_size + hidden_size, hidden_size)
        self.x2h_o = torch.nn.Linear(input_size + hidden_size, hidden_size)
        self.x2h_q = torch.nn.Linear(input_size + hidden_size, hidden_size)
        self.h2o   = torch.nn.Linear(hidden_size, output_size)
        self.sigmoid = torch.nn.Sigmoid()
        self.softmax = torch.nn.Softmax()
        self.tanh    = torch.nn.Tanh()

    def forward(self, input, h_t, c_t):

        combined_input = torch.cat((input,h_t),1)

        i_t = self.sigmoid(self.x2h_i(combined_input))
        f_t = self.sigmoid(self.x2h_f(combined_input))
        o_t = self.sigmoid(self.x2h_o(combined_input))
        q_t = self.tanh(self.x2h_q(combined_input))

        c_t_next = f_t*c_t + i_t*q_t
        h_t_next = o_t*self.tanh(c_t_next)

        output = self.softmax(h_t_next)
        return output, h_t, c_t

    def initHidden(self):
        return torch.autograd.Variable(torch.zeros(1, self.hidden_size))

    def weights_init(self,model):

        classname = model.__class__.__name__
        if classname.find('Linear') != -1:
            model.weight.data.normal_(0.0, 0.02)
            model.bias.data.fill_(0)

`

and this is the code for training the model :

`
input_tensor  = torch.autograd.Variable(torch.zeros(seq_length,n_vocab))
target_tensor = torch.autograd.Variable(torch.zeros(seq_length,n_vocab))

model   = CharRNN(input_size = n_vocab, hidden_size = hidden_size, output_size = output_size)
model.apply(model.weights_init)

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)

for i in range(n_epochs):
    print("Iteration", i)

    start_idx    = np.random.randint(0, n_chars-seq_length-1)
    train_data   = raw_text[start_idx:start_idx + seq_length + 1]

    input_tensor = torch.autograd.Variable(seq2tensor(train_data[:-1],n_vocab), requires_grad = True)
    target_tensor= torch.autograd.Variable(seq2tensor(train_data[1:],n_vocab), requires_grad = False).long()

    loss = 0

    h_t = torch.autograd.Variable(torch.zeros(1,hidden_size))
    c_t = torch.autograd.Variable(torch.zeros(1,hidden_size))

    for timestep in range(seq_length):

        output, h_t, c_t = model(input_tensor[timestep].view(1,n_vocab), h_t, c_t)

        loss += criterion(output,target_tensor[timestep].view(1,n_vocab))

    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

    x_t = input_tensor[0].view(1,n_vocab)
    h_t = torch.autograd.Variable(torch.zeros(1,hidden_size))
    c_t = torch.autograd.Variable(torch.zeros(1,hidden_size))

    gen_seq = []

    for timestep in range(100):
        output, h_t, c_t = model(x_t, h_t, c_t)
        ix = np.random.choice(range(n_vocab), p=output.data.numpy().ravel())
        x_t = torch.autograd.Variable(torch.zeros(1,n_vocab))
        x_t[0,ix] = 1
        gen_seq.append(idx2char[ix])

    txt = ''.join(gen_seq)
    print ('----------------------')
    print (txt)
    print ('----------------------')

Can you please help me ?

Thanks in advance.


title: TensorFlow consistently achieves smaller error than PyTorch for the same network, loss and initialization

body:
I have two implementations of same network for CIFAR10 in TensorFlow and PyTorch. Both have weight initializations from the same distributions (for weights - xiavier uniform and zeros for biases) but TensorFlow seems to consistently outperform PyTorch in terms of minimal test error (although PyTorch appears to be almost 9% faster!). Both are optimized using SGD. Even if equipped with exactly the same initialization values TensorFlow still achieves smaller error. These are  typical curves from the simulations:

The red curve is test-error of TensorFlow and orange curve is a test-error of PyTorch. Both had been initialized with the exact same values.

Since the code is relatively long, I provide here only implementations of architectures. The full reproducible code for both implementations in Jupyter format could be found here, on GitHub.

TF implementation of network:

def tf_model(graph, init=None):
    with graph.as_default():
        if init:
            conv1_init = init['conv1']
            conv2_init = init['conv2']
            logits_init = init['logits']
            conv1_init = tf.constant_initializer(conv1_init)
            conv2_init = tf.constant_initializer(conv2_init)
            logits_init = tf.constant_initializer(logits_init)
        else:
            conv1_init = tf.contrib.layers.xavier_initializer()
            conv2_init = tf.contrib.layers.xavier_initializer()
            logits_init = tf.contrib.layers.xavier_initializer()

        with tf.name_scope('Input'):
            x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
            y = tf.placeholder(tf.int32, shape=[None], name='y')
            keep_prob = tf.placeholder_with_default(1.0 - dropout_rate, shape=())
        with tf.device('/device:GPU:0'):
            with tf.name_scope('conv1'):
                conv1 = tf.layers.conv2d(x,
                                         filters=6,
                                         kernel_size=5,
                                         strides=1,
                                         padding='valid',
                                         kernel_initializer=conv1_init,
                                         bias_initializer=tf.initializers.zeros,
                                         activation=tf.nn.relu,
                                         name='conv1'
                                         )

                max_pool1 = tf.nn.max_pool(value=conv1,
                                           ksize=(1, 2, 2, 1),
                                           strides=(1, 2, 2, 1),
                                           padding='SAME',
                                           name='max_pool1')

                dropout1 = tf.nn.dropout(max_pool1, keep_prob=keep_prob)

            with tf.name_scope('conv2'):
                conv2 = tf.layers.conv2d(dropout1,
                                         filters=12,
                                         kernel_size=3,
                                         strides=1,
                                         padding='valid',
                                         bias_initializer=tf.initializers.zeros,
                                         activation=tf.nn.relu,
                                         kernel_initializer=conv2_init,
                                         name='conv2')

                max_pool2 = tf.nn.max_pool(value=conv2,
                                           ksize=(1, 2, 2, 1),
                                           strides=(1, 2, 2, 1),
                                           padding='VALID',
                                           name='max_pool2')

                dropout2 = tf.nn.dropout(max_pool2, keep_prob=keep_prob)

            with tf.name_scope('logits'):
                flatten = tf.layers.Flatten()(max_pool2)
                logits = tf.layers.dense(flatten,
                                         units=10,
                                         kernel_initializer=logits_init,
                                         bias_initializer=tf.initializers.zeros,
                                         name='logits')

    return x, y, keep_prob, logits

PyTorch implementation:

class TorchModel(nn.Module):
    def __init__(self, dropout_rate=0.0, init=None):
        super(TorchModel, self).__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(in_channels=3,
                      out_channels=6,
                      kernel_size=5,
                      padding=0,
                      bias=True),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Dropout(p=dropout_rate))
        if init:
            conv1_init = init['conv1']
            self.conv1[0].weight = nn.Parameter(torch.FloatTensor(conv1_init))
        else:
            torch.nn.init.xavier_uniform_(self.conv1[0].weight)
        torch.nn.init.zeros_(self.conv1[0].bias)
        self.conv2 = nn.Sequential(
            nn.Conv2d(in_channels=6,
                      out_channels=12,
                      kernel_size=3,
                      bias=True),
            nn.ReLU(),
            nn.MaxPool2d(2),
            nn.Dropout(p=dropout_rate))
        if init:
            conv2_init = init['conv2']
            self.conv2[0].weight = nn.Parameter(torch.FloatTensor(conv2_init))
        else:
            torch.nn.init.xavier_uniform_(self.conv2[0].weight)
        torch.nn.init.zeros_(self.conv2[0].bias)

        self.logits = nn.Linear(432, 10)
        if init:
            logits_init = init['logits']
            logits_init = np.reshape(logits_init, [10, 432])
            self.logits.weight = nn.Parameter(torch.FloatTensor(logits_init))
        else:
            torch.nn.init.xavier_uniform_(self.logits.weight)
        torch.nn.init.zeros_(self.logits.bias)

    def forward(self, x):
        x = self.conv1(x)
        x = self.conv2(x)
        x = x.view(x.size(0), -1)
        x = self.logits(x)
        return x

I also would like to add that the reason that I'm asking is that for different optimization algorithm (which is very complicated therefore I provide here a simple example with SGD) that I currently testing, the opposite happens - PyTorch is consistently better than TF in terms of minimal error.

I would glad to hear what your thoughts are. Did you have the same experience? Do I miss something in implementations of SGD in TensorFlow and in PyTorch?

Thank you.


title: Patch classification from large image using pytorch

body:
I am using the following function to classify an image using a trained pytorch model. It works fine for a small input image.

def test(net, STRIDE-16, BATCH_SIZE=20, WINDOW_SIZE= (256,256)):

# Use the network on the test image
img = (1 / 255 * np.asarray(io.imread("C:/bd/R1C1.tif"), dtype='float32'))

all_preds = []

# Switch the network to inference mode
net.eval()

pred = np.zeros(img.shape[:2] + (N_CLASSES,))

for i, coords in enumerate(tqdm(grouper(batch_size, sliding_window(img,
STRIDE, WINDOW_SIZE)))):

# Build the tensor
image_patches = [np.copy(img[x:x + w, y:y + h]).transpose((2, 0, 1)) for
x, y, w, h in coords]

image_patches = np.asarray(image_patches)
image_patches = Variable(torch.from_numpy(image_patches).cpu(),
volatile=True)

 # Do the inference
 outs = net(image_patches)
 outs = outs.data.cpu().numpy()

 # Fill in the results array
 for out, (x, y, w, h) in zip(outs, coords):
     out = out.transpose((1, 2, 0))
     pred[x:x + w, y:y + h] += out
  del (outs)

pred = np.argmax(pred, axis=-1)

all_preds.append(pred)

return all_preds

When I load a large img let say (40k x 40k) I get MemoryError. How can I avoid this memory error. Maybe by using a batch of smaller images? How to efficiently implement that?


title: How to implement fusion layer technique in pytorch?

body:
Currently, I'm working on creating image colorization model. I want to use in it fusion layer, presented by Iizuki et al., but I have some problem with implementing it in Pytorch.

The basic idea is that we extract mid-level features and global features from image independtly, then combine them according to formula
where u,v is number of "pixel", y_global - global feature vector of size 64, y^mid_u_v - mid-level feature vector of size 64. So we concat

Output shape  of my midlevel layer is (bs, 64, 28, 28), output shape of my global feature extractor is (bs, 64).

I tried to calculate per-pixel, using nn.Embedding(), but it seems to work incorrectly, assuming bs is not 1.

How to implement this technique elegantly and effectively in Pytorch?

I tried to calculate per-pixel, using nn.Embedding(), but it seems to work bad


title: How to deploy Pytorch in Python via a REST API with Flask?

body:
I am working on AWS Sagemaker and my goal is to follow this tutorial from Pytorch's official documentation.

The original predict function from the tutorial above is the following:

@app.route('/predict', methods=['POST'])
def predict():
    if request.method == 'POST':
        file = request.files['file']
        img_bytes = file.read()
        class_id, class_name = get_prediction(image_bytes=img_bytes)
        return jsonify({'class_id': class_id, 'class_name': class_name})

I was getting this error, so I added 'GET' as a method as mentioned in here. I also simplified my example to its minimal expression:

from flask import Flask, jsonify, request

app = Flask(__name__)

@app.route('/predict', methods=['GET','POST'])
def predict():
    if request.method == 'POST':
        return jsonify({'class_name': 'cat'})
    return 'OK'

if __name__ == '__main__':
    app.run()

I perform requests with the following code:

import requests

resp = requests.post("https://catdogclassifier.notebook.eu-west-1.sagemaker.aws/proxy/5000/predict",
                     files={"file": open('/home/ec2-user/SageMaker/cat.jpg', 'rb')})

resp is <Response [200]> but resp.json() returns JSONDecodeError: Expecting value: line 1 column 1 (char 0) Finally, resp.url points me to a page saying 'OK'.

Moreover, this is the output of resp.content

<!DOCTYPE HTML>

What am I missing?


title: Having trouble with input dimensions for Pytorch LSTM with torchtext

body:
Problem

I'm trying to build a text classifier network using LSTM. The error I'm getting is:

RuntimeError: Expected hidden[0] size (4, 600, 256), got (4, 64, 256)

Details

The data is json and looks like this:

{"cat": "music", "desc": "I'm in love with the song's intro!", "sent": "h"}

I'm using torchtext to load the data.

from torchtext import data
from torchtext import datasets

TEXT = data.Field(fix_length = 600)
LABEL = data.Field(fix_length = 10)

BATCH_SIZE = 64

fields = {
    'cat': ('c', LABEL),
    'desc': ('d', TEXT),
    'sent': ('s', LABEL),
}

My LSTM looks like this

EMBEDDING_DIM = 64
HIDDEN_DIM = 256
N_LAYERS = 4

MyLSTM(
  (embedding): Embedding(11967, 64)
  (lstm): LSTM(64, 256, num_layers=4, batch_first=True, dropout=0.5)
  (dropout): Dropout(p=0.3, inplace=False)
  (fc): Linear(in_features=256, out_features=8, bias=True)
  (sig): Sigmoid()
)

I end up with the following dimensions for the inputs and labels

batch = list(train_iterator)[0]
inputs, labels = batch
print(inputs.shape) # torch.Size([600, 64])
print(labels.shape) # torch.Size([100, 2, 64])

And my initialized hidden tensor looks like:

hidden # [torch.Size([4, 64, 256]), torch.Size([4, 64, 256])]

Question

I'm trying to understand what the dimensions at each step should be.
Should the hidden dimension be initialized to (4, 600, 256) or (4, 64, 256)?


I am trying to train a model on UCF11 dataset in PyTorch.
I am trying to implement a random augmentation on the video data samples.
self.transforms_list = [transforms.ColorJitter(brightness=0.225),
                            transforms.ColorJitter(contrast=0.225),
                            transforms.ColorJitter(saturation=0.1),
                            transforms.ColorJitter(hue=0.1),
                            transforms.Grayscale(num_output_channels=3),
                            transforms.RandomHorizontalFlip(p=0.25),
                            transforms.RandomRotation(30.0),
                            transforms.GaussianBlur((5,5), sigma=(0.1, 2.0))]

transformations = random.sample(self.transforms_list,random.randint(1,8))+[transforms.RandomCrop(224)]
transformations = transforms.Compose(transformations)

But when I try to apply these transformations on each frame of a video clip, I get this error
RuntimeError: The expanded size of the tensor (320) must match the existing size (224) at non-singleton dimension 2.
Target sizes: [3, 240, 320].  Tensor sizes: [3, 224, 224]

Cannot understand why this error is occurring. I checked the source code but I cannot see the reason this error should occur.
I created a custom transform class for the RandomCrop but still the same error occurs.
Please help!!
EDIT:
Since, the input is a video, the input has shape (T,C,H,W)
I was applying the transformations on each frame in the video like this
vid[i]  = transformations(vid[i])
Created a new variable new_vid = [] and changed the code to
new_vid.append(transformations(vid[i]))
In the end,
vid = torch.cat(new_vid,dim = 0)
I got a video with shape (T,3,224,224)


title: Pytorch transfer learning predictions

body:
I have been following the pytorch transfer learning tutorial,and following the tutorial to my own dataset i have arrived at this

model_conv = train_model(model_conv, criterion, optimizer_conv,
                     exp_lr_scheduler, num_epochs=25)

Epoch 1/1
......
......
......
Epoch 24/24
train Loss: 0.8674 Acc: 0.5784
val Loss: 1.2930 Acc: 0.4583
Training complete in 43m 24s
Best val Acc: 0.486111

The tutorial ends here,but im not exaclty sure how do i get predictions for my new images?

Any suggestions would be really helpful.
Thanks in advance.


title: Is this a correct reimplementation of Pytorch Seq2Seq model?

body:
I made a code that sort of change the tutorial script of seq2seq provided by Pytorch. Here’s the model:

class Seq2Seq(nn.Module):
def __init__(self, encoder, batch_size, vocab_size, input_size, output_size, hidden_dim, embedding_dim, n_layers=2, dropout_p=0.5):
    super(Seq2Seq, self).__init__()

    self.hidden_dim = hidden_dim
    self.batch_size = batch_size
    self.input_length = input_size
    self.output_length = output_size
    self.vocab_size = vocab_size

    self.encoder = encoder
    self.dropout = nn.Dropout(dropout_p)
    self.selu = nn.SELU()
    self.decoder_embeddings = nn.Embedding(vocab_size, hidden_dim)
    self.decoder_gru = nn.GRU(hidden_dim, hidden_dim)
    self.out = nn.Linear(hidden_dim, vocab_size)
    self.softmax = nn.LogSoftmax()

def decode(self, SOS_token, encoder_hidden, target_output, teacher_forcing_ratio=0.8):
    decoder_output_full = autograd.Variable(torch.zeros(self.output_length, self.batch_size, self.vocab_size))
    decoder_output_full = decoder_output_full.cuda() if use_cuda else decoder_output_full
    target = target_output.permute(1,0)

    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False

    for idx in range(self.output_length):
        if idx == 0:
            decoder_input = SOS_token
            decoder_hidden = encoder_hidden.unsqueeze(0)
        output = self.decoder_embeddings(decoder_input).view(1, self.batch_size, -1)
        output = self.dropout(output)

        output = self.selu(output)

        if use_teacher_forcing:
            decoder_output, decoder_hidden = self.decoder_gru(output, decoder_hidden)
            temp = 1
            out = self.out(decoder_output[0])
            out = out + sample_gumbel(out.shape)
            decoder_output = F.softmax(out / temp, dim=1)
            # decoder_output = (self.decoder_embeddings.weight * decoder_output.unsqueeze(1)).sum(0).view(1, 1, -1)
            decoder_output_full[idx, :, :] = decoder_output
            decoder_input = target[idx-1]  # Teacher forcing

        else:
            decoder_output, decoder_hidden = self.decoder_gru(output, decoder_hidden)
            temp = 1
            out = self.out(decoder_output[0])
            out = out + sample_gumbel(out.shape)
            decoder_output = F.softmax(out / temp, dim=1)
            # decoder_output = (self.decoder_embeddings.weight * decoder_output.unsqueeze(1)).sum(0).view(1, 1, -1)
            topv, topi = decoder_output.data.topk(1)
            # print topi
            ni = topi
            # decoder_input_v = autograd.Variable(torch.LongTensor([[ni]]))
            decoder_input = autograd.Variable(ni)
            # decoder_input = decoder_input.cuda() if use_cuda else decoder_input
            # print decoder_input
            decoder_output_full[idx, :, :] = decoder_output

    decoder_output_full = decoder_output_full.permute(1,0,2)

    # gen_output = self.softmax(self.out(decoder_output_full))

    return decoder_output_full

def forward(self, input, target_output, teacher_forcing_ratio=0.8):
    encoder_feat, _ = self.encoder(input)

    SOS_token = np.zeros((self.batch_size,1), dtype=np.int32)
    SOS_token = torch.LongTensor(SOS_token.tolist())
    SOS_token = autograd.Variable(SOS_token)
    if use_cuda:
        SOS_token = SOS_token.cuda(gpu)

    gen_output = self.decode(SOS_token, encoder_feat, target_output, teacher_forcing_ratio)

    return gen_output

def initHidden(self):
    result = autograd.Variable(torch.zeros(1, self.batch_size, self.hidden_dim))
    if use_cuda:
        return result.cuda()
    else:
        return result

The way I calculate the NLL loss is by creating one whole sequence of output first and compare it with the target output. Here’s the loss function:

class batchNLLLoss(nn.Module):
def __init__(self):
    super(batchNLLLoss, self).__init__()

def forward(self, synt, target, claim_length=20):
    loss_fn = nn.NLLLoss()

    loss = 0

    for i in range(synt.shape[0]):
        for j in range(claim_length):
            loss += loss_fn(synt[i][j].unsqueeze(0), target[i][j])

    return loss

The current problem is the loss value is really small and seems like the network learns nothing (the output is the same word repeated again and again). Any thought about this? Thanks in advance!


title: My Image segmentation result map contains black lattice in in the white patch

body:
I'm doing an image segmentation with UNet-like CNN architecture by Pytorch 0.4.0.It mark foreground as 1 and background as 0 in the final segmentation result.I use a pre-trained VGG's feature extractor as my encoder, so I need to upsampling the encoder output many times.But the result shows a weird lattice parttern in the result like this:

I suspect these different shape of black parttern were caused by the deconvolutional layers.It's said that deconv layer add (s-1) zeros between the input pixel in the official documentation.the deconvolutional layer's hyperparameter are listed below:

kernel_size=3,stride=2,padding=1,output_padding=1

(the output_padding is used to fit the skip connection feature size, for instance, with a input size 40*30,I want it size enlarge twice in one deconvolutional layer,under my calculation with the formula:out=s(in-1)+k-2p,I must use padding 1 and output-padding 1 to keep size,or there might be other better choice I don’t know.)

I'm wondering the reason that generate the black lattice.They looks like a grid or square parttern.And how to solve it.Besides,I need to adjust all deconv layer hyperparamater to one uniform or modify them separately?

environment:Ubuntu 16.04,Pytorch 0.4.0,GTX 1080Ti,
the architecture of upsampling is three deconv layer,they are one after another.

Update

after I modified the deconv hyperparameter,(inspired by noise in image segmentation result use

kernel_size=4,stride=2,padding=1

thus to avoid output-padding and satisfy the size.But after 100 epoch I met similiar problem.

after another 30 epoch training, it seems like this,

the black points just scattered,It seems these black points just change their parttern and jump into another parttern,I don't know why it occurs.I'm don't know how to modify my network hyperparameter.

self.conv1=Basic(1024,512,kernel_size=3,stride=1,padding=1)
        self.conv2=Basic(512,512,kernel_size=3,stride=1,padding=1)
        self.deconv1=Basic(512,256,kernel_size=4,stride=2,conv=False,padding=1)
        self.deconv2=Basic(256,128,kernel_size=4,stride=2,conv=False,padding=1)
        self.deconv3=Basic(128,64,kernel_size=4,stride=2,conv=False,padding=1)
        #output  480*640 segmap
        self.conv4=Basic(64,2,kernel_size=1,stride=1)
        # self.avgpool=nn.AvgPool2d(kernel_size=2,stride=2)

    def forward(self, input):
        input=self.conv1(input)
        input=self.conv2(input)
        input=self.deconv1(input)
        input=self.deconv2(input)
        input=self.deconv3(input)
        input=self.conv4(input)
        # print(input.shape)
        #a channel-wise probability map
        raw=input
        return raw`


title: Using Upsample instead of ConvTranspose2d causes cuda out of memory on gradient calculating step

body:
Video card: gtx1070ti 8Gb, batchsize 64, input image size 128*128.
I had such UNET with resnet152 as encoder which worked pretty fine:

class UNetResNet(nn.Module):

def __init__(self, encoder_depth, num_classes, num_filters=32, dropout_2d=0.2,
             pretrained=False, is_deconv=False):
    super().__init__()
    self.num_classes = num_classes
    self.dropout_2d = dropout_2d

    if encoder_depth == 34:
        self.encoder = torchvision.models.resnet34(pretrained=pretrained)
        bottom_channel_nr = 512
    elif encoder_depth == 101:
        self.encoder = torchvision.models.resnet101(pretrained=pretrained)
        bottom_channel_nr = 2048
    elif encoder_depth == 152:
        self.encoder = torchvision.models.resnet152(pretrained=pretrained)
        bottom_channel_nr = 2048

    else:
        raise NotImplementedError('only 34, 101, 152 version of Resnet are implemented')

    self.pool = nn.MaxPool2d(2, 2)

    self.relu = nn.ReLU(inplace=True)

    self.conv1 = nn.Sequential(self.encoder.conv1,
                               self.encoder.bn1,
                               self.encoder.relu,
                               self.pool) #from that pool layer I would like to get rid off

    self.conv2 = self.encoder.layer1
    self.conv3 = self.encoder.layer2
    self.conv4 = self.encoder.layer3
    self.conv5 = self.encoder.layer4
    self.center = DecoderCenter(bottom_channel_nr, num_filters * 8 *2, num_filters * 8, False)

    self.dec5 =  DecoderBlockV(bottom_channel_nr + num_filters * 8, num_filters * 8 * 2, num_filters * 8,   is_deconv)
    self.dec4 = DecoderBlockV(bottom_channel_nr // 2 + num_filters * 8, num_filters * 8 * 2, num_filters * 8, is_deconv)
    self.dec3 = DecoderBlockV(bottom_channel_nr // 4 + num_filters * 8, num_filters * 4 * 2, num_filters * 2, is_deconv)
    self.dec2 = DecoderBlockV(bottom_channel_nr // 8 + num_filters * 2, num_filters * 2 * 2, num_filters * 2 * 2,
                               is_deconv)
    self.dec1 = DecoderBlockV(num_filters * 2 * 2, num_filters * 2 * 2, num_filters, is_deconv)
    self.dec0 = ConvRelu(num_filters, num_filters)
    self.final = nn.Conv2d(num_filters, num_classes, kernel_size=1)

def forward(self, x):
    conv1 = self.conv1(x)
    conv2 = self.conv2(conv1)
    conv3 = self.conv3(conv2)
    conv4 = self.conv4(conv3)
    conv5 = self.conv5(conv4)
    center = self.center(conv5)
    dec5 = self.dec5(torch.cat([center, conv5], 1))
    dec4 = self.dec4(torch.cat([dec5, conv4], 1))
    dec3 = self.dec3(torch.cat([dec4, conv3], 1))
    dec2 = self.dec2(torch.cat([dec3, conv2], 1))
    dec1 = self.dec1(dec2)
    dec0 = self.dec0(dec1)

    return self.final(F.dropout2d(dec0, p=self.dropout_2d))
# blocks
    class DecoderBlockV(nn.Module):
        def __init__(self, in_channels, middle_channels, out_channels, is_deconv=True):
            super(DecoderBlockV2, self).__init__()
            self.in_channels = in_channels

            if is_deconv:
                self.block = nn.Sequential(
                    ConvRelu(in_channels, middle_channels),
                    nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2,
                                       padding=1),
                    nn.BatchNorm2d(out_channels),
                    nn.ReLU(inplace=True)

                )
            else:

                self.block = nn.Sequential(
                    nn.Upsample(scale_factor=2, mode='bilinear'),
                    ConvRelu(in_channels, middle_channels),
                    ConvRelu(middle_channels, out_channels),
                )

        def forward(self, x):
            return self.block(x)

class DecoderCenter(nn.Module):
    def __init__(self, in_channels, middle_channels, out_channels, is_deconv=True):
        super(DecoderCenter, self).__init__()
        self.in_channels = in_channels

        if is_deconv:
            """
                Paramaters for Deconvolution were chosen to avoid artifacts, following
                link https://distill.pub/2016/deconv-checkerboard/
            """

            self.block = nn.Sequential(
                ConvRelu(in_channels, middle_channels),
                nn.ConvTranspose2d(middle_channels, out_channels, kernel_size=4, stride=2,
                                   padding=1),
        nn.BatchNorm2d(out_channels),
                nn.ReLU(inplace=True)
            )
        else:
            self.block = nn.Sequential(
                ConvRelu(in_channels, middle_channels),
                ConvRelu(middle_channels, out_channels)

            )

    def forward(self, x):
        return self.block(x)

Then I wanted to use Upsample instead of ConvTranspose2d causes cuda out of memory on gradient calculating step, just by setting is_deconv = False in DecoderBlock - that caused problem on gradient calculating step. Why? Got it fixed only by reducing batch size from 64 to 40

~/anaconda3/lib/python3.6/site-packages/steppy/base.py in fit_transform(self, *args, **kwargs)
    603             dict: outputs
    604         """
--> 605         self.fit(*args, **kwargs)
    606         return self.transform(*args, **kwargs)
    607

~/Desktop/ml/salt/open-solution-salt-identification-master/common_blocks/models.py in fit(self, datagen, validation_datagen, meta_valid)
     76             for batch_id, data in enumerate(batch_gen):
     77                 self.callbacks.on_batch_begin()
---> 78                 metrics = self._fit_loop(data)
     79                 self.callbacks.on_batch_end(metrics=metrics)
     80                 if batch_id == steps:

~/Desktop/ml/salt/open-solution-salt-identification-master/common_blocks/models.py in _fit_loop(self, data)
    113             batch_loss = sum(partial_batch_losses.values())
    114         partial_batch_losses['sum'] = batch_loss
--> 115         batch_loss.backward()
    116         self.optimizer.step()
    117

~/anaconda3/lib/python3.6/site-packages/torch/autograd/variable.py in backward(self, gradient, retain_graph, create_graph, retain_variables)
    165                 Variable.
    166         """
--> 167         torch.autograd.backward(self, gradient, retain_graph, create_graph, retain_variables)
    168
    169     def register_hook(self, hook):

~/anaconda3/lib/python3.6/site-packages/torch/autograd/__init__.py in backward(variables, grad_variables, retain_graph, create_graph, retain_variables)
     97
     98     Variable._execution_engine.run_backward(
---> 99         variables, grad_variables, retain_graph)
    100
    101

RuntimeError: cuda runtime error (2) : out of memory at /opt/conda/conda-bld/pytorch_1518244421288/work/torch/lib/THC/generic/THCStorage.cu:58


title: Pytorch PPO implementation is not learning

body:
This PPO implementation has a bug somewhere and I can't figure out what's wrong. The network returns a normal distribution and a value estimate from the critic. The last layer of the actor provides four F.tanhed action values, which are used as mean value for the distribution. nn.Parameter(torch.zeros(action_dim)) is the standard deviation.

The trajectories for 20 parallel agents are added to the same memory. Episode length is 1000 and memory.sample() returns a np.random.permutation of the 20k memory entries as tensors with batches of size 64. Before stacking the batch tensors, the values are stored as (1,-1) tensors in collection.deques. The returned tensors are detach()ed.

environment

brain_name = envs.brain_names[0]
env_info = envs.reset(train_mode=True)[brain_name]
env_info = envs.step(actions.cpu().detach().numpy())[brain_name]
next_states = env_info.vector_observations
rewards = env_info.rewards
dones = env_info.local_done

update step

def clipped_surrogate_update(policy, memory, num_epochs=10, clip_param=0.2, gradient_clip=5, beta=0.001, value_loss_coeff=0.5):

    advantages_batch, states_batch, log_probs_old_batch, returns_batch, actions_batch = memory.sample()

    advantages_batch = (advantages_batch - advantages_batch.mean()) / advantages_batch.std()

    for _ in range(num_epochs):
        for i in range(len(advantages_batch)):

            advantages_sample = advantages_batch[i]
            states_sample = states_batch[i]
            log_probs_old_sample = log_probs_old_batch[i]
            returns_sample = returns_batch[i]
            actions_sample = actions_batch[i]

            dist, values = policy(states_sample)

            log_probs_new = dist.log_prob(actions_sample.to(device)).sum(-1).unsqueeze(-1)
            entropy = dist.entropy().sum(-1).unsqueeze(-1).mean()

            ratio = (log_probs_new - log_probs_old_sample).exp()

            clipped_ratio = torch.clamp(ratio, 1-clip_param, 1+clip_param)
            clipped_surrogate_loss = -torch.min(ratio*advantages_sample, clipped_ratio*advantages_sample).mean()
            value_function_loss = (returns_sample - values).pow(2).mean()

            Loss = clipped_surrogate_loss - beta * entropy + value_loss_coeff * value_function_loss

            optimizer_policy.zero_grad()
            Loss.backward()
            torch.nn.utils.clip_grad_norm_(policy.parameters(), gradient_clip)
            optimizer_policy.step()
            del Loss

data sampling

def collect_trajectories(envs, env_info, policy, memory, tmax=200, nrand=0, gae_tau = 0.95, discount = 0.995):

    next_episode = False

    states = env_info.vector_observations
    n_agents = len(env_info.agents)

    state_list=[]
    reward_list=[]
    prob_list=[]
    action_list=[]
    value_list=[]

    if nrand > 0:
        # perform nrand random steps
        for _ in range(nrand):
            actions = np.random.randn(num_agents, action_size)
            actions = np.clip(actions, -1, 1)
            env_info = envs.step(actions)[brain_name]
            states = env_info.vector_observations

    for t in range(tmax):

        states = torch.FloatTensor(states).to(device)
        dist, values = policy(states)

        actions = dist.sample()
        probs = dist.log_prob(actions).sum(-1).unsqueeze(-1)

        env_info = envs.step(actions.cpu().detach().numpy())[brain_name]
        next_states = env_info.vector_observations
        rewards = env_info.rewards
        dones = env_info.local_done

        state_list.append(states)
        reward_list.append(rewards)
        prob_list.append(probs)
        action_list.append(actions)
        value_list.append(values)

        states = next_states

        if np.any(dones):
            next_episode = True
            break

    _, next_value = policy(torch.FloatTensor(states).to(device))

    reward_arr = np.array(reward_list)
    undiscounted_rewards = np.sum(reward_arr, axis=0)

    state_arr = torch.stack(state_list)
    prob_arr = torch.stack(prob_list)
    action_arr = torch.stack(action_list)
    value_arr = torch.stack(value_list)

    reward_arr = torch.FloatTensor(reward_arr[:, :, np.newaxis])

    advantage_list = []
    return_list = []

    returns = next_value.detach()
    advantages = torch.FloatTensor(np.zeros((n_agents, 1)))
    for i in reversed(range(state_arr.shape[0])):

        returns = reward_arr[i] + discount * returns

        td_error = reward_arr[i] + discount * next_value - value_arr[i]
        advantages = advantages * gae_tau * discount + td_error
        next_value = value_arr[i]

        advantage_list.append(advantages.detach())
        return_list.append(returns.detach())

    advantage_arr = torch.stack(advantage_list)
    return_arr = torch.stack(return_list)

    for i in range(state_arr.shape[0]):
        memory.add({'advantages': advantage_arr[i],
                    'states': state_arr[i],
                    'log_probs_old': prob_arr[i],
                    'returns': return_arr[i],
                    'actions': action_arr[i]})

    return undiscounted_rewards, next_episode


title: Set torch.cuda.IntTensor to Long for embedding layer

body:
Right now I am having trouble telling my network to understand my data because I have created an error with the embedding layer. It is expecting a Long but I am passing it an torch.cuda.IntTensor
My Model

class LSTMClassification(torch.nn.Module):

    def __init__(self, n_vocab, n_embed, n_hidden, n_output, n_layers, drop_p = 0.5):
        super().__init__()
        # params: "n_" means dimension
        self.n_vocab = n_vocab     # number of unique words in vocabulary
        self.n_layers = n_layers   # number of LSTM layers
        self.n_hidden = n_hidden   # number of hidden nodes in LSTM

        self.embedding = torch.nn.Embedding(n_vocab, n_embed)
        self.lstm = torch.nn.LSTM(n_embed, n_hidden, n_layers, batch_first = True, dropout = drop_p)
        self.dropout = torch.nn.Dropout(drop_p)
        self.fc = torch.nn.Linear(n_hidden, n_output)
        self.sigmoid = torch.nn.Sigmoid()

    def forward (self, input_words):
                                             # INPUT   :  (batch_size, seq_length)
        embedded_words = self.embedding(input_words)    # (batch_size, seq_length, n_embed)
        lstm_out, h = self.lstm(embedded_words)         # (batch_size, seq_length, n_hidden)
        lstm_out = self.dropout(lstm_out)
        lstm_out = lstm_out.contiguous().view(-1, self.n_hidden) # (batch_size*seq_length, n_hidden)
        fc_out = self.fc(lstm_out)                      # (batch_size*seq_length, n_output)
        sigmoid_out = self.sigmoid(fc_out)              # (batch_size*seq_length, n_output)
        sigmoid_out = sigmoid_out.view(batch_size, -1)  # (batch_size, seq_length*n_output)

        # extract the output of ONLY the LAST output of the LAST element of the sequence
        sigmoid_last = sigmoid_out[:, -1]               # (batch_size, 1)

        return sigmoid_last, h

    def init_hidden (self, batch_size):  # initialize hidden weights (h,c) to 0

        device = "cuda" if torch.cuda.is_available() else "cpu"
        weights = next(self.parameters()).data
        h = (weights.new(self.n_layers, batch_size, self.n_hidden).zero_().to(device),
             weights.new(self.n_layers, batch_size, self.n_hidden).zero_().to(device))

        return h

**Model Initialization & Training **

num_epochs_first = 1
for epoch in range(n_epochs):
    h = model.init_hidden(batch_size)

    for i, data in enumerate(train_loader, 0):
        step += 1
        inputs, labels = data[0].to(device), data[1].to(device)
        # making requires_grad = False for the latest set of h
        h = tuple([each.data for each in h])

        model.zero_grad()
        output, h = model(inputs)
        #loss = criterion(output, labels)
        #loss.backward()
        #optimizer.step()

My Error

RuntimeError: Expected tensor for argument #1 'indices' to have scalar type Long; but got torch.cuda.IntTensor instead (while checking arguments for embedding)

**
The Error occurs on line:
output, h = model(inputs)
**


title: Pytorch - select region of a tensor using torch function

body:
I am looking for a way to select a region of a PyTorch tensor using a torch function (without using numpy). Do you have suggestions on how to proceed?

In other words, I'm looking for a way to crop a region of a matrix. Using numpy, it would be something like

import numpy as np
A = np.random.rand(16,16)
B = A[0:8, 0:8]

The approach I am trying is the following:

from torchvision import transforms
A = torch.randn([1,3,64,64])
B = torch.split(A, [16,32,16], dim =2)
C = torch.split(B, [16,32,16], dim =3)

Which gives the error

  'tuple' object has no attribute 'split'


title: Imagelist.from_folder() in Fastai returns different number of classes when re-running the same cell

body:
I have been trying to create a databunch using Imagelist.from_folder() function in Fastai (in Google Colab) . I have 750 folders(classes). However when I ran Imagelist.from_folder() function the first time, it returned 749 number of classes! 1 class less. So I re-ran the code cell again, this time it returned 748 number of classes and again running third time it returned 746! I checked the number of sub-folders in the main folder again by using 'glob' in python and there were 750 folders(the exact). So why is the Fastai function returning different number of classes (folders) everytime i run it. I am attaching the screenshot below for your reference.
After executing the cell for the first time:
Number of classes : 749

After executing the cell for the second time:
Number of classes : 748

After executing the cell for the third time:
Number of classes : 746

How can this be possible when I have exact 750 folders (as classes)!
I suspect is this due to using Imagelist.from_folder() and maybe do I need to use ImageDataBunch.from_folder() ?? Or is there any issue with google colab trying to fetch the data from Gdrive