title: GPU memory doesn't get freed up after evaluating data on PyTorch model in parallel process

body:
For my optimization algorithm, I need to evaluate a few hundred images every iteration. To speed up the process, I wanted to take full advantage of my 3 GPUs.

My process:

Load an instance of my deep learning model on each one of my GPUs
Then split the workload into as many parts as I have GPUs
pair each workload in a tuple with the instance of the GPU loaded model it should be processed on
run starmap(_runDataThroughModel, sub_workload) to process all sub_workload in parallel

Now there is no problem with doing this once and ending the problem, however, when I do this repeatedly the GPU memory starts to fill up with each iteration until I get a "RuntimeError: CUDA error: out of memory"

My Question:

What is the correct way of going about this?
Why is the GPU memory
not freed? Since I pre-instantiate the GPU model outside the
"starmap" command and always pass the same instances, why would there
be a buildup?

Update
I re-wrote the code taking into account the issue presented in this thread. Instantiating Pool() outside of any loop in the program didn't solve the GPU memory overflow, however, it stopped the CPU memory from building up over time.

'''
Test GPU Memory Leak
Description: Tests how the memory doesn't get freed up when running multiprocessing with PyTorch Model forward pass
'''
import torch
import torch.multiprocessing as mp
import importlib
from PIL import Image
from skimage import io, transform
from skimage.color import rgb2gray
from skimage.io._plugins.pil_plugin import *
import torch
import torch.nn as nn

# Convolutional neural network (twohttps://duckduckgo.com/?q=install+gmsh+conda&t=canonical convolutional layers)
class ConvNet(nn.Module):
    def __init__(self, num_classes=10, num_img_layers = 1, img_res = 128):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential(
            #torch.nn.Conv2d(in_channels, out_channels, kernel_size, stride=1,
            # padding=0, dilation=1, groups=1, bias=True, padding_mode='zeros')
            nn.Conv2d(num_img_layers, 64, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(64),
            nn.LeakyReLU())
        self.layer2 = nn.Sequential(
            nn.Conv2d(64, 32, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(32),
            nn.LeakyReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))
        self.fc1 = nn.Linear(32*int(img_res/2)*int(img_res/2), 32*32)
        self.fc2 = nn.Linear(32*32, num_classes)

    def forward(self, x):
        #print(x.shape)
        out = self.layer1(x)
        #print(out.shape)
        out = self.layer2(out)
        #print(out.shape)
        out = out.reshape(out.size(0), -1)
        out = self.fc1(out)
        out = self.fc2(out)
        return out

class NNEvaluator:
    def __init__(self, model_dict, GPU, img_res = 128, num_img_layers = 1, num_classes = None):
        # Load the model checkpoint
        gpu_id = 'cuda:' + str(GPU)
        self.device = torch.device(gpu_id if torch.cuda.is_available() else 'cpu')
        self.model_state_dict = model_dict['model_state_dict']
        self.model = ConvNet(num_classes = num_classes, num_img_layers = num_img_layers, img_res = img_res).to(self.device)
        self.model.to(self.device)
        self.model.load_state_dict(self.model_state_dict)

        self.epsilon = torch.tensor(1e-12, dtype = torch.float)
    def evaluate(self, img):
        self.model.eval()
        with torch.no_grad():
            img = img.to(self.device)
            out = self.model(img)
            out = out.to('cpu')
        return out
def loadImage(filename):
    im = Image.open("test.jpg")
    im = io._plugins.pil_plugin.pil_to_ndarray(im)
    im = rgb2gray(im)
    image = im.transpose((0, 1))
    im = torch.from_numpy(image).float()
    im = torch.unsqueeze(im,0)
    im = torch.unsqueeze(im,1)
    return im

def _worker(workload, evaluator):
    results = []
    for img in workload:
        results.append(evaluator.evaluate(img))
def main():
    # load a model for each GPU
    model_dict = torch.load('model_dict.ckpt')
    GPUs = [0,1,2] # available GPUs in the system
    evaluators = []
    for gpu_id in GPUs:
        evaluators.append(NNEvaluator(model_dict, gpu_id, num_classes=3))
    # instantiate multiprocessing pool
    mp.set_start_method('spawn')
    mypool = mp.Pool()

    # evaluate all datapoints 20 times
    im = loadImage('test.jpg')
    total_nr_iterations = 20
    for i in range(total_nr_iterations):
        # run a subset of the workload on each GPU in a separate process
        nr_datapoints = 99
        dp_per_evaluator = int(nr_datapoints/len(evaluators))
        workload = [im for i in range(dp_per_evaluator)]
        jobslist = [(workload, evaluator) for evaluator in evaluators]
        mypool.starmap(_worker, jobslist)
        print("Finished iteration {}".format(i))
if __name__ == '__main__':
    main()

Output when running the code:

Finished iteration 0
Finished iteration 1
Finished iteration 2
Process SpawnPoolWorker-10:
Process SpawnPoolWorker-12:
Traceback (most recent call last):
Traceback (most recent call last):
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/pool.py", line 110, in worker
    task = get()
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/queues.py", line 354, in get
    return _ForkingPickler.loads(res)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/pool.py", line 110, in worker
    task = get()
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/site-packages/torch/multiprocessing/reductions.py", line 119, in rebuild_cuda_tensor
    event_sync_required)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/queues.py", line 354, in get
    return _ForkingPickler.loads(res)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/site-packages/torch/multiprocessing/reductions.py", line 119, in rebuild_cuda_tensor
    event_sync_required)
RuntimeError: CUDA error: out of memory
RuntimeError: CUDA error: out of memory
Process SpawnPoolWorker-11:
Traceback (most recent call last):
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
    self.run()
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/process.py", line 99, in run
    self._target(*self._args, **self._kwargs)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/pool.py", line 110, in worker
    task = get()
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/multiprocessing/queues.py", line 354, in get
    return _ForkingPickler.loads(res)
  File "/home/ron/miniconda3/envs/PyTorchNN/lib/python3.7/site-packages/torch/multiprocessing/reductions.py", line 119, in rebuild_cuda_tensor
    event_sync_required)
RuntimeError: CUDA error: out of memory


title: HDF5 dataloading very slow. Causes zero % volatility in GPUs

body:
I am using a custom PyTorch dataclass to load instances from a H5 dataset I created. However, it appears to be incredibly slow when loading samples. I have followed several bits of advice on dealing with large HDF5 datasets, but I am wondering whether I am doing something that is obviously wrong. I am deploying my code on Linux if that makes a difference. I am running the code on 4 GPUs with nn.dataparallel in place for my model. As the dataloading is very slow, the GPU Volatility is at 0%.  Here is my dataclass loader:

import h5py
from torch.utils import data

class Features_Dataset(data.Dataset):
    def __init__(self, archive, phase):
        self.archive = archive
        self.phase = phase

    def __getitem__(self, index):
        with h5py.File(self.archive, 'r', libver='latest', swmr=True) as archive:
            datum = archive[str(self.phase) + '_all_arrays'][index]
            label = archive[str(self.phase) + '_labels'][index]
            path = archive[str(self.phase) +  '_img_paths'][index]
            return datum, label, path

    def __len__(self):
        with h5py.File(self.archive, 'r', libver='latest', swmr=True) as archive:
            datum = archive[str(self.phase) + '_all_arrays']
            return len(datum)

if __name__ == '__main__':
    train_dataset = Features_Dataset(archive= "featuresdata/train.hdf5", phase= 'train')
    trainloader = data.DataLoader(train_dataset, num_workers=8, batch_size=128)
    print(len(trainloader))
    for i, (data, label, path) in enumerate(trainloader):
        print(path)

Am I missing something obvious? Is there a better way of loading instances rapidly?

EDIT:

Here is the updated dataclass however I now get a picling error when trying to use multiprocessing.

import h5py
from torch.utils import data
import torch.multiprocessing as mp
mp.set_start_method('spawn')

class Features_Dataset(data.Dataset):
    def __init__(self, archive, phase):
        self.archive = h5py.File(archive, 'r')
        self.labels = self.archive[str(phase) + '_labels']
        self.data = self.archive[str(phase) + '_all_arrays']
        self.img_paths = self.archive[str(phase) + '_img_paths']

    def __getitem__(self, index):
        datum = self.data[index]
        label = self.labels[index]
        path = self.img_paths[index]
        return datum, label, path

    def __len__(self):
        return len(self.data)

    def close(self):
        self.archive.close()

if __name__ == '__main__':
    train_dataset = Features_Dataset(archive= "featuresdata/train.hdf5", phase= 'train')
    trainloader = data.DataLoader(train_dataset, num_workers=2, batch_size=4)
    print(len(trainloader))
    for i, (data, label, path) in enumerate(trainloader):
        print(path)


title: pytorch to Onnx(OCR model)

body:
I am trying to convert pytorch model in the given repo https://github.com/clovaai/deep-text-recognition-benchmark
to onnx.

I am facing the issue while doing so.

Failed to export an ONNX attribute 'onnx::Gather', since it's not constant, please try to make things (e.g., kernel size) static if possible

Link to the git issue https://github.com/clovaai/deep-text-recognition-benchmark/issues/76

Any suggestion?

Thanks.


Like title. Does someone knows the difference between xavier_normal_ and kaiming_normal_?  Only Xavier have an argument 'gain' more than kaiming?


title: Pytorch: How plot the prediction output from segmentation task when batch size is larger than 1?

body:
I have a doubt and a question about plot output of different batches in segmentation subject.

The snippet below plot the probability of each class and the prediction output.

I am sure the prob plots is plotting one batch, but not sure about prediction when I got the torch.argmax(outputs, 1). Am I plotted the argmax of one batch while the output of the network has the size of [10,4,256,256].

Also, I am wondering how can I plot the prediction of all batches while my batch size is 10.

outputs = model(t_image)

fig, (ax1, ax2, ax3, ax4, ax5) = plt.subplots(nrows=1, ncols=5, sharex=True, sharey=True, figsize=(6,6))

img1 = ax1.imshow(torch.exp(outputs[0,0,:,:]).detach().cpu(), cmap = 'jet')
ax1.set_title("prob class 0")

img2 = ax2.imshow(torch.exp(outputs[0,1,:,:]).detach().cpu(), cmap = 'jet')
ax2.set_title("prob class 1")

img3 = ax3.imshow(torch.exp(outputs[0,2,:,:]).detach().cpu(), cmap = 'jet')
ax3.set_title("prob class 2")

img4 = ax4.imshow(torch.exp(outputs[0,3,:,:]).detach().cpu(), cmap = 'jet')
ax4.set_title("prob class 3")

img5 = ax5.imshow(torch.argmax(outputs, 1).detach().cpu().squeeze(), cmap = 'jet')
ax5.set_title("predicted")


title: Cannot Iterate through PyTorch MNIST dataset

body:
I am trying load the MNIST dataset in Pytorch and use the built-in dataloader to iterate through the training examples. However I get an error when calling next() on the iterator. I don't have this problem with CIFAR10.

import torch
import torchvision
import torchvision.transforms as transforms

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

batch_size = 128

dataset = torchvision.datasets.MNIST(root='./data', train=True, transform=transform, download=True)
dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=4)
dataiter = iter(dataloader)
dataiter.next() # ERROR
# RuntimeError: output with shape [1, 28, 28] doesn't match the broadcast shape [3, 28, 28]

I am using Python 3.7.3 with PyTorch 1.1.0


title: Can pytorch optimize sequential operations (like a tensorflow graph or JAX's jit)?

body:
Originally, tensorflow and pytorch had a fundamental difference:

tensorflow is based on a computional graph. Building this graph and evaluating it in a session are two separate steps. While it is being used, the graph doesn't change, which allows for optimizations.
torch eagerly evaluates operations on a tensor. This makes the API more convenient (no sessions) but also looses the potential to recognize and optimize operations that always occur in sequence.

Now this difference is becoming less clear. Tensorflow has answered to the popularity of torch with tf eager. There is also the JAX project, which builds on the same underlying framework as tensorflow (XLA). JAX has no concept of a session. But it allows you to compile multiple operations together by simply calling jit.

Since Tensorflow has moved to cover PyTorch functionality, is PyTorch also working on integrating Tensorflow advantages? Is there something like a session or jit functionality in PyTorch (or on its roadmap) ?

The API docs have a jit section, but as far as I can see, that is more about exporting your models.


title: Torch getting stuck transfering model to GPU in multiple GPU setting

body:
While using torch multiprocessing pool to paralelize the run of multiple equal experiences in multiple GPUs I ran into the fact that a lot of times the processes block in the transfer of the model from CPU to the assigned GPU. I don't understand the reason of the blocking.

I've tried manually deleting the model at the end of each run and collecting the not used GPU memory to no avail. I would like to understand why.

I'm running this on python 3.7 in a linux system with pytorch 1.2.0 and CUDA 10.0.130 with 4 NVIDIA 1080.

My main code is the following. VGG16 is just a big network. I can add the code of it too if you feel it's necessary.

import torch

from torch.multiprocessing import Process, Manager, Pool
from VGG16 import VGG16

def run_experiment(name, manager_dict, manager_lock):
    device_id = None
    manager_lock.acquire()
    for device_id in manager_dict.keys():
        if device_id == "n_processes":
            continue
        if manager_dict[device_id] < manager_dict["n_processes"]:
            manager_dict[device_id] += 1
            device_id = device_id
            break
    manager_lock.release()
    print(manager_dict, flush=True)
    device = torch.device(device_id)
    print(str(name)+ "selected device: " + str(device_id), flush=True)

    model = VGG16()
    print(str(name), "Initialized model", )
    model = model.to(device)
    print(str(name), "LOADED MODEL to the device")
    manager_dict[device_id] -= 1
    model_cpu = model.to(torch.device("cpu"))
    print(str(name), "Should have returned", flush=True)
    del model
    torch.cuda.ipc_collect()
    #return 1
    return model_cpu.state_dict()

def normal_experiments():
    n_processes_per_gpu = 2
    manager = Manager()
    manager_dict = manager.dict()
    manager_dict["n_processes"] = n_processes_per_gpu
    n_gpu = torch.cuda.device_count()
    print("Running on %d"%n_gpu, flush=True)
    for i in range(n_gpu):
        manager_dict[i] = 0
    manager_lock = manager.Lock()

    with Pool(processes=n_processes_per_gpu*n_gpu) as pool:
        #We call run experiment with seeds with its type of arguments
        iterable_arguments = [(str(i), manager_dict, manager_lock) for i in range(30)]
        res = pool.starmap(run_experiment, iterable_arguments)
    print("Done")

def main():
    torch.multiprocessing.set_start_method('spawn')
    normal_experiments()

if __name__ == "__main__":
    main()

In this code some processes block in

model = model.to(device)


title: Activate Conda Environment From Script

body:
I am using a server that runs on slurm. It requires the use of scripts to launch jobs. Particularly, I have to use the following command:

sbatch script.sh

Inside this script, I have to specify a couple of things, including the conda environment I'm using, which I do in the following way:

source activate /path.../envs/
python job_to_run.py

The process keeps crushing because it fails to load Pytorch (even if it loads Numpy, for example). The problem is that Pytorch is installed in /env/lib, and it has no problem loading it if I run the job without a script, using the command

python job_to_run.py

directly from the command line.

How can I solve this problem? Thank You!


title: Flattening Efficientnet model

body:
I am trying to remove the top layer in the efficientnet-pytorch implementation. However, if I simply replace the final _fc layer with my own fully connected layer, as suggested by the author in this github comment, I am worried that there is still a swish activation even after this layer, as opposed to having nothing as I expected. When I print the model, the final lines is as follows:
(_bn1): BatchNorm2d(1280, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)
    (_avg_pooling): AdaptiveAvgPool2d(output_size=1)
    (_dropout): Dropout(p=0.2, inplace=False)
    (_fc): Sequential(
      (0): Linear(in_features=1280, out_features=512, bias=True)
      (1): ReLU()
      (2): Dropout(p=0.25, inplace=False)
      (3): Linear(in_features=512, out_features=128, bias=True)
      (4): ReLU()
      (5): Dropout(p=0.25, inplace=False)
      (6): Linear(in_features=128, out_features=1, bias=True)
    )
    (_swish): MemoryEfficientSwish()
  )
)

where _fc is my replaced module.
What I hoped to do was:
base_model = EfficientNet.from_pretrained('efficientnet-b3')
model = nn.Sequential(*list(base_model.children()[:-3]))

where in my mind base_model.children() flattens the model from the nested structure. However, now I cannot seem to be able to use the model as if I use a dummy input, x=torch.randn(1,3,255,255) I get the error: TypeError: forward() takes 1 positional argument but 2 were given.
It should be noted that model[:2](x) works, but not model[:3](x). model[2] seems to be the mobile blocks.
Here is a colab notebook with the above code.


title: Normalizing images passed to torch.transforms.Compose function

body:
How to find the values to pass to the transforms.Normalize function in PyTorch? Also, where in my code, should I exactly do the transforms.Normalize?
Since normalizing the dataset is a pretty well-known task, I was hoping there should be some sort of script for doing that automatically. At least I couldn't find it in PyTorch forum.
transformed_dataset = MothLandmarksDataset(csv_file='moth_gt.csv',
                                           root_dir='.',
                                           transform=transforms.Compose([
                                           Rescale(256),
                                           RandomCrop(224),
                                           transforms.Normalize(mean = [ 0.485, 0.456, 0.406 ],
                                           std = [ 0.229, 0.224, 0.225 ]),
                                           ToTensor()
                                               ]))

for i in range(len(transformed_dataset)):
    sample = transformed_dataset[i]
    print(i, sample['image'].size(), sample['landmarks'].size())
    if i == 3:
       break

I know these current values don't pertain to my dataset and pertain to ImageNet but using them I actually get an error:
    TypeError                                 Traceback (most recent call last)
    <ipython-input-81-eb8dc46e0284> in <module>
         10
         11 for i in range(len(transformed_dataset)):
    ---> 12     sample = transformed_dataset[i]
         13
         14     print(i, sample['image'].size(), sample['landmarks'].size())

    <ipython-input-48-9d04158922fb> in __getitem__(self, idx)
         30
         31         if self.transform:
    ---> 32             sample = self.transform(sample)
         33
         34         return sample

    ~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/transforms.py in __call__(self, img)
         59     def __call__(self, img):
         60         for t in self.transforms:
    ---> 61             img = t(img)
         62         return img
         63

    ~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/transforms.py in __call__(self, tensor)
        210             Tensor: Normalized Tensor image.
        211         """
    --> 212         return F.normalize(tensor, self.mean, self.std, self.inplace)
        213
        214     def __repr__(self):

    ~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/functional.py in normalize(tensor, mean, std, inplace)
        278     """
        279     if not torch.is_tensor(tensor):
    --> 280         raise TypeError('tensor should be a torch tensor. Got {}.'.format(type(tensor)))
        281
        282     if tensor.ndimension() != 3:

    TypeError: tensor should be a torch tensor. Got <class 'dict'>.

So basically three questions:

How can I find the similar values as in ImageNet mean and std for my own custom dataset?
How to pass these values and where? I assume I should do it in transforms.Compose method but I might be wrong.
I assume I should apply Normalize to my entire dataset not just the training set, am I right?

Update:
Trying the provided solution here didn't work for me: https://discuss.pytorch.org/t/about-normalization-using-pre-trained-vgg16-networks/23560/6?u=mona_jalal
mean = 0.
std = 0.
nb_samples = 0.
for data in dataloader:
    print(type(data))
    batch_samples = data.size(0)

    data.shape(0)
    data = data.view(batch_samples, data.size(1), -1)
    mean += data.mean(2).sum(0)
    std += data.std(2).sum(0)
    nb_samples += batch_samples

mean /= nb_samples
std /= nb_samples

error is:
<class 'dict'>

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-51-e8ba3c8718bb> in <module>
      5 for data in dataloader:
      6     print(type(data))
----> 7     batch_samples = data.size(0)
      8
      9     data.shape(0)

AttributeError: 'dict' object has no attribute 'size'

this is print(data) result:
{'image': tensor([[[[0.2961, 0.2941, 0.2941,  ..., 0.2460, 0.2456, 0.2431],
          [0.2953, 0.2977, 0.2980,  ..., 0.2442, 0.2431, 0.2431],
          [0.2941, 0.2941, 0.2980,  ..., 0.2471, 0.2471, 0.2448],
          ...,
          [0.3216, 0.3216, 0.3216,  ..., 0.2482, 0.2471, 0.2471],
          [0.3216, 0.3241, 0.3253,  ..., 0.2471, 0.2471, 0.2450],
          [0.3216, 0.3216, 0.3216,  ..., 0.2471, 0.2452, 0.2431]],

         [[0.2961, 0.2941, 0.2941,  ..., 0.2460, 0.2456, 0.2431],
          [0.2953, 0.2977, 0.2980,  ..., 0.2442, 0.2431, 0.2431],
          [0.2941, 0.2941, 0.2980,  ..., 0.2471, 0.2471, 0.2448],
          ...,
          [0.3216, 0.3216, 0.3216,  ..., 0.2482, 0.2471, 0.2471],
          [0.3216, 0.3241, 0.3253,  ..., 0.2471, 0.2471, 0.2450],
          [0.3216, 0.3216, 0.3216,  ..., 0.2471, 0.2452, 0.2431]],

         [[0.2961, 0.2941, 0.2941,  ..., 0.2460, 0.2456, 0.2431],
          [0.2953, 0.2977, 0.2980,  ..., 0.2442, 0.2431, 0.2431],
          [0.2941, 0.2941, 0.2980,  ..., 0.2471, 0.2471, 0.2448],
          ...,
          [0.3216, 0.3216, 0.3216,  ..., 0.2482, 0.2471, 0.2471],
          [0.3216, 0.3241, 0.3253,  ..., 0.2471, 0.2471, 0.2450],
          [0.3216, 0.3216, 0.3216,  ..., 0.2471, 0.2452, 0.2431]]],

        [[[0.3059, 0.3093, 0.3140,  ..., 0.3373, 0.3363, 0.3345],
          [0.3059, 0.3093, 0.3165,  ..., 0.3412, 0.3389, 0.3373],
          [0.3098, 0.3131, 0.3176,  ..., 0.3450, 0.3412, 0.3412],
          ...,
          [0.2931, 0.2966, 0.2931,  ..., 0.2549, 0.2539, 0.2510],
          [0.2902, 0.2902, 0.2902,  ..., 0.2510, 0.2510, 0.2502],
          [0.2864, 0.2900, 0.2863,  ..., 0.2510, 0.2510, 0.2510]],

         [[0.3059, 0.3093, 0.3140,  ..., 0.3373, 0.3363, 0.3345],
          [0.3059, 0.3093, 0.3165,  ..., 0.3412, 0.3389, 0.3373],
          [0.3098, 0.3131, 0.3176,  ..., 0.3450, 0.3412, 0.3412],
          ...,
          [0.2931, 0.2966, 0.2931,  ..., 0.2549, 0.2539, 0.2510],
          [0.2902, 0.2902, 0.2902,  ..., 0.2510, 0.2510, 0.2502],
          [0.2864, 0.2900, 0.2863,  ..., 0.2510, 0.2510, 0.2510]],

         [[0.3059, 0.3093, 0.3140,  ..., 0.3373, 0.3363, 0.3345],
          [0.3059, 0.3093, 0.3165,  ..., 0.3412, 0.3389, 0.3373],
          [0.3098, 0.3131, 0.3176,  ..., 0.3450, 0.3412, 0.3412],
          ...,
          [0.2931, 0.2966, 0.2931,  ..., 0.2549, 0.2539, 0.2510],
          [0.2902, 0.2902, 0.2902,  ..., 0.2510, 0.2510, 0.2502],
          [0.2864, 0.2900, 0.2863,  ..., 0.2510, 0.2510, 0.2510]]],

        [[[0.2979, 0.2980, 0.3015,  ..., 0.2825, 0.2784, 0.2784],
          [0.2980, 0.2980, 0.2980,  ..., 0.2830, 0.2764, 0.2795],
          [0.2980, 0.2980, 0.3012,  ..., 0.2827, 0.2814, 0.2797],
          ...,
          [0.3282, 0.3293, 0.3294,  ..., 0.2238, 0.2235, 0.2235],
          [0.3255, 0.3255, 0.3255,  ..., 0.2240, 0.2235, 0.2229],
          [0.3225, 0.3255, 0.3255,  ..., 0.2216, 0.2235, 0.2223]],

         [[0.2979, 0.2980, 0.3015,  ..., 0.2825, 0.2784, 0.2784],
          [0.2980, 0.2980, 0.2980,  ..., 0.2830, 0.2764, 0.2795],
          [0.2980, 0.2980, 0.3012,  ..., 0.2827, 0.2814, 0.2797],
          ...,
          [0.3282, 0.3293, 0.3294,  ..., 0.2238, 0.2235, 0.2235],
          [0.3255, 0.3255, 0.3255,  ..., 0.2240, 0.2235, 0.2229],
          [0.3225, 0.3255, 0.3255,  ..., 0.2216, 0.2235, 0.2223]],

         [[0.2979, 0.2980, 0.3015,  ..., 0.2825, 0.2784, 0.2784],
          [0.2980, 0.2980, 0.2980,  ..., 0.2830, 0.2764, 0.2795],
          [0.2980, 0.2980, 0.3012,  ..., 0.2827, 0.2814, 0.2797],
          ...,
          [0.3282, 0.3293, 0.3294,  ..., 0.2238, 0.2235, 0.2235],
          [0.3255, 0.3255, 0.3255,  ..., 0.2240, 0.2235, 0.2229],
          [0.3225, 0.3255, 0.3255,  ..., 0.2216, 0.2235, 0.2223]]]],
       dtype=torch.float64), 'landmarks': tensor([[[160.2964,  98.7339],
         [223.0788,  72.5067],
         [ 82.4163,  70.3733],
         [152.3213, 137.7867]],

        [[198.3194,  74.4341],
         [273.7188, 118.7733],
         [117.7113,  80.8000],
         [182.0750, 107.2533]],

        [[137.4789,  92.8523],
         [174.9463,  40.3467],
         [ 57.3013,  59.1200],
         [129.3375, 131.6533]]], dtype=torch.float64)}

dataloader = DataLoader(transformed_dataset, batch_size=3,
                        shuffle=True, num_workers=4)

and
transformed_dataset = MothLandmarksDataset(csv_file='moth_gt.csv',
                                           root_dir='.',
                                           transform=transforms.Compose(
                                               [
                                               Rescale(256),
                                               RandomCrop(224),

                                               ToTensor()#,
                                               ##transforms.Normalize(mean = [ 0.485, 0.456, 0.406 ],
                                               ##         std = [ 0.229, 0.224, 0.225 ])
                                               ]
                                                                        )
                                           )

and
class MothLandmarksDataset(Dataset):
    """Face Landmarks dataset."""

    def __init__(self, csv_file, root_dir, transform=None):
        """
        Args:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.landmarks_frame = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.landmarks_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_name = os.path.join(self.root_dir, self.landmarks_frame.iloc[idx, 0])
        image = io.imread(img_name)
        landmarks = self.landmarks_frame.iloc[idx, 1:]
        landmarks = np.array([landmarks])
        landmarks = landmarks.astype('float').reshape(-1, 2)
        sample = {'image': image, 'landmarks': landmarks}

        if self.transform:
            sample = self.transform(sample)

        return sample


I tried to run a Python script that uses the download_url_to_file method inside the torch hub, but I got the following error:
Traceback (most recent call last):
  File "crop-video.py", line 1, in <module>
    import face_alignment
  File "$PROGRAMS_PATH$\Python\Python36\lib\site-packages\face_alignment-1.3.3-py3.6.egg\face_alignment\__init__.py", line 7, in <module>
  File "$PROGRAMS_PATH$\Python\Python36\lib\site-packages\face_alignment-1.3.3-py3.6.egg\face_alignment\api.py", line 7, in <module>
  File "$PROGRAMS_PATH$\Python\Python36\lib\site-packages\face_alignment-1.3.3-py3.6.egg\face_alignment\utils.py", line 13, in <module>
ImportError: cannot import name 'download_url_to_file'

I am using Python version 3.6.2. I also have the packages installed with the following versions:
certifi (2020.12.5)
chardet (4.0.0)
cycler (0.10.0)
dataclasses (0.8)
decorator (4.4.2)
face-alignment (1.3.3)
future (0.18.2)
idna (2.10)
imageio (2.9.0)
imageio-ffmpeg (0.4.3)
kiwisolver (1.3.1)
llvmlite (0.36.0)
matplotlib (3.3.4)
networkx (2.5)
numba (0.53.0)
numpy (1.19.5)
opencv-python (4.5.1.48)
Pillow (8.1.2)
pip (9.0.1)
pyparsing (2.4.7)
python-dateutil (2.8.1)
PyWavelets (1.1.1)
requests (2.25.1)
rocketchat-API (0.6.9)
scikit-image (0.17.2)
scipy (1.5.4)
setuptools (28.8.0)
six (1.15.0)
tifffile (2020.9.3)
torch (1.0.0)
torchvision (0.2.1)
tqdm (4.24.0)
typing-extensions (3.7.4.3)
unknown (0.0.0)
urllib3 (1.26.4)

The download_url_to_file method does not appear to be found inside the hub file, but I checked the torch hub, and made sure it was defined! How can this error be fixed?


title: PyTorch weights and outputs aren't changing

body:
I'm really enjoying using PyTorch for classification and regression. I have an interesting new problem to solve and I can't quite figure out the solution, I feel like I'm really close.

My problem:
I have created a network with three outputs, let's call them x, y and z
I have a function F(x, y, z) that returns a value between 0.0 and 100.0 where 100 is better
My custom loss thus is 100-F(x,y,z) at each step
The goal is to figure out the best combination of outputs for problem F(...)
(I know a genetic algorithm will outperform this, that's my project right now to prove it on an array of problems)

To implement the above, I force the network to have 1 piece of input data and a batch size of 1, and then in the loss we just completely ignore the 'true' and 'predicted' values and replace the loss with 100-F(x,y,z). Basically our weights and outputs will lead to one solution at every epoch, and the fitness of this solution is inverse from the maximum possible fitness to give a loss (ie. fitness 100 will result in loss 0, 100-100).

Outputs are rounded to integers since F(...) requires them. To prevent this from being an issue, I have a large momentum and learning rate.

The issue I'm having is that, although the loss function is running and my first [x,y,z] is being evaluated, the values never change. The network isn't learning from the results produced.

My code is as follows:
Note testnetwork() is too long to paste but it is the F(x,y,z) mentioned above - any dummy function can replace it eg. 'return x+zy/2' etc. to minimise this function (100 - x+zy/2)

import torch
import torch.nn as nn

from testnetwork import *

n_in, n_h, n_out, batch_size = 10, 5, 3, 5

x = torch.randn(batch_size, n_in)
y = torch.tensor([[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [0.0], [1.0], [1.0]])

model = nn.Sequential(nn.Linear(n_in, n_h),
                     nn.ReLU(),
                     nn.ReLU()
                     )

def fitness(string):
    print(string)
    list = string.split(",")
    list[0] = (int(round(float(list[0]))))
    list[1] = (int(round(float(list[1]))))
    list[2] = (int(round(float(list[2]))))
    print(list)
    loss = 100 - testnetwork(list[0], list[1], list[2])
    return loss

def my_loss(output, target):
    table = str.maketrans(dict.fromkeys('tensor()'))
    ftn = fitness(str(output.data[0][0]).translate(table) + ", " + str(output.data[0][1]).translate(table) + ", " + str(output.data[0][2]).translate(table))

    loss = torch.mean((output - output)+ftn)

    return loss

#optimizer = torch.optim.SGD(model.parameters(), lr=1, momentum=2)
optimizer = torch.optim.Adam(model.parameters(), lr=1, momentum=2)

for epoch in range(10):
    # Forward Propagation
    y_pred = model(x)
    # Compute and print loss
    loss = my_loss(y_pred, y)
    print('epoch: ', epoch,' loss: ', loss.item())
    # Zero the gradients
    optimizer.zero_grad()

    # perform a backward pass (backpropagation)
    loss.backward(retain_graph=True)

    # Update the parameters
    optimizer.step()

Thank you so much for reading my post!

epoch:  0  loss:  50.339725494384766
0., 0.0200, 0.6790
[0, 0, 1]
testing: [0, 0, 1]
epoch:  1  loss:  50.339725494384766
0., 0.0200, 0.6790
[0, 0, 1]
testing: [0, 0, 1]
epoch:  2  loss:  50.339725494384766
0., 0.0200, 0.6790
[0, 0, 1]
testing: [0, 0, 1]
epoch:  3  loss:  50.339725494384766
0., 0.0200, 0.6790
[0, 0, 1]
testing: [0, 0, 1]
epoch:  4  loss:  50.339725494384766
0., 0.0200, 0.6790
[0, 0, 1]

..and so on, nothing seems to change from epoch to epoch.


title: how to fix it ''Given input size: (64x176x64). Calculated output size: (64x1x0). Output size is too small''

body:
I am trying to use colab to train a segdecNet  which looks like this,github https://github.com/mengcius/Surface-Defect-Detection
        # SE layers
        self.fc1 = nn.Conv2d(64, 64//16, kernel_size=1)
        self.fc2 = nn.Conv2d(64//16, 64, kernel_size=1)

        self.layer4 = nn.Sequential(
                            nn.Conv2d(64, 1024, 15, stride=1, padding=7),
                            nn.BatchNorm2d(1024),
                            nn.ReLU(inplace=True)
                        )

        self.layer5 = nn.Sequential(
                            nn.Conv2d(1024, 1, 1),
                            nn.ReLU(inplace=True)
                        )

        if init_weights == True:
            pass

    def forward(self, x):
        x1 = self.layer1(x)
        x2 = self.layer2(x1)
        x3 = self.layer3(x2)

        # Squeeze
        w = F.avg_pool2d(x3, x3.size(2))
        w = F.relu(self.fc1(w))
        w = torch.sigmoid(self.fc2(w))
        # Excitation
        x3 = x3 * w

        x4 = self.layer4(x3)
        x5 = self.layer5(x4)
        # print('x:', x4.shape, x5.shape)

        return {"f":x4, "seg":x5}

I get this error:
Namespace(b1=0.5, b2=0.999, batch_size=2, begin_epoch=0, cuda=True, end_epoch=101, gpu_num=1, img_height=1408, img_width=512, lr=0.0001, need_save=True, need_test=True, save_interval=10, test_interval=10, worker_num=4)
Traceback (most recent call last):
  File "train_segment.py", line 140, in <module>
    rst = segment_net(img)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/content/models.py", line 87, in forward
    w = F.avg_pool2d(x3, x3.size(2))
RuntimeError: Given input size: (64x176x64). Calculated output size: (64x1x0). Output size is too small

I'm guessing I'm making a mistake in my channel size or pooling size but i don't know how to fix it.i did't change the dataset in github,but i got error
i try to change torch1.2 it work.but i want to know if i did't change torch how can i fix my code


title: calculating the mean and std on an array of torch tensors

body:
I am trying to calculate to mean and std for an array of torch tensors. My dataset has 720 training images and each of these images has 4 landmarks with X and Y representing a 2D point on the image.
to_tensor = transforms.ToTensor()

landmarks_arr = []

for i in range(len(train_dataset)):
    landmarks_arr.append(to_tensor(train_dataset[i]['landmarks']))

mean = torch.mean(torch.stack(landmarks_arr, dim=0))#, dim=(0, 2, 3))
std = torch.std(torch.stack(landmarks_arr, dim=0)) #, dim=(0, 2, 3))

print(mean.shape)
print("mean is {} and std is {}".format(mean, std))

Result:
torch.Size([])
mean is nan and std is nan

There is a couple of problems above:

Why to_tensor is not converting the values between 0 and 1?
how to calculate mean correctly?
Should I divide by 255 and where?

I have:
len(landmarks_arr)

720

and
landmarks_arr[0].shape

torch.Size([1, 4, 2])

and
landmarks_arr[0]

tensor([[[502.2869, 240.4949],
         [688.0000, 293.0000],
         [346.0000, 317.0000],
         [560.8283, 322.6830]]], dtype=torch.float64)


title: os.mknod returns [error38] function not implemented in google colab

body:
I am trying to run the following piece of code on google colab.

dir_path = '/content/drive/My Drive/Colab Notebooks'
log_loss_path =os.path.join(dir_path, 'log_loss.txt')
if not os.path.isfile(log_loss_path):
    os.mknod(log_loss_path)

but i get the error [Errno 38] Function not implemented

    OSError                                   Traceback (most recent call last)
<ipython-input-15-bd3880e6bb8b> in <module>()
      2 log_loss_path = os.path.join(dir_path, 'log_loss.txt')
      3 if not os.path.isfile(log_loss_path):
----> 4     os.mknod(log_loss_path)

OSError: [Errno 38] Function not implemented

can anyone help to solve it?


title: Efficient way of selectively replacing vectors from a tensor in pytorch

body:
Given a batch of text sequences, the same is converted into a tensor with each word represented using word embeddings or vectors (of 300 dimensions). I need to selectively replace vectors for certain specific words with a new set of embeddings. Further, this replacement will occur only for not all occurrences of the specific word, but only randomly. Currently, I have the following code to achieve this. It traverses through every word using 2 for loops, check if the word is in a specified list, splIndices. Then it checks if the word needs to be replaced or not, based on T or F value in selected_.

But could this be done in a more efficient manner?

The below code may not be an MWE, but I have tried to simplify the code by removing the specifics, so as to focus on the problem. Please ignore the semantics or purpose of the code as it may not have been appropriately represented in this snippet. The question is about improving performance.

splIndices = [45, 62, 2983, 456, 762]  # vocabulary indices which needs to be replaced
splFreqs = 2000  # assuming the words in splIndices occurs 2000 times
selected_ = Torch.Tensor(2000).uniform_(0, 1) > 0.2  # Tensor with 20% of the entries True
replIndexCtr = 0  # counter for selected_

# Dictionary with vectors to be replaced. This is a dummy function.
# Original function depends on some property of the word
diffVector = {45: Torch.Tensor(300).uniform_(0, 1), ...... 762: Torch.Tensor(300).uniform_(0, 1) }

embeding = nn.Embedding.from_pretrained(embedding_matrix, freeze=False)
tempVals = x  # shape [32, 41] - batch of 32 sequences with 41 words each
x = embeding(x) # shape [32, 41, 300] - the sequence now has replaced vocab indices with embeddings

# iterate through batch for sequences
for i, item in enumerate(x):
    # iterate sequences for words
    for j, stuff in enumerate(item):
        if tempVals[i][j].item() in splIndices:
            if self.selected_[replIndexCtr] == True:
                x[i,j] = diffVector[tempVals[i][j].item()]
                replIndexCtr += 1


title: PyTorch to ONNX conversion problem - torch.randn

body:
Using this reimplementation of StyleGAN in PyTorch, I am trying to export the generator as an .onnx file using the following code:

import model
import torch

Gen = model.StyledGenerator(512)
dummy_input = torch.randn(1,512)
model1 = Gen

torch.onnx.export(model1, dummy_input, "disc.onnx", verbose=True)

When I run the script, I got a type error (see below) due to the translation of torch.randn. Do you have suggestions on how to fix this?

TypeError: i_(): incompatible function arguments. The following argument types are supported:
    1. (self: torch._C.Node, arg0: str, arg1: int) -> torch._C.Node

Invoked with: %227 : Tensor = onnx::RandomNormal(), scope: StyledGenerator
, 'shape', 222 defined in (%222 : int[] = prim::ListConstruct(%218, %219, %220, %221), scope: StyledGenerator
) (occurred when translating randn)

To duplicate the problem, you need to download/clone the github repo and cd to that folder.


title: PyTorch Multi Class Classification using CrossEntropyLoss - not converging

body:
I am trying to get a simple network to output the probability that a number is in one of three classes. These are, smaller than 1.1, between 1.1 and 1.5 and bigger than 1.5. I am using cross entropy loss with class labels of 0, 1 and 2, but cannot solve the problem.
Every time I train, the network outputs the maximum probability for class 2, regardless of input. The lowest loss I seem to be able to achieve is 0.9ish. Any advice on where I am going wrong would be greatly appreciated!! All code is below.
class gating_net(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(gating_net, self).__init__()
        self.linear1 = nn.Linear(input_dim, 32)
        self.linear2 = nn.Linear(32, output_dim)

    def forward(self, x):
        # The original input (action) is used as the residual.
        x = F.relu(self.linear1(x))
        x = F.sigmoid(self.linear2(x))
        return x

learning_rate = 0.01
batch_size = 64
epochs = 500
test = 1

gating_network = gating_net(1,3)

optimizer = torch.optim.SGD(gating_network.parameters(), lr=learning_rate, momentum=0.9)
scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.5, patience=20, verbose=True)

for epoch in range (epochs):
    input_ = []
    label_ = []

    for i in range (batch_size):
        scale = random.randint(10,20)/10

        input = scale
        if scale < 1.1:
            label = np.array([0])
        elif 1.1 < scale < 1.5:
            label = np.array([1])
        else:
            label = np.array([2])

        input_.append(np.array([input]))
        label_.append(label)

    optimizer.zero_grad()

    # get output from the model, given the inputs
    output = gating_network.forward(torch.FloatTensor(input_))
    old_label  = torch.FloatTensor(label_)

    # get loss for the predicted output
    loss = nn.CrossEntropyLoss()(output, old_label.squeeze().long())

    # get gradients w.r.t to parameters
    loss.backward()
    # update parameters
    optimizer.step()
    scheduler.step(loss)

    print('epoch {}, loss {}'.format(epoch, loss.item()))

    if loss.item() < 0.01:
        print("########## Solved! ##########")
        torch.save(mod_network.state_dict(), './supervised_learning/run_{}.pth'.format(test))
        break

    # save every 500 episodes
    if epoch % 100 == 0:
        torch.save(gating_network.state_dict(), './run_{}.pth'.format(test))


title: RuntimeError: Expected 4-dimensional input for 4-dimensional weight [1024, 64, 3, 3], but got input of size [32, 10] instead

body:
This line works fine
self.conv = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False)

I introduced ResNet18
self.conv = ResNet18()

**ResNet Class**

    '''ResNet in PyTorch.

For Pre-activation ResNet, see 'preact_resnet.py'.

Reference:
[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
    Deep Residual Learning for Image Recognition. arXiv:1512.03385
'''
import torch
import torch.nn as nn
import torch.nn.functional as F

class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(
            in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out

class Bottleneck(nn.Module):
    expansion = 4

    def __init__(self, in_planes, planes, stride=1):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, self.expansion *
                               planes, kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(self.expansion*planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = F.relu(self.bn2(self.conv2(out)))
        out = self.bn3(self.conv3(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out

class ResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out

def ResNet18():
    return ResNet(BasicBlock, [2, 2, 2, 2])

def ResNet34():
    return ResNet(BasicBlock, [3, 4, 6, 3])

def ResNet50():
    return ResNet(Bottleneck, [3, 4, 6, 3])

def ResNet101():
    return ResNet(Bottleneck, [3, 4, 23, 3])

def ResNet152():
    return ResNet(Bottleneck, [3, 8, 36, 3])

def test():
    net = ResNet18()
    y = net(torch.randn(1, 3, 3, 64))
    print(y.size())

# test()


I am working on creating an image generator using conditional GAN as the base model. I've run across an error that I don't understand how to debug, even after searching for solutions online. I'm not sure if I should change the settings for training or do some adjustment to my model, or something else. Any help on what to do would be appreciated.
The CGAN model I am using:
class Generator(nn.Module):
    def __init__(self, classes, channels, img_size, latent_dim):
        super(Generator, self).__init__()
        self.classes = classes
        self.channels = channels
        self.img_size = img_size
        self.latent_dim = latent_dim
        self.img_shape = (self.channels, self.img_size, self.img_size)
        self.label_embedding = nn.Embedding(self.classes, self.classes) # process label information, behave as a lookup table

        self.model = nn.Sequential(
            *self._create_layer_1(self.latent_dim + self.classes, 128, False),
            *self._create_layer_2(128, 256),
            *self._create_layer_2(256, 512),
            *self._create_layer_2(512, 1024),
            nn.Linear(1024, int(np.prod(self.img_shape))),
            nn.Tanh()
        )

    def _create_layer_1(self, size_in, size_out, normalize=True):
        layers = [nn.Linear(size_in, size_out)]
        if normalize:
            layers.append(nn.BatchNorm1d(size_out))
        layers.append(nn.LeakyReLU(0.2, inplace=True))
        return layers

    def _create_layer_2(self, size_in, size_out, normalize=True):
        layers = [nn.ConvTranspose2d(size_in, size_out, 4, 2, 1, bias=False)]
        if normalize:
            layers.append(nn.BatchNorm1d(size_out))
        layers.append(nn.LeakyReLU(0.2, inplace=True))
        return layers

    def forward(self, noise, labels):
        z = torch.cat((self.label_embedding(labels), noise), -1)
        x = self.model(z)
        x = x.view(x.size(0), *self.img_shape)
        return x

class Discriminator(nn.Module):
    def __init__(self, classes, channels, img_size, latent_dim):
        super(Discriminator, self).__init__()
        self.classes = classes
        self.channels = channels
        self.img_size = img_size
        self.latent_dim = latent_dim
        self.img_shape = (self.channels, self.img_size, self.img_size)
        self.label_embedding = nn.Embedding(self.classes, self.classes)
        self.adv_loss = torch.nn.BCELoss()

        self.model = nn.Sequential(
            *self._create_layer_1(self.classes + int(np.prod(self.img_shape)), 1024, False, True),
            *self._create_layer_2(1024, 512, True, True),
            *self._create_layer_2(512, 256, True, True),
            *self._create_layer_2(256, 128, False, False),
            *self._create_layer_1(128, 1, False, False),
            nn.Sigmoid()
        )

    def _create_layer_1(self, size_in, size_out, drop_out=True, act_func=True):
        layers = [nn.Linear(size_in, size_out)]
        if drop_out:
            layers.append(nn.Dropout(0.4))
        if act_func:
            layers.append(nn.LeakyReLU(0.2, inplace=True))
        return layers

    def _create_layer_2(self, size_in, size_out, drop_out=True, act_func=True):
        layers = [nn.Conv2d(size_in, size_out, 4, 2, 1, bias=False)]
        if drop_out:
            layers.append(nn.Dropout(0.4))
        if act_func:
            layers.append(nn.LeakyReLU(0.2, inplace=True))
        return layers

    def forward(self, image, labels):
        x = torch.cat((image.view(image.size(0), -1), self.label_embedding(labels)), -1)
        return self.model(x)

    def loss(self, output, label):
        return self.adv_loss(output, label)

Code for initializing the model:
class Model(object):
    def __init__(self,
                 name,
                 device,
                 data_loader,
                 classes,
                 channels,
                 img_size,
                 latent_dim,
                 style_dim=3):
        self.name = name
        self.device = device
        self.data_loader = data_loader
        self.classes = classes
        self.channels = channels
        self.img_size = img_size
        self.latent_dim = latent_dim
        self.style_dim = style_dim
        self.netG = cganG(self.classes, self.channels, self.img_size, self.latent_dim)
        self.netG.to(self.device)
        self.netD = cganD(self.classes, self.channels, self.img_size, self.latent_dim)
        self.netD.to(self.device)
        self.optim_G = None
        self.optim_D = None

    @property
    def generator(self):
        return self.netG

    @property
    def discriminator(self):
        return self.netD

    def create_optim(self, lr, alpha=0.5, beta=0.999):
        self.optim_G = torch.optim.Adam(filter(lambda p: p.requires_grad,
                                        self.netG.parameters()),
                                        lr=lr,
                                        betas=(alpha, beta))
        self.optim_D = torch.optim.Adam(filter(lambda p: p.requires_grad,
                                        self.netD.parameters()),
                                        lr=lr,
                                        betas=(alpha, beta))

    def _to_onehot(self, var, dim):
        res = torch.zeros((var.shape[0], dim), device=self.device)
        res[range(var.shape[0]), var] = 1.
        return res

    def train(self,
              epochs,
              log_interval=100,
              out_dir='',
              verbose=True):
        self.netG.train()
        self.netD.train()
        viz_z = torch.zeros((self.data_loader.batch_size, self.latent_dim), device=self.device)
        viz_noise = torch.randn(self.data_loader.batch_size, self.latent_dim, device=self.device)
        nrows = self.data_loader.batch_size // 8
        viz_label = torch.LongTensor(np.array([num for _ in range(nrows) for num in range(8)])).to(self.device)
        viz_onehot = self._to_onehot(viz_label, dim=self.classes)
        viz_style = torch.zeros((self.data_loader.batch_size, self.style_dim), device=self.device)
        total_time = time.time()
        for epoch in range(epochs):
            batch_time = time.time()
            for batch_idx, (data, target) in enumerate(self.data_loader):
                data, target = data.to(self.device), target.to(self.device)
                batch_size = data.size(0)
                real_label = torch.full((batch_size, 1), 1., device=self.device)
                fake_label = torch.full((batch_size, 1), 0., device=self.device)

                # Train G
                self.netG.zero_grad()
                z_noise = torch.randn(batch_size, self.latent_dim, device=self.device)
                x_fake_labels = torch.randint(0, self.classes, (batch_size,), device=self.device)
                x_fake = self.netG(z_noise, x_fake_labels)
                y_fake_g = self.netD(x_fake, x_fake_labels)
                g_loss = self.netD.loss(y_fake_g, real_label)
                g_loss.backward()
                self.optim_G.step()

                # Train D
                self.netD.zero_grad()
                y_real = self.netD(data, target)
                d_real_loss = self.netD.loss(y_real, real_label)
                y_fake_d = self.netD(x_fake.detach(), x_fake_labels)
                d_fake_loss = self.netD.loss(y_fake_d, fake_label)
                d_loss = (d_real_loss + d_fake_loss) / 2
                d_loss.backward()
                self.optim_D.step()

                if verbose and batch_idx % log_interval == 0 and batch_idx > 0:
                    print('Epoch {} [{}/{}] loss_D: {:.4f} loss_G: {:.4f} time: {:.2f}'.format(
                            epoch, batch_idx, len(self.data_loader),
                            d_loss.mean().item(),
                            g_loss.mean().item(),
                            time.time() - batch_time))
                    vutils.save_image(data, os.path.join(out_dir, 'real_samples.png'), normalize=True)
                    with torch.no_grad():
                        viz_sample = self.netG(viz_noise, viz_label)
                        vutils.save_image(viz_sample, os.path.join(out_dir, 'fake_samples_{}.png'.format(epoch)), nrow=8, normalize=True)
                    batch_time = time.time()

            torch.save(self.netG.state_dict(), os.path.join(out_dir, 'netG_{}.pth'.format(epoch)))
            torch.save(self.netD.state_dict(), os.path.join(out_dir, 'netD_{}.pth'.format(epoch)))

            self.save_to(path=out_dir, name=self.name, verbose=False)
        if verbose:
            print('Total train time: {:.2f}'.format(time.time() - total_time))

Code for setting up the training:
def main():
    device = torch.device("cuda:0" if FLAGS.cuda else "cpu")
    if FLAGS.train:
        dataloader = torch.utils.data.DataLoader(
            dset.ImageFolder(FLAGS.data_dir, transforms.Compose([
                transforms.Resize(FLAGS.img_size),
                transforms.CenterCrop(FLAGS.img_size),
                transforms.ToTensor()
                ])),
                batch_size=FLAGS.batch_size,
                shuffle=True,
                num_workers=4,
                pin_memory=True
                )
        model = Model(FLAGS.model, device, dataloader, FLAGS.classes, FLAGS.channels, FLAGS.img_size, FLAGS.latent_dim)
        model.create_optim(FLAGS.lr)

        # Train
        print("Start training...\n")
        model.train(FLAGS.epochs, FLAGS.log_interval, FLAGS.out_dir, True)

if __name__ == '__main__':
    from utils import boolean_string
    parser.add_argument('--cuda', type=boolean_string, default=True, help='enable CUDA.')
    parser.add_argument('--train', type=boolean_string, default=True, help='train mode or eval mode.')
    parser.add_argument('--data_dir', type=str, default='../datasets', help='Directory for dataset.')
    parser.add_argument('--out_dir', type=str, default='output', help='Directory for output.')
    parser.add_argument('--epochs', type=int, default=800, help='number of epochs')
    parser.add_argument('--batch_size', type=int, default=32, help='size of batches')
    parser.add_argument('--lr', type=float, default=0.0002, help='learning rate')
    parser.add_argument('--latent_dim', type=int, default=62, help='latent space dimension')
    parser.add_argument('--classes', type=int, default=25, help='number of classes')
    parser.add_argument('--img_size', type=int, default=128, help='size of images')
    parser.add_argument('--channels', type=int, default=3, help='number of image channels')

Settings:
PyTorch version: 1.1.0
CUDA version: 9.0.176

         Args         |    Type    |    Value
--------------------------------------------------
  cuda                |  bool      |  True
  train               |  bool      |  True
  resume              |  bool      |  False
  data_dir            |  str       |  ../datasets
  out_dir             |  str       |  output
  epochs              |  int       |  800
  batch_size          |  int       |  32
  lr                  |  float     |  0.0002
  latent_dim          |  int       |  62
  classes             |  int       |  25
  img_size            |  int       |  128
  channels            |  int       |  3

Image size as input:
torch.Size([32, 3, 128, 128])

Model structure:
Generator(
  (label_embedding): Embedding(25, 25)
  (model): Sequential(
    (0): Linear(in_features=87, out_features=128, bias=True)
    (1): LeakyReLU(negative_slope=0.2, inplace)
    (2): ConvTranspose2d(128, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (3): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (4): LeakyReLU(negative_slope=0.2, inplace)
    (5): ConvTranspose2d(256, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (7): LeakyReLU(negative_slope=0.2, inplace)
    (8): ConvTranspose2d(512, 1024, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (9): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (10): LeakyReLU(negative_slope=0.2, inplace)
    (11): Linear(in_features=1024, out_features=49152, bias=True)
    (12): Tanh()
  )
)

Discriminator(
  (label_embedding): Embedding(25, 25)
  (adv_loss): BCELoss()
  (model): Sequential(
    (0): Linear(in_features=49177, out_features=1024, bias=True)
    (1): LeakyReLU(negative_slope=0.2, inplace)
    (2): Conv2d(1024, 512, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (3): Dropout(p=0.4)
    (4): LeakyReLU(negative_slope=0.2, inplace)
    (5): Conv2d(512, 256, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (6): Dropout(p=0.4)
    (7): LeakyReLU(negative_slope=0.2, inplace)
    (8): Conv2d(256, 128, kernel_size=(4, 4), stride=(2, 2), padding=(1, 1), bias=False)
    (9): Linear(in_features=128, out_features=1, bias=True)
    (10): Sigmoid()
  )
)

The error I got:
File "main.py", line 121, in <module>
    main()
  File "main.py", line 56, in main
    model.train(FLAGS.epochs, FLAGS.log_interval, FLAGS.out_dir, True)
  File "build_gan.py", line 123, in train
    x_fake = self.netG(z_noise, x_fake_labels)
  File "anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "cgan.py", line 42, in forward
    x = self.model(z)
  File "anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "anaconda3/lib/python3.6/site-packages/torch/nn/modules/container.py", line 92, in forward
    input = module(input)
  File "anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "anaconda3/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 796, in forward
    output_padding, self.groups, self.dilation)
RuntimeError: Expected 4-dimensional input for 4-dimensional weight 128 256, but got 2-dimensional input of size [32, 128] instead

I am using my own image dataset with 3 channels and 25 classes. I have tried to change the image size and kernel size but still got the same error. Any help on what should I do to debug would be highly appreciated.


title: Image Captioning Example input size of Decoder LSTM Pytorch

body:
I'm new to Pytorch, there is a doubt that am having in the Image Captioning example code . In DcoderRNN class the lstm is defined as ,

self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)

in the forward function ,

embeddings = self.embed(captions)
embeddings = torch.cat((features.unsqueeze(1), embeddings), 1)

we first embed the captions and then concat the embeddings with the context feature from the EncoderCNN, but the concat increases the size from embed size how we can forward that to the lstm? as the input size of lstm is already defined as embed_size.

Am I missing something here? Thanks in advance .


title: Pytorch feeding dataloader batch with custom dataset and collate_fn() to model is not working

body:
Hi I have created a custom dataset using dataset class that original and target are images, like semantic segmentation and pix2pix,I am loading dataset using Imagefolder and a custom collate function and I am trying to load my custom dataset using dataloader for training the neural network, but an error occurred saying that input should be tensor not list:

my collate function

def my_collate(batch):
data = [item[0] for item in batch]
target = [item[1] for item in batch]
return [data, target]

Dataset class:

class bsds_dataset(Dataset):
def __init__(self, ds_main, ds_energy):
    self.dataset1 = ds_main
    self.dataset2 = ds_energy

def __getitem__(self, index):
    x1 = self.dataset1[index]
    x2 = self.dataset2[index]

    return x1, x2

def __len__(self):
    return len(self.dataset1)

Loading dataset:

generic_transform = transforms.Compose([
transforms.ToTensor(),
transforms.ToPILImage(),
#transforms.CenterCrop(size=128),
#transforms.Lambda(lambda x: myimresize(x, (128, 128))),
transforms.ToTensor(),
#transforms.Normalize((0., 0., 0.), (6, 6, 6))
])
original_imagefolder = './images/whole'
target_imagefolder = './results/whole'

original_ds = ImageFolder(original_imagefolder,
transform=generic_transform)
energy_ds = ImageFolder(target_imagefolder, transform=generic_transform)

dataset = bsds_dataset(original_ds, energy_ds)
loader = DataLoader(dataset, batch_size=16, collate_fn=my_collate)

epochs = 2
model = UNet(1, depth=5, merge_mode='concat')
model.cuda()
loss = torch.nn.MSELoss()
criterion_pixelwise = torch.nn.L1Loss()

loss.cuda()
criterion_pixelwise.cuda()

optimizer = optim.SGD(model.parameters(), lr=0.001)

Tensor = torch.cuda.FloatTensor
#main loop
for epoch in range(epochs):
   for i, batch in enumerate(loader):
       original, target = batch
       out = model(original)

And this error occurred:

  TypeError: conv2d(): argument 'input' (position 1) must be Tensor, not
  list

I am trying to forward each instance of batch into the model. The batch in iteration is list but it should be tensor I don't know how to convert it to tensor or load each instance of batch. Please help. Thanks a lot.

Full Traceback:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-147-d1dea1bc00f8> in <module>
     15     for i, batch in enumerate(loader):
     16         original, target = batch
---> 17         out = model(original)

C:\Anaconda3\envs\torchgpu\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    491             result = self._slow_forward(*input, **kwargs)
    492         else:
--> 493             result = self.forward(*input, **kwargs)
    494         for hook in self._forward_hooks.values():
    495             hook_result = hook(self, input, result)

<ipython-input-7-5f743c3455c4> in forward(self, x)
     89         # encoder pathway, save outputs for merging
     90         for i, module in enumerate(self.down_convs):
---> 91             x, before_pool = module(x)
     92             encoder_outs.append(before_pool)
     93

C:\Anaconda3\envs\torchgpu\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    491             result = self._slow_forward(*input, **kwargs)
    492         else:
--> 493             result = self.forward(*input, **kwargs)
    494         for hook in self._forward_hooks.values():
    495             hook_result = hook(self, input, result)

<ipython-input-5-26a0f7e21ea6> in forward(self, x)
     14
     15     def forward(self, x):
---> 16         x = F.relu(self.conv1(x))
     17         x = F.relu(self.conv2(x))
     18         before_pool = x

C:\Anaconda3\envs\torchgpu\lib\site-packages\torch\nn\modules\module.py in __call__(self, *input, **kwargs)
    491             result = self._slow_forward(*input, **kwargs)
    492         else:
--> 493             result = self.forward(*input, **kwargs)
    494         for hook in self._forward_hooks.values():
    495             hook_result = hook(self, input, result)

C:\Anaconda3\envs\torchgpu\lib\site-packages\torch\nn\modules\conv.py in forward(self, input)
    336                             _pair(0), self.dilation, self.groups)
    337         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 338                         self.padding, self.dilation, self.groups)
    339
    340

TypeError: conv2d(): argument 'input' (position 1) must be Tensor, not list


title: Singularity-Container + Python + PyTorch: Why does 'import torch' work on Arch Linux host but fails on Centos 7 host?

body:
I am trying to build a singularity container, to run a Python script on a CentOS 7-based cluster.
The container runs as expected on my host, which I also used to create the container, but fails on the cluster as soon as PyTorch is imported.

The problem can be reproduced with a container build from this minimal definition file:

debug.def:

Bootstrap: arch

%runscript
    exec /usr/bin/python3 -c 'import torch; print(torch.__version__)'

%post
    #--------------------------------------------------------------------------
    # Basic setup from
    # https://github.com/sylabs/singularity/blob/master/examples/arch/Singularity
    #--------------------------------------------------------------------------
    # Set time zone. Use whatever you prefer instead of UTC.
    ln -s /usr/share/zoneinfo/Europe/Berlin /etc/localtime

    # Set the package mirror server(s). This is only for the output image's
    # mirrorlist. `pacstrap' can only use your hosts's package mirrors.
    echo 'Server = https://mirrors.kernel.org/archlinux/$repo/os/$arch' > /etc/pacman.d/mirrorlist

    pacman -Sy --noconfirm gawk sed grep

    # Set locale. Use whatever you prefer instead of en_US.
    echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen
    locale-gen
    echo 'LANG=en_US.UTF-8' > /etc/locale.conf

    pacman -S --noconfirm python python-pytorch

    pacman -S --noconfirm pacman-contrib
    paccache -r -k0

It is build with sudo singularity build debug.sif debug.def.
Both, the container and my host run on Arch Linux.

Executing the container on my host outputs the PyTorch version:

schellsn@host $ singularity run debug.sif
1.3.1

Running it on the cluster results in the following error:

schellsn@cluster tmp$ singularity run debug.sif
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/usr/lib/python3.8/site-packages/torch/__init__.py", line 81, in <module>
    from torch._C import *
ImportError: libQt5Core.so.5: cannot open shared object file: No such file or directory

I don't understand why the file is not found as it should be included in the container:

schellsn@cluster tmp$ singularity shell debug.sif
Singularity debug.sif:~/tmp> ls -l /usr/lib | grep libQt5Core
-rw-r--r--  1 root root      1166 Nov 11 23:40 libQt5Core.prl
lrwxrwxrwx  1 root root        20 Nov 11 23:40 libQt5Core.so -> libQt5Core.so.5.13.2
lrwxrwxrwx  1 root root        20 Nov 11 23:40 libQt5Core.so.5 -> libQt5Core.so.5.13.2
lrwxrwxrwx  1 root root        20 Nov 11 23:40 libQt5Core.so.5.13 -> libQt5Core.so.5.13.2
-rwxr-xr-x  1 root root   5275240 Nov 11 23:40 libQt5Core.so.5.13.2

I assume that the according path is not included within the search while importing and that this problem does not occur on my host because some environment setting is leaking into the container.
I also tried using the Sylabs Remote Builder but it seems that it cannot construct Arch containers (pacstrap not found within $PATH).
Trying to build the container on one of the nodes leads to the same problem; Pacstrap and pacman are not available.

I am at my wits' end and would be very grateful for any hint to explain this behavior!
Why is the shared library not found and how could that possibly be fixed?

Update #1:

Here is the content of the LD_LIBRARY_PATH environment variable (in response to @tsnowlan).

Arch Linux Host:

schellsn@host tmp$ echo $LD_LIBRARY_PATH
:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64
schellsn@host tmp$ singularity shell evpt_debug.sif
Singularity evpt_debug.sif: ~/tmp> echo $LD_LIBRARY_PATH
:/usr/local/cuda/lib:/usr/local/cuda/lib64:/usr/local/cuda/lib:/usr/local/cuda/lib64:/.singularity.d/libs

CentOS 7 cluster node:

schellsn@cluster tmp$ echo $LD_LIBRARY_PATH
schellsn@cluster tmp$ singularity shell debug.sif
Singularity debug.sif:~/tmp> echo $LD_LIBRARY_PATH
/.singularity.d/libs

Update #2:

I did setup a new clean VM (also running arch) which also and rebuild the container there. This container shows the same problem; It runs on my host but not on the CentOS 7 cluster.


title: What does nnz in mean in the output of torch.sparse_coo_tensor(indices, values, size=None, dtype=None, device=None, requires_grad=False)?

body:
What does nnz mean in the output of below pytorch function

torch.sparse_coo_tensor(indices, values, size=None, dtype=None, device=None, requires_grad=False)

It can be found at this link https://pytorch.org/docs/stable/torch.html

i = torch.tensor([[0, 1, 1],
                      [2, 0, 2]])

v = torch.tensor([3, 4, 5], dtype=torch.float32)

torch.sparse_coo_tensor(i, v, [2, 4],
                            dtype=torch.float64,
                            device=torch.device('cuda:0'))

tensor(indices=tensor([[0, 1, 1],
                       [2, 0, 2]]),
       values=tensor([3., 4., 5.]),
       device='cuda:0', size=(2, 4), nnz=3, dtype=torch.float64,
       layout=torch.sparse_coo)


title: How to seemlessly blend (B x C x H x W) tensor tiles together to hide tile boundaries?

body:
For completeness this is a text summary of what I am trying to do:

Split the image into tiles.
Run each tile through a new copy of the model for a set number of iterations.
Feather tiles and put them into rows.
Feather rows and put them back together into the original image/tensor.
Maybe save the output, then split the output into tiles again.
Repeat steps 2 and 3 for a set number of iterations.

I only need help with steps 1, 3, and 4. The style transfer process causes some slight differences to form in the processed tiles, so I need to blend them back together. By feathering I basically mean fading a tile into another to blur the boundaries (like in ImageMagick, Photoshop, etc...). I am trying to accomplish this blending by using Torch.linspace() to create masks, though I'm not sure if there's a better approach.

What I am trying to accomplish is based on/inspired by: https://github.com/VaKonS/neural-style/blob/Multi-resolution/neural_style.lua, though I'm working with PyTorch. The code I am trying to implement tiling with can be found here: https://gist.github.com/ProGamerGov/e64fcb309274c2946f5a9a679ed45669, though you shouldn't need to look at it as everything you need can be found below.

In essence this is what I am trying to do (red areas overlap with another tile):

.

This is what I have for code so far. Feathering and adding rows together is not yet implemented as I can't get the individual tile feathering working yet.

import torch
from PIL import Image
import torchvision.transforms as transforms

def tile_calc(tile_size, v, d):
    max_val = max(min(tile_size*v+tile_size, d), 0)
    min_val = tile_size*v
    if abs(min_val - max_val) < tile_size:
        min_val = max_val-tile_size
    return min_val, max_val

def split_tensor(tensor, tile_size=256):
    tiles, tile_idx = [], []
    tile_size_y, tile_size_x = tile_size+8, tile_size +5 # Make H and W different for testing
    h, w = tensor.size(2), tensor.size(3)
    h_range, w_range = int(-(h // -tile_size_y)), int(-(w // -tile_size_x))

    for y in range(h_range):
        for x in range(w_range):
            ty, y_val = tile_calc(tile_size_y, y, h)
            tx, x_val = tile_calc(tile_size_x, x, w)

            tiles.append(tensor[:, :, ty:y_val, tx:x_val])
            tile_idx.append([ty, y_val, tx, x_val])

    w_overlap = tile_idx[0][3] - tile_idx[1][2]
    h_overlap = tile_idx[0][1] - tile_idx[w_range][0]

    if tensor.is_cuda:
        base_tensor = torch.zeros(tensor.squeeze(0).size(), device=tensor.get_device())
    else:
        base_tensor = torch.zeros(tensor.squeeze(0).size())
    return tiles, base_tensor.unsqueeze(0), (h_range, w_range), (h_overlap, w_overlap)

 # Feather vertically
def feather_tiles(tensor_list, hxw, w_overlap):
    print(len(tensor_list))
    mask_list = []
    if w_overlap > 0:
        for i, tile in enumerate(tensor_list):
            if i % hxw[1] != 0:
                lin_mask = torch.linspace(0,1,w_overlap).repeat(tile.size(2),1)
                mask_part = torch.ones(tile.size(2), tile.size(3)-w_overlap)
                mask = torch.cat([lin_mask, mask_part], 1)
                mask = mask.repeat(3,1,1).unsqueeze(0)
                mask_list.append(mask)
            else:
                mask = torch.ones(tile.squeeze().size()).unsqueeze(0)
                mask_list.append(mask)
    return mask_list

def build_row(tensor_tiles, tile_masks, hxw, w_overlap, bt, tile_size):
    print(len(tensor_tiles), len(tile_masks))
    if bt.is_cuda:
        row_base = torch.ones(bt.size(1),tensor_tiles[0].size(2),bt.size(3), device=bt.get_device()).unsqueeze(0)
    else:
        row_base = torch.ones(bt.size(1),tensor_tiles[0].size(2),bt.size(3)).unsqueeze(0)
    row_list = []
    for v in range(hxw[1]):
      row_list.append(row_base.clone())

    num_tiles = 0
    row_val = 0
    tile_size_y, tile_size_x = tile_size+8, tile_size +5
    h, w = bt.size(2), bt.size(3)
    h_range, w_range = hxw[0], hxw[1]
    for y in range(h_range):
        for x in range(w_range):
            ty, y_val = tile_calc(tile_size_y, y, h)
            tx, x_val = tile_calc(tile_size_x, x, w)

            if num_tiles % hxw[1] != 0:
                new_mean = (row_list[row_val][:, :, :, tx:x_val].mean() + tensor_tiles[num_tiles])/2
                row_list[row_val][:, :, :, tx:x_val] = row_list[row_val][:, :, :, tx:x_val] - row_list[row_val][:, :, :, tx:x_val].mean()
                tensor_tiles[num_tiles] = tensor_tiles[num_tiles] - tensor_tiles[num_tiles].mean()

                row_list[row_val][:, :, :, tx:x_val] = (row_list[row_val][:, :, :, tx:x_val] + ( tensor_tiles[num_tiles] * tile_masks[num_tiles])) + new_mean

            else:
                row_list[row_val][:, :, :, tx:x_val] = tensor_tiles[num_tiles]
            num_tiles+=1
        row_val+=1
    return row_list

def preprocess(image_name, image_size):
    image = Image.open(image_name).convert('RGB')
    if type(image_size) is not tuple:
        image_size = tuple([int((float(image_size) / max(image.size))*x) for x in (image.height, image.width)])
    Loader = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor()])
    tensor = (Loader(image) * 256).unsqueeze(0)
    return tensor

def deprocess(output_tensor):
    output_tensor = output_tensor.squeeze(0).cpu() / 256
    output_tensor.clamp_(0, 1)
    Image2PIL = transforms.ToPILImage()
    image = Image2PIL(output_tensor.cpu())
    return image

input_tensor = preprocess('test.jpg', 256)

tile_tensors, base_t, hxw, ovlp = split_tensor(input_tensor, 128)
tile_masks = feather_tiles(tile_tensors, hxw, ovlp[1])
row_tensors = build_row(tile_tensors, tile_masks, hxw, ovlp[1], base_t, 128)

ft = deprocess(row_tensors[0]) # save tensor to view it
ft.save('ft_row_0.png')


title: Hyper-parameter tuning and Over-fitting with Feed-Forward Neural Network - Mini-Batch Epoch and Cross Validation

body:
I am looking at implementing a hyper-parameter tuning method for a feed-forward neural network (FNN) implemented using PyTorch. My original FNN , the model is named net, has been implemented using a mini-batch learning approach with epochs:

#Parameters
batch_size = 50 #larger batch size leads to over fitting
num_epochs = 1000
learning_rate = 0.01 #was .01-AKA step size - The amount that the weights are updated during training
batch_no = len(x_train) // batch_size

criterion = nn.CrossEntropyLoss()  #performance of a classification model whose output is a probability value between 0 and 1
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)

for epoch in range(num_epochs):
    if epoch % 20 == 0:
        print('Epoch {}'.format(epoch+1))
    x_train, y_train = shuffle(x_train, y_train)
    # Mini batch learning - mini batch since batch size < n(batch gradient descent), but > 1 (stochastic gradient descent)
    for i in range(batch_no):
        start = i * batch_size
        end = start + batch_size
        x_var = Variable(torch.FloatTensor(x_train[start:end]))
        y_var = Variable(torch.LongTensor(y_train[start:end]))
        # Forward + Backward + Optimize
        optimizer.zero_grad()
        ypred_var = net(x_var)
        loss =criterion(ypred_var, y_var)
        loss.backward()
        optimizer.step()

I lastly test my model on a separate test set.

I came across an approach using randomised search to tune the hyper-parameters as well as implementing K-fold cross-validation (RandomizedSearchCV).

My question is two-fold(no pun intended!) and firstly is theoretical: Is k-fold validation is necessary or could add any benefit to mini-batch feed-forward neural network? From what I can see, the mini-batch approach should do roughly the same job, stopping over-fitting.

I also found a good answer here but I'm not sure this addresses a mini-batch approach approach specifically.

Secondly, if k-fold is not necessary, is there another hyper-parameter tuning function for PyTorch to avoid manually creating one?


title: How to implement tf.gather_nd in Pytorch with the argument batch_dims?

body:
I have been doing a project on image matching, so I need to find correspondences between 2 images. To get descriptors, I will need a interpolate function. However, when I read about a equivalent function which is done in Tensorflow, I still don’t get how to implement tf.gather_nd(parmas, indices, barch_dims) in Pytorch. Especially when there is a argument: batch_dims. I have gone through stackoverflow and there is no perfect equivalence yet.

The referred interpolate function in Tensorflow is below and I have been trying to implement this in Pytorch Arguments' information is below:

inputs is a dense feature map[i] from a for loop of batch size, which means it is 3D[H, W, C](in pytorch is [C, H, W])

pos is a set of random point coordinate shapes like [[i, j], [i, j],...,[i, j]], so it is 2D when it goes in interpolate function(in pytorch is [[i,i,...,i], [j,j,...,j]])

and it then expands both of their dimensions when they get into this function

I just want a perfect implement of tf.gather_nd with argument batch_dims. Thank you!
And here's a simple example of using it:

pos = tf.ones((12, 2)) ## stands for a set of coordinates [[i, i,…, i], [j, j,…, j]]
inputs = tf.ones((4, 4, 128)) ## stands for [H, W, C] of dense feature map
outputs = interpolate(pos, inputs, batched=False)
print(outputs.get_shape()) # We get (12, 128) here

interpolate function (tf version):

def interpolate(pos, inputs, nd=True):

    pos = tf.expand_dims(pos, 0)
    inputs = tf.expand_dims(inputs, 0)

    h = tf.shape(inputs)[1]
    w = tf.shape(inputs)[2]

    i = pos[:, :, 0]
    j = pos[:, :, 1]

    i_top_left = tf.clip_by_value(tf.cast(tf.math.floor(i), tf.int32), 0, h - 1)
    j_top_left = tf.clip_by_value(tf.cast(tf.math.floor(j), tf.int32), 0, w - 1)

    i_top_right = tf.clip_by_value(tf.cast(tf.math.floor(i), tf.int32), 0, h - 1)
    j_top_right = tf.clip_by_value(tf.cast(tf.math.ceil(j), tf.int32), 0, w - 1)

    i_bottom_left = tf.clip_by_value(tf.cast(tf.math.ceil(i), tf.int32), 0, h - 1)
    j_bottom_left = tf.clip_by_value(tf.cast(tf.math.floor(j), tf.int32), 0, w - 1)

    i_bottom_right = tf.clip_by_value(tf.cast(tf.math.ceil(i), tf.int32), 0, h - 1)
    j_bottom_right = tf.clip_by_value(tf.cast(tf.math.ceil(j), tf.int32), 0, w - 1)

    dist_i_top_left = i - tf.cast(i_top_left, tf.float32)
    dist_j_top_left = j - tf.cast(j_top_left, tf.float32)
    w_top_left = (1 - dist_i_top_left) * (1 - dist_j_top_left)
    w_top_right = (1 - dist_i_top_left) * dist_j_top_left
    w_bottom_left = dist_i_top_left * (1 - dist_j_top_left)
    w_bottom_right = dist_i_top_left * dist_j_top_left

    if nd:
        w_top_left = w_top_left[..., None]
        w_top_right = w_top_right[..., None]
        w_bottom_left = w_bottom_left[..., None]
        w_bottom_right = w_bottom_right[..., None]

    interpolated_val = (
        w_top_left * tf.gather_nd(inputs, tf.stack([i_top_left, j_top_left], axis=-1), batch_dims=1) +
        w_top_right * tf.gather_nd(inputs, tf.stack([i_top_right, j_top_right], axis=-1), batch_dims=1) +
        w_bottom_left * tf.gather_nd(inputs, tf.stack([i_bottom_left, j_bottom_left], axis=-1), batch_dims=1) +
        w_bottom_right * tf.gather_nd(inputs, tf.stack([i_bottom_right, j_bottom_right], axis=-1), batch_dims=1)
    )

    interpolated_val = tf.squeeze(interpolated_val, axis=0)
    return interpolated_val


title: How to use non-square padding for deconvnet in PyTorch

body:
Thank you for you attention. I hopes to use nn.ConvTranspose2d to expand dimension of a tensor in PyTorch.(from (N,C,4,4) to (N,C,8,8)) However, I find that if I want to keep the kernel size to 3 and stride to 2. I need to set the padding to [[0,0],[0,0],[0,1],[0,1]](only one side to H and W), which is not a square.

I learnt that in tensorflow we can set the padding like [[0,0],[0,0],[0,1],[0,1]], but in torch, the padding can only be a value for each dimension, which means the padding is on both sides.

So, I am wondering is there any way to do this in PyTorch?

Here's some code of details

import torch
import torch.nn as nn

a = torch.rand([100, 80, 4, 4])
a.shape
nn.ConvTranspose2d(80, 40, kernel_size=3, stride=2, padding=0)(a).shape
nn.ConvTranspose2d(80, 40, kernel_size=3, stride=2, padding=1)(a).shape
nn.ConvTranspose2d(80, 40, kernel_size=3, stride=2, padding=(0,1,0,1))(a).shape

>>> torch.Size([100, 80, 4, 4])
>>> torch.Size([100, 40, 9, 9])
>>> torch.Size([100, 40, 7, 7])
>>> RuntimeError: expected padding to be a single integer value or a list of 2 values to match the convolution dimensions, but got padding=[0, 1, 0, 1]


title: PyTorch inside Docker only sees GPUs when root

body:
I want to use PyTorch on a remote machine. The way our system is set up, is that we are only supposed to work from within docker containers. The container I am currently in was launched with:

nvidia-docker run -it --rm --shm-size=2g -v /home/user/:/home/user/ -v /home/user/storage:/storage
-v /storage/user:/storage/data_storage -v /etc/passwd:/etc/passwd -v /etc/group:/etc/group
--user 1009:1009 -p 8884:8884 --ulimit memlock=-1 --name user_test user/deepo:all-jupyter

If I type nvidia-smi I get output as expected:

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 415.27       Driver Version: 415.27       CUDA Version: 10.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce GTX 108...  On   | 00000000:04:00.0 Off |                  N/A |
| 64%   86C    P2   217W / 250W |   8578MiB / 11178MiB |     86%      Default |
+-------------------------------+----------------------+----------------------+
|   1  Tesla P40           On   | 00000000:06:00.0 Off |                    0 |
| N/A   89C    P0   123W / 250W |   8593MiB / 22919MiB |     99%      Default |
+-------------------------------+----------------------+----------------------+
|   2  GeForce GTX 108...  On   | 00000000:07:00.0 Off |                  N/A |
| 45%   75C    P2    94W / 250W |   6956MiB / 11178MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
|   3  Tesla P40           On   | 00000000:08:00.0 Off |                    0 |
| N/A   52C    P0   131W / 250W |   6965MiB / 22919MiB |     85%      Default |
+-------------------------------+----------------------+----------------------+
|   4  TITAN V             On   | 00000000:0C:00.0 Off |                  N/A |
| 33%   48C    P8    29W / 250W |   1043MiB / 12036MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
|   5  GeForce GTX 108...  On   | 00000000:0D:00.0 Off |                  N/A |
| 38%   65C    P2    80W / 250W |   6956MiB / 11178MiB |     14%      Default |
+-------------------------------+----------------------+----------------------+
|   6  Quadro GV100        On   | 00000000:0E:00.0 Off |                  Off |
| 50%   64C    P2    55W / 250W |   7516MiB / 32478MiB |     14%      Default |
+-------------------------------+----------------------+----------------------+
|   7  GeForce GTX 108...  On   | 00000000:0F:00.0 Off |                  N/A |
| 46%   78C    P2    98W / 250W |   6956MiB / 11178MiB |     14%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+

If I now try to use PyTorch:

[R1]user@docked:/$ python
Python 3.6.8 (default, Jan 14 2019, 11:02:34)
[GCC 8.0.1 20180414 (experimental) [trunk revision 259383]] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.cuda.is_available()
False

So in this case, my PyTorch is unable to find the GPUs or the CUDA install. However, if I now disconnect from the docker container and then reconnect as root with

[R1]user@remote-machine:~$ docker exec -it -u root 1122a334b455 bash
root@1122a334b455:/# python
Python 3.6.8 (default, Jan 14 2019, 11:02:34)
[GCC 8.0.1 20180414 (experimental) [trunk revision 259383]] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
>>> torch.cuda.is_available()
True

CUDA is suddenly available.
Does anyone know what is going on here? Why would my python see the GPUs only when I connect to the docker as root?

Many thanks!


title: What does the copy_initial_weights documentation mean in the higher library for Pytorch?

body:
I was trying to use the higher library for meta-learning and I was having issues understanding what the copy_initial_weights mean. The docs say:

  copy_initial_weights – if true, the weights of the patched module are copied to form the initial weights of the patched module, and thus are not part of the gradient tape when unrolling the patched module. If this is set to False, the actual module weights will be the initial weights of the patched module. This is useful when doing MAML, for example.

but that doesn't make much sense to me because of the following:

For example, "the weights of the patched module are copied to form the initial weights of the patched module" doesn't make sense to me because when the context manager is initiated a patched module does not exist yet. So it is unclear what we are copying from and to where (and why copying is something we want to do).

Also, "unrolling the patched module" does not make sense to me. We usually unroll a computaiton graph caused by a for loop. A patched module is just a neural net that has been modified by this library. Unrolling is ambiguous.

Also, there isn't a technical definition for "gradient tape".

Also, when describing what false is, saying that it's useful for MAML isn't actually useful because it doesn't even hint why it's useful for MAML.

Overall, it's impossible to use the context manager.

Any explanations and examples of what the that flag does in more precise terms would be really valuable.

Related:

gitissue: https://github.com/facebookresearch/higher/issues/30
new gitissue: https://github.com/facebookresearch/higher/issues/54
pytorch forum: https://discuss.pytorch.org/t/why-does-maml-need-copy-initial-weights-false/70387
pytorch forum: https://discuss.pytorch.org/t/what-does-copy-initial-weights-do-in-the-higher-library/70384
important question related to this on how the fmodel parameters are copied so that the optimizers work (and the use of deep copy): Why does higher need to deep copy the parameters of the base model to create a functional model?


title: I am installing the Detecto library in anaconda environment but it fails while importing

body:
As a source, I apply the guide here:
Object Detection

I applied the pip3 install detecto command as shown here and it was successfully installed. But when I run python and try to run the detecto command, I get an error. How can I solve it?

Operation:


title: Pytorch Custom Optimizer got an empty parameter list

body:
new here. I am trying to create a custom optimizer in PyTorch, where the backprop takes place in a meta RL policy, with the policy receiving the model parameters, and outputting the desired model parameters. However, I am seeing the above error. My models work fine on Adam and SGD, but not my optimizer.

Code:

class MetaBackProp(torch.optim.Optimizer):
    def __init__(self, params):

        self.param_shape_list = np.array([])
        for param in list(params):
            np.append(self.param_shape_list, list(param.size()))

        pseudo_lr = 1e-4
        pseudo_defaults = dict(lr=pseudo_lr)
        length = 100 #TODO: get shape, flatten, multiply...
        self.policy = AEPolicy(length)
        self.policy_optim = torch.optim.Adam(self.policy.parameters(), lr=pseudo_lr)
        super(MetaBackProp, self).__init__(params, pseudo_defaults)

    def step(self, closure=None):
        params = torch.cat([p.view(-1) for p in self.param_groups])
        self.policy_optim.zero_grad()
        quit()

Traceback:

Traceback (most recent call last):
  File "main.py", line 6, in <module>
    gan = CycleGAN()
  File "/home/ai/Projects_v2/R/cycle_gan.py", line 32, in __init__
    self.discriminator2_optim = MetaBackProp(self.discriminator2.parameters())
  File "/home/ai/Projects_v2/R/lr_schedule.py", line 34, in __init__
    super(MetaBackProp, self).__init__(params, pseudo_defaults)
  File "/home/ai/anaconda3/lib/python3.7/site-packages/torch/optim/optimizer.py", line 46, in __init__
    raise ValueError("optimizer got an empty parameter list")
ValueError: optimizer got an empty parameter list


title: Why pandas.core.series.Series sometimes cannot convert to torch tensor in Python?

body:
I have a dataframe in which I pick two columns:

X_train, X_test, y_train, y_test = train_test_split(df["EnergyFront"], df["particle"], test_size=0.2)

the type of both X_train and X_test is pandas.core.series.Series, the results are quite similar:

IMAGE

I can transform X_train to torch tensor:

X_train = torch.Tensor(X_train)

but, when I try to do the same with X_test:

X_test = torch.Tensor(X_test)

I get the following error:

ValueError                                Traceback (most recent call last)
<ipython-input-174-14117eb3ce4e> in <module>()
----> 1 X_test = torch.Tensor(X_test)

ValueError: could not determine the shape of object type 'Series'

How can I solve it?
By the way, I am running on Google Colaboratory.


title: Caffe Generated LMDB Data Load Crashes in Pytorch

body:
I am trying to load LMDB dataset that was prepared in Caffe. I use LSUNClass as reference to load the data.

dataloader = Data.DataLoader(LSUNClass(db_path=path),batch_size=50,
                             shuffle=True, num_workers=6, pin_memory=False)

The dataLoader object is created without anyissue. But when I iterate through

for step,(x,y) in enumerate(dataloader):

The program crashes @ getitem

def __getitem__(self, index):
    img, target = None, None
    env = self.env
    with env.begin(write=False) as txn:
        imgbuf = txn.get(self.keys[index])
        print('The buffer information :',len(imgbuf),index,self.keys[index])
    buf = six.BytesIO()
    buf.write(imgbuf)
    buf.seek(0)
    img = Image.open(buf).convert('RGB')

File "", line 30, in getitem
img = Image.open(buf).convert('RGB')
File "/anaconda/envs/py35/lib/python3.5/site-packages/PIL/Image.py", line 2319, in open
% (filename if filename else fp))
OSError: cannot identify image file <_io.BytesIO object at 0x7f677db0b570>

However, if I load LSUN LMDB dataset default from Pytorch then it works fine.Same set of code is used but with Caffe Lmdb is crashes.

Any inputs will be appreciated.


I am trying to train my neural networks on dog breeds data set. After feed-forward, during the loss computation it throws this error :

RuntimeError: Assertion `THIndexTensor_(size)(target, 0) == batch_size' failed.  at d:\projects\pytorch\torch\lib\thnn\generic/ClassNLLCriterion.c:54

Code:

criterion =nn.CrossEntropyLoss()
optimizer=optim.Adam(net.parameters(),lr=0.001)

for epoch in range(10):  # loop over the dataset multiple times
        running_loss = 0.0
        print(len(trainloader))
        for i, data in enumerate(trainloader, 0):
            # get the inputs
            inputs, labels  = data

            # wrap them in Variable
            inputs, labels = Variable(inputs).float(), Variable(labels).float().type(torch.LongTensor)

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = net(inputs)

            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            # print statistics
            running_loss += loss.data[0]
            if i % 2000 == 1999:    # print every 2000 mini-batches
                print('[%d, %5d] loss: %.3f' %
                      (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0

print('Finished Training')

Error is generated in this line :

loss = criterion(outputs, labels)

What's the issue ??