I am trying to computer a loss of policy target network in Deep Deterministic Policy Gradient Algorithms with pytorch 1.5, and I get the following error.
File "F:\agents\ddpg.py", line 128, in train_model
    policy_loss.backward()
  File "E:\conda\envs\pytorch\lib\site-packages\torch\tensor.py", line 198, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "E:\conda\envs\pytorch\lib\site-packages\torch\autograd\__init__.py", line 100, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [128, 1]], which is output 0 of TBackward, is at version 2; expected version 1 instead
. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

There are my networks and training progress. In the actor network, the lenght of output vector is 20 which means a continuous action. The input of critic net consists of state vector and action vector.
"""
ddpg actor
"""
class MLP(nn.Module):
    def __init__(self,
                 input_size,
                 output_size,
                 output_limit=1.0,
                 hidden_sizes=(64, 64),
                 activation=torch.relu,
                 output_activation=identity,
                 use_output_layer=True,
                 use_actor=False,
                 ):
        super(MLP, self).__init__()

        self.input_size = input_size
        self.output_size = output_size
        self.output_limit = output_limit
        self.hidden_sizes = hidden_sizes
        self.activation = activation
        self.output_activation = output_activation
        self.use_output_layer = use_output_layer
        self.use_actor = use_actor

        # Set hidden layers
        self.hidden_layers = nn.ModuleList()
        in_size = self.input_size
        for next_size in self.hidden_sizes:
            fc = nn.Linear(in_size, next_size)
            in_size = next_size
            self.hidden_layers.append(fc)

        # Set output layers
        if self.use_output_layer:
            self.output_layer1 = nn.Linear(in_size, self.output_size // 2)
            self.output_layer2 = nn.Linear(in_size, self.output_size // 2)
        else:
            self.output_layer = identity

    def forward(self, x):

        for hidden_layer in self.hidden_layers:
            x = self.activation(hidden_layer(x))
        x1 = torch.sigmoid(self.output_layer1(x))
        x2 = F.softmax(self.output_layer2(x), dim=0)
        out = torch.cat((x1, x2), dim=-1)

        # If the network is used as actor network, make sure output is in correct range
        out = out * self.output_limit if self.use_actor else out
        return out

"""
DDPG critic, TD3 critic, SAC qf, TAC qf
"""

class critic(nn.Module):
    def __init__(self,
                 input_size,
                 output_size,
                 output_limit=1.0,
                 hidden_sizes=(64, 64),
                 activation=torch.relu,
                 output_activation=identity,
                 use_output_layer=True,
                 use_actor=False,
                 ):
        super().__init__()

        self.input_size = input_size
        self.output_size = output_size
        self.output_limit = output_limit
        self.hidden_sizes = hidden_sizes
        self.activation = activation
        self.output_activation = output_activation
        self.use_output_layer = use_output_layer
        self.use_actor = use_actor

        # Set hidden layers
        self.hidden_layers = nn.ModuleList()
        in_size = self.input_size
        for next_size in self.hidden_sizes:
            fc = nn.Linear(in_size, next_size)
            in_size = next_size
            self.hidden_layers.append(fc)

        # Set output layers
        if self.use_output_layer:
            self.output_layer = nn.Linear(in_size, self.output_size)
        else:
            self.output_layer = identity

    def forward(self, x, a):
        q= torch.cat([x, a], dim=1)

        for hidden_layer in self.hidden_layers:
            q = self.activation(hidden_layer(q))
        q = torch.tanh(self.output_layer(q))

        return q

    def train_model(self):
        batch = self.replay_buffer.sample(self.batch_size)
        obs1 = batch['obs1']
        obs2 = batch['obs2']
        acts = batch['acts']
        rews = batch['rews']
        done = batch['done']
        # Check shape of experiences

        # Prediction Q(s,(s)), Q(s,a), Q‾(s',‾(s'))
        with torch.autograd.set_detect_anomaly(True):
            print("obs1",obs1.shape) #(64,22)
            print("a1",self.policy(obs1).shape) #(64,20)
            q_pi = self.qf(obs1, self.policy(obs1))
            q = self.qf(obs1, acts).squeeze(1)
            q_pi_target = self.qf_target(obs2, self.policy_target(obs2)).squeeze(1)

            # Target for Q regression
            q_backup = rews + self.gamma * (1 - done) * q_pi_target
            q_backup.to(self.device)

            # DDPG losses
            policy_loss = -q_pi.mean()
            qf_loss = F.mse_loss(q, q_backup.detach())

            # Update Q-function network parameter

            self.qf_optimizer.zero_grad()
            qf_loss.backward()
            nn.utils.clip_grad_norm_(self.qf.parameters(), self.gradient_clip_qf)
            self.qf_optimizer.step()

            # Update policy network parameter
            self.policy_optimizer.zero_grad()
            # here is the error
            policy_loss.backward()
            nn.utils.clip_grad_norm_(self.policy.parameters(), self.gradient_clip_policy)
            self.policy_optimizer.step()

            # Polyak averaging for target parameter
            soft_target_update(self.policy, self.policy_target)
            soft_target_update(self.qf, self.qf_target)

            # Save losses
            self.policy_losses.append(policy_loss.item())
            self.qf_losses.append(qf_loss.item())

I also take the advice given by the hint which uses  with torch.autograd.set_detect_anomaly(True).
The result is
  File "main.py", line 31, in <module>
    agent.run(100)
  File "F:\agents\ddpg.py", line 184, in run
    self.train_model()
  File "F:\agents\ddpg.py", line 109, in train_model
    q_pi = self.qf(obs1, self.policy(obs1))
  File "E:\conda\envs\pytorch\lib\site-packages\torch\nn\modules\module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "F:\agents\common\networks.py", line 115, in forward
    q = torch.tanh(self.output_layer(q))
  File "E:\conda\envs\pytorch\lib\site-packages\torch\nn\modules\module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "E:\conda\envs\pytorch\lib\site-packages\torch\nn\modules\linear.py", line 87, in forward
    return F.linear(input, self.weight, self.bias)
  File "E:\conda\envs\pytorch\lib\site-packages\torch\nn\functional.py", line 1610, in linear
    ret = torch.addmm(bias, input, weight.t())
 (print_stack at ..\torch\csrc\autograd\python_anomaly_mode.cpp:60)
Traceback (most recent call last):
  File "main.py", line 31, in <module>
    agent.run(100)
  File "F:/agents\ddpg.py", line 184, in run
    self.train_model()
  File "F:/agents\ddpg.py", line 130, in train_model
    policy_loss.backward()
  File "E:\conda\envs\pytorch\lib\site-packages\torch\tensor.py", line 198, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "E:\conda\envs\pytorch\lib\site-packages\torch\autograd\__init__.py", line 100, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [128, 1]], which is output 0 of TBackward, is at version 2; expected version 1 instead
. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!

I can't find what causes failure to compute its gradient in my code.


title: How to parallelize a training loop ever samples of a batch when CPU is only available in pytorch?

body:
I want to parallelize over single examples or batch of example (in my situation is that I only have cpus, I have up to 112). I tried it but I get a bug that the losses cannot have the gradient out of separate processes (which entirely ruins my attempt). I still want to do it and it essential that after the multiproessing happens that I can do an optimizer step. How do I get around it? I made a totally self contained example:

import torch
import torch.nn as nn
from torch.optim.lr_scheduler import StepLR
from torch.utils.data import Dataset, DataLoader

from torch.multiprocessing import Pool

class SimpleDataSet(Dataset):

    def __init__(self, Din, num_examples=23):
        self.x_dataset = [torch.randn(Din) for _ in range(num_examples)]
        # target function is x*x
        self.y_dataset = [x**2 for x in self.x_dataset]

    def __len__(self):
        return len(self.x_dataset)

    def __getitem__(self, idx):
        return self.x_dataset[idx], self.y_dataset[idx]

def get_loss(args):
    x, y, model = args
    y_pred = model(x)
    criterion = nn.MSELoss()
    loss = criterion(y_pred, y)
    return loss

def get_dataloader(D, num_workers, batch_size):
    ds = SimpleDataSet(D)
    dl = DataLoader(ds, batch_size=batch_size, num_workers=num_workers)
    return dl

def train_fake_data():
    num_workers = 2
    Din, Dout = 3, 1
    model = nn.Linear(Din, Dout).share_memory()

    optimizer = torch.optim.Adam(model.parameters(), lr=0.1)

    batch_size = 2
    num_epochs = 10
    # num_batches = 5
    num_procs = 5
    dataloader = get_dataloader(Din, num_workers, batch_size)
    scheduler = StepLR(optimizer, step_size=1, gamma=0.7)
    for epoch in range(num_epochs):
        for _, batch in enumerate(dataloader):
            batch = [(torch.randn(Din), torch.randn(Dout), model) for _ in batch]
            with Pool(num_procs) as pool:
                optimizer.zero_grad()

                losses = pool.map(get_loss, batch)
                loss = torch.mean(losses)
                loss.backward()

                optimizer.step()
            # scheduler
            scheduler.step()

if __name__ == '__main__':
    # start = time.time()
    # train()
    train_fake_data()
    # print(f'execution time: {time.time() - start}')

Error:
Traceback (most recent call last):
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/site-packages/IPython/core/interactiveshell.py", line 3427, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-ea57e03ba088>", line 1, in <module>
    runfile('/Users/brando/ML4Coq/playground/multiprocessing_playground/multiprocessing_cpu_pytorch.py', wdir='/Users/brando/ML4Coq/playground/multiprocessing_playground')
  File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_bundle/pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "/Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/Users/brando/ML4Coq/playground/multiprocessing_playground/multiprocessing_cpu_pytorch.py", line 95, in <module>
    train_fake_data()
  File "/Users/brando/ML4Coq/playground/multiprocessing_playground/multiprocessing_cpu_pytorch.py", line 83, in train_fake_data
    losses = pool.map(get_loss, batch)
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/multiprocessing/pool.py", line 290, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/multiprocessing/pool.py", line 683, in get
    raise self._value
multiprocessing.pool.MaybeEncodingError: Error sending result: '[tensor(0.5237, grad_fn=<MseLossBackward>)]'. Reason: 'RuntimeError('Cowardly refusing to serialize non-leaf tensor which requires_grad, since autograd does not support crossing process boundaries.  If you just want to transfer the data, call detach() on the tensor before serializing (e.g., putting it on the queue).')'

I am sure I want to do this. How should I be doing this?

New attempt using DDP
"""
Based on: https://pytorch.org/tutorials/intermediate/ddp_tutorial.html

Note: as opposed to the multiprocessing (torch.multiprocessing) package, processes can use
different communication backends and are not restricted to being executed on the same machine.
"""
import torch
from torch import nn, optim
import torch.distributed as dist
import torch.multiprocessing as mp
from torch.nn.parallel import DistributedDataParallel as DDP

import os

num_epochs = 5
batch_size = 8
Din, Dout = 10, 5
data_x = torch.randn(batch_size, Din)
data_y = torch.randn(batch_size, Dout)
data = [(i*data_x, i*data_y) for i in range(num_epochs)]

class OneDeviceModel(nn.Module):
    """
    Toy example for a model ran in parallel but not distributed accross gpus
    (only processes with their own gpu or hardware)
    """
    def __init__(self):
        super().__init__()
        self.net1 = nn.Linear(Din, Din)
        self.relu = nn.ReLU()
        self.net2 = nn.Linear(Din, Dout)

    def forward(self, x):
        return self.net2(self.relu(self.net1(x)))

def setup_process(rank, world_size, backend='gloo'):
    """
    Initialize the distributed environment (for each process).

    gloo: is a collective communications library (https://github.com/facebookincubator/gloo). My understanding is that
    it's a library/API for process to communicate/coordinate with each other/master. It's a backend library.
    """
    # set up the master's ip address so this child process can coordinate
    # os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_ADDR'] = 'localhost'
    os.environ['MASTER_PORT'] = '12355'

    # - use NCCL if you are using gpus: https://pytorch.org/tutorials/intermediate/dist_tuto.html#communication-backends
    if torch.cuda.is_available():
        backend = 'nccl'
    # Initializes the default distributed process group, and this will also initialize the distributed package.
    dist.init_process_group(backend, rank=rank, world_size=world_size)

def cleanup():
    """ Destroy a given process group, and deinitialize the distributed package """
    dist.destroy_process_group()

def run_parallel_training_loop(rank, world_size):
    """
    Distributed function to be implemented later.

    This is the function that is actually ran in each distributed process.

    Note: as DDP broadcasts model states from rank 0 process to all other processes in the DDP constructor,
    you don’t need to worry about different DDP processes start from different model parameter initial values.
    """
    print()
    print(f"Start running DDP with model parallel example on rank: {rank}.")
    print(f'current process: {mp.current_process()}')
    print(f'pid: {os.getpid()}')
    setup_process(rank, world_size)

    # create model and move it to GPU with id rank
    model = OneDeviceModel().to(rank) if torch.cuda.is_available() else OneDeviceModel().share_memory()
    # ddp_model = DDP(model, device_ids=[rank])
    ddp_model = DDP(model)

    for batch_idx, batch in enumerate(data):
        x, y = batch
        loss_fn = nn.MSELoss()
        optimizer = optim.SGD(ddp_model.parameters(), lr=0.001)

        optimizer.zero_grad()
        outputs = ddp_model(x)
        labels = y.to(rank) if torch.cuda.is_available() else y
        # Gradient synchronization communications take place during the backward pass and overlap with the backward computation.
        loss_fn(outputs, labels).backward()  # When the backward() returns, param.grad already contains the synchronized gradient tensor.
        optimizer.step()  # TODO how does the optimizer know to do the gradient step only once?

    print()
    print(f"Start running DDP with model parallel example on rank: {rank}.")
    print(f'current process: {mp.current_process()}')
    print(f'pid: {os.getpid()}')
    # Destroy a given process group, and deinitialize the distributed package
    cleanup()

def main():
    print()
    print('running main()')
    print(f'current process: {mp.current_process()}')
    print(f'pid: {os.getpid()}')
    # args
    world_size = mp.cpu_count()
    mp.spawn(run_parallel_training_loop, args=(world_size,), nprocs=world_size)

if __name__ == "__main__":
    print('starting __main__')
    main()
    print('Done!\a\n')

it seems it works but my question is in line 74 do I need to do this
    model = OneDeviceModel().to(rank) if torch.cuda.is_available() else OneDeviceModel().share_memory()

or
    model = OneDeviceModel().to(rank) if torch.cuda.is_available() else OneDeviceModel()

for it to work properly in multiple CPUs?

Serial is faster than parallel even if I have 112 cpu cores?1
My current issue is that the cpu parallel job is slower than the serially running one when only cpus are available.

related links from research:

https://discuss.pytorch.org/t/multiprocessing-for-loop-on-cpu/59836
How to use multiprocessing in PyTorch?
https://discuss.pytorch.org/t/how-to-parallelize-a-loop-over-the-samples-of-a-batch/32698/7


title: Installing packages but still python interpreter doesn't recognize using pycharm

body:
I'm trying to install pytorch, but when I'm trying to import, my pycharm doesn't recognize this package although I'm sure I've installed this package on the same interpreter.

What I am missing?!

Thanks for your help.


title: pytorch for gpu installation error on ubuntu 14.04(cuda 8.0, python2.7.6, pip 19.0.1) - wheel not supported

body:
I follow the tutorial command on pytorch site, but such errors came out.

    error: torch-1.0.0-cp27m-linux_x86_64.whl is not a valid wheel filename.

I used following commands for installation.

pip install https://download.pytorch.org/whl/cu80/torch-1.0.0-cp27-cp27m-linux_x86_64.whl


title: Pytorch: Illegal instruction core dumped when loading ResNet50 model

body:
I'm new to pytorch and I'm using ResNet50 model to train.

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = models.resnet50(pretrained=True)

for param in model.parameters():
    param.requires_grad = False

model.fc = nn.Sequential(nn.Linear(2048, 512),nn.ReLU(),nn.Dropout(0.2),nn.Linear(512, 10),nn.LogSoftmax(dim=1))
criterion = nn.NLLLoss()

optimizer = optim.Adam(model.fc.parameters(), lr=0.003)
model.to(device)

epochs = 1
steps = 0
running_loss = 0
print_every = 10
train_losses, test_losses = [], []
for epoch in range(epochs):
    for inputs, labels in trainloader:
        steps += 1
        inputs, labels = inputs.to(device), labels.to(device)
        optimizer.zero_grad()

        logps = model.forward(inputs)
        loss = criterion(logps, labels)
        loss.backward()
        optimizer.step()
        running_loss += loss.item()

        if steps % print_every == 0:
            test_loss = 0
            accuracy = 0
            model.eval()
            with torch.no_grad():
                for inputs, labels in testloader:
                    inputs, labels = inputs.to(device),labels.to(device)
                    logps = model.forward(inputs)
                    batch_loss = criterion(logps, labels)
                    test_loss += batch_loss.item()

                    ps = torch.exp(logps)
                    top_p, top_class = ps.topk(1, dim=1)
                    equals = top_class == labels.view(*top_class.shape)
                    accuracy += torch.mean(equals.type(torch.FloatTensor)).item()
            train_losses.append(running_loss/len(trainloader))
            test_losses.append(test_loss/len(testloader))
            print(f"Epoch {epoch+1}/{epochs}.. "
                  f"Train loss: {running_loss/print_every:.3f}.. "
                  f"Test loss: {test_loss/len(testloader):.3f}.. "
                  f"Test accuracy: {accuracy/len(testloader):.3f}")
            running_loss = 0
            model.train()

This code works fine in Google Colaboratory but in my local machine(CPU), on execution of "model.forward(inputs)", it gives an error "Illegal instruction core dumped". I've tried updating my pytorch version, but the problem persists.


title: Is it required to clear GPU tensors in PyTorch?

body:
I am new to PyTorch, and I am exploring the functionality of .to() method. As per the documentation for the CUDA tensors, I see that it is possible to transfer the tensors between the CPU and GPU memory.

# let us run this cell only if CUDA is available
if torch.cuda.is_available():

    # creates a LongTensor and transfers it to GPU as torch.cuda.LongTensor
    a = torch.full((10,), 3, device=torch.device("cuda"))
    # transfers it to CPU, back to being a torch.LongTensor
    b = a.to(torch.device("cpu"))

In this context, I would like to know if it always necessary to transfer back the tensors from the GPU to CPU, perhaps to free the GPU memory? Doesn't, the runtime automatically clear the GPU memory?

Apart from its use of transferring data between CPU and GPU, I would like to know the recommended usage for .to() method (from the memory perspective). Thanks in advance.


title: how to combine the images id images and convert into tensors using pytorch

body:
I am newbie to PyTorch , I am  doing image classification with PyTorch. I have a separate  images folder and train and test csv file with images ids and labels.  I don't have any an idea about how to combine those images and ides and converting into tensors.
I tried , But It's not working will, please help how to prepare the data for CNN model.

Here My data:

train.head()

image_names emergency_or_not
0   1503.jpg    0
1   1420.jpg    0
2   1764.jpg    0
3   1356.jpg    0
4   1117.jpg    0

This my data folder

!ls train_SOaYf6m/
images  test_vc2kHdQ.csv  train.csv

Here my code :

batch_size = 128

validation_split = .3
shuffle_dataset = True
random_seed= 42

dataset_size = len(train)
indices = list(range(dataset_size))
split = int(np.floor(validation_split * dataset_size))
if shuffle_dataset :
    np.random.seed(random_seed)
    np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]

class clf_Dataset(Dataset):
  def __init__(self, img_data,img_path,transfrom=None):
    self.img_path=img_path
    self.transfrom=transform
    self.img_data=img_data

  def __len__(self):
    return len(self.img_data)

  def __getitem__(self, index):
    img_name = os.path.join(self.img_path,self.img_data.loc[index, 'labels'],
                                self.img_data.loc[index, 'Images'])
    image = Image.open(img_name)
        #image = image.convert('RGB')
    image = image.resize((300,300))
    label = torch.tensor(self.img_data.loc[index, 'encoded_labels'])
    if self.transform is not None:
          image = self.transform(image)
    return image, label

dataset = clf_Dataset(train,Base_Path,transform)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-33-e7543361b139> in <module>()
----> 1 dataset = clf_Dataset(train,Base_Path,transform)

NameError: name 'Base_Path' is not defined


title: Training Loss and Accuracy both decreasing for my transformer model for Time Series Prediction

body:
I am using a transformer model for predicting the forex market. I transformed the open price data and calculated the difference between each 30 min interval. And converted the difference into tokens. The tokens are obtained by applying log1.5 to the difference. I obtained 28 types of tokens for 6 years. 14-27 represents a bull market and 0-13 tokens represent bear market.
I created a transformer model in PyTorch and applied the data.
import torch
import math
import numpy as np
import copy
from torch import nn
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader
import ast
from numpy import load
import torch.nn as nn
import random
import time
import matplotlib.pyplot as plt

class Embedder(nn.Module):
    def __init__(self, vocab_size, d_model):
        super().__init__()
        # print(vocab_size,d_model)
        self.embed = nn.Embedding(vocab_size+1, d_model,padding_idx=0)
    def forward(self, x):
        # print(x.shape)
        # print("Embed",self.embed(x).shape)
        return self.embed(x)

class PositionalEncoder(nn.Module):
    def __init__(self, d_model, max_seq_len = 500):
        super().__init__()
        self.d_model = d_model

        # create constant 'pe' matrix with values dependant on
        # pos and i
        pe = torch.zeros(max_seq_len, d_model)
        for pos in range(max_seq_len):
            for i in range(0, d_model, 2):
                pe[pos, i] = \
                math.sin(pos / (10000 ** ((2 * i)/d_model)))
                pe[pos, i + 1] = \
                math.cos(pos / (10000 ** ((2 * (i + 1))/d_model)))

        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x * math.sqrt(self.d_model)
        seq_len = x.size(1)
        x = x + torch.autograd.Variable(self.pe[:,:seq_len],requires_grad=False)
        return x

def attention(q, k, v, d_k, mask=None, dropout=None):

    scores = torch.matmul(q, k.transpose(-2, -1)) /  math.sqrt(d_k)
    if mask is not None:
        mask = mask.unsqueeze(1)
        scores = scores.masked_fill(mask == 0, -1e9)
        scores = torch.nn.functional.softmax(scores, dim=-1)

    if dropout is not None:
        scores = dropout(scores)

    output = torch.matmul(scores, v)
    return output

class MultiHeadAttention(nn.Module):
    def __init__(self, heads, d_model, dropout = 0.1):
        super().__init__()

        self.d_model = d_model
        self.d_k = d_model // heads
        self.h = heads

        self.q_linear = nn.Linear(d_model, d_model)
        self.v_linear = nn.Linear(d_model, d_model)
        self.k_linear = nn.Linear(d_model, d_model)
        self.dropout = nn.Dropout(dropout)
        self.out = nn.Linear(d_model, d_model)

    def forward(self, q, k, v, mask=None):

        bs = q.size(0)

        # perform linear operation and split into h heads

        k = self.k_linear(k).view(bs, -1, self.h, self.d_k)
        q = self.q_linear(q).view(bs, -1, self.h, self.d_k)
        v = self.v_linear(v).view(bs, -1, self.h, self.d_k)

        # transpose to get dimensions bs * h * sl * d_model

        k = k.transpose(1,2)
        q = q.transpose(1,2)
        v = v.transpose(1,2)
        # calculate attention using function we will define next
        scores = attention(q, k, v, self.d_k, mask, self.dropout)

        # concatenate heads and put through final linear layer
        concat = scores.transpose(1,2).contiguous()\
        .view(bs, -1, self.d_model)

        output = self.out(concat)

        return output

class FeedForward(nn.Module):
    def __init__(self, d_model, d_ff=512, dropout = 0.1):
        super().__init__()
        # We set d_ff as a default to 2048
        self.linear_1 = nn.Linear(d_model, d_ff)
        self.dropout = nn.Dropout(dropout)
        self.linear_2 = nn.Linear(d_ff, d_model)
    def forward(self, x):
        x = self.dropout(torch.nn.functional.relu(self.linear_1(x)))
        x = self.linear_2(x)
        return x

class Norm(nn.Module):
    def __init__(self, d_model, eps = 1e-6):
        super().__init__()

        self.size = d_model
        # create two learnable parameters to calibrate normalisation
        self.alpha = nn.Parameter(torch.ones(self.size))
        self.bias = nn.Parameter(torch.zeros(self.size))
        self.eps = eps
    def forward(self, x):
        norm = self.alpha * (x - x.mean(dim=-1, keepdim=True)) \
        / (x.std(dim=-1, keepdim=True) + self.eps) + self.bias
        return norm

class EncoderLayer(nn.Module):
    def __init__(self, d_model, heads, dropout = 0.1):
        super().__init__()
        self.norm_1 = Norm(d_model)
        self.norm_2 = Norm(d_model)
        self.attn = MultiHeadAttention(heads, d_model)
        self.ff = FeedForward(d_model)
        self.dropout_1 = nn.Dropout(dropout)
        self.dropout_2 = nn.Dropout(dropout)

    def forward(self, x, mask):
        x2 = self.norm_1(x)
        x = x + self.dropout_1(self.attn(x2,x2,x2,mask))
        x2 = self.norm_2(x)
        x = x + self.dropout_2(self.ff(x2))
        return x

class DecoderLayer(nn.Module):
    def __init__(self, d_model, heads, dropout=0.1):
        super().__init__()
        self.norm_1 = Norm(d_model)
        self.norm_2 = Norm(d_model)
        self.norm_3 = Norm(d_model)

        self.dropout_1 = nn.Dropout(dropout)
        self.dropout_2 = nn.Dropout(dropout)
        self.dropout_3 = nn.Dropout(dropout)

        self.attn_1 = MultiHeadAttention(heads, d_model)
        self.attn_2 = MultiHeadAttention(heads, d_model)
        self.ff = FeedForward(d_model).cuda()
        # self.ff = FeedForward(d_model)

    def forward(self, x, e_outputs, src_mask, trg_mask):
        x2 = self.norm_1(x)
        x = x + self.dropout_1(self.attn_1(x2, x2, x2, trg_mask))
        x2 = self.norm_2(x)
        x = x + self.dropout_2(self.attn_2(x2, e_outputs, e_outputs,
        src_mask))
        x2 = self.norm_3(x)
        x = x + self.dropout_3(self.ff(x2))
        return x
# We can then build a convenient cloning function that can generate multiple layers:
def get_clones(module, N):
    return nn.ModuleList([copy.deepcopy(module) for i in range(N)])

class Encoder(nn.Module):
    def __init__(self, vocab_size, d_model, N, heads):
        super().__init__()
        self.N = N
        self.embed = Embedder(vocab_size, d_model)
        self.pe = PositionalEncoder(d_model)
        self.layers = get_clones(EncoderLayer(d_model, heads), N)
        self.norm = Norm(d_model)

    def forward(self, src, mask):
        x = self.embed(src)
        x = self.pe(x)
        for i in range(self.N):
            x = self.layers[i](x, mask)
        return self.norm(x)

class Decoder(nn.Module):
    def __init__(self, vocab_size, d_model, N, heads):
        super().__init__()
        self.N = N
        self.embed = Embedder(vocab_size, d_model)
        self.pe = PositionalEncoder(d_model)
        self.layers = get_clones(DecoderLayer(d_model, heads), N)
        self.norm = Norm(d_model)
    def forward(self, trg, e_outputs, src_mask, trg_mask):
        x = self.embed(trg)
        x = self.pe(x)
        for i in range(self.N):
            x = self.layers[i](x, e_outputs, src_mask, trg_mask)
        return self.norm(x)

class Transformer(nn.Module):
    def __init__(self, src_vocab, trg_vocab, d_model, N, heads):
        super().__init__()
        self.encoder = Encoder(src_vocab, d_model, N, heads)
        self.decoder = Decoder(trg_vocab, d_model, N, heads)
        self.out = nn.Linear(d_model, trg_vocab)

    def forward(self, src, trg, src_mask, trg_mask):
        e_outputs = self.encoder(src, src_mask)
        d_output = self.decoder(trg, e_outputs, src_mask, trg_mask)
        output = self.out(d_output)
        return output

def batchify(data, bsz):
    nbatch = data.size(0) // bsz
    data = data.narrow(0, 0, nbatch * bsz)
    data = data.view(bsz, -1).t().contiguous()
    return data

bptt = 128
class CustomDataLoader:
    def __init__(self,source):
        print("Source",source.shape)
        self.batches = list(range(0, source.size(0) - 2*bptt))
        # random.shuffle(self.batches)
        # print(self.batches)
        self.data = source
        self.sample = random.sample(self.batches,120)

    def batchcount(self):
        return len(self.batches)

    def shuffle_batches(self):
        random.shuffle(self.batches)

    def get_batch_from_batches(self,i):
        if i==0:
            random.shuffle(self.batches)
        ind = self.batches[i]
        seq_len = min(bptt,len(self.data)-1-ind)
        src = self.data[ind:ind+seq_len]
        tar = self.data[ind+seq_len-3:ind+seq_len-3+seq_len+1]
        return src,tar

    def get_batch(self,i):
        # print(i,len(self.batches))
        ind = self.sample[i]
        seq_len = min(bptt,len(self.data)-1-ind)
        src = self.data[ind:ind+seq_len]
        tar = self.data[ind+seq_len-3:ind+seq_len-3+seq_len+1]
        # tar = tar.view(-1)
        if(i==len(self.sample)-1):
            random.sample(self.batches,60)
            # print("Data shuffled",self.batches[:10])
        return src,tar

def get_batch(source, i):
    seq_len = min(bptt, len(source) - 1 - i)
    data = source[i:i+seq_len]
    target = source[i+seq_len-3:i+seq_len-3+seq_len]
    return data, target

def plot_multiple(data,legend):
    fig,ax = plt.subplots()
    for line in data:
        plt.plot(list(range(len(line))),line)
    plt.legend(legend)
    plt.show()

def plot_subplots(data,legends,name):
    names = ['Accuracy', 'Loss']
    plt.figure(figsize=(10, 5))
    for i in range(len(data)):
        plt.subplot(121+i)
        plt.plot(list(range(0,len(data[i])*3,3)),data[i])
        plt.title(legends[i])
        plt.xlabel("Epochs")
    plt.savefig(name)

def evaluate(eval_model, data_source):
    eval_model.eval() # Turn on the evaluation mode
    total_loss = 0.
    ntokens = 28
    count = 0
    with torch.no_grad():
        cum_loss = 0
        acc_count = 0
        accs = 0
        print(data_source.shape)
        for batch, i in enumerate(range(0, data_source.size(0) - bptt*2, bptt)):
            data, targets = get_batch(data_source, i)
            # data,targets = dataLoader.get_batch(i)
            data = data.transpose(0,1).contiguous()
            targets= targets.transpose(0,1).contiguous()
            trg_input = targets[:,:-1]
            trg_output = targets[:,1:].contiguous().view(-1)
            src_mask , trg_mask = create_masks(data,trg_input)
            output = model(data,trg_input,src_mask,trg_mask)
            output = output.view(-1,output.size(-1))
            loss = torch.nn.functional.cross_entropy(output,trg_output-1)
            accs += ((torch.argmax(output,dim=1)==trg_output).sum().item()/output.size(0))
            # accs += ((torch.argmax(output,dim=1)==targets).sum().item()/output.size(0))
            cum_loss += loss
            count+=1
        # print(epoch,"Loss: ",(cum_loss/count),"Accuracy ",accs/count)

    return cum_loss/ (count), accs/count

def nopeak_mask(size,cuda_enabled):
    np_mask = np.triu(np.ones((1, size, size)),
    k=1).astype('uint8')
    np_mask =  torch.autograd.Variable(torch.from_numpy(np_mask) == 0)

    if cuda_enabled:
      np_mask = np_mask.cuda()
    return np_mask

def create_masks(src, trg):
    src_mask = (src != 0).unsqueeze(-2)
    if trg is not None:
        trg_mask = (trg != 0).unsqueeze(-2)
        size = trg.size(1) # get seq_len for matrix
        # print("Sequence lenght in mask ",size)
        np_mask = nopeak_mask(size,True)
        # print(np_mask.shape,trg_mask.shape)
        if trg.is_cuda:
            np_mask.cuda()
        trg_mask = trg_mask & np_mask
    else:
        trg_mask = None
    return src_mask, trg_mask

def create_padding_mask(seq):
  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)

  # add extra dimensions to add the padding
  # to the attention logits.
  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)

if __name__ == '__main__':
    data = []
    dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    procsd_data = load("Eavg_open.npy")
    print(set(procsd_data[:,0]))
    train_data =torch.tensor(procsd_data)[:30000*2]
    print(train_data.shape)
    val_data = torch.tensor(procsd_data)[30000*2:35000*2]
    test_data = torch.tensor(procsd_data)[35000*2:]
    train_data = train_data.to(dev)
    val_data = val_data.to(dev)
    test_data = test_data.to(dev)

    # train_data = train_data.transpose(1,0).contiguous()
    # val_data = val_data.transpose(1,0).contiguous()

    batch_size = 32
    ntokens = 28
    train_data = batchify(train_data,batch_size)
    # print(train_data.shape)
    val_data = batchify(val_data,batch_size)
    test_data = batchify(train_data,batch_size)
    # model = Transformer(n_blocks=3,d_model=256,n_heads=8,d_ff=256,dropout=0.5)
    model = Transformer(28,28,64,3,4)
    # model = torch.load("modela")
    for p in model.parameters():
        if p.dim() > 1:
            nn.init.xavier_uniform_(p)

    model.to(dev)
    criterion = nn.CrossEntropyLoss()
    lr = 0.00001 # learning rate

    optim = torch.optim.Adam(model.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)
    #########training starts###########

    accuracies = []
    lossies = []
    val_loss = []
    val_accuracy = []
    dataLoader = CustomDataLoader(train_data)
    _onehot = torch.eye(29)
    for epoch in range(500):
        count = 0
        cum_loss = 0
        acc_count = 0
        accs = 0
        s = time.time()
        # for i in range(len(range(0, train_data.size(0) - bptt))):
        model.train()
        # dataLoader.shuffle_batches()
        for i in range(300):
            # data, targets = get_batch(train_data, i)
            # d = time.time()
            hh = time.time()
            data,targets = dataLoader.get_batch_from_batches(i)
            data = data.transpose(0,1).contiguous()
            targets= targets.transpose(0,1).contiguous()
            # print(data.shape,targets.shape)
            trg_input = targets[:,:-1]
            trg_output = targets[:,1:].contiguous().view(-1)
            # print(data.shape,trg_input.shape)
            src_mask , trg_mask = create_masks(data,trg_input)
            # print("Source Mask",src_mask)
            # print("Target Mask",trg_mask)
            output = model(data,trg_input,src_mask,trg_mask)
            # output = output.view(-1,28)
            output = output.view(-1,output.size(-1))

            loss = torch.nn.functional.cross_entropy(output,trg_output-1)
            accuracy = ((torch.argmax(output,dim=1)==trg_output).sum().item()/output.size(0))
            accs += accuracy
            cum_loss += loss.item();
            loss.backward()
            optim.step()
            model.zero_grad()
            optim.zero_grad()
            print(i," Batch Loss", loss.item()," Batch Accuracy ",accuracy," Time taken ",time.time()-hh)

            count+=1

        data,targets = None,None
        print(epoch,"Loss: ",(cum_loss/count),"Accuracy ",accs/count," Time Taken: ",time.time()-s)
        if(epoch%3==0):
            lossies.append(cum_loss/count)
            accuracies.append(accs/count)
            legend = ["accuracy","Loss"]
            plot_subplots([accuracies,lossies],legend,"A&L_v1")
            print("Valdata",val_data.shape)
            eval_loss,eval_acc = evaluate(model,val_data)
            val_accuracy.append(eval_acc)
            val_loss.append(eval_loss)
            plot_subplots([val_accuracy,val_loss],legend,"Val A&L_v1")
            print(epoch,"Loss: ",(cum_loss/count),"Accuracy ",accs/count," Valid_loss: ",eval_loss," Valid_accuracy: ",eval_acc)
            if len(val_loss)>0 and eval_loss < val_loss[-1]:
                val_loss.append(eval_loss)
                torch.save(model,"evalModel")
            else:
                val_loss.append(eval_loss)
                torch.save(model,"evalModel")
        if(epoch%5==0):
            torch.save(model,"modela")

I got the following loss and accuracy while training:

What is causing this behaviour?
Am I wrong in my tokenization method?
Is it necessary to add any time embedding to the data?


title: PyTorch to CoreML, input issue

body:
I am trying to run my PyTorch network on an iOS device.
However, I face the issue of an incorrect input format when I put my network in Apple's demo (https://developer.apple.com/documentation/vision/classifying_images_with_vision_and_core_ml), my input is a MultiArray (Double 3 x 224 x 224) and the one used in the demo is Image (Color 224 x 224).

I didn't find a way to get an input Image with all the research I have done.
To convert my network to .mlmodel, I save it as a .onnx with torch.onnx._export and then I use this library https://github.com/onnx/onnx-coreml to get the .mlmodel.

Do anyone of you have experienced the same issue and have a possible solution?

Thank you!


title: How can I add a feature using torchtext?

body:
torchtext is able to read a file with some columns, each one corresponding to a field. What if I want to create a new column (which I will use as a feature)? For example, imagine the file has two columns, text and target, and I want to extract some information from the text and generate a new feature (e.g. if it contains certain words), can I do this directly with torchtext or do I need to do it in the file before?

Thanks!


title: pdb cannot debug into backward hooks

body:
Here is my code.

import torch
v = torch.tensor([0., 0., 0.], requires_grad=True)
x = 1
def f(grad):
    global x
    x = 2
    return grad * 2
h = v.register_hook(f)  # double the gradient
v.backward(torch.tensor([1., 2., 3.]))
h.remove()
print(v.grad)

When I debug with pdb, I find that I cannot break in function f (I set a breakpoint inside f at the statement x = 2).

Does anyone know how to solve this?

Note: if I use pycharm, I can break into the function. But on the remote server, I would like to use pdb.


title: Optimize Custom Parameters in Pytorch

body:
I'm trying to create some custom parameters to optimize and came across this helpful link here. However, I'm a bit confused as to why this code works.

Here is the code with some slight modifications from the original post to clearly see the behavior of the optimization.

import torch
import torch.nn as nn
import torch.nn.functional as F
import numpy as np

class Mask(nn.Module):
    def __init__(self):
        super(Mask, self).__init__()
        self.weight = torch.nn.Parameter(data=torch.Tensor(5, 5), requires_grad=True)

        self.weight.data.uniform_(-1, 1)

    def forward(self, x):
        masked_wt = self.weight.mul(1)
        return masked_wt

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.mask = Mask()

    def forward(self,x):
        x = self.mask(x)
        return x

model = Model()
indata = torch.ones(5,5)
optimizer = torch.optim.Adam(model.parameters(),lr=0.001)

while True:
    x = model(indata)
    optimizer.zero_grad()
    loss = F.l1_loss(indata,x)
    loss.backward()
    optimizer.step()

    print(model.mask.weight)

So two questions:

In the forward function of the Mask class, why do I need to do self.weight.mul(1) and why doesn't x get used?
x = model(indata) clearly does per element multiplication between the 5x5 matrix indata and the weight, but how does this happen if we didn't use x in the forward function of Mask?


title: PyTorch-YOLOv3 Generating Training and Validation Curves

body:
Hello again stackoverflow!  I greatly appreciate this community and the helpful feedback.

I have some other questions that I hope someone can help me with.  I am working with an implementation of PyTorch-YOLOv3 from https://github.com/eriklindernoren/PyTorch-YOLOv3

I have been able to train the model, but now I would like to generate training/validation curves. During training, I get back metrics on each epoch that look like this:

Epoch 1/3001 batch 0/8
Epoch 1/3001 batch 7/8

I'm trying to generate a graph of a metric (loss, recall, precision, accuacy, mAP) versus epoch by logging these metrics to an external .csv file and plotting those values.

Question 1: Does anyone with experience with this YOLOv3 know where the relevant information is?  I know the AP and mAP for each epoch are at the bottom of the second image (epoch 1/3001 batch 7/8).  I'm not sure where to look for the relevant loss, recall, and precision metrics.

Question 2: Does anyone know the difference between the YOLO layer 0 and layer 1.  Would plotting the metrics from each of these layers yield the training and validation curves, respectively?

Question 3: As of two to three months ago, I started receiving the warning the following warning instead of getting the epoch outputs.

/pytorch/aten/src/ATen/native/IndexingUtils.h:20: UserWarning: indexing with dtype torch.uint8 is now deprecated, please use a dtype torch.bool instead.

It doesn't affect the training, but I would like to update the code so that this warning disappears.  Any suggestions?

Thank you all in advance.


title: How can I import/use two different versions of a library (pytorch) in one program in Python?

body:
I need to use two different versions of pytorch in different parts of the same python webserver. Unfortunately, I can't install them both on the same conda environment that I'm using. I've tried importing one of them from the path itself:
MODULE_PATH = "/home/abc/anaconda3/envs/env/lib/python3.7/site-packages/torch/__init__.py"
MODULE_NAME = "torch"
import importlib
import sys
spec = importlib.util.spec_from_file_location(MODULE_NAME, MODULE_PATH)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)

Which works fine for importing a different version than the one in the active environment, but then I run into an error when trying to import the second one (I've tried simply 'import torch' and also the same as above):
File "/home/abc/anaconda3/envs/env2/lib/python3.7/site-packages/torch/__init__.py", line 82, in <module>
    __all__ += [name for name in dir(_C)
NameError: name '_C' is not defined

Any ideas on how I can use both versions? Thanks!


title: Normal distribution sampling in pytorch-lightning

body:
In Pytorch-Lightning you usually never have to specify cuda or gpu. But when I want to create a gaussian sampled Tensor using torch.normal I get
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!

so, how do I have to change torch.normal such that pytorch-lightning works properly? Since I use the code on different machines on cpu and on gpu
centers = data["centers"] #already on GPU... sometimes...

lights = torch.normal(0, 1, size=[100, 3])
lights += centers


title: AttributeError: module 'torch' has no attribute '__version__'

body:
I try to check if torch have been properly installed by script:
    import torch
    print(torch.__version__)

and have the result:
AttributeError: module 'torch' has no attribute 'version'
I've checked if torch is installed in my venv by:
    pip freeze

and it is installed(I tried to install as via conda as via pip as suggested at https://pytorch.org/):

torch==1.7.1+cu110
torchaudio==0.7.2
torchvision==0.8.2+cu110

Environment:

OS: Ubuntu 18.04
GPU Geforce RTX 3070
Driver version 460.39
Cuda Toolkit V11.0.194
Cudnn V8.0.5.39
Python V3.6.9

Do not understand what's the problem


title: How to stack 1-dimensional vectors in pytorch

body:
I am trying to stack 1-dimensional tensors in pytorch but the stack function seems to be interpreting them as 2-d square matrices. Any ideas how to stack 1-d tensors into a new 1-d tensor?

Reproducibility:

a = torch.randn([2])
b = torch.randn([3])
c = torch.stack([a, b]) # want a (5,) tensor

RuntimeError: invalid argument 0: Sizes of tensors must match except in dimension 0. Got 2 and 3 in dimension 1 at c:\new-builder_3\win-wheel\pytorch\aten\src\th\generic/THTensorMath.cpp:3616

If I unsqueeze a and b to (2,1) and (3,1) tensors or squeeze them (should have no effect) the error is still present. This seems odd too though because they should both be size 1 in dimension 1 when unsqueezed (and printing their sizes will accurately reflect this), but the error still appears the same, character-for-character.

Additionally, stacking in dimension 1 just makes it say "...2 and 3 in dimension 0...".

This is all in python 3.5.4, pytorch 0.4.1


title: How to train a L2-SVM classifier on top of a flattened vector of representations as per DCGAN paper

body:
In the original DCGAN paper, the GAN is partly evaluated by being used as a feature extractor to classify CIFAR-10, after having been trained on Imagenet.

From the paper:

  To evaluate the quality of the representations learned by DCGANs for supervised tasks,
  we train on Imagenet-1k and then use the discriminator’s convolutional features from all layers,
  maxpooling each layers representation to produce a 4 × 4 spatial grid. These features are then
  flattened and concatenated to form a 28672 dimensional vector and a regularized linear L2-SVM
  classifier is trained on top of them.

I have tried to replicate this using PyTorch to train the official PyTorch DCGAN and then use scikit-learn to classify using their linear SVC, but find the wording of the paper confusing and am not sure where to go from here. I've been able to maxpool each layer and then concatenate them, but am stumped on how to proceed with the classification of CIFAR-10.

In e.g. sklearn, you use model.fit(x,y) to fit the model according to the given training data, and then model.predict([X]) to predict the class labels for the samples in X. In model.fit(x,y), x is the (2D) features (e.g. images) and y is the labels. But it feels like they’re saying in the above quote make this 28672 dimensional vector the x. But that’s a 1D vector, and they use it to classify CIFAR-10, which has 50k images, and 50000 > 28672. Am I missing something obvious?

Do I use e.g. model.fit with x being the CIFAR-10 images (using e.g. torchvision.datasets.cifar10) (although how to make 50k Tensors of RGB images a 2D array is another story) and y being their labels, and then somehow predict using the 28672 dimensional vector?

Apologies if this is super obvious; unfortunately that’s all they say about it in the paper, and no one seems to have reproduced it (at least on GitHub etc.). Any help would be greatly appreciated!


title: Efficient implementation of Mixture of Expert Layer in Pytorch

body:
I am trying to implement the a mixture of expert layer, similar to the one described in:
https://arxiv.org/abs/1701.06538

Basically this layer have a number of sub-layers F_i(x_i) which process a projected version of the input. There is also a gating layer G_i(x_i) which is basically an attention mechanism over all sub-expert-layers:

sum(G_i(x_i)*F_i(x_i).

My Naive approach is to build a list for the sub-layers:

sublayer_list = nn.ModuleList()
for i in range(num_of_layer):
    sublayer_list.append(self.make_layer())

Then when applying this I use another for loop

out_list= []
for i,l in enumerate(sublayer_list):
    out_list.appned(l(input[i]))

However the addition of this Mixture-of-Expert layer slows training by almost 7 times (against one with MoE layer swapped for a similar-sized MLP). I am wondering if there are more efficient ways to implement this in pytorch? Many thanks!


title: Add padding in pytorch c++ API

body:
I have a tensor with dimensions (1,3, 375, 1242). I want to reshape it to (1, 3, 384, 1248) by adding padding into it. How do i do that in Pytorch c++ API. Thank you in advance.

target = torch.zeros(1, 3, 384, 1248)
source = torch.ones(1, 3, 375, 1242)
target[: , : , :375, :1242] = source


title: PyTorch GPU out of memory

body:
I am running an evaluation script in PyTorch. I have a number of trained models (*.pt files), which I load and move to the GPU, taking in total 270MB of GPU memory. I am using a batch size of 1. For every sample, I load a single image and also move it to the GPU. Then, depending on the sample, I need to run a sequence of these trained models. Some models have a tensor as input and as output. Other models have a tensor as input, but a string as output. The final model in a sequence always has a string as output. The intermediary tensors are temporarily stored in a dictionary. When a model has consumed a tensor input, it is deleted using del. Still, I notice that after every sample, the GPU memory keeps increasing until the entire memory is full.
Below is some pseudocode to give you a better idea of what is going on:
with torch.no_grad():
    trained_models = load_models_from_pt() # Loaded and moved to GPU, taking 270MB
    model = Model(trained_models) # Keeps the trained_models in a dictionary by name
    for sample in data_loader:
        # A sample contains a single image and is moved to the GPU
        # A sample also has some other information, but no other tensors
        model.forward(sample)

class Model(nn.Module)
    def __init__(self, trained_models):
        self.trained_models = trained_models
        self.intermediary = {}

    def forward(sample):
        for i, elem in enumerate(sample['sequence']):
             name = elem['name']
             in = elem['input']
             if name == 'a':
                model = self.trained_models['a']
                out = model(self.intermediary[in])
                del self.intermediary[in]
                self.intermediary[i] = out
             elif name == 'b':
                model self.trained_models['b']
                out = model(self.intermediary[in])
                del self.intermediary[in]
                self.intermediary[i] = out
             elif ...

I have no idea why the GPU is out of memory. Any ideas?


title: Python multiprocessing.pool's interaction with a class objective function and neuro-evolution

body:
Warning, this is gonna be long since I want to be as specific as I can be.

Exact problem: This is a multi-processing  problem. I have ensured that my classes all behave as built/expected in previous experiments.

edit: said threading beforehand.

When I run toy example of my problem in a threaded environment, everything behaves; however, when I transition into my real problem, the code breaks. Specifically, I get a TypeError: can't pickle _thread.lock objects error. Full stack is at the bottom.

My threading needs here are bit different than the example I adapted my code from -- https://github.com/CMA-ES/pycma/issues/31. In this example we have one fitness function that can be independently called by each evaluation and none of the function calls can interact with each other. However, in my real problem we are trying to optimize neural network weights using a genetic algorithm. The GA will suggest potential weights and we need to evaluate these NN controller-weights in our environment. In a single threaded case, we can have just one environment where we evaluate the weights with a simple for-loop: [nn.evaluate(weights) for weights in potential_candidates], find the best-performing individual, and use those weights in the next mutation round. However, we cannot simply have one simulation in a threaded environment.

So, instead of passing in a single function to evaluate I am passing in a list of function (one for each individual, where the environment is the same, but we have forked the processes so that the communication streams don't interact between individuals.)

One further thing of immediate note:
I am using a build-for-parallel evaluation data-structure from neat

from neat.parallel import ParallelEvaluator  # uses multiprocessing.Pool

Toy example code:

NPARAMS = nn.flat_init_weights.shape[0]    # make this a 1000-dimensional problem.
NPOPULATION = 5                            # use population size of 5.
MAX_ITERATION = 100                        # run each solver for 100 function calls.

import time
from neat.parallel import ParallelEvaluator  # uses multiprocessing.Pool
import cma

def fitness(x):
    time.sleep(0.1)
    return sum(x**2)

# # serial evaluation of all solutions
# def serial_evals(X, f=fitness, args=()):
#     return [f(x, *args) for x in X]

# parallel evaluation of all solutions
def _evaluate2(self, weights, *args):
    """redefine evaluate without the dependencies on neat-internal data structures
    """
    jobs = []
    for i, w in enumerate(weights):
        jobs.append(self.pool.apply_async(self.eval_function[i], (w, ) + args))

    return [job.get() for job in jobs]

ParallelEvaluator.evaluate2 = _evaluate2
parallel_eval = ParallelEvaluator(12, [fitness]*NPOPULATION)

# time both
for eval_all in [parallel_eval.evaluate2]:
    es = cma.CMAEvolutionStrategy(NPARAMS * [1], 1, {'maxiter': MAX_ITERATION,
                                                     'popsize': NPOPULATION})
    es.disp_annotation()
    while not es.stop():
        X = es.ask()
        es.tell(X, eval_all(X))
    es.disp()

Necessary background:

When I switch from the toy example to my real code, the above fails.

My classes are:

LevelGenerator (simple GA class that implements mutate, etc)
GridGame (OpenAI wrapper; launches a Java server in which to run the simulation;
          handles all communication between the Agent and the environment)
Agent    (neural-network class, has an evaluate fn which uses the NN to play a single rollout)
Objective (handles serializing/de-serializing weights: numpy <--> torch; launching the evaluate function)

# The classes get composed to get the necessary behavior:
env   = GridGame(Generator)
agent = NNAgent(env)                # NNAgent is a subclass of (Random) Agent)
obj   = PyTorchObjective(agent)

# My code normally all interacts like this in the single-threaded case:

def test_solver(solver): # Solver: CMA-ES, Differential Evolution, EvolutionStrategy, etc
    history = []
    for j in range(MAX_ITERATION):
        solutions = solver.ask() #2d-numpy array. (POPSIZE x NPARAMS)
        fitness_list = np.zeros(solver.popsize)
        for i in range(solver.popsize):
            fitness_list[i] = obj.function(solutions[i], len(solutions[i]))
        solver.tell(fitness_list)
        result = solver.result() # first element is the best solution, second element is the best fitness
        history.append(result[1])

        scores[j] = fitness_list

    return history, result

So, when I attempt to run:

NPARAMS = nn.flat_init_weights.shape[0]
NPOPULATION = 5
MAX_ITERATION = 100

_x = NNAgent(GridGame(Generator))

gyms = [_x.mutate(0.0) for _ in range(NPOPULATION)]
objs = [PyTorchObjective(a) for a in gyms]

def evaluate(objective, weights):
    return objective.fun(weights, len(weights))

import time
from neat.parallel import ParallelEvaluator  # uses multiprocessing.Pool
import cma

def fitness(agent):
    return agent.evalute()

# # serial evaluation of all solutions
# def serial_evals(X, f=fitness, args=()):
#     return [f(x, *args) for x in X]

# parallel evaluation of all solutions
def _evaluate2(self, X, *args):
    """redefine evaluate without the dependencies on neat-internal data structures
    """
    jobs = []
    for i, x in enumerate(X):
        jobs.append(self.pool.apply_async(self.eval_function[i], (x, ) + args))

    return [job.get() for job in jobs]

ParallelEvaluator.evaluate2 = _evaluate2
parallel_eval = ParallelEvaluator(12, [obj.fun for obj in objs])
# obj.fun takes in the candidate weights, loads them into the NN, and then evaluates the NN in the environment.

# time both
for eval_all in [parallel_eval.evaluate2]:
    es = cma.CMAEvolutionStrategy(NPARAMS * [1], 1, {'maxiter': MAX_ITERATION,
                                                     'popsize': NPOPULATION})
    es.disp_annotation()
    while not es.stop():
        X = es.ask()
        es.tell(X, eval_all(X, NPARAMS))
    es.disp()

I get the following error:

TypeError                            Traceback (most recent call last)
<ipython-input-57-3e6b7bf6f83a> in <module>
      6     while not es.stop():
      7         X = es.ask()
----> 8         es.tell(X, eval_all(X, NPARAMS))
      9     es.disp()

<ipython-input-55-2182743d6306> in _evaluate2(self, X, *args)
     14         jobs.append(self.pool.apply_async(self.eval_function[i], (x, ) + args))
     15
---> 16     return [job.get() for job in jobs]

<ipython-input-55-2182743d6306> in <listcomp>(.0)
     14         jobs.append(self.pool.apply_async(self.eval_function[i], (x, ) + args))
     15
---> 16     return [job.get() for job in jobs]

~/miniconda3/envs/thesis/lib/python3.7/multiprocessing/pool.py in get(self, timeout)
    655             return self._value
    656         else:
--> 657             raise self._value
    658
    659     def _set(self, i, obj):

~/miniconda3/envs/thesis/lib/python3.7/multiprocessing/pool.py in _handle_tasks(taskqueue, put, outqueue, pool, cache)
    429                         break
    430                     try:
--> 431                         put(task)
    432                     except Exception as e:
    433                         job, idx = task[:2]

~/miniconda3/envs/thesis/lib/python3.7/multiprocessing/connection.py in send(self, obj)
    204         self._check_closed()
    205         self._check_writable()
--> 206         self._send_bytes(_ForkingPickler.dumps(obj))
    207
    208     def recv_bytes(self, maxlength=None):

~/miniconda3/envs/thesis/lib/python3.7/multiprocessing/reduction.py in dumps(cls, obj, protocol)
     49     def dumps(cls, obj, protocol=None):
     50         buf = io.BytesIO()
---> 51         cls(buf, protocol).dump(obj)
     52         return buf.getbuffer()
     53

TypeError: can't pickle _thread.lock objects

I also read here that this might be being caused by the fact that this is a class function -- TypeError: can't pickle _thread.lock objects -- so I created the global scoped fitness function def fitness(agent): return agent.evalute(), but that didn't work either.

I thought this error might be coming from the fact that originally, I had the evaluate function in the PyTorchObjective class as a lambda function, but when I changed that it still broke.

Any insight would be greatly appreciated, and thanks for reading this giant wall of text.


title: CNN pytorch : How are parameters selected and flow between layers

body:
I'm pretty new to CNN and have been following the below code. I'm not able to understand how and why have we selected the each argument of Conv2d() and nn.Linear ()  as they are i.e. the output, filter, channels, weights,padding and stride. I do understand the meaning of each though.  Can someone very succinctly explain the flow for each layer?  (Input Image Size is 32*32*3)

import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
        self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
        self.conv3 = nn.Conv2d(32, 64, 3, padding=1)
        self.pool = nn.MaxPool2d(2, 2)
        self.fc1 = nn.Linear(64 * 4 * 4, 500)
        self.fc2 = nn.Linear(500, 10)
        self.dropout = nn.Dropout(0.25)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = self.pool(F.relu(self.conv3(x)))
        x = x.view(-1, 64 * 4 * 4)
        x = self.dropout(x)
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        return x


title: How to pass input dim from fit method to skorch wrapper?

body:
I am trying to incorporate PyTorch functionalities into a scikit-learn environment (in particular Pipelines and GridSearchCV) and therefore have been looking into skorch. The standard documentation example for neural networks looks like

import torch.nn.functional as F
from torch import nn
from skorch import NeuralNetClassifier

class MyModule(nn.Module):
    def __init__(self, num_units=10, nonlin=F.relu):
        super(MyModule, self).__init__()

        self.dense0 = nn.Linear(20, num_units)
        self.nonlin = nonlin
        self.dropout = nn.Dropout(0.5)
        ...
        ...
        self.output = nn.Linear(10, 2)
    ...
    ...

where you explicitly pass the input and output dimensions by hardcoding them into the constructor. However, this is not really how scikit-learn interfaces work, where the input and output dimensions are derived by the fit method rather than being explicitly passed to the constructors. As a practical example consider

# copied from the documentation
net = NeuralNetClassifier(
    MyModule,
    max_epochs=10,
    lr=0.1,
    # Shuffle training data on each epoch
    iterator_train__shuffle=True,
)

# any general Pipeline interface
pipeline = Pipeline([
        ('transformation', AnyTransformer()),
        ('net', net)
        ])

gs = GridSearchCV(net, params, refit=False, cv=3, scoring='accuracy')
gs.fit(X, y)

besides the fact that nowhere in the transformers must one specify the input and output dimensions, the transformers that are applied before the model may change the dimentionality of the training set (think at dimensionality reductions and similar), therefore hardcoding input and output in the neural network constructor just will not do.

Did I misunderstand how this is supposed to work or otherwise what would be a suggested solution (I was thinking of specifying the constructors into the forward method where you do have X available for fit already, but I am not sure this is good practice)?


title: how to get/print the regularization loss/ l2 loss /weight decay value from optimizer in pytorch?

body:
described as the title.
I know the regularization loss in pytorch usually defined through the defination of the optimizer (weight_decay):
torch.optim.SGD(params, lr=<required parameter>, momentum=0, dampening=0, weight_decay=1e-5, nesterov=False)

how can I get the regularization loss value so that I can print it?


title: Way too much resources are used with Pytorch

body:
I am using pytorch to train a DQN model. With ubuntu, if I use htop, I get

As you can see, all resources are used and I am a bit worried about that.
Here is my code.

Is there a way to use less resource? Do I have to add my requirement using pytorch?

Be aware that there's no GPUs on my machine, just CPUs


title: BERT encoding layer produces same output for all inputs during evaluation (PyTorch)

body:
I don't understand why my BERT model returns the same output during evaluation. The output of my model during training seems correct, as the values were different, but is totally the same during evaluation.

Here is my BERT model class

class BERTBaseUncased(nn.Module):
    def __init__(self):
        super(BERTBaseUncased, self).__init__()
        self.bert = BertModel.from_pretrained("bert-base-uncased")
        self.bert_drop = nn.Dropout(0.3)
        self.out = nn.Linear(768, 4)

    def forward(self, ids, mask, token_type_ids):
        _, o2 = self.bert(ids, attention_mask=mask, token_type_ids=token_type_ids) # Use one of the outputs
        bo = self.bert_drop(o2)
        return self.out(bo)

My dataset class

class BERTDataset:
    def __init__(self, review, target, tokenizer, classes=4):
        self.review = review
        self.target = target
        self.tokenizer = tokenizer
        self.max_len = max_len
        self.classes = classes

    def __len__(self):
        return len(self.review)

    def __getitem__(self, item):
        review = str(self.review)
        review = " ".join(review.split())

        inputs = self.tokenizer.encode_plus(review, None, add_special_tokens=True, max_length= self.max_len,
                                            pad_to_max_length=True, return_token_type_ids=True,
                                            return_attention_masks=True)

        ids = inputs["input_ids"]
        mask = inputs["attention_mask"]
        token_type_ids = inputs["token_type_ids"]

        return {
            'ids': torch.tensor(ids, dtype=torch.long),
            'mask': torch.tensor(mask, dtype=torch.long),
            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),
            'targets': torch.tensor(to_categorical(self.target[item], self.classes), dtype=torch.float)
        }

My evaluation function

def eval_fn(data_loader, model, device):
    model.eval()

    total_loss = 0.0

    with torch.no_grad():
        for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
            ids = d['ids']
            token_type_ids = d['token_type_ids']
            mask = d['mask']
            targets = d['targets']

            ids = ids.to(device, dtype=torch.long)
            token_type_ids = token_type_ids.to(device, dtype=torch.long)
            mask = mask.to(device, dtype=torch.long)
            targets = targets.to(device, dtype=torch.float)

            outputs = model(
                ids=ids,
                mask=mask,
                token_type_ids=token_type_ids
            )

            loss = loss_fn(outputs, targets)
            total_loss += loss.item()

And my training function

def train_fn(data_loader, model, optimizer, device, scheduler):
    model.train()

    total_loss = 0.0

    for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):
        ids = d['ids']
        token_type_ids = d['token_type_ids']
        mask = d['mask']
        targets = d['targets']

        ids = ids.to(device, dtype=torch.long)
        token_type_ids = token_type_ids.to(device, dtype=torch.long)
        mask = mask.to(device, dtype=torch.long)
        targets = targets.to(device, dtype=torch.float)

        optimizer.zero_grad()

        outputs = model(
            ids=ids,
            mask=mask,
            token_type_ids=token_type_ids
        )

        loss = loss_fn(outputs, targets)
        total_loss += loss.item()
        loss.backward()

        optimizer.step()
        scheduler.step()

    return total_loss/len(data_loader)

Thanks!


title: torchvision.io cannot find reference read_image() in __init.py__

body:
I am trying to import read_image from torchvision.io
and when i hover the error it says
torchvision.io cannot find reference 'read_image' in '__init.py__'
from torchvision.io import read_image

I am following this example
https://pytorch.org/tutorials/beginner/basics/data_tutorial.html
from torchvision.io import read_image ImportError: cannot import name 'read_image' from 'torchvision.io' (C:\Users\X\.conda\envs\Pytorch37\lib\site-packages\torchvision\io\__init__.py)
This is the error I'm getting.


title: How to solve the RuntimeError when using torch.utils.tensorboard to add a graph

body:
I am trying to use tensorboard to visualize my pytorch model and encounter a problem. The input tensor's shape is (-1, 1, 20, 15) and the output tensor's shape is (-1, 6). My model combines a list of 5 convolutional networks.

packages:

python: 3.7.6
pytorch: 1.4.0
tensorboard: 2.1.0

The pytorch model is as below:

import torch
from torch import nn
from torch.nn import functional as F
class MyModel(nn.Module):
    """example"""

    def __init__(self, nchunks=[2, 5, 3, 2, 3], resp_size=6):
        super().__init__()
        self.nchunks = nchunks
        self.conv = [nn.Conv2d(1, 2, (2, x)) for x in nchunks]
        self.pool = nn.Sequential(
            nn.AdaptiveMaxPool1d(output_size=10), nn.Flatten(start_dim=1)
        )
        self.bn = nn.BatchNorm1d(100)
        self.fc1 = nn.Linear(100, 100)
        self.fc2 = nn.Linear(100, 100)
        self.fc3 = nn.Linear(100, resp_size)

    def forward(self, x):
        xi = torch.split(x, self.nchunks, dim=3)
        xi = [f(subx.float()).view(-1, 2, 19) for f, subx in zip(self.conv, xi)]
        xi = [self.pool(subx) for subx in xi]
        xi = torch.cat(xi, dim=1)
        xi = self.bn(xi)
        xi = F.relu(self.fc1(xi))
        xi = F.relu(self.fc2(xi))
        xi = self.fc3(xi)
        return xi

Here is the code for the tensorboard summary writer:

from torch.utils.tensorboard import SummaryWriter
x = torch.rand((5,1,20,15))
model = MyModel()
writer = SummaryWriter('logs')
writer.add_graph(model, x)

Such an error is returned:

RuntimeError: Cannot insert a Tensor that requires grad as a constant. Consider making it a parameter or input, or detaching the gradient
Tensor:
(1,1,.,.) =
 -0.2108 -0.4986
 -0.4009 -0.1910

(2,1,.,.) =
  0.2383 -0.4147
  0.2642  0.0456
[ torch.FloatTensor{2,1,2,2} ]

I guess the model has some issues, but I am not sure what happens.

This similar github issue does not relate to my problem because I am not using multi GPUs.


title: Reading h5py files into tensors

body:
So I have a training set and a test set both in h5py format. I also have a data_load function that loads the files and returns NumPy arrays. The main problem is I don't need NumPy as I am working with Tensors. I am expecting to have an x&y tensor of size N(batch size) and D_in(input size for each image) and D_out(Output size of each tensor).

The problem:

x&y do not get converted to tensors of dimensions mentioned below.If anything their types remain to be numpy.ndarray. Any help is appreciated.

def load_data(train_file, test_file):
# Load the training data

train_dataset =h5py.File(train_file, 'r')

# Separate features(x) and labels(y) for training set
train_set_x_orig =np.array(train_dataset["train_set_x"][:])
train_set_y_orig =np.array(train_dataset["train_set_y"][:])

# Load the test data
test_dataset =h5py.File(test_file,'r')

# Separate features(x) and labels(y) for training set
test_set_x_orig =np.array(test_dataset["test_set_x"][:])
test_set_y_orig =np.array(test_dataset["test_set_y"][:])

classes = np.array(test_dataset["list_classes"][:]) # the list of classes

train_set_y_orig = torch.from_numpy(train_set_y_orig.reshape((1, train_set_y_orig.shape[0])))
test_set_y_orig = torch.from_numpy(test_set_y_orig.reshape((1, test_set_y_orig.shape[0])))

return train_set_x_orig, train_set_y_orig, test_set_x_orig, test_set_y_orig, classes

x = torch.Tensor(N, D_in)
y = torch.Tensor(N, D_out)
train_file="data/train_catvnoncat.h5"
test_file="data/test_catvnoncat.h5"
x,y,_,_,_=load_data(train_file,test_file)


title: How to load your image dataset in python

body:
I have a folder (on my windows desktop) containing the images I want to use to build my deep learning classifier. I also have one .csv file which has the image number (for example img_1035) and the corresponding class label. How do I load the dataset with the labels into python/jupyter notebooks?
This is the link to the dataset on kaggle (https://www.kaggle.com/debdoot/bdrw).

I would preferably like to use PyTorch to do this but any other ways would also be highly appreciated.


title: PyTorch AttributeError: 'UNet3D' object has no attribute 'size'

body:
I am making an image segmentation transfer learning project using Pytorch. I am using the weights of this pre-trained model and class UNet3D.
https://github.com/MrGiovanni/ModelsGenesis
When I run the following codes I get this error at the line which MSELoss is called: "AttributeError: 'DataParallel' object has no attribute 'size' ".
When I delete the first line I get a similar error: "AttributeError: 'UNet3D' object has no attribute 'size'
"
How can I convert DataParallel or UNet3D class to an object which MSELoss can use? I do not need DataParallel for now. I need to run the UNet3D() class for transfer learning.
model = nn.DataParallel(model, device_ids = [i for i in range(torch.cuda.device_count())])
criterion = nn.MSELoss()
optimizer = torch.optim.SGD(model.parameters(), conf.lr, momentum=0.9, weight_decay=0.0, nesterov=False)
scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
initial_epoch=10
for epoch in range(initial_epoch, conf.nb_epoch):
    scheduler.step(epoch)
    model.train()
    for batch_ndx, (x,y) in enumerate(train_loader):
        x, y = x.float().to(device), y.float().to(device)
        pred = model
        loss = criterion(pred, y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-46-20d1943b3498> in <module>
     25         x, y = x.float().to(device), y.float().to(device)
     26         pred = model
---> 27         loss = criterion(pred, y)
     28         optimizer.zero_grad()
     29         loss.backward()

/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    430
    431     def forward(self, input, target):
--> 432         return F.mse_loss(input, target, reduction=self.reduction)
    433
    434

/opt/anaconda3/lib/python3.7/site-packages/torch/nn/functional.py in mse_loss(input, target, size_average, reduce, reduction)
   2528                 mse_loss, tens_ops, input, target, size_average=size_average, reduce=reduce,
   2529                 reduction=reduction)
-> 2530     if not (target.size() == input.size()):
   2531         warnings.warn("Using a target size ({}) that is different to the input size ({}). "
   2532                       "This will likely lead to incorrect results due to broadcasting. "

/opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in __getattr__(self, name)
    592                 return modules[name]
    593         raise AttributeError("'{}' object has no attribute '{}'".format(
--> 594             type(self).__name__, name))
    595
    596     def __setattr__(self, name, value):

AttributeError: 'UNet3D' object has no attribute 'size'


title: Getting error when using print() or summary() in pytorch to see the layers and weight dimensions in a Pytorch model

body:
When using print on an existing model,
it doesn't print the model. Instead it shows:
<function resnext101_32x8d at 0x00000178CC26BA68>

>>> import torch
>>> import torchvision.models as models
>>> m1 = models.resnext101_32x8d
>>> print(m1)
<function resnext101_32x8d at 0x00000178CC26BA68>
>>>

When using summary, it gives the following error:

  AttributeError: 'function' object has no attribute 'apply'

>>> import torch
>>> import torchvision.models as models
>>> from torchvision import summary
>>> m1 = models.resnext101_32x8d
>>>
>>> summary(m1, (3, 224, 224))
 Traceback(most recent call last):
   File "<stdin>", line 1, in <module>
   File torchsummary.py, line 68, in summary
      model.apply(register_hook)
AttributeError: 'function' object has no attribute 'apply'

How to fix these issues related to print and summary? Any other ways to easily see all pytorch layers and model topology?


title: RuntimeError: Trying to backward through the graph a second time, but the buffers have already been freed pytorch

body:
How do I clear the gradient before calling the .backward() for the second time.

RuntimeError: Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling backward the first time.

a = torch.tensor([2.0], requires_grad = True)
b = torch.tensor([2.0], requires_grad = True)
d = torch.tensor([2.0], requires_grad = True)
c=a*b
c.backward()
e = d*e
e.backward(retain_graph=True)

I tried doing this: c.zero_grad() but I got the error c has no method zero_grad()


title: Check if the expected object is of backend CUDA or CPU?

body:
I am trying to run a code on both CPU and CUDA.
  The problem arise when I create objects, as I need to know what's expected.

I need to determine if the computer is expecting a CUDA or CPU tensor, before it is created.

Code:

def initilize(self, input):
     self.x = torch.nn.Parameter(torch.zeros((1,M))

def run(self,x,state):
  B = torch.cat((self.x,h)

This outputs:

Error: 'Expected object of backend CUDA but got backend CPU for argument #1'

Code idea:

def initilize(self, input):
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
  if (expecting_cuda == True):
     self.x = torch.nn.Parameter(torch.zeros((1,M)).to(device))
  else
     self.x = torch.nn.Parameter(torch.zeros((1,M))

def run(self,h):
  B = torch.cat((self.x,h)

Question:
How to figure out what the computer expects?
Limitations:
I am running on a pre-defined "check" procedure, so I can not send an argument into the function 'initilize' with information about CUDA or CPU.


title: Sum all diagonals in feature maps in parallel in PyTorch

body:
Let's say I have a tensor shaped (1, 64, 128, 128) and I want to create a tensor of shape (1, 64, 255) holding the sums of all diagonals for every (128, 128) matrix (there are 1 main, 127 below, 127 above diagonals so in total 255). What I am currently doing is the following:

x = torch.rand(1, 64, 128, 128)

diag_sums = torch.zeros(1, 64, 255)
j = 0
for k in range(-127, 128):
    diag_sums[j, :, k + 127] = torch.diagonal(x, offset=k, dim1=-2, dim2=-1).sum(dim=2)

This is obviously very slow, since it is using Python loops and is not done in parallel with respect to k.

I don't think this can be done using torch.diagonal since the function explicitly uses a single int for the offset parameter. If I could pass a list there, this would work, but I guess it would be complicated to implement (requiring changes in PyTorch itself).

I think it could be possible to implement this using torch.einsum, but I cannot think of a way to do it.

So this is my question: how do I get the tensor described above?


title: Failling installing NeuralProphet

body:
I'm trying to install NeuralProphet package which as many dependancies.
I'm running it on Python 3.8.5  May it's  an python version issue  ?
Here are my errors:
$ pip install neuralprophet
Collecting neuralprophet
  Downloading neuralprophet-0.2.7-py3-none-any.whl (56 kB)
Collecting attrdict>=2.0.1
  Downloading attrdict-2.0.1-py2.py3-none-any.whl (9.9 kB)
Requirement already satisfied: matplotlib>=2.0.0 in c:\users\gmu\desktop\programmation\lib\site-packages (from neuralprophet) (3.3.2)
Requirement already satisfied: pandas>=1.0.4 in c:\users\gmu\desktop\programmation\lib\site-packages (from neuralprophet) (1.1.4)
Collecting convertdate>=2.1.2
  Downloading convertdate-2.3.0-py3-none-any.whl (45 kB)
Collecting holidays>=0.10.2
  Downloading holidays-0.10.4.tar.gz (120 kB)
Requirement already satisfied: python-dateutil>=2.8.0 in c:\users\gmu\desktop\programmation\lib\site-packages (from neuralprophet) (2.8.1)
Collecting torch-lr-finder>=0.2.1
  Downloading torch_lr_finder-0.2.1-py3-none-any.whl (11 kB)
Collecting ipywidgets>=7.5.1
  Downloading ipywidgets-7.6.3-py2.py3-none-any.whl (121 kB)
Collecting neuralprophet
  Downloading neuralprophet-0.2.6-py3-none-any.whl (55 kB)
  Downloading neuralprophet-0.2.5-py3-none-any.whl (54 kB)
ERROR: Cannot install neuralprophet==0.2.5, neuralprophet==0.2.6 and neuralprophet==0.2.7 because these package versions have conflicting dependencies.

The conflict is caused by:
    neuralprophet 0.2.7 depends on torch<1.7.0 and >=1.4.0
    neuralprophet 0.2.6 depends on torch<1.7.0 and >=1.4.0
    neuralprophet 0.2.5 depends on torch<1.7.0 and >=1.4.0

To fix this you could try to:
1. loosen the range of package versions you've specified
2. remove package versions to allow pip attempt to solve the dependency conflict

ERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/user_guide/#fixing-conflicting-dependencies

Moreover I have download the repository from Github and did the install step from the package documentation as follow : http://neuralprophet.com/
I have these errors  when I write the pip install command:
GMU@GMU-PORT MINGW64 ~/desktop/programmation_1/Tests_Articles/neural_prophet (master)
$ pip install .
Processing c:\users\gmu\desktop\programmation_1\tests_articles\neural_prophet
  Installing build dependencies: started
  Installing build dependencies: finished with status 'done'
  Getting requirements to build wheel: started
  Getting requirements to build wheel: finished with status 'done'
    Preparing wheel metadata: started
    Preparing wheel metadata: finished with status 'done'
Collecting torch-lr-finder>=0.2.1
  Using cached torch_lr_finder-0.2.1-py3-none-any.whl (11 kB)
Collecting LunarCalendar>=0.0.9
  Downloading LunarCalendar-0.0.9-py2.py3-none-any.whl (18 kB)
Requirement already satisfied: python-dateutil>=2.8.0 in c:\users\gmu\desktop\programmation\lib\site-packages (from neuralprophet==0.2.8) (2.8.1)
Requirement already satisfied: numpy>=1.15.4 in c:\users\gmu\desktop\programmation\lib\site-packages (from neuralprophet==0.2.8) (1.19.2)
Collecting ipywidgets>=7.5.1
  Using cached ipywidgets-7.6.3-py2.py3-none-any.whl (121 kB)
Requirement already satisfied: matplotlib>=2.0.0 in c:\users\gmu\desktop\programmation\lib\site-packages (from neuralprophet==0.2.8) (3.3.2)
ERROR: Could not find a version that satisfies the requirement torch>=1.4.0 (from neuralprophet)
ERROR: No matching distribution found for torch>=1.4.0

Thank you for your help.


title: How to save and use a trained neural network developed in PyTorch / TensorFlow / Keras?

body: Are there ways to save a model after training and sharing just the model with others? Like a regular script? Since the network is a collection of float matrices, is it possible to just extract these trained weights and run it on new data to make predictions, instead of requiring the users to install these frameworks too? I am new to these frameworks and will make any clarifications as needed.


title: Training frcnn on pytorch 1.0 with cuda: Segmentation fault (core dumped)

body:
I try to use this repo to train FRCNN on my custom VOC dataset:

 ~/miniconda3/bin/python3 trainval_net.py --dataset pascal_voc --net res101 --cag --bs 1 --nw 1  --lr 1e-3 --lr_decay_step 5

Called with args:
Namespace(batch_size=1, checkepoch=1, checkpoint=0, checkpoint_interval=10000, checksession=1, class_agnostic=True, cuda=False, dataset='pascal_voc', disp_interval=100, large_scale=False, lr=0.001, lr_decay_gamma=0.1, lr_decay_step=5, mGPUs=False, max_epochs=20, net='res101', num_workers=1, optimizer='sgd', resume=False, save_dir='models', session=1, start_epoch=1, use_tfboard=False)
/home/stiv/faster-rcnn.pytorch/lib/model/utils/config.py:374: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.
  yaml_cfg = edict(yaml.load(f))
Using config:{ ... some long stuff here ...},
 'USE_GPU_NMS': True}
WARNING: You have a CUDA device, so you should probably run with --cuda
Loaded dataset `voc_2007_trainval` for training
Set proposal method: gt
Appending horizontally-flipped training examples...
voc_2007_trainval gt roidb loaded from /home/stiv/faster-rcnn.pytorch/data/cache/voc_2007_trainval_gt_roidb.pkl
done
Preparing training data...
done
before filtering, there are 5170 images...
after filtering, there are 5170 images...
5170 roidb entries
Loading pretrained weights from data/pretrained_model/resnet101_caffe.pth
Segmentation fault (core dumped)

I have installed required dependencies and lastest g++. And also I have cuda working (I was able to run mmdetection package). How can I diagnose and fix such a crash?


title: Alternate optimization with two different optimizers in pytorch

body:
I have two loss functions l1 and l2, each optimized by two separate ADAM optimizers opt1 and opt2.
The current value of my parameters is x.
I want to update x using opt1 and opt2 separately, and then "merge" the resulting new value of x depending on the magnitude of the gradients.

Pseudocode

grad1 = get_grad(l1)
grad2 = get_grad(l2)
n1 = norm(grad1)
n2 = norm(grad2)
x1 = opt1(grad1)
x2 = opt2(grad2)
w = n1 / (n1 + n2)
x = w*x1 + (1-w)*x2

How can I do it in pytorch? I am not sure how to use backward() and step().


title: Pytorch gather function to get current Q values for Deep Q

body:
I am trying to compute the current Q for deep Q learning but am running into this error:

RuntimeError: invalid argument 4: Index tensor must have same dimensions as input tensor at C:/w/1/s/windows/pytorch/aten/src\THC/generic/THCTensorScatterGather.cu:16

The code triggering this error is this:

curr_Q = self.model.forward(states).gather(1, actions.unsqueeze(1))

The shape of self.model.forward(states) is [32, 640, 10], and the shape of actions is [32].

The rest of the code in this function is here:

 def compute_loss(self, batch):
    states, actions, rewards, next_states, dones = batch
    states = torch.FloatTensor(states).to(self.device)
    actions = torch.LongTensor(actions).to(self.device)
    rewards = torch.FloatTensor(rewards).to(self.device)
    next_states = torch.FloatTensor(next_states).to(self.device)
    dones = torch.FloatTensor(dones)

    curr_Q = self.model.forward(states).gather(1, actions.unsqueeze(1))
    curr_Q = curr_Q.squeeze(1)
    next_Q = self.model.forward(next_states)
    max_next_Q = torch.max(next_Q, 1)[0]
    expected_Q = rewards.squeeze(1) + self.gamma * max_next_Q

    loss = self.MSE_loss(curr_Q, expected_Q)
    return loss


title: running pytorch geometric get CUDA error: CUBLAS_STATUS_NOT_INITIALIZED when calling `cublasCreate(handle)`

body:
I am running the code in this notebook.
https://colab.research.google.com/github/zaidalyafeai/Notebooks/blob/master/Deep_GCN_Spam.ipynb#scrollTo=UjoTbUQVnCz8
I get an error when I change the data set to my own data set. I know this might be an error of my code. Then I cleared all the code to generate data sets. I saved the two data sets as file and reload it. I really cannot see the difference between these two data sets.
The shape and type of these two data sets is provided below. I can provide any information that is needed. Can anyone help me fix this?
This is my data set
data = torch.load("dataset.pt")
data
>>>Data(edge_attr=[3585, 1], edge_index=[2, 3585], x=[352, 1], y=[352])
data.x.dtype, data.y.dtype, data.edge_attr.dtype, data.edge_index.dtype
>>>(torch.float32, torch.int64, torch.float32, torch.int64)
data.edge_index.T.numpy().shape
>>>(3585, 2)
np.unique(data.edge_index.T.numpy(), axis=0).shape
>>>(3585, 2)
np.unique(data.edge_index.T.numpy(), axis=0).shape
>>>(3585, 2)
data.edge_index.unique().shape
>>>torch.Size([352])
data.edge_index
>>>tensor([[ 13,  13,  13,  ..., 103, 103, 103],
           [  1,   2,   3,  ...,   6,   9,  10]])

This is the data set mentioned in the notebook
data2 = torch.load("spam.pt")
data2
>>>Data(edge_attr=[50344, 1], edge_index=[2, 50344], x=[1000, 1], y=[1000])
data2.x.dtype, data2.y.dtype, data2.edge_attr.dtype, data2.edge_index.dtype
>>>(torch.float32, torch.int64, torch.float32, torch.int64)
data2.edge_index
>>>tensor([[  0,   1,   1,  ..., 999, 999, 999],
           [455, 173, 681,  ..., 377, 934, 953]])

Python version: 3.8
PyTorch geometric version: 1.6.2
CUDA version: 10.2
System: Windows 10
Screeshot


title: How to convert a pandas dataframe into a numpy array with the column names

body:
This must use vectorized methods, nothing iterative

I would like to create a numpy array from pandas dataframe.
My code:
import pandas as pd
_df = pd.DataFrame({'itme': ['book', 'book' , 'car', ' car', 'bike', 'bike'], 'color': ['green', 'blue' , 'red', 'green' , 'blue', 'red'], 'val' : [-22.7, -109.6, -57.19, -11.2, -25.6, -33.61]})

item     color    val
book    green   -22.70
book    blue    -109.60
car     red     -57.19
car     green   -11.20
bike    blue    -25.60
bike    red     -33.61

There are about 12k million rows.
I need to create a numpy array like :
item    green    blue     red
book    -22.70  -109.60   null
car     -11.20   null     -57.19
bike    null    -25.60    -33.16

each row is the item name and each col is color name.
The order of the items and colors are not important.
But, in numpy array, there are no row and column names, I need to keep the item and color name for each value, so that I know what the value represents in the numpy array.
For example
 how to know that -57.19 is for "car" and "red" in numpy array ?

So, I need to create a dictionary to keep the mapping between :
  item <--> row index in the numpy array
  color <--> col index in the numpy array

I do not want to use iteritems and itertuples because they are not efficient for large dataframe due to How to iterate over rows in a DataFrame in Pandas and How to iterate over rows in a DataFrame in Pandas and Python Pandas iterate over rows and access column names and Does pandas iterrows have performance issues?
I prefer numpy vectorization solution for this.
How to efficiently convert the pandas dataframe to numpy array ?
The array will also be transformed to torch.tensor.
thanks


title: Pytorch Argrelmax function (or C++)

body:
I'm trying to find the equivalent pytorch (or C++) for scipy.signal.argrelmax(), which finds the peaks in a 1D array with some padding. https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.signal.argrelmax.html

Here's what I've come up with and it is faster than scipy.signal.argrelmax - but I'm missing a fast solution to the last step which deletes peaks within some window.

import torch

# initalize an array (not the one in plot below)
gpu_max = torch.rand(100000)

# find peaks and troughs by subtracting shifted versions
gpu_temp1 = gpu_max[1:-1]-gpu_max[:-2]
gpu_temp2 = gpu_max[1:-1]-gpu_max[2:]

# and checking where both shifts are positive;
out1 = torch.where(gpu_temp1>0, gpu_temp1*0+1, gpu_temp1*0)
out2 = torch.where(gpu_temp2>0, out1, gpu_temp2*0)

# argrelmax containing all peaks
argrelmax_gpu = torch.nonzero(out2, out=None)+1

So the top plot which marks every single relmaxpeak is pretty fast. But need the bottom one - which comes from scipy.signal.argrelmax() and uses a 30sample time window (i.e. it only returns maxima within a 60 time point window).

EDIT: I've updated code to reflect updated torch.nonzero() search.  But still need to figure out how to do the last step (in figure below).