title: Fixing incorrect dimensions in PyTorch neural network

body:
I am trying to train my neural network, which is written in PyTorch, but I got the following traceback because of incorrect dimensions. Got the following traceback

Traceback (most recent call last):
  File "plot_parametric_pytorch.py", line 139, in <module>
    ops = opfun(X_train[smpl])
  File "plot_parametric_pytorch.py", line 92, in <lambda>
    opfun = lambda X: model.forward(Variable(torch.from_numpy(X)))
  File "/mnt_home/klee/LBSBGenGapSharpnessResearch/deepnet.py", line 77, in forward
    x = self.features(x)
  File "/home/klee/anaconda3/envs/sharpenv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/klee/anaconda3/envs/sharpenv/lib/python3.7/site-packages/torch/nn/modules/container.py", line 100, in forward
    input = module(input)
  File "/home/klee/anaconda3/envs/sharpenv/lib/python3.7/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/klee/anaconda3/envs/sharpenv/lib/python3.7/site-packages/torch/nn/modules/pooling.py", line 141, in forward
    self.return_indices)
  File "/home/klee/anaconda3/envs/sharpenv/lib/python3.7/site-packages/torch/_jit_internal.py", line 209, in fn
    return if_false(*args, **kwargs)
  File "/home/klee/anaconda3/envs/sharpenv/lib/python3.7/site-packages/torch/nn/functional.py", line 539, in _max_pool2d
    input, kernel_size, stride, padding, dilation, ceil_mode)
RuntimeError: Given input size: (512x1x1). Calculated output size: (512x0x0). Output size is too small

This is all when trying to run a forward pass.
I'm pretty sure this is a small bug but I myself am new to writing PyTorch code so I am not sure if I know where it is. For reference, when I checked the dimensions of the Keras model version of this by using model.summary(), the final dimensions before flattening and adding dense layers(which I think should happen in self.classifier in pytorch, although I am not sure too) were 512 x 1 x 1.

This is my model in PyTorch:

class VGG(nn.Module):
    def __init__(self, num_classes=10):
        super(VGG, self).__init__()
        self.features = nn.Sequential(
            nn.Conv2d(3, 64, kernel_size=3, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.Dropout(0.3),
            nn.Conv2d(64, 64, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),

            nn.Conv2d(64, 128, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(128, 128, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(128),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),

            nn.Conv2d(128, 256, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(256, 256, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(256, 256, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(256),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),

            nn.Conv2d(256, 512, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(512, 512, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(512, 512, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),

            nn.Conv2d(512, 512, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(512, 512, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.Dropout(0.4),
            nn.Conv2d(512, 512, kernel_size=3, padding = 1, bias=False),
            nn.BatchNorm2d(512),
            nn.ReLU(inplace=True),
            nn.MaxPool2d(kernel_size=2, stride=2),
        )
        self.classifier = nn.Sequential(
            nn.Linear(512, 512, bias=False),
            nn.Dropout(0.5),
            nn.BatchNorm1d(512),
            nn.ReLU(inplace=True),
            nn.Dropout(0.5),
            nn.Linear(512, num_classes)
        )

    def forward(self, x):
        x = self.features(x)
        x = x.view(-1, 512)
        x = self.classifier(x)
        return F.log_softmax(x)

def cifar10_deep(**kwargs):
    num_classes = getattr(kwargs, 'num_classes', 10)
    return VGG(num_classes)

def cifar100_deep(**kwargs):
    num_classes = getattr(kwargs, 'num_classes', 100)
    return VGG(num_classes)

And I am trying to run the following code:

cudnn.benchmark = True
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
X_train = X_train.astype('float32')
X_train = np.transpose(X_train, axes=(0, 3, 1, 2))
X_test = X_test.astype('float32')
X_test = np.transpose(X_test, axes=(0, 3, 1, 2))
X_train /= 255
X_test /= 255
device = torch.device('cuda:0')

# This is where you can load any model of your choice.
# I stole PyTorch Vision's VGG network and modified it to work on CIFAR-10.
# You can take this line out and add any other network and the code
# should run just fine.
model = cifar_shallow.cifar10_shallow()
#model.to(device)

# Forward pass
opfun = lambda X: model.forward(Variable(torch.from_numpy(X)))

# Forward pass through the network given the input
predsfun = lambda op: np.argmax(op.data.numpy(), 1)

# Do the forward pass, then compute the accuracy
accfun = lambda op, y: np.mean(np.equal(predsfun(op), y.squeeze()))*100

# Initial point
x0 = deepcopy(model.state_dict())

# Number of epochs to train for
# Choose a large value since LB training needs higher values
# Changed from 150 to 30
nb_epochs = 30
batch_range = [25, 40, 50, 64, 80, 128, 256, 512, 625, 1024, 1250, 1750, 2048, 2500, 3125, 4096, 4500, 5000]

# parametric plot (i.e., don't train the network if set to True)
hotstart = False

if not hotstart:
    for batch_size in batch_range:
        optimizer = torch.optim.Adam(model.parameters())
        model.load_state_dict(x0)
        #model.to(device)
        average_loss_over_epoch = '-'
        print('Optimizing the network with batch size %d' % batch_size)
        np.random.seed(1337) #So that both networks see same sequence of batches
        for e in range(nb_epochs):
            model.eval()
            print('Epoch:', e, ' of ', nb_epochs, 'Average loss:', average_loss_over_epoch)
            average_loss_over_epoch = 0

            # Checkpoint the model every epoch
            torch.save(model.state_dict(), "./models/ShallowNetCIFAR10BatchSize" + str(batch_size) + ".pth")
            array = np.random.permutation(range(X_train.shape[0]))
            slices = X_train.shape[0] // batch_size
            beginning = 0
            end = 1

            # Training loop!
            for _ in range(slices):
                start_index = batch_size * beginning
                end_index = batch_size * end
                smpl = array[start_index:end_index]
                model.train()
                optimizer.zero_grad()
                ops = opfun(X_train[smpl]) <<----- error in this line
                tgts = Variable(torch.from_numpy(y_train[smpl]).long().squeeze())
                loss_fn = F.nll_loss(ops, tgts)
                average_loss_over_epoch += loss_fn.data.numpy() / (X_train.shape[0] // batch_size)
                loss_fn.backward()
                optimizer.step()
                beginning += 1
                end += 1

I am wondering where in my model I went wrong. I was writing the PyTorch version of the following Keras model. Any help in fixing the small bug would be appreciated!

def deepnet(nb_classes):
    global img_size
    model = Sequential()
    model.add(Conv2D(64, (3, 3), input_shape=img_size))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu'))
    model.add(Dropout(0.3))
    model.add(Conv2D(64, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    model.add(Conv2D(128, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(128, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    model.add(Conv2D(256, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(256, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(256, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    model.add(Conv2D(512, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(512, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(512, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    model.add(Conv2D(512, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(512, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu')); model.add(Dropout(0.4))
    model.add(Conv2D(512, (3, 3), padding='same'))
    model.add(BatchNormalization(axis=1))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2,2), strides=(2,2), padding='same'))

    model.add(Flatten()); model.add(Dropout(0.5))
    model.add(Dense(512))
    model.add(BatchNormalization())
    model.add(Activation('relu')); model.add(Dropout(0.5))
    model.add(Dense(nb_classes, activation='softmax'))
    return model

Please let me know if there is an issue with the way I converted the neural network model from PyTorch to Keras. From what I understand, padding should always equal 1 in pytorch because of padding=same setting in Keras.


title: How to add a learnable bias to one of the network output channel in pytorch

body:
class pu_fc(nn.Module):

    def __init__(self, input_dim):
        super(pu_fc, self).__init__()
        self.input_dim = input_dim

        self.fc1 = nn.Linear(input_dim, 50)
        self.fc2 = nn.Linear(50, 2)

        self.loss_fn = custom_NLL()

        device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
        self.bias = torch.autograd.Variable(torch.rand(1,1), requires_grad=True).to(device)

    def forward(self, x):
        out = self.fc1(x)
        out = F.relu(out, inplace=True)
        out = self.fc2(out)
        out[..., 1] = out[..., 1] + self.bias
        print('bias: ', self.bias)

        return out

As you can see from the code, I wanted to add a bias term to the second output channel. However, my implementation does not work. The bias term is not updated at all. It kept the same during training which I assume that it is not learnable during training. So the question is that how I can make the bias term learnable? Is it possible to do this? Below is some output of the bias during training. Any hint is grateful, thanks in advance!
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
Current Epoch: 1
Epoch loss:  0.4424589276313782
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
Current Epoch: 2
Epoch loss:  0.3476297199726105
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)
bias:  tensor([[0.0930]], device='cuda:0', grad_fn=<CopyBackwards>)


title: Does low GPU utilization indicate bad fit for GPU acceleration?

body:
I'm running some GPU-accelerated PyTorch code and training it against a custom dataset, but while monitoring the state of my workstation during the process, I see GPU usage along the following lines:

I have never written my own GPU primitives, but I have a long history of doing low-level optimizations for CPU-intensive workloads and my experience there makes me concerned that while pytorch/torchvision are offloading the work to the GPU, it may not be an ideal workload for GPU acceleration.
When optimizing CPU-bound code, the goal is to try and get the CPU to perform as much (meaningful) work as possible in a unit of time: a supposedly CPU-bound task that shows only 20% CPU utilization (of a single core or of all cores, depending on whether the task is parallelizable or not) is a task that is not being performed efficiently because the CPU is sitting idle when ideally it would be working towards your goal. Low CPU usage means that something other than number crunching is taking up your wall clock time, whether it's inefficient locking, heavy context switching, pipeline flushes, locking IO in the main loop, etc. which prevents the workload from properly saturating the CPU.
When looking at the GPU utilization in the chart above, and again speaking as a complete novice when it comes to GPU utilization, it strikes me that the GPU usage is extremely low and appears to be limited by the rate at which data is being copied into the GPU memory. Is this assumption correct? I would expect to see a spike in copy (to GPU) followed by an extended period of calculations/transforms, followed by a brief copy (back from the GPU), repeated ad infinitum.
I notice that despite the low (albeit constant) copy utilization, the GPU memory is constantly peaking at the 8GB limit. Can I assume the workload is being limited by the low GPU memory available (i.e. not maxing out the copy bandwidth because there's only so much that can be copied)?
Does that mean this is a workload better suited for the CPU (in this particular case with this RTX 2080 and in general with any card)?


title: What is the correct way to implement gradient accumulation in pytorch?

body:
Broadly there are two ways:

Call loss.backward() on every batch, but only call optimizer.step() and optimizer.zero_grad() every N batches. Is it the case that the gradients of the N batches are summed up? Hence to maintain the same learning rate per effective batch, we have to divide the learning rate by N?

Accumulate loss instead of gradient, and call (loss / N).backward() every N batches. This is easy to understand, but does it defeat the purpose of saving memory (because the gradients of the N batches are computed at once)? The learning rate doesn't need adjusting to maintain the same learning rate per effective batch, but should be multiplied by N if you want to maintain the same learning rate per example.

Which one is better, or more commonly used in packages such as pytorch-lightning? It seems that optimizer.zero_grad() is a prefect fit for gradient accumulation, therefore (1) should be recommended.


I'm having a little trouble using my code on a google compute engine VM.
I am trying to run a small flask API that detects tables in images.
Initializing the detector model works, but when i try to detect tables this error occurs:
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 2447, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 1952, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 1821, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python3.5/dist-packages/flask/_compat.py", line 39, in reraise
    raise value
  File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 1950, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python3.5/dist-packages/flask/app.py", line 1936, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "ElvyCascadeTabNetAPI.py", line 36, in detect_tables
    result = inference_detector(model, "temp.jpg")
  File "/SingleModelTest/src/mmdet/mmdet/apis/inference.py", line 86, in inference_detector
    result = model(return_loss=False, rescale=True, **data)
  File "/usr/local/lib/python3.5/dist-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/core/fp16/decorators.py", line 49, in new_func
    return old_func(*args, **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/models/detectors/base.py", line 149, in forward
    return self.forward_test(img, img_metas, **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/models/detectors/base.py", line 130, in forward_test
    return self.simple_test(imgs[0], img_metas[0], **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/models/detectors/cascade_rcnn.py", line 342, in simple_test
    x[:len(bbox_roi_extractor.featmap_strides)], rois)
  File "/usr/local/lib/python3.5/dist-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/core/fp16/decorators.py", line 127, in new_func
    return old_func(*args, **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/models/roi_extractors/single_level.py", line 105, in forward
    roi_feats_t = self.roi_layers[i](feats[i], rois_)
  File "/usr/local/lib/python3.5/dist-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/SingleModelTest/src/mmdet/mmdet/ops/roi_align/roi_align.py", line 144, in forward
    self.sample_num, self.aligned)
  File "/SingleModelTest/src/mmdet/mmdet/ops/roi_align/roi_align.py", line 36, in forward
    spatial_scale, sample_num, output)
RuntimeError: cuda runtime error (48) : no kernel image is available for execution on the device at mmdet/ops/roi_a
lign/src/roi_align_kernel.cu:139

When i searched for possible solutions i came across a couple stackoverflow questions with the same error where the problem was an unsupported old gpu, so I changed the GPU on my google compute engine VM to a newer one from Nvidia Tesla K80 to Nvidia Tesla T4. The K80 has a cuda compute capability of 3.7 and the new T4 has 7.5 so I thought that would fix the problem, but it didn't.
Output nvidia-smi:
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.80.02    Driver Version: 450.80.02    CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   72C    P8    12W /  70W |    106MiB / 15109MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A       918      G   /usr/lib/xorg/Xorg                 95MiB |
|    0   N/A  N/A       974      G   /usr/bin/gnome-shell                9MiB |
+-----------------------------------------------------------------------------+

nvcc --version:
nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2019 NVIDIA Corporation
Built on Sun_Jul_28_19:07:16_PDT_2019
Cuda compilation tools, release 10.1, V10.1.243

torch version: 1.4.0+cu100
torchvision version 0.5.0+cu100
I am running the API in a docker container, my Dockerfile:
# Dockerfile
FROM nvidia/cuda:10.0-devel

RUN nvidia-smi

RUN set -xe \
    && apt-get update \
    && apt-get install python3-pip -y \
    && apt-get install git -y \
    && apt-get install libgl1-mesa-glx -y
RUN pip3 install --upgrade pip

WORKDIR /SingleModelTest

COPY requirements /SingleModelTest/requirements

RUN export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64

RUN pip3 install -r requirements/requirements1.txt
RUN pip3 install -r requirements/requirements2.txt

COPY . /SingleModelTest

ENTRYPOINT ["python3"]

CMD ["TabNetAPI.py"]

EDIT:
I got confused by the output of nvidia-smi because of the higher cuda version than i installed, but it turns out that is normal according to: https://medium.com/@brianhourigan/if-different-cuda-versions-are-shown-by-nvcc-and-nvidia-smi-its-necessarily-not-a-problem-and-311eda26856c
If anyone has a solution I'd be very grateful.
If I need to supply more information I'd be happy to.
Thank you in advance.


title: PyToch: ValueError: Expected input batch_size (256) to match target batch_size (128)

body:
I've faced a ValueError while training a BiLSTM part of speech tagger using pytorch. ValueError: Expected input batch_size (256) to match target batch_size (128).
def train(model, iterator, optimizer, criterion, tag_pad_idx):

    epoch_loss = 0
    epoch_acc = 0

    model.train()

    for batch in iterator:

        text = batch.p
        tags = batch.t

        optimizer.zero_grad()

        #text = [sent len, batch size]

        predictions = model(text)

        #predictions = [sent len, batch size, output dim]
        #tags = [sent len, batch size]

        predictions = predictions.view(-1, predictions.shape[-1])
        tags = tags.view(-1)

        #predictions = [sent len * batch size, output dim]
        #tags = [sent len * batch size]

        loss = criterion(predictions, tags)

        acc = categorical_accuracy(predictions, tags, tag_pad_idx)

        loss.backward()

        optimizer.step()

        epoch_loss += loss.item()
        epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

def evaluate(model, iterator, criterion, tag_pad_idx):

    epoch_loss = 0
    epoch_acc = 0

    model.eval()

    with torch.no_grad():

        for batch in iterator:

            text = batch.p
            tags = batch.t

            predictions = model(text)

            predictions = predictions.view(-1, predictions.shape[-1])
            tags = tags.view(-1)

            loss = criterion(predictions, tags)

            acc = categorical_accuracy(predictions, tags, tag_pad_idx)

            epoch_loss += loss.item()
            epoch_acc += acc.item()

    return epoch_loss / len(iterator), epoch_acc / len(iterator)

class BiLSTMPOSTagger(nn.Module):
        def __init__(self,
                     input_dim,
                     embedding_dim,
                     hidden_dim,
                     output_dim,
                     n_layers,
                     bidirectional,
                     dropout,
                     pad_idx):

            super().__init__()

            self.embedding = nn.Embedding(input_dim, embedding_dim, padding_idx = pad_idx)

            self.lstm = nn.LSTM(embedding_dim,
                                hidden_dim,
                                num_layers = n_layers,
                                bidirectional = bidirectional,
                                dropout = dropout if n_layers > 1 else 0)

            self.fc = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)

            self.dropout = nn.Dropout(dropout)

        def forward(self, text):
            embedded = self.dropout(self.embedding(text))
            outputs, (hidden, cell) = self.lstm(embedded)
            predictions = self.fc(self.dropout(outputs))
            return predictions

...........................
...........................
...........................
...........................
INPUT_DIM = len(POS.vocab)
EMBEDDING_DIM = 100
HIDDEN_DIM = 128
OUTPUT_DIM = len(TAG.vocab)
N_LAYERS = 2
BIDIRECTIONAL = True
DROPOUT = 0.25
PAD_IDX = POS.vocab.stoi[POS.pad_token]

print(INPUT_DIM)  #output 22147
print(OUTPUT_DIM) #output 42

model = BiLSTMPOSTagger(INPUT_DIM,
                        EMBEDDING_DIM,
                        HIDDEN_DIM,
                        OUTPUT_DIM,
                        N_LAYERS,
                        BIDIRECTIONAL,
                        DROPOUT,
                        PAD_IDX)

...........................
...........................
...........................
...........................
N_EPOCHS = 10

best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):

    start_time = time.time()

    train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)
    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)

    end_time = time.time()

    epoch_mins, epoch_secs = epoch_time(start_time, end_time)

    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'tut1-model.pt')

    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
    print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')

ValueError                                Traceback (most recent call last)
<ipython-input-55-83bf30366feb> in <module>()
      7     start_time = time.time()
      8
----> 9     train_loss, train_acc = train(model, train_iterator, optimizer, criterion, TAG_PAD_IDX)
     10     valid_loss, valid_acc = evaluate(model, valid_iterator, criterion, TAG_PAD_IDX)
     11

4 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2260     if input.size(0) != target.size(0):
   2261         raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'
-> 2262                          .format(input.size(0), target.size(0)))
   2263     if dim == 2:
   2264         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (256) to match target batch_size (128).


title: Tensor format issue from converting Pytorch -> Onnx -> Tensorflow

body:
I have an issue with Tensorflow model that is converted from Pytorch -> Onnx -> Tensorflow. The issue is the converted Tensorflow model expects the input in Pytorch format that is (batch size, number channels, height, width) but not in Tensorflow format (batch size, height, width, number channel). Therefore, I cannot use the model to process further with Vitis AI.
So I would like to ask is there is any ways to convert this Pytorch input format to Tensorflow format by using tools from Onnx, Tensorflow 1, or others?
My code is as below:
Pytorch -> Onnx
from hardnet import hardnet
import torch
import onnx

ckpt = torch.load('../hardnet.pth')
model_state_dict = ckpt['model_state_dict']
optimizer_state_dict = ckpt['optimizer_state_dict']

model = hardnet(11)
model.load_state_dict(model_state_dict)
model.eval()

dummy_input = torch.randn(1, 3, 1080, 1920)
input_names = ['input0']
output_names = ['output0']

output_file = 'hardnet.onnx'
torch.onnx.export(model, dummy_input, output_file, verbose=True,
    input_names=input_names, output_names=output_names,
    opset_version=11, keep_initializers_as_inputs=True)

onnx_model = onnx.load(output_file)
onnx.checker.check_model(onnx_model)
print('Passed Onnx')

Onnx -> Tensorflow 1 (using Tensorflow 1.15)
import cv2
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import onnx
from onnx_tf.backend import prepare

output_file = 'hardnet.onnx'
onnx_model = onnx.load(output_file)
output = prepare(onnx_model)
output.export_graph('hardnet.pb')
tf.compat.v1.disable_eager_execution()

def load_pb(path_to_pb: str):
    """From: https://stackoverflow.com/questions/51278213/what-is-the-use-of-a-pb-file-in-tensorflow-and-how-does-it-work
    """
    with tf.gfile.GFile(path_to_pb, "rb") as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
    with tf.Graph().as_default() as graph:
        tf.import_graph_def(graph_def, name='')
        return graph

graph = load_pb('hardnet.pb')
input = graph.get_tensor_by_name('input0:0')
output = graph.get_tensor_by_name('output0:0')
mean = [0.485, 0.456, 0.406]
std = [0.229, 0.224, 0.225]
img = cv2.imread('train_0.jpg', cv2.IMREAD_COLOR)
img = cv2.resize(img, (1920,  1080))

img = img/255
img = img - mean
img = img/std
img = np.expand_dims(img, -1)
# To Pytorch format.
img = np.transpose(img, (3, 2, 0, 1))
img = img

with tf.Session(graph=graph) as sess:
    pred = sess.run(output, {input: img})


title: What does the following error mean, when I tried to import torch library on iPython?

body:
RuntimeError                              Traceback (most recent call last)

RuntimeError: module compiled against API version 0xb but this version of numpy is 0xa

ImportError                               Traceback (most recent call last)
 in ()
----> 1 import torch

~/anaconda3/lib/python3.6/site-packages/torch/init.py in ()
     51 sys.setdlopenflags(_dl_flags.RTLD_GLOBAL | _dl_flags.RTLD_NOW)
     52
---> 53 from torch._C import *
     54
     55 all += [name for name in dir(_C)

ImportError: numpy.core.multiarray failed to import


title: How to add parameters from self.some_dictionary_of_modules to self.parameters?

body:
Consider this simple example:

import torch

class MyModule(torch.nn.Module):
    def __init__(self):
        super(MyModule, self).__init__()
        self.conv_0=torch.nn.Conv2d(3,32,3,stride=1,padding=0)
        self.blocks=torch.nn.ModuleList([
            torch.nn.Conv2d(3,32,3,stride=1,padding=0),
            torch.nn.Conv2d(32,64,3,stride=1,padding=0)])

        #the problematic part
        self.dict_block={"key_1": torch.nn.Conv2d(64,128,3,1,0),
                "key_2": torch.nn.Conv2d(56,1024,3,1,0)}

if __name__=="__main__":
    my_module=MyModule()
    print(my_module.parameters)

The output I get here is (notice that the parmeters from self.dict_block are missing)

<bound method Module.parameters of MyModule(
  (conv_0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1))
  (blocks): ModuleList(
    (0): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1))
    (1): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))
  )
)>

This means that if I want the parameters of self.dict_block to be optimised, I will need to use something  like

my_optimiser.add_param_group({"params": params_from_self_dict})

before using my optimiser. However, I think there might be a more straight-forward alternative that will add the parameters of self.dict_block to the parameters of the my_module_object. Something that comes close is nn.Parameter(...) as explained here, but this requires the input to be a tensor.


title: PyTorch & skorch: How to fix my nn.Module to work with skorch's GridSearchCV

body:
Using PyTorch, I have an ANN model (for a classification task) below:
import torch
import torch.nn as nn

# Setting up artifical neural net model which separates out categorical
# from continuous features, so that embedding could be applied to
# categorical features
class TabularModel(nn.Module):
    # Initialize parameters embeds, emb_drop, bn_cont and layers
    def __init__(self, emb_szs, n_cont, out_sz, layers, p=0.5):
        super().__init__()
        self.embeds = nn.ModuleList([nn.Embedding(ni, nf) for ni, nf in emb_szs])
        self.emb_drop = nn.Dropout(p)
        self.bn_cont = nn.BatchNorm1d(n_cont)

        # Create empty list for each layer in the neural net
        layerlist = []
        # Number of all embedded columns for categorical features
        n_emb = sum((nf for ni, nf in emb_szs))
        # Number of inputs for each layer
        n_in = n_emb + n_cont

        for i in layers:
            # Set the linear function for the weights and biases, wX + b
            layerlist.append(nn.Linear(n_in, i))
            # Using ReLu activation function
            layerlist.append(nn.ReLU(inplace=True))
            # Normalised all the activation function output values
            layerlist.append(nn.BatchNorm1d(i))
            # Set some of the normalised activation function output values to zero
            layerlist.append(nn.Dropout(p))
            # Reassign number of inputs for the next layer
            n_in = i
        # Append last layer
        layerlist.append(nn.Linear(layers[-1], out_sz))
        # Create sequential layers
        self.layers = nn.Sequential(*layerlist)

    # Function for feedforward
    def forward(self, x_cat_cont):
        x_cat = x_cat_cont[:,0:cat_train.shape[1]].type(torch.int64)
        x_cont = x_cat_cont[:,cat_train.shape[1]:].type(torch.float32)

        # Create empty list for embedded categorical features
        embeddings = []
        # Embed categorical features
        for i, e in enumerate(self.embeds):
            embeddings.append(e(x_cat[:,i]))
        # Concatenate embedded categorical features
        x = torch.cat(embeddings, 1)
        # Apply dropout rates to categorical features
        x = self.emb_drop(x)

        # Batch normalize continuous features
        x_cont = self.bn_cont(x_cont)

        # Concatenate categorical and continuous features
        x = torch.cat([x, x_cont], 1)

        # Feed categorical and continuous features into neural net layers
        x = self.layers(x)
        return x

I am trying to use this model with skorch's GridSearchCV, as below:
from skorch import NeuralNetBinaryClassifier

# Random seed chosen to ensure results are reproducible by using the same
# initial random weights and biases, and applying dropout rates to the same
# random embedded categorical features and neurons in the hidden layers
torch.manual_seed(0)

net = NeuralNetBinaryClassifier(module=TabularModel,
                                module__emb_szs=emb_szs,
                                module__n_cont=con_train.shape[1],
                                module__out_sz=2,
                                module__layers=[30],
                                module__p=0.0,
                                criterion=nn.CrossEntropyLoss,
                                criterion__weight=cls_wgt,
                                optimizer=torch.optim.Adam,
                                optimizer__lr=0.001,
                                max_epochs=150,
                                device='cuda'
                                )

from sklearn.model_selection import GridSearchCV

param_grid = {'module__layers': [[30], [50,20]],
              'module__p': [0.0, 0.2, 0.4],
              'max_epochs': [150, 175, 200, 225]
             }

models = GridSearchCV(net, param_grid, scoring='roc_auc').fit(cat_con_train.cpu(), y_train.cpu())

models.best_params_

But when I ran the code, I am getting this error message below:
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: Expected module output to have shape (n,) or (n, 1), got (128, 2) instead

  FitFailedWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: Expected module output to have shape (n,) or (n, 1), got (128, 2) instead

  FitFailedWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: Expected module output to have shape (n,) or (n, 1), got (128, 2) instead

  FitFailedWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: Expected module output to have shape (n,) or (n, 1), got (128, 2) instead

  FitFailedWarning)
/usr/local/lib/python3.6/dist-packages/sklearn/model_selection/_validation.py:536: FitFailedWarning: Estimator fit failed. The score on this train-test partition for these parameters will be set to nan. Details:
ValueError: Expected module output to have shape (n,) or (n, 1), got (128, 2) instead

  FitFailedWarning)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-86-c408d65e2435> in <module>()
     98
---> 99 models = GridSearchCV(net, param_grid, scoring='roc_auc').fit(cat_con_train.cpu(), y_train.cpu())
    100
    101 models.best_params_

11 frames
/usr/local/lib/python3.6/dist-packages/skorch/classifier.py in infer(self, x, **fit_params)
    303             raise ValueError(
    304                 "Expected module output to have shape (n,) or "
--> 305                 "(n, 1), got {} instead".format(tuple(y_infer.shape)))
    306
    307         y_infer = y_infer.reshape(-1)

ValueError: Expected module output to have shape (n,) or (n, 1), got (128, 2) instead

I am not sure what is wrong or how to fix this. Any help on this would really be appreciated.
Many thanks in advance!


title: pip to install Pytorch for usr/bin/python

body:
I have set up Malmo on my mac and got into a trouble. I am using Malmo-0.17.0-Mac-64bit, python 2.7, macOS Sierra.

As it is explained on the Malmo installation instruction, mac users require to run the python scripts with the default mac python located on /usr/bin/python. I am aiming to implement a neural nets with Pytorch and doing some experiments. However, when I run the command /usr/bin/python -m pip install http://download.pytorch.org/whl/torch-0.2.0.post3-cp27-none-macosx_10_7_x86_64.whl to install Pytorch I am getting this error:

Command "/usr/bin/python -u -c "import setuptools, tokenize;__file__='/private/var/folders/c7/24z_lgxx4b7107ynf4cy34qh0000gn/T/pip-build-kLOmTh/pyyaml/setup.py';f=getattr(tokenize, 'open', open)(__file__);code=f.read().replace('\r\n', '\n');f.close();exec(compile(code, __file__, 'exec'))" install --record /var/folders/c7/24z_lgxx4b7107ynf4cy34qh0000gn/T/pip-YxAJnq-record/install-record.txt --single-version-externally-managed --compile" failed with error code 1 in /private/var/folders/c7/24z_lgxx4b7107ynf4cy34qh0000gn/T/pip-build-kLOmTh/pyyaml/.

I was wondering if anyone has faced the same problem.

Many Thanks,


title: Does torch.utils.tensorboard need installation of Tensorflow?

body: I'm new to pytorch and I wonder if using tensorboard on pytorch needs tensorflow as a dependency. Moreover, except for tensorboard, what are other options for training curve visualization?


title: Decrease the maximum learning rate after every restart

body: I'm training a neural network for a computer vision-based task. For the optimizer, I found out that it isn't ideal to use a single learning rate for the entire training, and what people do is that they use learning rate schedulers to decay the learning rate in a specific manner. So to do this, I tried out PyTorch's CosineAnnealingWarmRestarts(). What this does is that it anneals/decreases the initial learning rate (set by us) in a cosine manner until it hits a restart. After this "restart," the learning rate is set back to the initial learning rate, and the cycle happens again. This worked pretty well for me, but I wanted to make a few changes in it. I wanted to change the learning rate, the optimizer is assigned after each restart, so that after every restart the maximum learning rate for the optimizer also decreases. Can this be done in PyTorch?


title: Validation loss is constant and training loss decreasing

body:
I have a model training and I got this plot. It is over audio (about 70K of around 5-10s) and no augmentation is being done. I have tried the following to avoid overfitting:

Reduce complexity of the model by reducing number of GRU cells and hidden dimensions.
Add dropout in each layer.
I have tried with higher dataset.

What I am not sure is if my calculation of training loss and validation loss is correct. It is something like this. I am using drop_last=True and I am using the CTC loss criterion.
train_data_len = len(train_loader.dataset)
valid_data_len = len(valid_loader.dataset)
epoch_train_loss = 0
epoch_val_loss = 0
train_losses = []
valid_losses = []

    model.train()
    for e in range(n_epochs):
        t0 = time.time()
        #batch loop
        running_loss = 0.0
        for batch_idx, _data in enumerate(train_loader, 1):
            # Calculate output ...
             # bla bla
            loss = criterion(output, labels.float(), input_lengths, label_lengths)
            loss.backward()
            optimizer.step()
            scheduler.step()
            # loss stats
            running_loss += loss.item() * specs.size(0)

        t_t = time.time() - t0

        ######################
        # validate the model #
        ######################
        with torch.no_grad():
            model.eval()
            tv = time.time()
            running_val_loss = 0.0
            for batch_idx_v, _data in enumerate(valid_loader, 1):
                #bla, bla
                val_loss = criterion(output, labels.float(), input_lengths, label_lengths)
                running_val_loss += val_loss.item() * specs.size(0)

            print("Epoch {}: Training took {:.2f} [s]\tValidation took: {:.2f} [s]\n".format(e+1, t_t, time.time() - tv))

        epoch_train_loss = running_loss / train_data_len
        epoch_val_loss = running_val_loss / valid_data_len
        train_losses.append(epoch_train_loss)
        valid_losses.append(epoch_val_loss)
        print('Epoch: {} Losses\tTraining Loss: {:.6f}\tValidation Loss: {:.6f}'.format(
                e+1, epoch_train_loss, epoch_val_loss))
        model.train()


title: Find n smallest values in a list of tensors

body:
I am trying to find the indices of the n smallest values in a list of tensors in pytorch. Since these tensors might contain many non-unique values, I cannot simply compute percentiles to obtain the indices. The ordering of non-unique values does not matter however.
I came up with the following solution but am wondering if there is a more elegant way of doing it:
import torch

n = 10

tensor_list = [torch.randn(10, 10), torch.zeros(20, 20), torch.ones(30, 10)]
all_sorted, all_sorted_idx = torch.sort(torch.cat([t.view(-1) for t in tensor_list]))

cum_num_elements = torch.cumsum(torch.tensor([t.numel() for t in tensor_list]), dim=0)
cum_num_elements = torch.cat([torch.tensor([0]), cum_num_elements])

split_indeces_lt = [all_sorted_idx[:n] < cum_num_elements[i + 1] for i, _ in enumerate(cum_num_elements[1:])]
split_indeces_ge = [all_sorted_idx[:n] >= cum_num_elements[i] for i, _ in enumerate(cum_num_elements[:-1])]
split_indeces = [all_sorted_idx[:n][torch.logical_and(lt, ge)] - c for lt, ge, c in zip(split_indeces_lt, split_indeces_ge, cum_num_elements[:-1])]

n_smallest = [t.view(-1)[idx] for t, idx in zip(tensor_list, split_indeces)]

EDIT: Ideally a solution would pick a random subset of the non-unique values instead of picking the entries of the first tensor of the list.


title: What is tape-based autograd in Pytorch?

body:
I understand autograd is used to imply automatic differentiation. But what exactly is tape-based autograd in Pytorch and why there are so many discussions that affirm or deny it.
For example:
this

In pytorch, there is no traditional sense of tape

and this

We don’t really build gradient tapes per se. But graphs.

but not this

Autograd is now a core torch package for automatic differentiation. It
uses a tape based system for automatic differentiation.

And for further reference, please compare it with GradientTape in Tensorflow.


title: Error when loading torch.hub.load('pytorch/fairseq', 'roberta.large.mnli') on AWS EC2

body:
I'm trying to run some code using Torch (and Roberta language model) on an EC2 instance on AWS.
The compilation seems to fail, does anyone have a pointer to fix?
Confirm that Torch is correctly installed
import torch
a = torch.rand(5,3)
print (a)

Return this: tensor([[0.7494, 0.5213, 0.8622],...
Attempt to load Roberta
roberta = torch.hub.load('pytorch/fairseq', 'roberta.large.mnli')

Using cache found in /home/ubuntu/.cache/torch/hub/pytorch_fairseq_master
/home/ubuntu/.local/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)
  return torch._C._cuda_getDeviceCount() > 0
fatal: not a git repository (or any of the parent directories): .git
running build_ext
/home/ubuntu/.local/lib/python3.8/site-packages/torch/utils/cpp_extension.py:352: UserWarning: Attempted to use ninja as the BuildExtension backend but we could not find ninja.. Falling back to using the slow distutils backend.
  warnings.warn(msg.format('we could not find ninja.'))
skipping 'fairseq/data/data_utils_fast.cpp' Cython extension (up-to-date)
skipping 'fairseq/data/token_block_utils_fast.cpp' Cython extension (up-to-date)
building 'fairseq.libnat' extension
x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/TH -I/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/THC -I/usr/include/python3.8 -c fairseq/clib/libnat/edit_dist.cpp -o build/temp.linux-x86_64-3.8/fairseq/clib/libnat/edit_dist.o -DTORCH_API_INCLUDE_EXTENSION_H -DPYBIND11_COMPILER_TYPE="_gcc" -DPYBIND11_STDLIB="_libstdcpp" -DPYBIND11_BUILD_ABI="_cxxabi1011" -DTORCH_EXTENSION_NAME=libnat -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++14
In file included from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/ATen/Parallel.h:149,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/utils.h:3,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/nn/cloneable.h:5,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/nn.h:3,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/all.h:12,
                 from /home/ubuntu/.local/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/torch.h:3,
                 from fairseq/clib/libnat/edit_dist.cpp:9:
/home/ubuntu/.local/lib/python3.8/site-packages/torch/include/ATen/ParallelOpenMP.h:84: warning: ignoring #pragma omp parallel [-Wunknown-pragmas]
   84 | #pragma omp parallel for if ((end - begin) >= grain_size)

It then ends, after a long while.
x86_64-linux-gnu-gcc: fatal error: Killed signal terminated program cc1plus compilation terminated.
error: command 'x86_64-linux-gnu-gcc' failed with exit status 1


title: coverting roi pooling in pytorch to nn layer

body:
I have a an mlmodel  using ROI pooling for which I am using this
(adapted from here) (non NN layer version)

def forward(self, features, rois):
        batch_size, num_channels, data_height, data_width = features.size()
        num_rois = rois.size()[0]
        outputs = Variable(torch.zeros(num_rois, num_channels, self.pooled_height, self.pooled_width)).cuda()

        for roi_ind, roi in enumerate(rois):
            batch_ind = int(roi[0].data[0])
            roi_start_w, roi_start_h, roi_end_w, roi_end_h = np.round(
                roi[1:].data.cpu().numpy() * self.spatial_scale).astype(int)
            roi_width = max(roi_end_w - roi_start_w + 1, 1)
            roi_height = max(roi_end_h - roi_start_h + 1, 1)
            bin_size_w = float(roi_width) / float(self.pooled_width)
            bin_size_h = float(roi_height) / float(self.pooled_height)

            for ph in range(self.pooled_height):
                hstart = int(np.floor(ph * bin_size_h))
                hend = int(np.ceil((ph + 1) * bin_size_h))
                hstart = min(data_height, max(0, hstart + roi_start_h))
                hend = min(data_height, max(0, hend + roi_start_h))
                for pw in range(self.pooled_width):
                    wstart = int(np.floor(pw * bin_size_w))
                    wend = int(np.ceil((pw + 1) * bin_size_w))
                    wstart = min(data_width, max(0, wstart + roi_start_w))
                    wend = min(data_width, max(0, wend + roi_start_w))

                    is_empty = (hend <= hstart) or(wend <= wstart)
                    if is_empty:
                        outputs[roi_ind, :, ph, pw] = 0
                    else:
                        data = features[batch_ind]
                        outputs[roi_ind, :, ph, pw] = torch.max(
                            torch.max(data[:, hstart:hend, wstart:wend], 1)[0], 2)[0].view(-1)

        return outputs

I want to convert the pytorch model to caffe and hence I need to convert the above to an NN layer for which I am using the below (adapted from here)

def forward(self, input, rois):
    output = []
    rois = rois.data.float()
    num_rois = rois.size(0)

    rois[:,1:].mul_(self.spatial_scale)
    rois = rois.long()
    for i in range(num_rois):
        roi = rois[i]
        im_idx = roi[0]
        im = input.narrow(0, im_idx, 1)[..., roi[2]:(roi[4]+1), roi[1]:(roi[3]+1)]
        op = nn.functional.adaptive_max_pool2d(input = im, output_size = self.size)
        output.append(op)

    return torch.cat(tuple(output), dim=0)

The output returned doesn't seem to match for the above methods even though they are performing the same function. I seem to hit a deadlock. Could anyone point if I am making any obvious mistake in the above ?


title: Inconsistency when comparing scipy, torch and fourier periodic convolution

body:
I'm implementing a 2d periodic convolution on a synthetic image in three different ways: using scipy, using torch and using the Fourier transform (also under torch framework).

However, I've got different results. Performing the operation by hand I can see that scipy's convolution yields the correct results. torch's spatial version, on the other hand, yields the expected result inverted. Finally, the Fourier version returns something unexpected.

The code is the following:

import torch
import numpy as np
import scipy.signal as sig
import torch.nn.functional as F
import matplotlib.pyplot as plt

def numpy_periodic_conv(f, k):
    H, W = f.shape
    periodic_f = np.hstack([f, f])
    periodic_f = np.vstack([periodic_f, periodic_f])
    conv = sig.convolve2d(periodic_f, k, mode='same')
    conv = conv[H // 2:-H // 2, W // 2:-W // 2]
    return periodic_f, conv

def torch_periodic_conv(f, k):
    H, W = f.shape[-2:]
    periodic_f = f.repeat(1, 1, 2, 2)
    conv = F.conv2d(periodic_f, k, padding=1)
    conv = conv[:, :, H // 2:-H // 2, W // 2:-W // 2]
    return periodic_f.squeeze().numpy(), conv.squeeze().numpy()

def torch_fourier_conv(f, k):
    pad_x = f.shape[-2] - k.shape[-2]
    pad_y = f.shape[-1] - k.shape[-1]
    expanded_kernel = F.pad(k, [0, pad_x, 0, pad_y])
    fft_x = torch.rfft(f, 2, onesided=False)
    fft_kernel = torch.rfft(expanded_kernel, 2, onesided=False)
    real = fft_x[:, :, :, :, 0] * fft_kernel[:, :, :, :, 0] - \
           fft_x[:, :, :, :, 1] * fft_kernel[:, :, :, :, 1]
    im = fft_x[:, :, :, :, 0] * fft_kernel[:, :, :, :, 1] + \
         fft_x[:, :, :, :, 1] * fft_kernel[:, :, :, :, 0]
    fft_conv = torch.stack([real, im], -1) # (a+bj)*(c+dj) = (ac-bd)+(ad+bc)j
    ifft_conv = torch.irfft(fft_conv, 2, onesided=False)
    return expanded_kernel.squeeze().numpy(), ifft_conv.squeeze().numpy()

if __name__ == '__main__':
    f = np.concatenate([np.ones((10, 5)), np.zeros((10, 5))], 1)
    k = np.array([[1, 0, -1], [2, 0, -2], [1, 0, -1]])

    f_tensor = torch.from_numpy(f).unsqueeze(0).unsqueeze(0).float()
    k_tensor = torch.from_numpy(k).unsqueeze(0).unsqueeze(0).float()

    np_periodic_f, np_periodic_conv = numpy_periodic_conv(f, k)
    tc_periodic_f, tc_periodic_conv = torch_periodic_conv(f_tensor, k_tensor)
    tc_fourier_k, tc_fourier_conv = torch_fourier_conv(f_tensor, k_tensor)

    print('Spatial numpy conv shape= ', np_periodic_conv.shape)
    print('Spatial torch conv shape= ', tc_periodic_conv.shape)
    print('Fourier torch conv shape= ', tc_fourier_conv.shape)

    r_np = dict(name='numpy', im=np_periodic_f, k=k, conv=np_periodic_conv)
    r_torch = dict(name='torch', im=tc_periodic_f, k=k, conv=tc_periodic_conv)
    r_fourier = dict(name='fourier', im=f, k=tc_fourier_k, conv=tc_fourier_conv)
    titles = ['{} im', '{} kernel', '{} conv']
    results = [r_np, r_torch, r_fourier]
    fig, axs = plt.subplots(3, 3)
    for i, r_dict in enumerate(results):
        axs[i, 0].imshow(r_dict['im'], cmap='gray')
        axs[i, 0].set_title(titles[0].format(r_dict['name']))
        axs[i, 1].imshow(r_dict['k'], cmap='gray')
        axs[i, 1].set_title(titles[1].format(r_dict['name']))
        axs[i, 2].imshow(r_dict['conv'], cmap='gray')
        axs[i, 2].set_title(titles[2].format(r_dict['name']))
    plt.show()

The results I'm obtaining:

Note: The image for both numpyand torch versions show the periodic image, which is required to perform the periodic convolution. The kernel for the Fourier version shows the original kernel zero-padded to the image size, which is required to compute the element-wise multiplication in the frequency domain.

-Edit1: There was an error when in the multiplication in the Fourier version, I was doing (ac-bd)+(ad-bc)j instead of (ac-bd)+(ad+bc)j. But now, I get the convolution shifted by one column.

-Edit2: torch's spatial convolution results is inverted because the operation is actually a cross-correlation. This was confirmed in the pytorch's official forum here. Furthermore, after fixing the kernel padding as Cris Luengo's answer, the frequency method yielded the same results as the correlations. This is rather strange for me because, as far as I know, the frequency property hold for convolution, not correlation.

New-results after fixing the kernel:


title: Initializing convolutional layers in CNN

body:
Is there a function to initialize weights of a convolutional layer to focus more on information closer to the center of input images?

All my input images are centered, so pixels further away from the center of an image matter less than pixels closer to the center.


title: Why casting input and model to float16 doesn't work?

body:
I'm trying to change inputs and a deep learning model to flaot16, since I'm using T4 GPU and they work much faster with fp16.
Here's part of the code: I first have my model and then made some dummy data point for the sake of figuring the data casting figured out first (I ran it with the whole batch and got the same error).

model = CRNN().to(device)
model = model.type(torch.cuda.HalfTensor)

data_recon = torch.from_numpy(data_recon)
data_truth = torch.from_numpy(data_truth)

dummy = data_recon[0:1,:,:,:,:] # Gets just one batch
dummy = dummy.to(device)
dummy = dummy.type(torch.cuda.HalfTensor)

model(dummy)

And here's the error I get:

> ---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call
> last) <ipython-input-27-1fe8ecc524aa> in <module>
> ----> 1 model(dummy)
>
> /opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py
> in __call__(self, *input, **kwargs)
>     491             result = self._slow_forward(*input, **kwargs)
>     492         else:
> --> 493             result = self.forward(*input, **kwargs)
>     494         for hook in self._forward_hooks.values():
>     495             hook_result = hook(self, input, result)
>
> <ipython-input-12-06f39f9304a1> in forward(self, inputs, test)
>      57
>      58             net['t%d_x0'%(i-1)] = net['t%d_x0'%(i-1)].view(times, batch, self.filter_size, width,
> height)
> ---> 59             net['t%d_x0'%i] = self.bcrnn(inputs, net['t%d_x0'%(i-1)], test)
>      60             net['t%d_x0'%i] = net['t%d_x0'%i].view(-1, self.filter_size, width, height)
>      61
>
> /opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py
> in __call__(self, *input, **kwargs)
>     491             result = self._slow_forward(*input, **kwargs)
>     492         else:
> --> 493             result = self.forward(*input, **kwargs)
>     494         for hook in self._forward_hooks.values():
>     495             hook_result = hook(self, input, result)
>
> <ipython-input-11-b687949e9ce5> in forward(self, inputs,
> input_iteration, test)
>      31         hidden = initial_hidden
>      32         for i in range(times):
> ---> 33             hidden = self.CRNN(inputs[i], input_iteration[i], hidden)
>      34             output_forward.append(hidden)
>      35         output_forward = torch.cat(output_forward)
>
> /opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py
> in __call__(self, *input, **kwargs)
>     491             result = self._slow_forward(*input, **kwargs)
>     492         else:
> --> 493             result = self.forward(*input, **kwargs)
>     494         for hook in self._forward_hooks.values():
>     495             hook_result = hook(self, input, result)
>
> <ipython-input-10-15c0b221226b> in forward(self, inputs,
> hidden_iteration, hidden)
>      23     def forward(self, inputs, hidden_iteration, hidden):
>      24         in_to_hid = self.i2h(inputs)
> ---> 25         hid_to_hid = self.h2h(hidden)
>      26         ih_to_ih = self.ih2ih(hidden_iteration)
>      27
>
> /opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py
> in __call__(self, *input, **kwargs)
>     491             result = self._slow_forward(*input, **kwargs)
>     492         else:
> --> 493             result = self.forward(*input, **kwargs)
>     494         for hook in self._forward_hooks.values():
>     495             hook_result = hook(self, input, result)
>
> /opt/anaconda3/lib/python3.7/site-packages/torch/nn/modules/conv.py in
> forward(self, input)
>     336                             _pair(0), self.dilation, self.groups)
>     337         return F.conv2d(input, self.weight, self.bias, self.stride,
> --> 338                         self.padding, self.dilation, self.groups)
>     339
>     340
>
> RuntimeError: Input type (torch.cuda.FloatTensor) and weight type
> (torch.cuda.HalfTensor) should be the same


title: How can I change the following code from pytorch to tensorflow?

body:
I want to change the follow pytorch network (v1.2) to tensorflow. I am confusing between tf.nn.conv2d and tf.keras.layers.Conv2D what should I choose?

import torch.nn as nn
nn.Sequential(nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=True),
    nn.BatchNorm2d(out_planes),
    nn.ReLU(inplace=True))


title: model.parameters() not updating in Linear Regression with Pytorch

body:
I'm a newbie in Deep Learning with Pytorch. I am using the Housing Prices dataset from Kaggle here. I tried sampling with first 50 rows. But the model.parameters() is not updating as I perform the training. Can anyone help?
import torch
import numpy as np
from torch.utils.data import TensorDataset
import torch.nn as nn
from torch.utils.data import DataLoader
import torch.nn.functional as F

inputs = np.array(label_X_train[:50])
targets = np.array(train_y[:50])

# Tensors
inputs = torch.from_numpy(inputs)
targets = torch.from_numpy(targets)
targets = targets.view(-1, 1)
train_ds = TensorDataset(inputs, targets)
batch_size = 5
train_dl = DataLoader(train_ds, batch_size, shuffle=True)

model = nn.Linear(10, 1)
# Define Loss func
loss_fn = F.mse_loss
# Optimizer
opt = torch.optim.SGD(model.parameters(), lr = 1e-5)

num_epochs = 100
model.train()
for epoch in range(num_epochs):
    # Train with batches of data
    for xb, yb in train_dl:

        # 1. Generate predictions
        pred = model(xb.float())

        # 2. Calculate loss
        loss = loss_fn(pred, yb.float())

        # 3. Compute gradients
        loss.backward()

        # 4. Update parameters using gradients
        opt.step()

        # 5. Reset the gradients to zero
        opt.zero_grad()

    if (epoch+1) % 10 == 0:
        print('Epoch [{}/{}], Loss: {:.4f}'.format(epoch +
                                                   1, num_epochs,
                                                   loss.item()))


title: 'bert-base-multilingual-uncased' dataloader RuntimeError : stack expects each tensor to be equal size

body:
I am a begineer in nlp, as I was giving this competition https://www.kaggle.com/c/contradictory-my-dear-watson I am using the model 'bert-base-multilingual-uncased' and using BERT tokenizer from the same. I am also using kaggle tpu. This is the custom dataloader I created.
class SherlockDataset(torch.utils.data.Dataset):

def __init__(self,premise,hypothesis,tokenizer,max_len,target = None):
    super(SherlockDataset,self).__init__()
    self.premise = premise
    self.hypothesis = hypothesis
    self.tokenizer = tokenizer
    self.max_len = max_len
    self.target = target

def __len__(self):
    return len(self.premise)

def __getitem__(self,item):
    sen1 = str(self.premise[item])
    sen2 = str(self.hypothesis[item])

    encode_dict = self.tokenizer.encode_plus(sen1,
                                        sen2,
                                        add_special_tokens = True,
                                        max_len = self.max_len,
                                        pad_to_max_len = True,
                                        return_attention_mask = True,
                                        return_tensors = 'pt'
                                       )
    input_ids = encode_dict["input_ids"][0]
    token_type_ids = encode_dict["token_type_ids"][0]
    att_mask = encode_dict["attention_mask"][0]

    if self.target is not None:
        sample = {
        "input_ids":input_ids,
        "token_type_ids":token_type_ids,
        "att_mask":att_mask,
        "targets": self.target[item]
        }
    else:
        sample = {
        "input_ids":input_ids,
        "token_type_ids":token_type_ids,
        "att_mask":att_mask
        }

    return sample

and during the time of loading data in dataloader
def train_fn(model,dataloader,optimizer,criterion,scheduler = None):
model.train()
print("train")
for idx, sample in enumerate(dataloader):
    '''
    input_ids = sample["input_ids"].to(config.DEVICE)
    token_type_ids = sample["token_type_ids"].to(config.DEVICE)
    att_mask = sample["att_mask"].to(config.DEVICE)
    targets = sample["targets"].to(config.DEVICE)
    '''
    print("train_out")
    input_ids = sample[0].to(config.DEVICE)
    token_type_ids = sample[1].to(config.DEVICE)
    att_mask = sample[2].to(config.DEVICE)
    targets = sample[3].to(config.DEVICE)

    optimizer.zero_grad()
    output = model(input_ids,token_type_ids,att_mask)
    output = np.argmax(output,axis = 1)
    loss = criterion(outputs,targets)
    accuracy = accuracy_score(output,targets)
    loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(),1.0)
    xm.optimizer_step(optimizer, barrier=True)
    if scheduler is not None:
        scheduler.step()
    if idx%50==0:
        print(f"idx : {idx}, TRAIN LOSS : {loss}")

I am facing this error again and again
RuntimeError: Caught RuntimeError in DataLoader worker process 0. Original Traceback (most recent
call last): File "/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line
178,
in _worker_loop data = fetcher.fetch(index) File "/opt/conda/lib/python3.7/site-
packages/torch/utils/data/_utils/fetch.py", line 47, in fetch return self.collate_fn(data) File
"/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 79, in
 default_collate return [default_collate(samples) for samples in transposed] File
"/opt/conda/lib/python3.7/site-packages/torch/utils/data/_utils/collate.py", line 79, in return
 [default_collate(samples) for samples in transposed] File "/opt/conda/lib/python3.7/site-
 packages/torch/utils/data/_utils/collate.py", line 55, in default_collate return torch.stack(batch,
 0, out=out) RuntimeError: stack expects each tensor to be equal size, but got [47] at entry 0 and
 [36] at entry 1

I have tried changing num_workers values,changing batch sizes. I have checked the data and none of the text in it is null, 0 or corrupt in any sense. I have also tried changing max_len in  tokenizer but I am not able to find out solution to this problem. Please check and let me know how can I fix it.


title: AND-gate with Pytorch

body:
I'm new to PyTorch and deep learning generally.
The code I wrote can be seen longer down.
I'm trying to learn the simple 'And' problem, which is linearby separable.
The problem is, that I'm getting poor results. Only around 2/10 times it gets to the correct answer.
Sometimes the loss.item() values is stuck at 0.250.

Just to clear up

Why does it only work 2/10 times?

.

import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torch.autograd as autog

data_x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
data_y = np.array([[0, 0, 0, 1]]).T
data_x = autog.Variable(torch.FloatTensor(data_x))
data_y = autog.Variable(torch.FloatTensor(data_y), requires_grad=False)

in_dim = 2
out_dim = 1
epochs = 15000
epoch_print = epochs / 5
l_rate = 0.001

class NeuralNet(nn.Module):
    def __init__(self, input_size, output_size):
        super(NeuralNet, self).__init__()
        self.lin1 = nn.Linear(input_size, output_size)
        self.relu = nn.ReLU()

    def forward(self, x):
        out = x
        out = self.lin1(out)
        out = self.relu(out)
        return out

model = NeuralNet(in_dim, out_dim)
criterion = nn.L1Loss()
optimizer = optim.Adam(model.parameters(), lr=l_rate)

for epoch in range(epochs):
    pred = model(data_x)
    loss = criterion(pred, data_y)
    loss.backward()
    optimizer.step()
    if (epoch + 1) % epoch_print == 0:
        print("Epoch %d  Loss %.3f" %(epoch + 1, loss.item()))

for x, y in zip(data_x, data_y):
    pred = model(x)
    print("Input", list(map(int, x)), "Pred", int(pred), "Output", int(y))


title: how to deal with multi label classfication in LSTM model in pytorch

body:
I am doing a multi label classfication(4 labels) task, specially a text classfication. I get 1000 txts and every one has 50 word and a labels, each word is embedded 100 dimension, and I use pytorch. I have a question,
every txt has diffentent length, e.g. ten words, five words or eight word. I first embedding the words into 100 dimension and I know LSTM is Sequence length independent. So I can deal the every txt success, but how can I deal with many txts at the same time??

When I deal with many txts, I just pad the txt to the same length(20len), and I train torch.nn.LSTM in batch_first, the input is (1000, 20, 100) and output is (1000, 20, 100), and the labels is (1000), I use crossEntropy...I know it is wrong, so how to do it


title: torch.max slower with GPU than with CPU when specifying dimension

body:
t1_h = torch.tensor(np.arange(100000), dtype=torch.float32)
cuda0 = torch.device('cuda:0')
t1_d = torch.tensor(np.arange(100000), dtype=torch.float32, device = cuda0)

%timeit -n 10000 max_h = torch.max(t1_h, 0)
%timeit -n 10000 max_d = torch.max(t1_d, 0)

10000 loops, best of 3: 144 µs per loop
10000 loops, best of 3: 985 µs per loop

As you can see above, GPU takes much more time than CPU. But if I don't specify dimension for calculating max, then GPU is faster.
%timeit -n 10000 max_h = torch.max(t1_h)
%timeit -n 10000 max_d = torch.max(t1_d)

10000 loops, best of 3: 111 µs per loop
10000 loops, best of 3: 41.8 µs per loop

I also tried with argmax instead of max but it is working correctly (GPU faster than CPU).
%timeit -n 10000 cs_h = torch.argmax(t1_h, 0)
%timeit -n 10000 cs_d = torch.argmax(t1_d, 0)

10000 loops, best of 3: 108 µs per loop
10000 loops, best of 3: 18.1 µs per loop

Is there any reason why torch.max is slow on GPU after specifying dimension?


title: Google Colab TensorBoard in another Chrome Tab

body:
I am going through the PyTorch tutorials and am currently on the TensorBoard one.
Through research, I have been able to get it to work inline and through another tab.
My preference is to have it persisent in another tab that will update automatically.

The method described below uses ngrok:
https://medium.com/@iamsdt/using-tensorboard-in-google-colab-with-pytorch-458f9bb95212

However, it is unstable because it gets to many connections and then drop:

Too many connections! The tunnel session '1Z8HZHdl5gLd1xJVdx5Vqqpl9dW' has violated the rate-limit policy of 20 connections per minute by initiating 133 connections in the last 60 seconds. Please decrease your inbound connection volume or upgrade to a paid plan for additional capacity.

The error encountered was: ERR_NGROK_702

When this happens i have to restart the whole kernal and start over. Very frustrating.

This really isn't the solution I am looking for.
Does anyone have a solution for this?

I figured google would have a solution for this.....


title: Using torchtext for inference

body:
I wonder what is the right way to use torchtext for inference.
Let's assume I've trained the model and dump all Fields with built vocabularies. It seems the next step is to use torchtext.data.Example to load one single example. Somehow I should numeralize it by using loaded Fields and create an Iterator.
I would appreciate any simple examples of using torchtext for inference.


title: Why the scale became zero when using torch.cuda.amp.GradScaler?

body:
I use  the following snippet of code to show the scale when using Pytorch's Automatic Mixed Precision Package(amp):
scaler = torch.cuda.amp.GradScaler(init_scale = 65536.0,growth_interval=1)
print(scaler.get_scale())

and This is the output that I get:
...
65536.0
32768.0
16384.0
8192.0
4096.0
...
1e-xxx
...
0
0
0

And all the loss after this step became Nan (the scale still is 0 in the meanwhile).
Whats wrong with my loss function or training data?


title: How to restructure the output tensor of a cnn layer for use by a linear layer in a simple pytorch model

body:
Given a pytorch input dataset  with dimensions:
dat.shape = torch.Size([128, 3, 64, 64])

This is a supervised learning problem: we have a separate labels.txt file containing one of C classes for each input observation.  The value of C is calculated by the number of distinct values in the labeles file and is presently in the single digits.
I could use assistance on how to mesh the layers of a simple mix of convolutional and linear layers network that is performing multiclass classification.  The intent is to pass through:

two cnn layers with maxpooling after each
a linear "readout" layer
softmax activation before the output/labels

Here is the core of my (faulty/broken) network.  I am unable to determine the proper size/shape required of:
 Output of Convolutional layer -> Input of Linear [Readout] layer

class CNNClassifier(torch.nn.Module):

    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 16, 3)
        self.maxpool = nn.MaxPool2d(kernel_size=3,padding=1)
        self.conv2 = nn.Conv2d(16, 32, 3)
        self.linear1 = nn.Linear(32*16*16, C)
        self.softmax1 = nn.LogSoftmax(dim=1)

    def forward(self, x):
        x = self.conv1(x)
        x = self.maxpool(F.leaky_relu(x))
        x = self.conv2(x)
        x = self.maxpool(F.leaky_relu(x))
        x = self.linear1(x)  # Size mismatch error HERE
        x = self.softmax1(x)
        return x

Training of the model is started by :
        Xout = model(dat)

This results in :

RuntimeError: size mismatch, m1: [128 x 1568], m2: [8192 x 6]

at the linear1 input.  What is needed here ? Note I have seen uses of wildcard input sizes e.g via a view:
    ..
    x = x.view(x.size(0), -1)
    x = self.linear1(x)  # Size mismatch error HERE

If that is included then the error changes to

RuntimeError: size mismatch, m1: [28672 x 7], m2: [8192 x 6]

Some pointers on how to think about and calculate the cnn layer / linear layer input/output sizes would be much appreciated.


title: 5.51 GiB already allocated; 417.00 MiB free; 5.53 GiB reserved in total by PyTorch CUDA out of memory

body:
I am not sure why running this cell throws CUDA out of memory error and how I could fix it? Everytime I have to do a kill -9 PID from the list of $ nvidi-smi jupyter notebook from which this cell is being run. And still after a fresh start I have the same problem.
#torch.autograd.set_detect_anomaly(True)
network = Network()
network.cuda()

criterion = nn.MSELoss()
optimizer = optim.Adam(network.parameters(), lr=0.0001)

loss_min = np.inf
num_epochs = 10

start_time = time.time()
for epoch in range(1,num_epochs+1):

    loss_train = 0
    loss_test = 0
    running_loss = 0

    network.train()
    print('size of train loader is: ', len(train_loader))

    for step in range(1,len(train_loader)+1):

        ##images, landmarks = next(iter(train_loader))
        ##print(type(images))

        batch = next(iter(train_loader))
        images, landmarks = batch['image'], batch['landmarks']
        images = images.permute(0,3,1,2)

        images = images.cuda()

        #RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[64, 600, 800, 3] to have 3 channels, but got 600 channels instead

        landmarks = landmarks.view(landmarks.size(0),-1).cuda()

        print('images shape: ', images.shape)
        print('landmarks shape: ', landmarks.shape)

        ##images = torchvision.transforms.Normalize(images)
        ##landmarks = torchvision.transforms.Normalize(landmarks)

        predictions = network(images)

        # clear all the gradients before calculating them
        optimizer.zero_grad()

        # find the loss for the current step
        loss_train_step = criterion(predictions.float(), landmarks.float())

        print("type(loss_train_step) is: ", type(loss_train_step))

        print("loss_train_step.dtype is: ",loss_train_step.dtype)

        ##loss_train_step = loss_train_step.to(torch.float32)

        # calculate the gradients
        loss_train_step.backward()

        # update the parameters
        optimizer.step()

        loss_train += loss_train_step.item()
        running_loss = loss_train/step

        print_overwrite(step, len(train_loader), running_loss, 'train')

    network.eval()
    with torch.no_grad():

        for step in range(1,len(test_loader)+1):

            batch = next(iter(train_loader))
            images, landmarks = batch['image'], batch['landmarks']
            images = images.cuda()
            landmarks = landmarks.view(landmarks.size(0),-1).cuda()

            predictions = network(images)

            # find the loss for the current step
            loss_test_step = criterion(predictions, landmarks)

            loss_test += loss_test_step.item()
            running_loss = loss_test/step

            print_overwrite(step, len(test_loader), running_loss, 'Testing')

    loss_train /= len(train_loader)
    loss_test /= len(test_loader)

    print('\n--------------------------------------------------')
    print('Epoch: {}  Train Loss: {:.4f}  Test Loss: {:.4f}'.format(epoch, loss_train, loss_test))
    print('--------------------------------------------------')

    if loss_test < loss_min:
        loss_min = loss_test
        torch.save(network.state_dict(), '../moth_landmarks.pth')
        print("\nMinimum Test Loss of {:.4f} at epoch {}/{}".format(loss_min, epoch, num_epochs))
        print('Model Saved\n')

print('Training Complete')
print("Total Elapsed Time : {} s".format(time.time()-start_time))

The complete log is:
size of train loader is:  12
images shape:  torch.Size([64, 3, 600, 800])
landmarks shape:  torch.Size([64, 8])

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-18-efa8f1a4056e> in <module>
     44         ##landmarks = torchvision.transforms.Normalize(landmarks)
     45
---> 46         predictions = network(images)
     47
     48         # clear all the gradients before calculating them

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

<ipython-input-11-46116d2a7101> in forward(self, x)
     10     def forward(self, x):
     11         x = x.float()
---> 12         out = self.model(x)
     13         return out

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

~/anaconda3/lib/python3.7/site-packages/torchvision/models/resnet.py in forward(self, x)
    218
    219     def forward(self, x):
--> 220         return self._forward_impl(x)
    221
    222

~/anaconda3/lib/python3.7/site-packages/torchvision/models/resnet.py in _forward_impl(self, x)
    206         x = self.maxpool(x)
    207
--> 208         x = self.layer1(x)
    209         x = self.layer2(x)
    210         x = self.layer3(x)

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
    115     def forward(self, input):
    116         for module in self:
--> 117             input = module(input)
    118         return input
    119

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

~/anaconda3/lib/python3.7/site-packages/torchvision/models/resnet.py in forward(self, x)
     57         identity = x
     58
---> 59         out = self.conv1(x)
     60         out = self.bn1(out)
     61         out = self.relu(out)

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    720             result = self._slow_forward(*input, **kwargs)
    721         else:
--> 722             result = self.forward(*input, **kwargs)
    723         for hook in itertools.chain(
    724                 _global_forward_hooks.values(),

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    417
    418     def forward(self, input: Tensor) -> Tensor:
--> 419         return self._conv_forward(input, self.weight)
    420
    421 class Conv3d(_ConvNd):

~/anaconda3/lib/python3.7/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight)
    414                             _pair(0), self.dilation, self.groups)
    415         return F.conv2d(input, weight, self.bias, self.stride,
--> 416                         self.padding, self.dilation, self.groups)
    417
    418     def forward(self, input: Tensor) -> Tensor:

RuntimeError: CUDA out of memory. Tried to allocate 470.00 MiB (GPU 0; 7.80 GiB total capacity; 5.51 GiB already allocated; 417.00 MiB free; 5.53 GiB reserved in total by PyTorch)

Here's $ nvidia-smi output right after running this cell:
$ nvidia-smi
Tue Oct 13 23:14:01 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.51.06    Driver Version: 450.51.06    CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  GeForce RTX 2070    Off  | 00000000:01:00.0 Off |                  N/A |
| N/A   47C    P8    13W /  N/A |   7609MiB /  7982MiB |      5%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      1424      G   /usr/lib/xorg/Xorg                733MiB |
|    0   N/A  N/A      1767      G   /usr/bin/gnome-shell              426MiB |
|    0   N/A  N/A      6420      G   /usr/lib/firefox/firefox            2MiB |
|    0   N/A  N/A      6949      G   /usr/lib/firefox/firefox            2MiB |
|    0   N/A  N/A      8888      G   /usr/lib/firefox/firefox            2MiB |
|    0   N/A  N/A     10610      G   /usr/lib/firefox/firefox            2MiB |
|    0   N/A  N/A     14943      G   /usr/lib/firefox/firefox            2MiB |
|    0   N/A  N/A     16181      C   ...mona/anaconda3/bin/python     6429MiB |
+-----------------------------------------------------------------------------+

I have an NVIDIA GeForce RTX 2070 GPU.
I also switched these two lines and still no chance and still same error:
images = images.permute(0,3,1,2)
images = images.cuda()

I also checked the nvidia-smi exactly after running all the cells above this cell, and none of them are causing this CUDA out of memory error.


title: pytorch with cuda 10.1 mac os 10.13.6 , UserWarning: CUDA initialization: Found no NVIDIA driver on your system

body:
import torch
print(torch.version.cuda)
10.1
torch.cuda.is_available()
/Users/djennelbaroudhadjcheikh/anaconda3/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  ../c10/cuda/CUDAFunctions.cpp:104.)
return torch._C._cuda_getDeviceCount() > 0
False

even if i installed the web driver for mac os version and for my nvidia geforce gt 650m
using a command to get the right version or patching a version
i have a macbook pro 2012 mi
so anyone can help me with ?


title: RuntimeError: inconsistent tensor sizes at /pytorch/torch/lib/TH/generic/THTensorMath.c:2864

body:
Im trying to build a dataloader, This is what it looks like

`class WhaleData(Dataset):
def __init__(self, data_file, root_dir , transform = None):
    self.csv_file = pd.read_csv(data_file)
    self.root_dir = root_dir
    self.transform = transforms.Resize(224)

def __len__(self):
    return len(os.listdir(self.root_dir))

def __getitem__(self, index):
    image = os.path.join(self.root_dir, self.csv_file['Image'][index])
    image = Image.open(image)
    image = self.transform(image)
    image = np.array(image)
    label  = self.csv_file['Image'][index]
    sample = {'image': image, 'label':label}
    return sample

trainset  = WhaleData(data_file = '/mnt/55-91e8-b2383e89165f/Ryan/1234/train.csv',
     root_dir = '/mnt/4d55-91e8-b2383e89165f/Ryan/1234/train')
train_loader = torch.utils.data.DataLoader(trainset , batch_size = 4, shuffle =True,num_workers= 2)
for i, batch in enumerate(train_loader):
      (i, batch)

When i try running this block of code, i get this error,I do get the nature of the error that all my images may not be of the same shape,and my images are not all of the same shape ,But if im not wrong the error should only arise when i feed them to the network because the images are all of different shapes,But why is it throwing an errror here?
Any suggestions on where i might have gone wrong will be extremly helpful,
I would be happy to provide any extra information if needed,

Thanks

RuntimeError: Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 42, in _worker_loop
samples = collate_fn([dataset[i] for i in batch_indices])
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 116, in default_collate
    return {key: default_collate([d[key] for d in batch]) for key in batch[0]}
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 116, in <dictcomp>
    return {key: default_collate([d[key] for d in batch]) for key in batch[0]}
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/dataloader.py", line 105, in default_collate
return torch.stack([torch.from_numpy(b) for b in batch], 0)
  File "/usr/local/lib/python3.5/dist-packages/torch/functional.py", line 64, in stack
    return torch.cat(inputs, dim)
RuntimeError: inconsistent tensor sizes at /pytorch/torch/lib/TH/generic    /THTensorMath.c:2864


title: Torch Dataset Looping too far

body:
Why does this dataset try to iterate past the final element

from torch.utils.data.dataset import Dataset
class DumbDataset(Dataset):
    def __init__(self, dct):
        self.dct = dct
        self.mapping = dict(enumerate(dct))
    def __getitem__(self, index):
        return self.dct[self.mapping[index]]

    def __len__(self):
        print('called')
        return len(self.dct)

ds = DumbDataset({'a': 'aword', 'b': 'another_words'})

for k in ds: print(k)

This raises KeyError: 2, which I don't understand since length of the object is 2. Shouldn't the iterator get StopIteration once it is exhausted?