title: PyTorch (GPU) slower than CPU slower than keras

body:
I'm just getting started with PyTorch and I wanted to run through a few toy problems. In the following case, I'm noticing a significant difference in how much time it takes for the model to train once over and issue one batch of predictions.
This is the PyTorch implementation. On the GPU, it takes ~17 seconds on my machine. The same model on the CPU takes ~11 seconds.
class LR(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.linear1 = torch.nn.Linear(2, 20)
        self.linear2 = torch.nn.Linear(20, 1)

    def forward(self, x):
        x = torch.nn.functional.relu(self.linear1(x))
        x = torch.nn.functional.relu(self.linear2(x))
        return x

def fit_torch(df_train, df_test):
    sampler_tr = torch.utils.data.SubsetRandomSampler(df_train.index)
    train = torch.utils.data.DataLoader(
        torch.tensor(df_train.values, dtype=torch.float),
        batch_size=batch_size, sampler=sampler_tr)

    sampler_te = torch.utils.data.SubsetRandomSampler(df_test.index)
    test = torch.utils.data.DataLoader(
        torch.tensor(df_test.values, dtype=torch.float),
        batch_size=batch_size, sampler=sampler_te)

    model = LR()
    model = model.to(device)

    loss = torch.nn.MSELoss()
    optim = torch.optim.Adam(model.parameters(), lr=0.001)

    model.train()
    for _ in range(1000):
        for train_data in train:
            train_data = train_data.to(device)

            x_train = train_data[:, :2]
            y_train = train_data[:, 2]

            optim.zero_grad()

            pred = model(x_train)
            loss_val = loss(pred.squeeze(), y_train)

            loss_val.backward()
            optim.step()

    model.eval()
    with torch.no_grad():
        for test_data in test:
            test_data = test_data.to(device)

            pred = model(test_data[:, :2].float())
            break

This is the keras implementation. It takes approximately 9 seconds to run.
def fit_tf(df_train, df_test):
    model = tf.keras.models.Sequential()
    model.add(tf.keras.layers.Dense(20, activation='relu'))
    model.add(tf.keras.layers.Dense(1, activation='relu'))

    model.compile(loss='mse', optimizer='adam')
    model.fit(
        df_train.values[:, :2],
        df_train.values[:, 2],
        batch_size=batch_size, epochs=1000, verbose=0)

    model.predict(df_test.iloc[:batch_size].values[:, :2])

The dataset and main functions.
device = torch.device('cuda:0')
scaler = MinMaxScaler()

batch_size = 64

def create_dataset():
    dataset = []
    random_x = np.random.randint(10, 1000, 1000)
    random_y = np.random.randint(10, 1000, 1000)

    for x, y in zip(random_x, random_y):
        dataset.append((x, y, 4 * x + 3 * y + 10))

    np.random.shuffle(dataset)
    df = pd.DataFrame(dataset)
    df = pd.DataFrame(scaler.fit_transform(df))

    return df

def __main__():
    df = create_dataset()
    df_train, df_test = train_test_split(df)

    start_time = time.time()
    fit_tf(df_train.reset_index(drop=True), df_test.reset_index(drop=True))
    print(time.time() - start_time)


title: PyTorch LSTM not learning in training

body:
I have the following simple LSTM network:
class LSTMModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):
        super().__init__()
        self.hidden_dim = hidden_dim
        self.layer_dim = layer_dim
        self.rnn = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)
        self.fc = nn.Linear(hidden_dim, output_dim)
        self.batch_size = None
        self.hidden = None

    def forward(self, x):
        h0, c0 = self.init_hidden(x)
        out, (hn, cn) = self.rnn(x, (h0, c0))
        out = self.fc(out[:, -1, :])
        return out

    def init_hidden(self, x):
        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)
        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim)
        return [t for t in (h0, c0)]

I am initialising this model as"
model = LSTMClassifier(28, 10, 6, 1)

i.e. each input instance has 6 time steps and the dimension of each time step is 28, and the hidden dimension is 10. The inputs are being mapped to an output dim of 1.
The training data is being prepared in batches of size 16, meaning that the data passed in the training loop has the shape:
torch.Size([16, 6, 28])

With labels of shape:
batches[1][0].size()

An example of the input is:
tensor([[-0.3674,  0.0347, -0.2169, -0.0821, -0.3673, -0.1773,  1.1840, -0.2669,
         -0.4202, -0.1473, -0.1132, -0.4756, -0.3565,  0.5010,  0.1274, -0.1147,
          0.2783,  0.0836, -1.3251, -0.8067, -0.6447, -0.7396, -0.3241,  1.3329,
          1.3801,  0.8198,  0.6098,  0.0697],
        [-0.2710,  0.1596, -0.2524, -0.0821, -0.3673, -0.1773,  0.0302, -0.2099,
         -0.4550,  0.1451, -0.4561, -0.5207, -0.5657, -0.5287, -0.2690, -0.1147,
         -0.0346, -0.1043, -0.7515, -0.8392, -0.4745, -0.7396, -0.3924,  0.8122,
         -0.1624, -1.2198,  0.0326, -0.9306],
        [-0.1746,  0.0972, -0.2702, -0.0821, -0.3673, -0.1773, -0.0468, -1.1225,
         -0.4480, -0.4397,  0.4011, -1.1073, -1.0536, -0.1855, -0.7502, -0.1147,
         -0.0146, -0.1545, -0.1919, -0.1674,  0.0930, -0.7396,  0.8106,  1.1594,
          0.4546, -1.2198, -0.5446, -1.2640],
        [-0.2710,  0.0660, -0.2524, -0.0821, -0.4210, -0.1773,  1.8251, -0.5236,
         -0.4410, -0.7321,  0.4011, -0.6110, -0.2171,  1.1875, -0.2973, -0.1147,
         -0.1278,  0.7728, -0.9334, -0.5141, -2.1202,  1.3521, -0.9393,  0.5085,
         -0.4709,  0.8198, -1.1218,  0.0697],
        [-0.3674, -0.0277, -0.2347, -0.0821, -0.0448, -0.1773,  0.2866, -0.1386,
         -0.4271,  0.4375, -0.2847, -0.1146, -0.4262, -0.3571, -0.0425, -0.1147,
         -0.4207, -0.4552, -0.5277, -0.9584, -0.4177, -0.7396, -0.2967,  0.5085,
          0.4546, -1.2198, -0.3522, -1.2640],
        [-0.3674, -0.1447, -0.1991, -0.0821,  0.1701, -0.1773,  0.0430,  0.1324,
         -0.4271,  0.7299, -0.4561,  0.2915, -0.5657, -0.1855, -0.2123, -0.1147,
         -0.0413, -0.8311, -0.6396, -1.0451, -0.4177, -0.7396, -0.2967, -0.4028,
          0.7631, -1.2198, -0.3522, -1.2640]])

When I train the model as:
Epochs = 10
batch_size = 32
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=5e-4)
for epoch in range(Epochs):
    print(f"Epoch {epoch + 1}")
    for n, (X, y) in enumerate(batches):
        model.train()
        optimizer.zero_grad()
        y_pred = model(X)
        loss = criterion(y_pred, y)
        loss.backward()
        optimizer.step()

    model.eval()
    accurate = 0
    for X_instance, y_instance in zip(test_X, test_y):
        if y_instance == round(model(X_instance.view(-1, 6, 28)).detach().item()):
            accurate += 1
    print(f"Accuracy test set: {accurate/len(test_X)}")

The accuracy does not converge:
Epoch 1
Accuracy test set: 0.23169107856191745
Sample params:
tensor([-0.3356, -0.0105, -0.3405, -0.0049,  0.0037,  0.1707,  0.2685, -0.3893,
        -0.4707, -0.2872, -0.1544, -0.1455,  0.0393,  0.0774, -0.4194,  0.0780,
        -0.2177, -0.3829, -0.4679,  0.0370, -0.0794,  0.0455, -0.1331, -0.0169,
        -0.1551, -0.0348,  0.1746, -0.5163], grad_fn=<SelectBackward>)
tensor([ 0.2137, -0.2558,  0.1509, -0.0975,  0.5591,  0.0907, -0.1249,  0.3095,
         0.2112,  0.3134, -0.1581, -0.3051, -0.3559, -0.0177,  0.1485,  0.4397,
        -0.1441,  0.1705,  0.3230, -0.3236,  0.0692,  0.0920, -0.2691, -0.3695,
        -0.0692,  0.3747,  0.0149,  0.5216], grad_fn=<SelectBackward>)
Epoch 2
Accuracy test set: 0.23049267643142476
Sample params:
tensor([-0.3483, -0.0144, -0.3512,  0.0213, -0.0081,  0.1777,  0.2674, -0.4031,
        -0.4628, -0.3041, -0.1651, -0.1511,  0.0216,  0.0513, -0.4320,  0.0839,
        -0.2602, -0.3629, -0.4541,  0.0398, -0.0768,  0.0432, -0.1150, -0.0160,
        -0.1346, -0.0727,  0.1801, -0.5253], grad_fn=<SelectBackward>)
tensor([ 0.1879, -0.2534,  0.1461, -0.1141,  0.5735,  0.0872, -0.1286,  0.3273,
         0.2084,  0.3037, -0.1535, -0.2934, -0.3870, -0.0252,  0.1492,  0.4752,
        -0.1709,  0.1776,  0.3390, -0.3318,  0.0734,  0.1077, -0.2790, -0.3777,
        -0.0518,  0.3726,  0.0228,  0.5404], grad_fn=<SelectBackward>)
Epoch 3
Accuracy test set: 0.22982689747003995
Sample params:
tensor([-0.3725, -0.0069, -0.3623,  0.0393, -0.0167,  0.1748,  0.2577, -0.4183,
        -0.4681, -0.3196, -0.1657, -0.1613,  0.0122,  0.0268, -0.4361,  0.0838,
        -0.2962, -0.3566, -0.4344,  0.0366, -0.0822,  0.0486, -0.1150, -0.0295,
        -0.1080, -0.1094,  0.1841, -0.5336], grad_fn=<SelectBackward>)
tensor([ 0.1664, -0.2456,  0.1477, -0.1332,  0.5820,  0.0819, -0.1228,  0.3426,
         0.2066,  0.2985, -0.1464, -0.2824, -0.4199, -0.0323,  0.1530,  0.5057,
        -0.1991,  0.1856,  0.3407, -0.3347,  0.0800,  0.1203, -0.2791, -0.3863,
        -0.0426,  0.3760,  0.0327,  0.5641], grad_fn=<SelectBackward>)
Epoch 4
Accuracy test set: 0.23249001331557922
Sample params:
tensor([-0.3945,  0.0032, -0.3765,  0.0600, -0.0248,  0.1713,  0.2442, -0.4297,
        -0.4741, -0.3311, -0.1653, -0.1667,  0.0029,  0.0066, -0.4373,  0.0738,
        -0.3320, -0.3530, -0.4136,  0.0390, -0.0731,  0.0552, -0.1117, -0.0517,
        -0.0871, -0.1455,  0.1841, -0.5359], grad_fn=<SelectBackward>)
tensor([ 0.1495, -0.2292,  0.1524, -0.1473,  0.5938,  0.0661, -0.1157,  0.3626,
         0.2013,  0.2927, -0.1350, -0.2661, -0.4558, -0.0411,  0.1562,  0.5381,
        -0.2279,  0.1927,  0.3319, -0.3431,  0.0852,  0.1402, -0.2747, -0.4026,
        -0.0297,  0.3757,  0.0396,  0.5856], grad_fn=<SelectBackward>)

Have I made a mistake in the model definition?


title: Defining Loss function in pytorch

body:
I have to define a huber loss function which is this:
This is my code
def huber(a, b):
   res = (((a-b)[abs(a-b) < 1]) ** 2 / 2).sum()
   res += ((abs(a-b)[abs(a-b) >= 1]) - 0.5).sum()
   res = res / torch.numel(a)
   return res

'''
yet, it is not working properly. Do you have any idea what is wrong?


title: Issue with pytorch LSTM source code

body:
I am using bidirectional LSTM with batch_first=True. However, it is throwing me an error regarding dimensions.

**Error:
Expected hidden[0] size (6, 5, 40), got (5, 6, 40)**

When I checked the source code, the error occurred due to below function

if is_input_packed:
            mini_batch = int(batch_sizes[0])
        else:
            mini_batch = input.size(0) if self.batch_first else input.size(1)

        num_directions = 2 if self.bidirectional else 1
        expected_hidden_size = (self.num_layers * num_directions,
                                mini_batch, self.hidden_size)

        def check_hidden_size(hx, expected_hidden_size, msg='Expected hidden size {}, got {}'):
            if tuple(hx.size()) != expected_hidden_size:
                raise RuntimeError(msg.format(expected_hidden_size, tuple(hx.size())))

By default expected_hidden_size is written with respect to sequence first. I believe it is causing the problem. Can someone advise if I am right and the issue needs to be fixed?


title: How to use an optimizer within a forward pass in PyTorch

body:
I want to use an optimizer within the forward pass of a custom defined Function, but it doesn't work. My code is as follows:

class MyFct(Function):

   @staticmethod
   def forward(ctx, *args):
       input, weight, bias = args[0], args[1], args[2]

       y = torch.tensor([[0]], dtype=torch.float, requires_grad=True) #initial guess
       loss_fn = lambda y_star: (input + weight - y_star)**2

       learning_rate = 1e-4
       optimizer = torch.optim.Adam([y], lr=learning_rate)
       for t in range(5000):
           y_star = y
           print(y_star)
           loss = loss_fn(y_star)
           if t % 100 == 99:
               print(t, loss.item())
           optimizer.zero_grad()
           loss.backward()
           optimizer.step()

       return y_star

And that's my test inputs:

x = torch.tensor([[2]], dtype=torch.float, requires_grad=True)
w = torch.tensor([[2]], dtype=torch.float, requires_grad=True)
y = torch.tensor([[6]], dtype=torch.float)

fct= MyFct.apply
y_hat = fct(x, w, None)

I always get the RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn.

Also, I've tested the optimization outside of the forward and it works, so I guess it's something with the context? According to the documentation "Tensor arguments that track history (i.e., with requires_grad=True) will be converted to ones that don’t track history before the call, and their use will be registered in the graph", see https://pytorch.org/docs/stable/notes/extending.html. Is this the problem? Is there a way to work around it?

I am new to PyTorch and I wonder what I'm overlooking. Any help and explanation is appreciated.


title: Can't train ResNet using gpu with pytorch

body:
I'm trying to use gpu to train a ResNet architecture on CIFAR10 dataset. Here's my code for ResNet :

import torch
import torch.nn as nn
import torch.nn.functional as F

class ResNetBlock(nn.Module):

    def __init__(self, in_planes, planes, stride=1):
        super(ResNetBlock, self).__init__()
        self.stride = stride
        self.in_planes=in_planes
        self.planes = planes
        if stride!=1:

          self.fx = nn.Sequential(nn.Conv2d(in_planes, planes, 3, stride=2,
                                            padding=1),
                                  nn.ReLU(),
                                  nn.Conv2d(planes, planes,3, padding=1))

        else:
          self.fx = nn.Sequential(nn.Conv2d(planes, planes, 3, padding = 1),
                                  nn.ReLU(),
                                  nn.Conv2d(planes, planes,3, padding=1))

    def forward(self, x):

      if self.stride ==1:
        fx = self.fx(x)
        id = nn.Sequential()
        out = fx + id(x)
        relu = nn.ReLU()
        return relu(out)

      else:

        fx = self.fx(x)
        id = nn.Conv2d(self.in_planes, self.planes, 2, stride = 2)
        out = fx + id(x)
        relu = nn.ReLU()
        return relu(out)

class ResNet(nn.Module):

  def __init__(self, block, num_blocks, num_classes=10, num_filters=16, input_dim=3):
      super(ResNet, self).__init__()
      self.in_planes = num_filters

      self.conv1 = nn.Conv2d(input_dim, num_filters, kernel_size=3, stride=1, padding=1, bias=False)
      self.bn1 = nn.BatchNorm2d(num_filters)

      layers = []
      plane = num_filters

      for nb in num_blocks:

        layer = self._make_layer(block,plane ,nb,2)
        layers.append(layer)
        plane*=2

      self.layers = nn.Sequential(*layers)

      self.linear = nn.Linear(2304, num_classes)

  def _make_layer(self, block, planes, num_blocks, stride):
      layers = []
      block1 = ResNetBlock(planes, 2*planes, stride = 2)
      planes *=2
      layers.append(block1)

      for i in range(1,num_blocks):
        block = ResNetBlock(planes, planes, stride =1)
        layers.append(block)

      return nn.Sequential(*layers)

  def forward(self, x):
      out = F.relu(self.bn1(self.conv1(x)))
      out = self.layers(out)

      out = F.avg_pool2d(out, 4)
      out = out.view(out.size(0), -1)
      out = self.linear(out)
      return out

# (1 + 2*(1 + 1) + 2*(1 + 1) + 2*(1 + 1) + 2*(1 + 1)) + 1 = 18
def ResNet18():
    return ResNet(ResNetBlock, [2,2,2,2])

Then I train the network using gpu :

net = ResNet18()
net = net.to('cuda')
train2(net, torch.optim.Adam(net.parameters(), lr=0.001), trainloader, criterion, n_ep=3)

And I get the error :

RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

which is annoying because my weights should be cuda as well because of the resnet.cuda().

With another network the train function works well, so it must come from the classes mentioned above.

Also, next(resnet.parameters()).is_cuda returns True.

Update : Here's my training function.

def train(net, optimizer, trainload, criterion, n_ep=10, cuda = True):
  if cuda:
    net = net.to('cuda')

  for epoch in range(n_ep):
    for data in trainload:

      inputs, labels = data
      if cuda:
        inputs = inputs.type(torch.cuda.FloatTensor)
        labels = labels.type(torch.cuda.LongTensor)

      optimizer.zero_grad()

      print(next(net.parameters()).is_cuda)
      ## this actually prints "True" !

      outputs = net.forward(inputs)
      loss = criterion(outputs, labels)
      loss.backward()
      optimizer.step()

  return net

The thing is, this training function works well with another type of net. For example is used this one (AlexNet) :

class AlexNet(nn.Module):

    def __init__(self, num_classes=1000):
        super(AlexNet, self).__init__()
        self.features = nn.Sequential(nn.Conv2d(3,64,11), nn.ReLU(),nn.MaxPool2d(2, stride = 2), nn.Conv2d(64,192,5),
                                     nn.ReLU(), nn.MaxPool2d(2, stride = 2), nn.Conv2d(192,384,3),
                                     nn.ReLU(),nn.Conv2d(384,256,3), nn.ReLU(), nn.Conv2d(256,256,3), nn.ReLU())
        self.avgpool = nn.AdaptiveAvgPool2d((6, 6))
        self.classifier = nn.Sequential(
            nn.Dropout(),
            nn.Linear(256 * 6 * 6, 4096),
            nn.ReLU(inplace=True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(inplace=True),
            nn.Linear(4096, num_classes),)

    def forward(self, x):
        x = self.features(x)
        x = self.avgpool(x)
        x = x.view(x.size(0), 256 * 6 * 6)
        x = self.classifier(x)
        return x

and with this one the gpu training works well.

There's something else I don't understand. I tried to train a network that I moved to GPU (using .cuda()  ) with training data that I did not move to GPU (on purpose). And this time I get the error that weights type is torch.cuda and data type isn't.

EDIT : I thought it had to do with using nn.ModuleList instead of regular python lists. However I tried that and it has not fixed the issue.


title: What is the appropriate way to use BCE loss with ResNet outputs?

body:
The title explains the overall problem, but for some elaboration:

I'm using torchvision.models.resnet18() to run an anomaly detection scheme. I initialize the model by doing:

net = torchvision.models.resnet18(num_classes=2)

since in my particular setting 0 equals normal samples and 1 equals anomalous samples.

The output from my model is of shape (16, 2) (batch size is 16) and labels are of size (16, 1). This gives me the error that the two input tensors are of inappropriate shape.

In order to solve this, I tried something like:

>>> new_output = torch.argmax(output, dim=1)

Which gives me the appropriate shape, but running loss = nn.BCELoss(new_output, labels) gives me the error:

RuntimeError: bool value of Tensor with more than one value is ambiguous

What is the appropriate way for me to approach this issue? Thanks.

Edit

I've also tried using nn.CrossEntropyLoss as well, but am getting the same RuntimeError.

More specifically, I tried nn.CrossEntropyLoss(output, label) and nn.CrossEntropyLoss(output, label.flatten()).


title: Pytorch: after embedding layer, Unable to get repr for

body:
I am new to PyTorch and trying to reproduce the project: https://github.com/eXascaleInfolab/ActiveLink
However, errors occur in the feedforward() which has been bothering me for days, here is part of the code (for complete code of the model, see https://github.com/eXascaleInfolab/ActiveLink/blob/master/models.py please):
def forward(self, e1, rel, batch_size=None, weights=None):
......
        e1_embedded = self.emb_e(e1).view(-1, 1, 10, 20)
        rel_embedded = self.emb_rel(rel).view(-1, 1, 10, 20)
        stacked_inputs = torch.cat([e1_embedded, rel_embedded], 2)  # out: (128L, 1L, 20L, 20L)

That gives me the error (I am using GPU):
THCudaCheck FAIL file=/pytorch/aten/src/THC/generic/THCTensorMath.cu line=196 error=710 : device-side assert triggered
Traceback (most recent call last):
  File "main.py", line 147, in <module>
    main()
  File "main.py", line 136, in main
    model = run_meta_incremental(config, model, train_batcher, test_rank_batcher)
  File "/home/yonghui/yt/meta_incr_training.py", line 158, in run_meta_incremental
    g = run_inner(config, model, task)
  File "/home/yonghui/yt/meta_incr_training.py", line 120, in run_inner
    pred = model.forward(e1, rel)
  File "/home/yonghui/yt/models.py", line 136, in forward
    stacked_inputs = torch.cat([e1_embedded, rel_embedded], 2)
RuntimeError: cuda runtime error (710) : device-side assert triggered at /pytorch/aten/src/THC/generic/THCTensorMath.cu:196
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [0,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [1,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [2,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [3,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [4,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [5,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [6,0,0] Assertion `srcIndex < srcSelectDimSize` failed.
/pytorch/aten/src/THC/THCTensorIndex.cu:361: void indexSelectLargeIndex(TensorInfo<T, IndexType>, TensorInfo<T, IndexType>, TensorInfo<long, IndexType>, int, int, IndexType, IndexType, long) [with T = float, IndexType = unsigned int, DstDim = 2, SrcDim = 2, IdxDim = -2, IndexIsMajor = true]: block: [189,0,0], thread: [7,0,0] Assertion `srcIndex < srcSelectDimSize` failed.

I use Debugger in an attempt to find out where goes wrong:
Before e1 and rel are embedded, they are both tensors in int64 with the shape of torch.Size([128, 1]).
e1 can be embedded as normal, converting into torch.float32 and torch.Size([128, 1, 10, 20]). However, after rel passed the embedding layer of emb_rel, Debugger shows all tenros as Unable to get repr for <class 'torch.Tensor'>.
What's going on? How can I fix that? Thank you for any possible help!!


title: Efficient way to compute Jacobian x Jacobian.T

body:
Assume J is the Jacobian of some function f with respect to some parameters. Are there efficient ways (in PyTorch or perhaps Jax) to have a function that takes two inputs (x1 and x2) and computes J(x1)*J(x2).transpose() without instantiating the entire J matrices in memory?
I have come across something like jvp(f, input, v=vjp(f, input)) but don't quite understand it and not sure is what I want.


title: java.lang.IllegalArgumentException: No OpKernel was registered to support Op 'GatherV2' with these attrs.

body:
We are trying to feed an image into our tensorflow model but we keep getting this error:

ERROR Part 1

ERROR Part 2

We used pytorch initially, converted it into an ONNX model, and then to a tensorflow model to be used in Android Studio. We are able to load the model successfully, but after that we run into errors.

Thanks in advance to those that respond!


title: Problem when converting Pytorch Image Classifier to mlmodel: Returns same softmax output regardless of img

body:
I trained and tested an image classifier (Resnet34, Fast.ai, 3 classes) using pytorch and learn.predict() works as expected. When I convert pytorch -> onnx -> mlmodel it predicts the same softmax values regardless of the image I submit.

Here's my pytorch model:

Sequential(
  (0): Sequential(
    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    (4): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (1): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (5): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (3): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (6): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (3): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (4): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (5): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (7): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
  )
  (1): Sequential(
    (0): AdaptiveConcatPool2d(
      (ap): AdaptiveAvgPool2d(output_size=1)
      (mp): AdaptiveMaxPool2d(output_size=1)
    )
    (1): Flatten()
    (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (3): Dropout(p=0.25, inplace=False)
    (4): Linear(in_features=1024, out_features=512, bias=True)
    (5): ReLU(inplace=True)
    (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (7): Dropout(p=0.5, inplace=False)
    (8): Linear(in_features=512, out_features=3, bias=True)
  )
)

To convert it to .onnx, I need to first normalize the image data and flatten it. I found this tutorial, which worked on a previous version of fastai/onnx-coreml. I do this with the following class:

sz = (960,540)
class ImageScale(nn.Module):
    def __init__(self):
        super().__init__()
        self.denominator = torch.full((3, sz[0], sz[1]), 255.0, device=torch.device("cuda"))

    def forward(self, x): return torch.div(x, self.denominator).unsqueeze(0)

To construct the entire model, I concatenate my ImageScale layer, the model, and a softmax function like this:

final_model = [ImageScale()] + [learn.model] + [nn.Softmax(dim=-1)]
final_model = nn.Sequential(*final_model)

Which ends up looking like this:

Sequential(
  (0): ImageScale()
  (1): Sequential(
    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (2): ReLU(inplace=True)
    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    (4): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (1): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (5): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (3): BasicBlock(
        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (6): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (3): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (4): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (5): BasicBlock(
        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (7): Sequential(
      (0): BasicBlock(
        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (downsample): Sequential(
          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        )
      )
      (1): BasicBlock(
        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
      (2): BasicBlock(
        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu): ReLU(inplace=True)
        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
  )
  (2): Sequential(
    (0): AdaptiveConcatPool2d(
      (ap): AdaptiveAvgPool2d(output_size=1)
      (mp): AdaptiveMaxPool2d(output_size=1)
    )
    (1): Flatten()
    (2): BatchNorm1d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (3): Dropout(p=0.25, inplace=False)
    (4): Linear(in_features=1024, out_features=512, bias=True)
    (5): ReLU(inplace=True)
    (6): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (7): Dropout(p=0.5, inplace=False)
    (8): Linear(in_features=512, out_features=3, bias=True)
  )
  (3): Softmax(dim=-1)
)

I convert to .onnx like this:

dummy_input = Variable(torch.randn(3, sz[0], sz[1])).cuda()
torch.onnx.export(final_model, dummy_input, 'model.onnx', input_names = ['input'], output_names =['output'], verbose=True)

And I convert from .onnx to .mlmodel like this:

model_file = open('model.onnx', 'rb')
model_proto = onnx_pb.ModelProto()
model_proto.ParseFromString(model_file.read())
coreml_model = convert(model_proto, image_input_names = ['image'], mode='classifier', class_labels="labels.txt")
coreml_model.save('model.mlmodel')

When I call predict using coremltools, I get the same output regardless of the image I input:

import coremltools
from PIL import Image
model =  coremltools.models.MLModel('model.mlmodel')
img = Image.open('img.jpg')
preds = model.predict({'image': img})
# preds: {'output': {'class1': 0.011085365898907185, 'class2': 0.9794686436653137, 'class2': 0.009446004405617714}, 'classLabel': 'class2'}
img2 = Image.open('img2.jpg')
preds = model.predict({'image': img2})
# preds: {'output': {'class1': 0.011085365898907185, 'class2': 0.9794686436653137, 'class2': 0.009446004405617714}, 'classLabel': 'class2'}

Possible issues:
1. I'm not setting up the Sequential correctly before converting
2. Coreml or onnx cannot handle non-square images

I've tried a bunch of different inputs, but keep getting the same so any help would be much appreciated!

Here are screen shots of my head and tail from netron:

Head:

Tail:


title: During Pytorch tutorial a ModuleNotFoundError: ‘pycocotools._mask’ occurs

body:
Hello I’m new to Pytorch and I’ve been trying to work through this tutorial.
[https://github.com/pytorch/tutorials/blob/master/intermediate_source/torchvision_tutorial.rst]

I’m using Visual Studio Code, conda installed the Pytorch. Initially the issue was that it was missing the module engine. But this problem was solved here:
So how do we get the references/detection/ folders? What should we download and install? I have installed the pytorch, torchvision in my environment, but I could not find those files. Thanks
With a step I misread.
So I downloaded and copied the pycocotools into the project directory and the vision/detection/ .py files in to it too.
That being done it fixed that issue but the error it is now giving is.

(base) C:\Users\Sean\Desktop\Project\Test\Tutorial>D:/Anaconda/python.exe c:/Users/Sean/Desktop/Project/Test/Tutorial/tv-training-code.py
Traceback (most recent call last):
  File "c:/Users/Sean/Desktop/Project/Test/Tutorial/tv-training-code.py", line 13, in <module>
    from engine import train_one_epoch, evaluate
  File "c:\Users\Sean\Desktop\Project\Test\Tutorial\engine.py", line 8, in <module>
    from coco_utils import get_coco_api_from_dataset
  File "c:\Users\Sean\Desktop\Project\Test\Tutorial\coco_utils.py", line 9, in <module>
    from pycocotools import mask as coco_mask
  File "c:\Users\Sean\Desktop\Project\Test\Tutorial\pycocotools\mask.py", line 3, in <module>
    import pycocotools._mask as _mask
ModuleNotFoundError: No module named 'pycocotools._mask'

I’m not quite sure what the issue here is or how to fix it. Given that the _mask.pyx is present and is what I think is being imported. As I said I’m not to sure what the issue is but I would appreciate the help.

Python Version : 3.7.4
Pytorch: 1.2.0 (Cuda 10)


title: Unable to allocate GPU memory, when there is enough of cached memory

body:
I am training vgg16 model from scratch on AWS EC2 Deep Learning AMI machine (Ubuntu 18.04.3 LTS (GNU/Linux 4.15.0-1054-aws x86_64v)) with Python3 (CUDA 10.1 and Intel MKL) (Pytorch 1.3.1) and facing below error while updating model parameters.

  RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 0; 11.17 GiB total capacity; 10.76 GiB already allocated; 4.81 MiB free; 119.92 MiB cached)

Code for updating parameters:

def _update_fisher_params(self, current_ds, batch_size, num_batch):
    dl = DataLoader(current_ds, batch_size, shuffle=True)
    log_liklihoods = []
    for i, (input, target) in enumerate(dl):
        if i > num_batch:
            break
        output = F.log_softmax(self.model(input.cuda().float()), dim=1)
        log_liklihoods.append(output[:, target])
    log_likelihood = torch.cat(log_liklihoods).mean()
    grad_log_liklihood = autograd.grad(log_likelihood, self.model.parameters())
    _buff_param_names = [param[0].replace('.', '__') for param in self.model.named_parameters()]
    for _buff_param_name, param in zip(_buff_param_names, grad_log_liklihood):
        self.model.register_buffer(_buff_param_name+'_estimated_fisher', param.data.clone() ** 2)

After debugging: log_liklihoods.append(output[:, target]) line throws error after 157 iterations

I have the required memory but it does not allocate, I am not getting why updating the gradients is causing the memory problem, as gradients should be de-referenced and released automatically on each iteration. Any idea?

I have tried following solutions but no luck.

Lowering batch size
Freeing cache with torch.cuda.empty_cache()
Reducing the number of filters to reduce the memory footprint

Machine Specs:


title: how "data" and "target" are choosen in a federated learning? (PySyft)

body:
i can't understand how in function train() below, the variable (data, target) are choosen.
def train(args, model, device, federated_train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(federated_train_loader): # <-- now it is a distributed dataset
        model.send(data.location) # <-- NEW: send the model to the right location`

i guess they are 2 tensor representing 2 random images of dataset train, but then the loss function
loss = F.nll_loss(output, target)

is calculated at every interaction with different target?
Also i have different question: i trained the network with images of cats, then i test it with images of cars and the accuracy reached is 97%. How is this possible? is a proper value or i'm doing something wrong?
here is the entire code:
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms

import syft as sy  # <-- NEW: import the Pysyft library
hook = sy.TorchHook(torch)  # <-- NEW: hook PyTorch ie add extra functionalities to support Federated Learning
bob = sy.VirtualWorker(hook, id="bob")  # <-- NEW: define remote worker bob
alice = sy.VirtualWorker(hook, id="alice")  # <-- NEW: and alice

class Arguments():
    def __init__(self):
        self.batch_size = 64
        self.test_batch_size = 1000
        self.epochs = 2
        self.lr = 0.01
        self.momentum = 0.5
        self.no_cuda = False
        self.seed = 1
        self.log_interval = 30
        self.save_model = False

args = Arguments()

use_cuda = not args.no_cuda and torch.cuda.is_available()

torch.manual_seed(args.seed)

device = torch.device("cuda" if use_cuda else "cpu")

kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}

federated_train_loader = sy.FederatedDataLoader( # <-- this is now a FederatedDataLoader
    datasets.MNIST("C:\\users...\\train", train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ]))
    .federate((bob, alice)), # <-- NEW: we distribute the dataset across all the workers, it's now a FederatedDataset
    batch_size=args.batch_size, shuffle=True, **kwargs)

test_loader = torch.utils.data.DataLoader(
    datasets.MNIST("C:\\Users...\\test", train=False, download=True, transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=args.test_batch_size, shuffle=True, **kwargs)

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5, 1)
        self.conv2 = nn.Conv2d(20, 50, 5, 1)
        self.fc1 = nn.Linear(4*4*50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(self.conv1(x))
        x = F.max_pool2d(x, 2, 2)
        x = F.relu(self.conv2(x))
        x = F.max_pool2d(x, 2, 2)
        x = x.view(-1, 4*4*50)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

def train(args, model, device, federated_train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(federated_train_loader): # <-- now it is a distributed dataset
        model.send(data.location) # <-- NEW: send the model to the right location
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data)
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        model.get() # <-- NEW: get the model back
        if batch_idx % args.log_interval == 0:
            loss = loss.get() # <-- NEW: get the loss back
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * args.batch_size, len(federated_train_loader) * args.batch_size,
                100. * batch_idx / len(federated_train_loader), loss.item()))

def test(args, model, device, test_loader):
    model.eval()
    test_loss = 0
    correct = 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            test_loss += F.nll_loss(output, target, reduction='sum').item() # sum up batch loss
            pred = output.argmax(1, keepdim=True) # get the index of the max log-probability
            correct += pred.eq(target.view_as(pred)).sum().item()

    test_loss /= len(test_loader.dataset)

    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

model = Net().to(device)
optimizer = optim.SGD(model.parameters(), lr=args.lr) # TODO momentum is not supported at the moment

for epoch in range(1, args.epochs + 1):
    train(args, model, device, federated_train_loader, optimizer, epoch)
    test(args, model, device, test_loader)

if (args.save_model):
    torch.save(model.state_dict(), "mnist_cnn.pt")


title: How to train faster-rcnn on dataset including negative data in pytorch

body:
I am trying to train the torchvision Faster R-CNN model for object detection on my custom data. I used the code in torchvision object detection fine-tuning tutorial. But getting this error:
Expected target boxes to be a tensor of shape [N, 4], got torch.Size([0])

This has to do with negative data (empty training images / no bounding boxes) in my custom dataset. How can we change the below Dataset class to enable training faster-rcnn on dataset including negative data?
class MyCustomDataset(Dataset):

    def __init__(self, root, transforms):
        self.root = root
        self.transforms = transforms
        # load all image files, sorting them to
        # ensure that they are aligned
        self.imgs = list(sorted(os.listdir(os.path.join(root, "PNGImages"))))
        self.masks = list(sorted(os.listdir(os.path.join(root, "PedMasks"))))

    def __len__(self):
        return len(self.imgs)

    def __getitem__(self, idx):
        # load images ad masks
        img_path = os.path.join(self.root, "PNGImages", self.imgs[idx])
        mask_path = os.path.join(self.root, "PedMasks", self.masks[idx])
        img = Image.open(img_path).convert("RGB")
        # note that we haven't converted the mask to RGB,
        # because each color corresponds to a different instance
        # with 0 being background
        mask = Image.open(mask_path)
        # convert the PIL Image into a numpy array
        mask = np.array(mask)
        # instances are encoded as different colors
        obj_ids = np.unique(mask)
        # first id is the background, so remove it
        obj_ids = obj_ids[1:]

        # split the color-encoded mask into a set of binary masks
        masks = mask == obj_ids[:, None, None]

        # get bounding box coordinates for each mask
        num_objs = len(obj_ids)

        boxes = []
        for i in range(num_objs):
            pos = np.where(masks[i])
            xmin = np.min(pos[1])
            xmax = np.max(pos[1])
            ymin = np.min(pos[0])
            ymax = np.max(pos[0])
            boxes.append([xmin, ymin, xmax, ymax])

        # convert everything into a torch.Tensor
        boxes = torch.as_tensor(boxes, dtype=torch.float32)
        # there is only one class
        labels = torch.ones((num_objs,), dtype=torch.int64)
        image_id = torch.tensor([idx])
        area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])
        # suppose all instances are not crowd
        iscrowd = torch.zeros((num_objs,), dtype=torch.int64)

        target = {}
        target["boxes"] = boxes
        target["labels"] = labels
        target["image_id"] =  torch.tensor([idx])
        target["area"] = area
        target["iscrowd"] = iscrowd

        return img, target


title: Pytorch: CNN don't learn anything after torch.cat()?

body:
I try to concatenate Variable in the network with code like this

    x = self.layer1(x)
    x = self.layer2(x)
    x = self.layer3(x)
    x = self.layer4(x)

    x = x.view(x.size(0), -1)
    x= torch.cat((x,angle),1) # from here I concat it.
    x = self.dropout1(self.relu1(self.bn1(self.fc1(x))))
    x = self.dropout2(self.relu2(self.bn2(self.fc2(x))))
    x = self.fc3(x)

And then I find my network learn nothing and give acc always around 50%. So I print param.grad and as I expected, they are all nan. Does anyone encounter this thing before?

I ran the code without concatenation before and it works out well. So I suppose this is where the rub is and the system doesn't throw any error or exception. if any other backup info is needed, please let me know.

Thank you.


title: How to optimize lower Cholesky Parameter in Pytorch?

body: Is there any way to create a parameter which is lower triangular with positive diagonal and enforce this constraint during optimization in Pytorch?


title: Pytorch Sigmoid Function what is e

body:
I have a question on setting up the sigmoid function in pytroch.

So I define it as this

# Sigmoid function
def sigmoid(x):
    return 1/(1 + torch.exp(-x))

But then looking at the sigmoid function from here http://mathworld.wolfram.com/SigmoidFunction.html

The sigmoid should be defined as

y = 1/(1 + e^-x)

I see the 1/(1+  part but I don't get the e^-x part. Can someone explain why

torch.exp(-x) == e^-x

What is e here? Is that the tensor. But i thought that x was the tensor


title: Gradient of the loss of DistilBERT for measuring token importance

body:
I am trying to access the gradient of the loss in DistilBERT with respect to each attention weight in the first layer. I could access the computed gradient value of the output weight matrix via the following code when requires_grad=True

loss.backward()
for name, param in model.named_parameters():
    if name == 'transformer.layer.0.attention.out_lin.weight':
       print(param.grad)  #shape is [768,768]

where model is the loaded distilbert model.
My question is how to get the gradient with respect to [SEP] or [CLS] or other tokens' attention? I need it to reproduce the figure about the "Gradient-based feature importance estimates for attention to [SEP]" in the following link:
https://medium.com/analytics-vidhya/explainability-of-bert-through-attention-7dbbab8a7062

A similar question for the same purpose has been asked in the following, but it is not my issue:
BERT token importance measuring issue. Grad is none


title: How to use pytorch's cat function for K-Fold Validation (i.e. concatenate a list of pytorch chunks together)

body:
I'm working on splitting up a data set for k-fold cross validation but having trouble with concatenating a list of tensors using Pytorch's stack/cat functions.
First, I split the training and test set into chunks using the .chunk method as follows
  x_train_folds = torch.chunk(x_train, num_folds)
  y_train_folds = torch.chunk(y_train, num_folds)

Where x_train is a tensor of torch.Size([5000, 3, 32, 32]) and y_train is a tensor of torch.Size([5000])
x_train_folds and y_train_folds are now tuples of num_folds tensors
Then, I need to setup a series of nested loops to iterate through the different values for K, and the various folds, while always excluding one fold from the training set to be used at test/validation time:
  for k in k_choices:
    k_to_accuracies[k] = [] # create empty space to append for a given k-value
    for fold in range(num_folds):
      # create training sets by excluding the current loop index fold and using that as the test set
      x_train_cross_val = torch.cat((x_train_folds[:fold], x_train_folds[fold+1:]), 0)
      y_train_cross_val = torch.cat((y_train_folds[:fold], y_train_folds[fold+1:]), 0)
      classifier = KnnClassifier(x_train_cross_val, y_train_cross_val)
      k_to_accuracies[k].append(classifier.check_accuracy(x_train_folds[fold], y_train_folds[fold], k=k))

As you can see, I'm always skipping one fold from the original training set to be used for validation. This is standard K-fold cross validation.
Unfortunately, I am getting the following error which I cannot seem to figure out:
TypeError: expected Tensor as element 0 in argument 0, but got tuple
As you can see in the API listing, .cat seems to need a tuple of tensors which is what I have.
https://pytorch.org/docs/stable/torch.html#torch.cat
Does anyone have any suggestions?
Much appreciated
-Drew


title: Cannot iterate through Pytorch Dataloader . Unequal tensor

body:
I am training a BERT base model on the imdb dataset. However i am unable to iterate throught the Pytorch Dataloader .
Here is the code fully reproducible. Training file is the above mentioned csv.
import transformers
from sklearn import model_selection
import torch
import pandas as pd

tokenizer = transformers.BertTokenizer.from_pretrained("bert-base-cased", do_lower_case=True)
max_len = 512
train_batch_size = 8

"""
This class takes reviews and targets as arguments
- Split the reviews and tokenizes
"""
class BERTDataset:
    def __init__(self, review, target):
        self.review = review
        self.target = target
        self.tokenizer = tokenizer
        self.max_len = max_len

    def __len__(self):
        return len(self.review)

    def __getitem__(self, item):
        review = str(self.review[item])
        review = " ".join(review.split())

        tokenized_inputs = self.tokenizer.encode_plus(
            review,
            None,
            add_special_tokens=True,
            max_length=self.max_len,
            padding=True,
            truncation=True
        )

        ids = tokenized_inputs["input_ids"]
        mask = tokenized_inputs["attention_mask"]
        token_type_ids = tokenized_inputs["token_type_ids"]

        return {
            "ids": torch.tensor(ids, dtype=torch.long),
            "mask": torch.tensor(mask, dtype=torch.long),
            "token_type_ids": torch.tensor(token_type_ids, dtype=torch.long),
            "targets": torch.tensor(self.target[item], dtype=torch.float),
        }

dfx = pd.read_csv(training_file).fillna("none")
dfx['sentiment'] = dfx['sentiment'].apply(lambda x: 1 if x == 'positive' else 0)

df_train, df_valid = model_selection.train_test_split(
                        dfx,
                        test_size=0.1,
                        random_state=42,
                        stratify=dfx['sentiment'].values
                        )

# reset indices
df_train = df_train.reset_index(drop=True)

# get ids, tokens, masks and targets
train_dataset = BERTDataset(review=df_train['review'], target=df_train['sentiment'])

# load into pytorch dataset object
# DataLoader inputs tensor dataset of Inputs and targets
train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=train_batch_size, num_workers=0)

# Iterating to the Data loader
train_iter = iter(train_data_loader)
print(type(train_iter))

review, labels = train_iter.next()

The following error throws up.
    RuntimeError Traceback (most recent call last)
    <ipython-input-19-c99d0829d5d9> in <module>()
          2 print(type(train_iter))
          3
    ----> 4 images, labels = train_iter.next()

    5 frames
    /usr/local/lib/python3.6/dist-packages/torch/utils/data/_utils/collate.py in default_collate(batch)
         53             storage = elem.storage()._new_shared(numel)
         54             out = elem.new(storage)
    ---> 55         return torch.stack(batch, 0, out=out)
         56     elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
         57             and elem_type.__name__ != 'string_':

    RuntimeError: stack expects each tensor to be equal size, but got [486] at entry 0 and [211] at entry 1

There was a memory issue which was i fixed by num_workers=0 in the Dataloader. Can someone help in figuring out what i am missing here ?


title: How to handle imbalanced classes in transformers pytorch binary classification

body:
I am working on a binary text classification problem. How do I apply SMOTE or WeightedRandomSampler for the imbalance in my dataset? My code currently looks like this:
class GDataset(Dataset):

  def __init__(self, passage, targets, tokenizer, max_len):
    self.passage = passage
    self.targets = targets
    self.tokenizer = tokenizer
    self.max_len = max_len

  def __len__(self):
    return len(self.passage)

  def __getitem__(self, item):
    passage = str(self.passage[item])
    target = self.targets[item]

    if (target == 1) and self.transform: # minority class
            x = self.transform(x)

    encoding = self.tokenizer.encode_plus(
      passage,
      add_special_tokens=True,
      max_length=self.max_len,
      return_token_type_ids=False,
      pad_to_max_length=True,
      return_attention_mask=True,
      return_tensors='pt',
    )
    return
      'passage_text': passage,
      'input_ids': encoding['input_ids'].flatten(),
      'attention_mask': encoding['attention_mask'].flatten(),
      'targets': torch.tensor(target, dtype=torch.long

How can I use other balancing techniques?


title: Installing pytorch on python 3.7 installed ubuntu gives an error

body:
Hi all am new to pytorch and trying to run a model based on pytorch.My workstation has python 3.7 installed.So as per pytorch official site when I try to install python with below command

pip3 install http://download.pytorch.org/whl/cpu/torch-0.4.1.post2-cp37-cp37m-linux_x86_64.whl

It gives error like below

torch-0.4.1.post2-cp37-cp37m-linux_x86_64.whl is not a supported wheel on this platform.

Can any one help me with this.


title: PyTorch MaxPool2D unexpected behavior with padding=1

body:
I was playing around with MaxPool2D in PyTorch and discovered strange behavior when setting padding=1. Here is what I got:

Code:

import torch
from torch.nn.functional import max_pool2d

TEST = 1

def test_maxpool(negative=False, tnsr_size=2, kernel_size=2, stride=2, padding=0):
    """Test MaxPool2D.
    """
    global TEST
    print(f'=== TEST {TEST} ===')
    print(*[f'{i[0]}: {i[1]}' for i in locals().items()], sep=' | ')

    inp = torch.arange(1., tnsr_size ** 2 + 1).reshape(1, tnsr_size, tnsr_size)
    inp = -inp if negative else inp

    print('In:')
    print(inp)

    out = max_pool2d(inp, kernel_size, stride, padding=padding)
    print('Out:')
    print(out)
    print()
    TEST += 1

test_maxpool()
test_maxpool(True)

test_maxpool(padding=1)
test_maxpool(True, padding=1)

Out:

=== TEST 1 ===
negative: False | tnsr_size: 2 | kernel_size: 2 | stride: 2 | padding: 0
In:
tensor([[[1., 2.],
         [3., 4.]]])
Out:
tensor([[[4.]]])

=== TEST 2 ===
negative: True | tnsr_size: 2 | kernel_size: 2 | stride: 2 | padding: 0
In:
tensor([[[-1., -2.],
         [-3., -4.]]])
Out:
tensor([[[-1.]]])

=== TEST 3 ===
negative: False | tnsr_size: 2 | kernel_size: 2 | stride: 2 | padding: 1
In:
tensor([[[1., 2.],
         [3., 4.]]])
Out:
tensor([[[1., 2.],
         [3., 4.]]])

=== TEST 4 ===
negative: True | tnsr_size: 2 | kernel_size: 2 | stride: 2 | padding: 1
In:
tensor([[[-1., -2.],
         [-3., -4.]]])
Out:
tensor([[[-1., -2.],
         [-3., -4.]]])

Tests 1, 2, 3 are fine but Test 4 is weird, I expected to get [[0 0], [0 0]] tensor:

In:
[[-1 -2]
 [-3 -4]]

+ padding ->

[[ 0  0  0  0]
 [ 0 -1 -2  0]
 [ 0 -3 -4  0]
 [ 0  0  0  0]]

-> kernel_size=2, stride=2 ->

[[0 0]
 [0 0]]

According to Test 3 zero padding was used but Test 4 produced controversial result.

What kind of padding (if any) was that? Why does MaxPool2D behave like that?

pytorch 1.3.1


title: how many epochs required for model with lstm training

body:
I am having nn Actor Critic TD3 model with LSTM in my AI.
For every training, I am creating batches of sequential data and training my AI.
Can someone expert please help to let know if I require epochs as well for this AI.And in general how many epochs can I run with this code, because I am creating many batches on one training step is it feasible to have epochs as well.
Below is training step code
def train(
        self,
        replay_buffer,
        iterations,
        batch_size=50,
        discount=0.99,
        tau=0.005,
        policy_noise=0.2,
        noise_clip=0.5,
        policy_freq=2,
        ):

        b_state = torch.Tensor([])
        b_next_state = torch.Tensor([])
        b_done = torch.Tensor([])
        b_reward = torch.Tensor([])
        b_action = torch.Tensor([])

        for it in range(iterations):

            # print ('it: ', it, ' iterations: ', iterations)

      # Step 4: We sample a batch of transitions (s, s’, a, r) from the memory

            (batch_states, batch_next_states, batch_actions,
             batch_rewards, batch_dones) = \
                replay_buffer.sample(batch_size)

            batch_states = batch_states.astype(float)
            batch_next_states = batch_next_states.astype(float)
            batch_actions = batch_actions.astype(float)
            batch_rewards = batch_rewards.astype(float)
            batch_dones = batch_dones.astype(float)

            state = torch.from_numpy(batch_states)
            next_state = torch.from_numpy(batch_next_states)
            action = torch.from_numpy(batch_actions)
            reward = torch.from_numpy(batch_rewards)
            done = torch.from_numpy(batch_dones)

            b_size = 1
            seq_len = state.shape[0]
            batch = b_size
            input_size = state_dim

            state = torch.reshape(state, ( 1,seq_len, state_dim))
            next_state = torch.reshape(next_state, ( 1,seq_len, state_dim))
            done = torch.reshape(done, ( 1,seq_len, 1))
            reward = torch.reshape(reward, ( 1, seq_len, 1))
            action = torch.reshape(action, ( 1, seq_len, action_dim))

            b_state = torch.cat((b_state, state),dim=0)
            b_next_state = torch.cat((b_next_state, next_state),dim=0)
            b_done = torch.cat((b_done, done),dim=0)
            b_reward = torch.cat((b_reward, reward),dim=0)
            b_action = torch.cat((b_action, action),dim=0)

            # state = torch.reshape(state, (seq_len, 1, state_dim))
            # next_state = torch.reshape(next_state, (seq_len, 1,
            #         state_dim))
            # done = torch.reshape(done, (seq_len, 1, 1))
            # reward = torch.reshape(reward, (seq_len, 1, 1))
            # action = torch.reshape(action, (seq_len, 1, action_dim))

            # b_state = torch.cat((b_state, state),dim=1)
            # b_next_state = torch.cat((b_next_state, next_state),dim=1)
            # b_done = torch.cat((b_done, done),dim=1)
            # b_reward = torch.cat((b_reward, reward),dim=1)
            # b_action = torch.cat((b_action, action),dim=1)

        print("dim state:",b_state.shape)

      # for h and c shape (num_layers * num_directions, batch, hidden_size)

        ha0 = torch.zeros(lstm_layers, b_state.shape[0], state_dim)
        ca0 = torch.zeros(lstm_layers, b_state.shape[0], state_dim)
        hc0 = torch.zeros(lstm_layers, b_state.shape[0], state_dim + action_dim)
        cc0 = torch.zeros(lstm_layers, b_state.shape[0], state_dim + action_dim)
        # Step 5: From the next state s’, the Actor target plays the next action a’

        b_next_action = self.actor_target(b_next_state, (ha0, ca0))
        b_next_action = b_next_action[0]

        # Step 6: We add Gaussian noise to this next action a’ and we clamp it in a range of values supported by the environment

        noise = torch.Tensor(b_next_action).data.normal_(0,
                policy_noise)
        noise = noise.clamp(-noise_clip, noise_clip)
        b_next_action = (b_next_action + noise).clamp(-self.max_action,
                self.max_action)

        # Step 7: The two Critic targets take each the couple (s’, a’) as input and return two Q-values Qt1(s’,a’) and Qt2(s’,a’) as outputs

        result = self.critic_target(b_next_state, b_next_action, (hc0,cc0))
        target_Q1 = result[0]
        target_Q2 = result[1]

        # Step 8: We keep the minimum of these two Q-values: min(Qt1, Qt2)

        target_Q = torch.min(target_Q1, target_Q2).double()

        # Step 9: We get the final target of the two Critic models, which is: Qt = r + γ * min(Qt1, Qt2), where γ is the discount factor

        target_Q = b_reward + (1 - b_done) * discount * target_Q

        # Step 10: The two Critic models take each the couple (s, a) as input and return two Q-values Q1(s,a) and Q2(s,a) as outputs

        b_action_reshape = torch.reshape(b_action, b_next_action.shape)
        result = self.critic(b_state, b_action_reshape, (hc0, cc0))
        current_Q1 = result[0]
        current_Q2 = result[1]

        # Step 11: We compute the loss coming from the two Critic models: Critic Loss = MSE_Loss(Q1(s,a), Qt) + MSE_Loss(Q2(s,a), Qt)

        critic_loss = F.mse_loss(current_Q1, target_Q) \
            + F.mse_loss(current_Q2, target_Q)

        # Step 12: We backpropagate this Critic loss and update the parameters of the two Critic models with a SGD optimizer

        self.critic_optimizer.zero_grad()
        critic_loss.backward()
        self.critic_optimizer.step()

        # Step 13: Once every two iterations, we update our Actor model by performing gradient ascent on the output of the first Critic model

        out = self.actor(b_state, (ha0, ca0))
        out = out[0]
        (actor_loss, hx, cx) = self.critic.Q1(b_state, out, (hc0,cc0))
        actor_loss = -1 * actor_loss.mean()
        self.actor_optimizer.zero_grad()
        actor_loss.backward()
        self.actor_optimizer.step()

        # Step 14: Still once every two iterations, we update the weights of the Actor target by polyak averaging

        for (param, target_param) in zip(self.actor.parameters(),
                self.actor_target.parameters()):
            target_param.data.copy_(tau * param.data + (1 - tau)
                    * target_param.data)

        # Step 15: Still once every two iterations, we update the weights of the Critic target by polyak averaging

        for (param, target_param) in zip(self.critic.parameters(),
                self.critic_target.parameters()):
            target_param.data.copy_(tau * param.data + (1 - tau)
                    * target_param.data)


title: How to do 2-layer nested FOR loop in PYTORCH?

body:
I am learning to implement the Factorization Machine in Pytorch.
And there should be some feature crossing operations.
For example, I've got three features [A,B,C], after embedding, they are [vA,vB,vC], so the feature crossing is "[vA·vB], [vA·vC], [vB·vc]".

I know this operation can be simplified by the following:

It can be implemented by MATRIX OPERATIONS.
But this only gives a final result, say, a single value.

The question is, how to get all cross_vec in the following without doing FOR loop:
note: size of "feature_emb" is [batch_size x feature_len x embedding_size]

    g_feature = 0
    for i in range(self.featurn_len):
        for j in range(self.featurn_len):
            if j <= i: continue
            cross_vec = feature_emb[:,i,:] * feature_emb[:,j,:]
            g_feature += torch.sum(cross_vec, dim=1)


title: Computational graph vs (computer algebra) symbolic expression

body:
I was reading Baydin et al, Automatic Differentiation in Machine Learning: a Survey, 2018 (Arxiv), which differentiates between symbolic differentiation and automatic differentiation (AD). It then says:

  AD Is Not Symbolic Differentiation.
  Symbolic differentiation is the automatic manipulation of [symbolic] expressions.

  AD can be thought of as performing a non-standard interpretation of a computer program
  where this interpretation involves augmenting the standard computation with the calculation of various derivatives.

  Evaluation traces form the basis of the AD techniques.
  [A computational graph (Bauer, 1974) visualizes dependency relations of (input, working, output) variables in evaluation traces.]

It then goes on by describing how to compute the derivative with AD (in forward or backward mode). The description is basically transforming the evaluation trace / computational graph.

  Autograd, Chainer, and PyTorch provide general-purpose reverse mode AD.

It also discusses Theano, TensorFlow, and others, but it basically compares define-and-run / static computational graph (Theano, TF) vs define-by-run / dynamic computational graph (PyTorch, TF Eager).
(This would be orthogonal in my understanding to the question of how AD is performed, or would mostly just change how AD is implemented, but not so much the concept of AD.)

  Theano is a computational graph optimizer and compiler [...] and it currently handles
  derivatives in a highly optimized form of symbolic differentiation. The result can be interpreted as a
  hybrid of symbolic differentiation and reverse mode AD, but Theano does not use the general-purpose
  reverse accumulation as we describe in this paper. (Personal communication with the authors.)

I'm not sure if the authors imply that Theano/TF do not provide general-purpose reverse mode AD (which would be wrong in my understanding).

I don't exactly understand how Theano does not use the general-purpose reverse accumulation.

Also, I don't understand how symbolic differentiation is different from AD, given this definition.

Or: How are symbolic expressions different from computational graphs?

Related is also differentiable programming

  differentiable directed graphs assembled from functional blocks

where I again do not see the difference to a computational graph.

And backpropagation (BP):

  The resulting algorithm is essentially equivalent to transforming the network evaluation function composed with the objective function under reverse mode AD, which, as we
  shall see, actually generalizes the backpropagation idea.

I don't see how reverse mode AD is more general than backpropagation. Is it? How?

Schmidhuber, Deep Learning in Neural Networks: An Overview, 2014 (section 5.5) (also) states:

  BP is also known as the reverse mode of automatic differentiation (Griewank, 2012).


title: Problems with RandomApply on PyTorch

body:
I'm trying to test my network (trained with the original training set) on a noisy test set.
To do so I wrote
valid_and_test_set = torchvision.datasets.MNIST("./mnist", train=False, download=True)
dataset_valid, dataset_test = torch.utils.data.random_split(valid_and_test_set,[5000, 5000])

class Convert(object):
    def __call__(self, img):
        return torch.unsqueeze(torch.from_numpy(np.array(img)), 0).float()

class Flatten(object):
    def __call__(self, img):
        return img.view(28*28)

class OneHot(object):
    def __call__(self, label):
        return label

class AddGaussianNoise(object):
def __init__(self, mean=0., std=1.):
    self.std = std
    self.mean = mean
def __call__(self, img):
    return img + torch.randn(img.size()) * self.std + self.mean

transform_gauss = transforms.Compose(
    [Convert(),
     transforms.Normalize(mean=[mu], std=[std]),
     Flatten(), torchvision.transforms.RandomApply([AddGaussianNoise()], p=prob)])

dataset_test.dataset.transform = transform_gauss
dataloader_test = torch.utils.data.DataLoader(dataset_test.dataset, batch_size=5000, num_workers=num_workers, pin_memory=True)

Am I missing something? Because I get the same accuracy with prob=1.0 and prob=0.5


title: I need help in installing tensorflow, keras, and pytorch

body:
When I import Tensorflow, Keras or Pytorch I face these errors:

I'm using Window 10 and my CPU supports AVX.
Also I was preparing my project on Python(Anaconda) and installed Keras, TensorFlow using pip(pip install keras) and Pytorch using conda
I activated my virtual environment(Test_01) before download, and I checked it again with conda list after download.
After checking those 3 programs are installed, I imported them with ipython on Anaconda prompt. and this what I faced:
========================================================================
(Test_01) C:\Users\User>ipython

Python 3.8.3 (default, Jul  2 2020, 17:30:36) [MSC v.1916 64 bit (AMD64)]
Type 'copyright', 'credits' or 'license' for more information
IPython 7.16.1 -- An enhanced Interactive Python. Type '?' for help.

In [1]: import tensorflow

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-1-d6579f534729> in <module>
----> 1 import tensorflow

ModuleNotFoundError: No module named 'tensorflow'

In [2]: import keras

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
C:\Anaconda3\lib\site-packages\keras\__init__.py in <module>
      2 try:
----> 3     from tensorflow.keras.layers.experimental.preprocessing import RandomRotation
      4 except ImportError:

ModuleNotFoundError: No module named 'tensorflow'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
<ipython-input-2-88d96843a926> in <module>
----> 1 import keras

C:\Anaconda3\lib\site-packages\keras\__init__.py in <module>
      3     from tensorflow.keras.layers.experimental.preprocessing import RandomRotation
      4 except ImportError:
----> 5     raise ImportError(
      6         'Keras requires TensorFlow 2.2 or higher. '
      7         'Install TensorFlow via `pip install tensorflow`')

ImportError: Keras requires TensorFlow 2.2 or higher. Install TensorFlow via `pip install tensorflow`

In [3]: import torch

---------------------------------------------------------------------------
ModuleNotFoundError                       Traceback (most recent call last)
<ipython-input-3-eb42ca6e4af3> in <module>
----> 1 import torch

ModuleNotFoundError: No module named 'torch'

===========================================================================

Keras error message says:

Keras requires TensorFlow 2.2 or higher. Install TensorFlow via
pip install TensorFlow

also, tensorflow and pytorch says

ModuleNotFoundError: No module named 'tensorflow'
ModuleNotFoundError: No module named 'torch'

So I installed TensorFlow with pip again, and it says:
(Test_01) C:\Users\User>pip install tensorflow

Requirement already satisfied: tensorflow in c:\anaconda3\envs\test_01\lib\site-packages (2.3.1)

Requirement already satisfied: grpcio>=1.8.6 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (1.33.2)

Requirement already satisfied: wrapt>=1.11.1 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (1.12.1)

Requirement already satisfied: gast==0.3.3 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (0.3.3)

Requirement already satisfied: protobuf>=3.9.2 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (3.13.0)

Requirement already satisfied: tensorboard<3,>=2.3.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (2.3.0)

Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (2.3.0)

Requirement already satisfied: google-pasta>=0.1.8 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (0.2.0)

Requirement already satisfied: absl-py>=0.7.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (0.11.0)

Requirement already satisfied: six>=1.12.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (1.15.0)

Requirement already satisfied: wheel>=0.26 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (0.35.1)

Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (1.1.2)
Collecting numpy<1.19.0,>=1.16.0
Using cached numpy-1.18.5-cp38-cp38-win_amd64.whl (12.8 MB)

Requirement already satisfied: termcolor>=1.1.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (1.1.0)

Requirement already satisfied: astunparse==1.6.3 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (1.6.3)

Requirement already satisfied: opt-einsum>=2.3.2 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (3.3.0)

Requirement already satisfied: h5py<2.11.0,>=2.10.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorflow) (2.10.0)

Requirement already satisfied: setuptools in c:\anaconda3\envs\test_01\lib\site-packages (from protobuf>=3.9.2->tensorflow) (50.3.0.post20201006)

Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (0.4.2)

Requirement already satisfied: markdown>=2.6.8 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (3.3.3)

Requirement already satisfied: requests<3,>=2.21.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (2.24.0)

Requirement already satisfied: google-auth<2,>=1.6.3 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (1.23.0)

Requirement already satisfied: werkzeug>=0.11.15 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (1.0.1)

Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in c:\anaconda3\envs\test_01\lib\site-packages (from tensorboard<3,>=2.3.0->tensorflow) (1.7.0)

Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\anaconda3\envs\test_01\lib\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow) (1.3.0)

Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\anaconda3\envs\test_01\lib\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (1.25.11)

Requirement already satisfied: chardet<4,>=3.0.2 in c:\anaconda3\envs\test_01\lib\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (3.0.4)

Requirement already satisfied: certifi>=2017.4.17 in c:\anaconda3\envs\test_01\lib\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (2020.6.20)

Requirement already satisfied: idna<3,>=2.5 in c:\anaconda3\envs\test_01\lib\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow) (2.10)

Requirement already satisfied: cachetools<5.0,>=2.0.0 in c:\anaconda3\envs\test_01\lib\site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow) (4.1.1)

Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\anaconda3\envs\test_01\lib\site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow) (0.2.8)

Requirement already satisfied: rsa<5,>=3.1.4; python_version >= "3.5" in c:\anaconda3\envs\test_01\lib\site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow) (4.6)

Requirement already satisfied: oauthlib>=3.0.0 in c:\anaconda3\envs\test_01\lib\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow) (3.1.0)

Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\anaconda3\envs\test_01\lib\site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow) (0.4.8)
Installing collected packages: numpy
  Attempting uninstall: numpy
    Found existing installation: numpy 1.19.2
    Uninstalling numpy-1.19.2:
      Successfully uninstalled numpy-1.19.2

ERROR: After October 2020 you may experience errors when installing or updating packages. This is because pip will change the way that it resolves dependency conflicts.

We recommend you use --use-feature=2020-resolver to test your packages with the new resolver before it becomes the default.

torch 1.6.0 requires future, which is not installed.
Successfully installed numpy-1.18.5

===============================================================================

I promise I checked that I installed and executed them on same environment(Test_01) so Can you tell me:
What Should I do ? and How can I check my 'CUDA' version? I tried it with 'nvxx --version' on cmd and nothing works


title: pytorch allocate memory for small size tensor on cpu and gpu but got error on a node with more than 400 GB

body:
I would like to build a torch.nn.embedding with tensors on databricks (the node is p2.8xlarge) by py3.
My code:
  import numpy as np
  import torch
  from torch import nn

  num_embedding, num_dim = 14000, 300
  embedding = nn.Embedding(num_embedding, num_dim)
  row, col = 800000, 302
  t = [[x for x in range(col)] for _ in range(row)]

  t1 = torch.tensor(t)
  print(t1.shape) # torch.Size([800000, 302])

  t1.dtype, t1.nelement() # torch.int64, 241600000

  type(t1), t1.device, (t1.nelement() * t1.element_size())/(1024**3) # (torch.Tensor, device(type='cpu'), 1.8000602722167969)

  tt = embedding(t1) # error [enforce fail at CPUAllocator.cpp:64] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 288,000,000,000 bytes. Error code 12 (Cannot allocate memory)

  t2 = t1.cuda()
  t2.device, t2.shape, t2.grad, t2.nelement(), t2.element_size(), (t2.nelement() * t2.element_size())/(1024**3) # (device(type='cuda', index=0), torch.Size([800000, 302]), None, 241600000, 8, 1.8000602722167969)

  embedding_cuda = embedding.cuda()
  embedding_cuda(t2) #  CUDA out of memory. Tried to allocate 270.01 GiB (GPU 0; 11.17 GiB total capacity; 7.16 GiB already allocated; 2.01 GiB free; 8.88 GiB reserved in total by PyTorch)

I do not understand why the size of the given tensor is less than 2GB (1.8 GB) but it cannot be located to cpu and gpu ?  Why cpu and gpu have to allocate so large size of 270.01 GiB  ?
Did I miss out something ?
thanks


title: Numerical differences between manually computed gradient and PyTorch chain rule

body:
I am computing the gradient myself manually using formulas for this very simple linear network with a MSE loss.

I then compare with the gradient that is computed by PyTorch and use the allclose function from PyTorch to check that the gradients are correctly computed by PyTorch (i.e. the relative difference between the manually computed gradients and pytorch is small enough).
All the tests should pass as the formulas are correct. But for some seeds it just doesn't.

So obviously PyTorch isn't doing anything wrong but as the formulas are correct it must comes from some numerical instabilities issues in the formulas.

import torch

class Network(torch.nn.Module):
    def __init__(self):
        super(Network, self).__init__()

        self.linear = torch.nn.Linear(10, 1)

    def forward(self, x):
        return self.linear(x)

loss = torch.nn.MSELoss()

for i in range(0, 1000):
    torch.manual_seed(i)
    X = torch.randn(100, 10)
    y = torch.randn(100, 1)

    model=Network()
    model.train()
    optimizer=torch.optim.SGD(model.parameters(),lr=1.)
    optimizer.zero_grad()
    output = loss(model(X), y)
    output.backward()

    torch_grads=[]
    for p in model.parameters():
        torch_grads.append(p.grad.detach().data)

    #df/dW = (-2X.T*y+2*X.T*b+2*X.T*X*W)/nsamples
    #df/db = (2*b-2*y+2*W.T*X.T).mean() (the mean comes from implicit broadcasting of b)

    theory_grad_w = (-2 * torch.matmul(X.t(), y)
                     +2 * torch.matmul(torch.t(X), torch.ones((X.shape[0], 1)))* list(model.parameters())[1]
                     +2 * torch.matmul(torch.matmul(X.t(), X), list(model.parameters())[0].t())
                     ) / float(X.shape[0])

    theory_grad_w = theory_grad_w.t()

    theory_grad_b = torch.mean(2 * list(model.parameters())[1]- 2 * y+ 2 * torch.matmul((list(model.parameters())[0]), torch.t(X)))

    theory_grads = [theory_grad_w, theory_grad_b]

    b=all([torch.allclose(u, d) for u, d in zip(torch_grads, theory_grads)])
    if not(b):

      print("i=%s, pass=%s"%(i, b))

What are the sources of the observed numerical instabilities and how to deal with them so that tests would pass all the time. Is it just a matter of ordering the operations differently ?


title: Reshape and pad a tensor given a list of lengths

body:
I have given a 2d tensor in of shape a x b like the following (where a = 9 and each of A1, A2, ..., C2 represents a b-dimensional vector):

Furthermore, I have an array of lengths, where sum(lengths) = a and each entry is a positive integer:

Then I would like to obtain a 3d output tensor out, where the first lengths[0] entries of in form the first row, the next lengths[1] entries of in form the second row, and so on. That is, the output tensor should have the shape len(lengths) x max(lengths) x b, and be padded with zeros (each 0 in the below picture represents a b-dimensional zero vector):

As this is part of a neural network that is trained using backpropagation, all operations used must be differentiable. How can this be achieved (ideally, with good performance) using PyTorch?


title: Problems setting up FastAi working on Colab

body:
I'm trying to work through a tutorial on CycleGANs using the Colab platform but I'm struggling to find a way through the 'simple' part of just importing libraries.

I'm just trying to import the following:

from fastai.conv_learner import *
from fastai.dataset import *
from cgan.options.train_options import *
from sklearn.model_selection import train_test_split
from cgan.options.train_options import TrainOptions
from cgan.data.data_loader import CreateDataLoader
from cgan.models.models import create_model
from cgan.util.visualizer import Visualizer
from google_images_download import google_images_download

I'm currently stuck with an error on the first line:

----> 7 class IntermediateLayerGetter(nn.ModuleDict):
      8     """
      9     Module wrapper that returns intermediate layers from a model

AttributeError: module 'torch.nn' has no attribute 'ModuleDict'

The code can be found online: https://colab.research.google.com/drive/1dMQWStzLfAHDTGuKaUNQn1aOBWeJw4FN


title: I have a pytorch image classifier training, and I want to pause training and save the weights at time of program pause. Can I do this?

body: I'm in the middle of training a classifier that's been training for a few days now, but my problem is that I  didn't code in to save .pt checkpoints throughout training, and so I'll only end up with a weights file when the program is done with all of its epochs. Is there a way to pause training (PAUSE BREAK) and save the model's weights right now?


title: 5 fold cross validation using pytorch

body:
Need to perform 5 fold cross validation on my dataset. I was able to find 2 examples of doing this but could not integrate to my current pipeline.Could anyone please help me with this.

###############################################################################################
class leukemiaClassifier(Dataset):
    def __init__(self, csv_file, transform):

        self.data = pd.read_csv(csv_file)
        self.data = self.data
        self.transform = transform

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        img_name = self.data.loc[idx][0]
        img = Image.open(img_name).convert('RGB')

        img = cv2.imread(img_name)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        image = self.transform(image=img)
        image = image['image']
        labels = torch.tensor(self.data.loc[idx][1])
        return image, labels

train_file = 'train.csv'
val_file = 'test.csv'
batch_size = 28

train_dataset = leukemiaClassifier(
    csv_file=train_file,transform = data_transforms)

val_dataset = leukemiaClassifier(
    csv_file=val_file, transform = data_transforms_test)

read_target = pd.read_csv('train.csv')
target = read_target['label'].values

data_loader_train = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True,num_workers=64)
data_loader_val = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=False, num_workers=64)
###############################################################################################
#Model Utils Hyperparameter etc
###############################################################################################

def get_lr(optimizer):
    for param_group in optimizer.param_groups:
        return param_group['lr']

def efficientnet(version,num_classes):
    model = EfficientNet.from_pretrained('efficientnet-b{}'.format(version), num_classes=num_classes)
    num_ftrs = model._fc.in_features
    model._fc = nn.Linear(num_ftrs, num_classes)
    return model.cuda()

target_names = ['Lymphocyte(atypical)', 'Monoblast', 'Promyelocyte(bilobed)', 'Metamyelocyte', 'Erythroblast', 'Neutrophil(segmented)', 'Myeloblast','Promyelocyte','Monocyte','Lymphocyte(typical)','Neutrophil(band)','Smudge cell', 'Eosinophil',  'Myelocyte',   'Basophil']
model= efficientnet(5,15)
model = nn.DataParallel(model)
wandb.watch(model)
# criterion = torch.nn.CrossEntropyLoss()
criterion = FocalLoss()
labels = torch.tensor((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14)).cuda()
no_of_classes = 15
optimizer = optim.Adam(model.parameters(), lr=0.01)
scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.01, patience=5, verbose=True)
global_validation_loss= 100
###############################################################################################
#Training and validation loop
###############################################################################################
for epoch in range(300000): # loop over the dataset multiple times
    running_loss = 0.0
    label_list =[]
    predicted_list=[]
    model = model.train()
    for batch_idx, data in enumerate( tqdm.tqdm(data_loader_train)):

        inputs,labels = data
        inputs,labels = inputs.cuda(), labels.cuda().long()

        # zero the parameter gradients
        optimizer.zero_grad()
        # forward + backward + optimize
        outputs = model(inputs)
        loss = criterion(outputs,labels)
        # print(loss)
        loss.backward()
        optimizer.step()
        for item in torch.argmax(outputs,1).detach().cpu().numpy():predicted_list.append(item)
        for item in labels.detach().cpu().numpy():label_list.append(item)
        # print statistics
        running_loss += loss.item()