title: LSTM for time-series prediction failing to learn (PyTorch)

body:
I'm currently working on building an LSTM network to forecast time-series data using PyTorch. I tried to share all the code pieces that I thought would be helpful, but please feel free to let me know if there's anything further I can provide. I added some comments at the end of the post regarding what the underlying issue might be.
From the univariate time-series data indexed by date, I created 3 date features and split the data into training and validation sets as below.
# X_train
             weekday    monthday    hour
timestamp
2015-01-08 17:00:00 3   8   17
2015-01-12 19:30:00 0   12  19
2014-12-01 15:30:00 0   1   15
2014-07-26 09:00:00 5   26  9
2014-10-17 20:30:00 4   17  20
... ... ... ...
2014-08-29 06:30:00 4   29  6
2014-10-13 14:30:00 0   13  14
2015-01-03 02:00:00 5   3   2
2014-12-06 16:00:00 5   6   16
2015-01-06 20:30:00 1   6   20
8256 rows × 3 columns

# y_train
                    value
timestamp
2015-01-08 17:00:00 17871
2015-01-12 19:30:00 20321
2014-12-01 15:30:00 16870
2014-07-26 09:00:00 11209
2014-10-17 20:30:00 26144
... ...
2014-08-29 06:30:00 9008
2014-10-13 14:30:00 17698
2015-01-03 02:00:00 12850
2014-12-06 16:00:00 18277
2015-01-06 20:30:00 19640
8256 rows × 1 columns

# X_val
             weekday    monthday    hour
timestamp
2015-01-08 07:00:00 3   8   7
2014-10-13 22:00:00 0   13  22
2014-12-07 01:30:00 6   7   1
2014-10-14 17:30:00 1   14  17
2014-10-25 09:30:00 5   25  9
... ... ... ...
2014-09-26 12:30:00 4   26  12
2014-10-08 16:00:00 2   8   16
2014-12-03 01:30:00 2   3   1
2014-09-11 08:00:00 3   11  8
2015-01-15 10:00:00 3   15  10
2064 rows × 3 columns

# y_val
                    value
timestamp
2014-09-13 13:00:00 21345
2014-10-28 20:30:00 23210
2015-01-21 17:00:00 17001
2014-07-20 10:30:00 13936
2015-01-29 02:00:00 3604
... ...
2014-11-17 11:00:00 15247
2015-01-14 00:00:00 10584
2014-09-02 13:00:00 17698
2014-08-31 13:00:00 16652
2014-08-30 12:30:00 15775
2064 rows × 1 columns

Then, I transformed the values in the datasets by using MinMaxScaler from the sklearn library.
scaler = MinMaxScaler()
X_train_arr = scaler.fit_transform(X_train)
X_val_arr = scaler.transform(X_val)
y_train_arr = scaler.fit_transform(y_train)
y_val_arr = scaler.transform(y_val)

After converting these NumPy arrays into PyTorch Tensors, I created iterable datasets using TensorDataset and DataLoader classes provided by PyTorch.
from torch.utils.data import TensorDataset, DataLoader
from torch.autograd import Variable

train_features = torch.Tensor(X_train_arr)
train_targets = torch.Tensor(y_train_arr)

val_features = torch.Tensor(X_val_arr)
val_targets = torch.Tensor(y_val_arr)

train = TensorDataset(train_features, train_targets)
train_loader = DataLoader(train, batch_size=64, shuffle=False)

val = TensorDataset(val_features, val_targets)
val_loader = DataLoader(train, batch_size=64, shuffle=False)

Then, I defined my LSTM Model and train_step functions as follows:
class LSTMModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, layer_dim, output_dim):
        super(LSTMModel, self).__init__()
        # Hidden dimensions
        self.hidden_dim = hidden_dim

        # Number of hidden layers
        self.layer_dim = layer_dim

        # Building your LSTM
        # batch_first=True causes input/output tensors to be of shape
        # (batch_dim, seq_dim, feature_dim)
        self.lstm = nn.LSTM(input_dim, hidden_dim, layer_dim, batch_first=True)

        # Readout layer
        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        # Initialize hidden state with zeros
        h0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()

        # Initialize cell state
        c0 = torch.zeros(self.layer_dim, x.size(0), self.hidden_dim).requires_grad_()

        # We need to detach as we are doing truncated backpropagation through time (BPTT)
        # If we don't, we'll backprop all the way to the start even after going through another batch
        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))

        # Index hidden state of last time step
        out = self.fc(out[:, -1, :])
        return out

def make_train_step(model, loss_fn, optimizer):
    # Builds function that performs a step in the train loop
    def train_step(x, y):
        # Sets model to TRAIN mode
        model.train()
        # Makes predictions
        yhat = model(x)
        # Computes loss
        loss = loss_fn(y, yhat)
        # Computes gradients
        loss.backward()
        # Updates parameters and zeroes gradients
        optimizer.step()
        optimizer.zero_grad()
        # Returns the loss
        return loss.item()

    # Returns the function that will be called inside the train loop
    return train_step

Finally, I start training my LSTM model in mini-batches with AdamOptimizer for 20 epochs, which is already long enough to see the model is not learning.
import torch.optim as optim

input_dim = n_features
hidden_dim = 64
layer_dim = 3
output_dim = 1

model = LSTMModel(input_dim, hidden_dim, layer_dim, output_dim)

criterion = nn.MSELoss(reduction='mean')
optimizer = optim.Adam(model.parameters(), lr=1e-2)

train_losses = []
val_losses = []
train_step = make_train_step(model, criterion, optimizer)
n_epochs = 20
device = 'cuda' if torch.cuda.is_available() else 'cpu'

for epoch in range(n_epochs):
    batch_losses = []
    for x_batch, y_batch in train_loader:
        x_batch = x_batch.unsqueeze(dim=0).to(device)
        y_batch = y_batch.to(device)
        loss = train_step(x_batch, y_batch)
        batch_losses.append(loss)
    training_loss = np.mean(batch_losses)
    train_losses.append(training_loss)
    with torch.no_grad():
        batch_val_losses = []
        for x_val, y_val in val_loader:
            x_val = x_val.unsqueeze(dim=0).to(device)
            y_val = y_val.to(device)
            model.eval()
            yhat = model(x_val)
            val_loss = criterion(y_val, yhat).item()
            batch_val_losses.append(val_loss)
        validation_loss = np.mean(batch_val_losses)
        val_losses.append(validation_loss)

    print(f"[{epoch+1}] Training loss: {training_loss:.4f}\t Validation loss: {validation_loss:.4f}")

And this is the output:
C:\Users\VS32XI\Anaconda3\lib\site-packages\torch\nn\modules\loss.py:446: UserWarning: Using a target size (torch.Size([1, 1])) that is different to the input size (torch.Size([64, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.
  return F.mse_loss(input, target, reduction=self.reduction)
[1] Training loss: 0.0505    Validation loss: 0.0315
[2] Training loss: 0.0317    Validation loss: 0.0315
[3] Training loss: 0.0317    Validation loss: 0.0315
[4] Training loss: 0.0317    Validation loss: 0.0315
[5] Training loss: 0.0317    Validation loss: 0.0315
[6] Training loss: 0.0317    Validation loss: 0.0315
[7] Training loss: 0.0317    Validation loss: 0.0315
[8] Training loss: 0.0317    Validation loss: 0.0315
[9] Training loss: 0.0317    Validation loss: 0.0315
[10] Training loss: 0.0317   Validation loss: 0.0315
[11] Training loss: 0.0317   Validation loss: 0.0315
[12] Training loss: 0.0317   Validation loss: 0.0315
[13] Training loss: 0.0317   Validation loss: 0.0315
[14] Training loss: 0.0317   Validation loss: 0.0315
[15] Training loss: 0.0317   Validation loss: 0.0315
[16] Training loss: 0.0317   Validation loss: 0.0315
[17] Training loss: 0.0317   Validation loss: 0.0315
[18] Training loss: 0.0317   Validation loss: 0.0315
[19] Training loss: 0.0317   Validation loss: 0.0315
[20] Training loss: 0.0317   Validation loss: 0.0315

Note 1: Looking at the warning given, I'm not sure if that's the real reason why the model is not learning well. After all, I'm trying to predict the future values in the time-series data; therefore, 1 would be a plausible output dimension.
Note 2: To train the model in mini-batches, I relied on the class DataLoader. When iterating over the X and Y batches in both train and validation DataLoaders, the dimensions of x_batches were 2, while the model expected 3. So, I used PyTorch's unsqueeze function to match the expected dimension as in x_batch.unsqueeze(dim=0) . I'm not sure if this is how I should have gone about it, which could also be the issue.


title: Is there a way to force any function to not be verbose in Python?

body:
While loading a pretrained model with pytorch via
model.load_state_dict(torch.load(MODEL_PATH))

the console is flooded with output containing information about the model (which is annoying). Afaik there is no verbosity option, neither in model.load_state_dict nor in torch.load. Please let me know if I overlooked this parameter.
However, this led me to the question if there is any general way to force a function to be non verbose. Maybe something like:
with os.nonverbose():
   model.load_state_dict(torch.load(MODEL_PATH))

Any ideas?


title: Underfitting a single batch: Can't cause autoencoder to overfit multi-sample batches of 1d data. How to debug?

body:
TL;DR
I am unable to overfit batches with multiple samples using autoencoder.
Fully connected decoder seems to handle more samples per batch than conv decoder, but then also fails when number of samples increases.
Why is this happening, and how to debug this?

In depth
I am trying to use an auto encoder on 1d data points of size (n, 1, 1024), where n is the number of samples in the batch.
I am trying to overfit to that single batch.
Using a convolutional decoder, I am only able to fit a single sample (n=1), and when n>1 I am unable to drop the loss (MSE) below 0.2.
In blue: expected output (=input), in orange: reconstruction.
Single sample, single batch:

Multiple samples, single batch, loss won't go down:

Using more than one sample, we can see the net learns the general shape of the input (=output) signal, but greatly misses the bias.

Using a fully connected decoder does manage to reconstruct batches of multiple samples:

Relevant code:
class Conv1DBlock(nn.Module):
    def __init__(self, in_channels, out_channels, kernel_size):
        super().__init__()
        self._in_channels = in_channels
        self._out_channels = out_channels
        self._kernel_size = kernel_size

        self._block = nn.Sequential(
                nn.Conv1d(
                        in_channels=self._in_channels,
                        out_channels=self._out_channels,
                        kernel_size=self._kernel_size,
                        stride=1,
                        padding=(self._kernel_size - 1) // 2,
                ),
                # nn.BatchNorm1d(num_features=out_channels),
                nn.ReLU(True),
                nn.MaxPool1d(kernel_size=2, stride=2),
        )

    def forward(self, x):
        for layer in self._block:
            x = layer(x)
        return x

class Upsample1DBlock(nn.Module):
    def __init__(self, in_channels, out_channels, factor):
        super().__init__()
        self._in_channels = in_channels
        self._out_channels = out_channels
        self._factor = factor

        self._block = nn.Sequential(
                nn.Conv1d(
                        in_channels=self._in_channels,
                        out_channels=self._out_channels,
                        kernel_size=3,
                        stride=1,
                        padding=1
                ),  # 'same'
                nn.ReLU(True),
                nn.Upsample(scale_factor=self._factor, mode='linear', align_corners=True),
        )

    def forward(self, x):
        x_tag = x
        for layer in self._block:
            x_tag = layer(x_tag)
        # interpolated = F.interpolate(x, scale_factor=0.5, mode='linear') # resnet idea
        return x_tag

encoder:
self._encoder = nn.Sequential(
            # n, 1024
            nn.Unflatten(dim=1, unflattened_size=(1, 1024)),
            # n, 1, 1024
            Conv1DBlock(in_channels=1, out_channels=8, kernel_size=15),
            # n, 8, 512
            Conv1DBlock(in_channels=8, out_channels=16, kernel_size=11),
            # n, 16, 256
            Conv1DBlock(in_channels=16, out_channels=32, kernel_size=7),
            # n, 32, 128
            Conv1DBlock(in_channels=32, out_channels=64, kernel_size=5),
            # n, 64, 64
            Conv1DBlock(in_channels=64, out_channels=128, kernel_size=3),
            # n, 128, 32
            nn.Conv1d(in_channels=128, out_channels=128, kernel_size=32, stride=1, padding=0),  # FC
            # n, 128, 1
            nn.Flatten(start_dim=1, end_dim=-1),
            # n, 128
        )

conv decoder:
self._decoder = nn.Sequential(
    nn.Unflatten(dim=1, unflattened_size=(128, 1)),  # 1
    Upsample1DBlock(in_channels=128, out_channels=64, factor=4),  # 4
    Upsample1DBlock(in_channels=64, out_channels=32, factor=4),  # 16
    Upsample1DBlock(in_channels=32, out_channels=16, factor=4),  # 64
    Upsample1DBlock(in_channels=16, out_channels=8, factor=4),  # 256
    Upsample1DBlock(in_channels=8, out_channels=1, factor=4),  # 1024
    nn.ReLU(True),
    nn.Conv1d(in_channels=1, out_channels=1, kernel_size=3, stride=1, padding=1),
    nn.ReLU(True),
    nn.Flatten(start_dim=1, end_dim=-1),
    nn.Linear(1024, 1024)
)

FC decoder:
self._decoder = nn.Sequential(
    nn.Linear(128, 256),
    nn.ReLU(True),
    nn.Linear(256, 512),
    nn.ReLU(True),
    nn.Linear(512, 1024),
    nn.ReLU(True),
    nn.Flatten(start_dim=1, end_dim=-1),
    nn.Linear(1024, 1024)
)

Another observation is that when the batch size increases more, to say, 16, the FC decoder also starts to fail.
In the image, 4 samples of a 16 sample batch I am trying to overfit

What could be wrong with the conv decoder?
How to debug this or make the conv decoder work?


title: How to convert output to 1/0

body:
I am Relatively new to building deep learrning models and I seem to be completely confused and stuck with errors related to shape and size.
Here’s the LSTM model and relevant code:
    class LSTMTagger(nn.Module):

        def __init__(self):
            super(LSTMTagger, self).__init__()
            self.embedding =
    nn.Embedding(wv.vectors.shape[0],512)#embedding_matrix.shape[1])
            self.lstm1 = nn.LSTM(input_size = 512, hidden_size = 64, dropout =
    0.1,batch_first=True,bidirectional = True)
            self.dropout = nn.Dropout(p = 0.25)
            self.linear1 = nn.Linear(in_features = 128, out_features = 64)
            self.dropout = nn.Dropout(p = 0.25)
            self.linear2 = nn.Linear(in_features = 64, out_features = 1)
            self.sigmoid = nn.Sigmoid()

        def forward(self, X):
            X_embed = self.embedding(X)
            outr1, _ = self.lstm1(X_embed)
            xr = self.dropout(outr1)
            xr= self.linear1(xr)
            xr = self.dropout(xr)
            xr= self.linear2(xr)
            outr4 = self.sigmoid(xr)
            outr4 = outr4.view(1,-1)

            return outr4

model = LSTMTagger()
torch.multiprocessing.set_sharing_strategy('file_system')
if torch.cuda.device_count() > 1:
  print("Using ", torch.cuda.device_count(), " GPUs")
  # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs

# model =model.load_state_dict(torch.load('best_model_state.bin'))
model = nn.DataParallel(model, device_ids=[0]) #py r
torch.cuda.empty_cache()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = model.to(device)

def train_epoch(
      model,
      data_loader,
      loss_fn,
      optimizer,
      device,
      scheduler,
      n_examples
    ):
      model = model.train()
      losses = []
      correct_predictions = 0
      for d in data_loader:
        print(f"Input ids: {np.shape(d['input_ids'])}\n len: {len(d['input_ids'][0])}")
        input_ids = d["input_ids"].to(device)
        targets = d["targets"].to(device)
        outputs = model(input_ids)
        _, preds = torch.max(outputs, dim=1)
        print(f"outputs is {np.shape(outputs)}")
        print(f"targets is {targets}")
        # continue
        loss = criterion(outputs.squeeze(), targets)
        # loss.backward()
        # nn.utils.clip_grad_norm_(model.parameters(), clip)
        # optimizer.step()
        # loss = loss_fn(outputs, targets)
        correct_predictions += torch.sum(preds == targets)
        losses.append(loss.item())
        loss.backward()
        nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()
      return correct_predictions.double() / n_examples, np.mean(losses)
EPOCHS = 6
optimizer = optim.Adam(model.parameters(), lr=2e-5)
total_steps = len(data_train) * EPOCHS
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1)
loss_fn = nn.CrossEntropyLoss().to(device)
history = defaultdict(list)
best_accuracy = 0
criterion = nn.BCELoss()

print('starting training')
# exit()

for epoch in range(EPOCHS):
      print(f'Epoch {epoch + 1}/{EPOCHS}')
      print('-' * 10)
      train_acc, train_loss = train_epoch(
        model,
        data_train,
        loss_fn,
        optimizer,
        device,
        scheduler,
        len(df_train)
      )

In this instance the sample input is a tensor of size: torch.Size([1, 512])
, that looks like this:
tensor([[44561, 972, 7891, 94, 2191, 131, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0]], device=‘cuda:0’)

and out put label (targets from train_epoch function) in case is just a simple 1 or 0 label in tensor form such as tensor([1], device=‘cuda:0’).
I have been facing issues consistently with this approach. Initially the output was 1x512x1. So, I added
outr4 = outr4.view(1,-1)

after the sigmoid layer. Then, the output shape was reduced to 1x512 and I used squeeze function but, still, I face errors such as this one:
ValueError: Using a target size (torch.Size([1])) that is different to the input size (torch.Size([512])) is deprecated. Please ensure they have the same size.

I have spent a lot of time trying to figure out what was going on but to no avail. Isn’t the output supposed to be either ?1 or 0, instead of being a 1x512 shaped tensor?
I am relatively new to building models, so please excuse my lack of knowledge.


title: Using self in init part of a class in Python

body:
Is there any difference between the following two codes related to initializing a class in Python?
class summation:
    def __init__(self, f, s):
        self.first = f
        self.second = s
        self.summ = self.first + self.second
    .
    .
    .

class summation:
    def __init__(self, f, s):
        self.first = f
        self.second = s
        self.summ = f + s
    .
    .
    .

If there exists any difference, what is that, and which code is preferable?
Edit: I am going to write an artificial neural network with Python (and Pytorch). In fact, the above two codes are just some examples. In the actual case, I have seen in various resources that when there exists self.input = input in the initialization of a class, in other parts it is used as self.input, not input.
My questions: What are the differences between these two approaches? Why is the use of self.input preferable, in my case?
Example: (from https://docs.dgl.ai/en/latest/tutorials/models/1_gnn/4_rgcn.html#sphx-glr-tutorials-models-1-gnn-4-rgcn-py)
import torch
import torch.nn as nn
import torch.nn.functional as F
from dgl import DGLGraph
import dgl.function as fn
from functools import partial

class RGCNLayer(nn.Module):
    def __init__(self, in_feat, out_feat, num_rels, num_bases=-1, bias=None,
                 activation=None, is_input_layer=False):
        super(RGCNLayer, self).__init__()
        self.in_feat = in_feat
        self.out_feat = out_feat
        self.num_rels = num_rels
        self.num_bases = num_bases
        self.bias = bias
        self.activation = activation
        self.is_input_layer = is_input_layer

        # sanity check
        if self.num_bases <= 0 or self.num_bases > self.num_rels:
            self.num_bases = self.num_rels

        # weight bases in equation (3)
        self.weight = nn.Parameter(torch.Tensor(self.num_bases, self.in_feat,
                                                self.out_feat))
        if self.num_bases < self.num_rels:
            # linear combination coefficients in equation (3)
            self.w_comp = nn.Parameter(torch.Tensor(self.num_rels, self.num_bases))

        # add bias
        if self.bias:
            self.bias = nn.Parameter(torch.Tensor(out_feat))

        # init trainable parameters
        nn.init.xavier_uniform_(self.weight,
                                gain=nn.init.calculate_gain('relu'))
        if self.num_bases < self.num_rels:
            nn.init.xavier_uniform_(self.w_comp,
                                    gain=nn.init.calculate_gain('relu'))
        if self.bias:
            nn.init.xavier_uniform_(self.bias,
                                    gain=nn.init.calculate_gain('relu'))

    def forward(self, g):
        if self.num_bases < self.num_rels:
            # generate all weights from bases (equation (3))
            weight = self.weight.view(self.in_feat, self.num_bases, self.out_feat)
            weight = torch.matmul(self.w_comp, weight).view(self.num_rels,
                                                        self.in_feat, self.out_feat)
        else:
            weight = self.weight

        if self.is_input_layer:
            def message_func(edges):
                # for input layer, matrix multiply can be converted to be
                # an embedding lookup using source node id
                embed = weight.view(-1, self.out_feat)
                index = edges.data['rel_type'] * self.in_feat + edges.src['id']
                return {'msg': embed[index] * edges.data['norm']}
        else:
            def message_func(edges):
                w = weight[edges.data['rel_type']]
                msg = torch.bmm(edges.src['h'].unsqueeze(1), w).squeeze()
                msg = msg * edges.data['norm']
                return {'msg': msg}

        def apply_func(nodes):
            h = nodes.data['h']
            if self.bias:
                h = h + self.bias
            if self.activation:
                h = self.activation(h)
            return {'h': h}

        g.update_all(message_func, fn.sum(msg='msg', out='h'), apply_func)


title: Is there a way to create a torch.utils.data.DataLoader object from torchtext.TabularDataset?

body:
I'm new to ML and I am struggling with a data preprocessing task.
I'm using PyTorch and what I want to do is to classify texts using a simple RNN model. I don't know if the architecture of the model is relevant at this point.
To give you a little bit of context, I have two json files from where I get the data: a train.json file and a test.json file.
Every line of these files is represented like this:
{"text" : ["word_1", "word_2", ..., "word_n"], "label" : "actual_label"},

where text key is a list of words and the label is the actual label for the current text.
So far, I have something like this:
from torchtext import data

TEXT = data.Field()
LABEL = data.LabelField()

fields = {'text': ('text', TEXT), 'label': ('label', LABEL)}

train_data, test_data = data.TabularDataset.splits(
                            path = 'data_path_folder',
                            train = 'train.json',
                            test = 'test.json',
                            format = 'json',
                            fields = fields)

Then, I create a vocabulary for my data:
TEXT.build_vocab(train_data, max_size = max_size)
LABEL.build_vocab(train_data)

and the iterators:
batch_size = 64
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iter, test_iter = data.BucketIterator.splits(
    (train_data, test_data),
    sort_key = lambda x: len(x.text),
    batch_size = batch_size,
    device = device)

Now, in the training loop, I can easily iterate through the data like this:
for batch in train_iter:
    batch.text # this is a tensor with a shape of [max words number from a sentence, number of sencentes]
    batch.label # this is a tensor with a shape of [number of sentences]

My first question is where exactly my data is transformed to tensors. Does the data change to tensors when I call the data.BucketIterator.splits function? Or does have anything to do when creating the vocabulary?
What I want to do is to process the data using torch.utils.data.DataLoader instead.
The goal is to iterate through the data like this:
for batch_idx, (text, label) in enumerate(dataloader):
    text # this should be a tensor with a shape of [max words number from a sentence, number of sencentes]
    label # this should be a tensor with a shape of [number of sentences]

I tried to keep the data.TabularDataset.splits part and from there to create a dataloader, but with no success.
Can anyone help me with this?
Thanks in advance!


title: Understanding input and output size for Conv2d

body:
I'm learning image classification using PyTorch (using CIFAR-10 dataset) following this link.
I'm trying to understand the input & output parameters for the given Conv2d code:
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Net()

My conv2d() understanding (Please correct if I am wrong/missing anything):

since image has 3 channels that's why first parameter is 3.
6 is no of filters (randomly chosen)
5 is kernel size (5, 5) (randomly chosen)
likewise we create next layer (previous layer output is input of this layer)
Now creating a fully connected layer using linear function:
self.fc1 = nn.Linear(16 * 5 * 5, 120)

16 * 5 * 5: here 16 is the output of last conv2d layer, But what is 5 * 5 in this?.
Is this kernel size ? or something else? How to know we need to multiply by 5*5 or 4*4 or 3*3.....
I researched & got to know that since image size is 32*32, applying max pool(2) 2 times, so image size would be 32 -> 16 -> 8, so we should multiply it by last_ouput_size * 8 * 8 But in this link its 5*5.
Could anyone please explain?


title: Same PyTorch script not running on different machines

body:
I have build a network with PyTorch that runs fine on my local machine. Now I wanted to move it to a machine with GPU/CUDA support to train on larger data. But here the training fails. More precisely, the calculation of the loss function fails

In my code, I have something along the line:

loss_function = nn.NLLLoss()
....

def train():
    ...
    loss = loss_function(log_probs, target)
    ...

On the GPU machine, the script stops at loss = loss_function(log_probs, target). No error, nothing. Just stops. Even if I enforce using the CPU instead of the GPU.

I use the exact same script with exact same training dataset. Using print statements, log_probs and target look the same/similar on both machines. Nothing in this line is None. I simply cannot see why the behaviors would differ on the two machines.


title: How to reshape a pytorch matrix without mixing elements of items in a batch

body:
In my Neural network model, I represent  an 8 word-sentence with a 8x256 dimensional embedding matrix. I want to give it to a LSTM as a input where LSTM takes a single word embedding at a time as input and process it. According to pytorch documentation, the input should be in the shape of  (seq_len, batch, input_size). What is the correct way to convert my input to desired shape ? I don't want to mixup the numbers by mistake. I am quite new in PyTorch and row-major calculations, therefore I wanted to ask it here. I do it as follows, is it correct ?

x = torch.rand(8,256)
lstm_input = torch.reshape(x,(8,1,256))


title: IndexError - Implementing the test of CBOW with pytorch

body:
I am not very used to python & machine learning code. I am testing on pytorch CBOW test, but it says something about Index Error. Can anyone help?

# model class
class CBOW(nn.Module):

    ...
    def get_word_embedding(self, word):
    word = torch.cuda.LongTensor([word_to_ix[word]])
    return self.embeddings(word).view(1, -1)

# test method
def test_cbow(model, train_words, word_to_ix):
    # test word similarity
    word_1 = train_words[2] #randomly chosen word
    word_2 = train_words[3] #randomly chosen word

    word_1_vec = model.get_word_embedding(word_1)[0].cpu()
    word_2_vec = model.get_word_embedding(word_2)[0].cpu()

    print(word_1_vec)
    print(word_2_vec)

    word_similarity = (word_1_vec.dot(word_2_vec) / (torch.norm(word_1_vec) * torch.norm(word_2_vec))).data.numpy()[0]
    print("Similarity between '{}' & '{}' : {:0.4f}".format(word_1, word_2, word_similarity))

# executing the test
test_cbow(model, train_words, word_to_ix)

BELOW IS THE RESULT:

tensor([ 0.8978,  1.0713, -1.6856, -1.0967, -0.0114,  0.4107, -0.4293, -0.7351,
         0.4410, -1.5937, -1.3773,  0.7744,  0.0739, -0.3263,  1.0342,  1.0420,
        -1.1333,  0.4158,  1.1316, -0.0141, -0.8383,  0.2544, -2.2409, -1.1858,
         0.2652, -0.3232,  0.1287, -1.5274,  0.3199, -2.1822,  0.9464, -0.6619,
         1.1549,  0.5276,  0.0849, -0.1594, -1.7922,  1.3567, -0.4376, -0.9093,
         1.0701,  1.5373, -1.3277, -1.1833,  1.8070, -0.0551, -0.8439,  1.5236,
        -0.3890, -0.2306, -0.7392, -1.6435,  0.4485,  0.8988, -0.5958, -0.6989,
         1.6123, -1.6668,  0.0583,  0.6698, -0.6998,  1.1942,  0.6355,  0.7437,
        -1.0006, -0.5398,  1.3197,  1.3696, -0.3221,  0.9004,  0.6268,  0.0221,
         0.0269, -1.7966, -1.6153, -0.1695, -0.0339, -0.5145,  1.5744, -0.3388,
        -0.9617,  0.6750, -1.1334,  0.0377,  1.1123,  1.1002, -0.3605,  0.2105,
        -1.6570,  1.3818,  0.9183,  0.0274,  0.9072,  0.8414,  0.3424,  0.2199,
         1.6546, -0.1357,  1.1291, -0.5309], grad_fn=<CopyBackwards>)
tensor([-0.6263, -0.5639,  2.1590, -0.3659,  0.2862, -0.4542, -0.4825, -0.1776,
        -0.4242,  0.9525,  0.7138, -0.3107,  1.8733, -0.3406,  0.0277,  1.6775,
         2.1893,  2.0332,  0.7185,  0.0050, -0.1627, -0.1113,  1.0444,  1.4057,
         0.2183,  0.3405,  0.0930,  1.2428, -0.0740,  0.3991, -0.2722,  1.4980,
         0.9207,  0.5008, -1.9297,  0.5600,  1.6416,  1.1550,  0.1440,  0.0739,
        -0.7465, -0.2458,  0.9217,  0.7156, -1.2558, -0.9891, -0.7313,  0.8501,
        -1.2851, -0.3068, -0.0796,  0.9361,  0.0927, -1.2988,  0.7422,  0.1388,
         1.3895, -0.7935,  0.4008, -0.1338,  1.5563,  0.5864,  0.6606, -0.2341,
         0.1218, -0.7313,  0.5073, -0.2941,  0.0316, -2.5356, -0.0885,  2.5765,
         0.2090,  0.2819, -0.0386,  0.7986,  2.1165, -0.0271, -0.2987,  0.2905,
         0.0149,  0.2403,  0.0752, -1.5535,  0.3794,  2.0638,  1.0603,  0.0703,
        -0.3643, -1.5671, -0.4736, -1.3035,  0.6583,  0.2531,  0.9829, -0.6025,
        -0.8148, -0.3457, -0.7339,  0.6758], grad_fn=<CopyBackwards>)

What I am also confused is that I have to convert cuda datatype to numpy, since I used cuda in get_word_embedding method. Is adding .cpu() to convert the datatype correct?

IndexError                                Traceback (most recent call last)
<ipython-input-68-39d73aa6e0de> in <module>()
     17     print("Similarity between '{}' & '{}' : {:0.4f}".format(word_1, word_2, word_similarity))
     18
---> 19 test_cbow(model, train_words, word_to_ix)

<ipython-input-68-39d73aa6e0de> in test_cbow(model, train_words, word_to_ix)
     14     print(type(word_1_vec))
     15
---> 16     word_similarity = (word_1_vec.dot(word_2_vec) / (torch.norm(word_1_vec) * torch.norm(word_2_vec))).data.numpy()[0]
     17     print("Similarity between '{}' & '{}' : {:0.4f}".format(word_1, word_2, word_similarity))
     18

**IndexError: too many indices for array**


title: Validation accuracy drops after loading model from checkpoint

body:
I use this code to save my VAE model every epoch:

torch.save({'epoch' : epoch,
            'encoder' : encoder.state_dict(),
            'decoder' : decoder.state_dict(),
            'property_predictor_model' : property_predictor_model.state_dict(),
            'train_ids' : train_ids,
            'valid_ids' : valid_ids,
            'validationQuality' : validationQuality
}, '{}/model_checkpoint{}.pt'.format(out_dir, epoch))

And when I restart training, I use this code to initialize the model and to load the previous state:

model_encode = VAE_encode(**encoder_parameter).to(device)
model_decode = VAE_decode(**decoder_parameter).to(device)
model_prop_predict = property_predictor_model(**prop_pred_parameter).to(device)

model_encode.load_state_dict(checkpoint['encoder'])
model_decode.load_state_dict(checkpoint['decoder'])
model_prop_predict.load_state_dict(checkpoint['property_predictor_model'])

The Validation accuracy saved in the checkpoint itself would be around 60% yet running validation immediately after loading the model returns an accuracy of <10%. Training continues as if I just initialized the model randomly. The Validation accuracy before and after i load the state is different, so it's loading something, just not the right thing. Any ideas why this is happening.

Interesting note:
If I use my debugger and load the checkpoint right after it gets saved, the validation accuracy is maintained, but if I cancel my script and load up the same checkpoint, accuracy is <10% again.


title: PyTorch training file - TypeError: a bytes-like object is required, not 'str'

body:
When I run the following code:

...
    if write and is_train:
        train_loss_file.write(
            "[%d], Loss:%.5f, Loss(VecMap):%.5f, Pixel Accuracy:%.3f, Mean Accuracy:%.3f, Mean IoU:%.3f, Class IoU:[%.5f/%.5f], Freq.Weighted Accuray:%.3f  \n"
            % (
                epoch,
                loss,
                loss_vec,
                100 * pixel_accuracy,
                100 * np.nanmean(mean_accuracy),
                100 * np.nanmean(mean_iou),
                mean_iou[0],
                mean_iou[1],
                (100 * fwavacc),
            )
        )
...

I get the following error:

[>....... 1/1 ............]S:25s968ms|T:0ms|Loss: -0.199331 | VecLoss: 3.783525 | road miou: 17.2746%(18.3798%) | angle miou: 0.2103%
Traceback (most recent call last):
  File "train_mtl.py", line 443, in <module>
    main()
  File "train_mtl.py", line 434, in main
    train(epoch)
  File "train_mtl.py", line 294, in train
    write=True,
  File "C:\Users\...\road_connectivity-master\utils\util.py", line 147, in performMetrics
    (100 * fwavacc),
TypeError: a bytes-like object is required, not 'str'

Can you please help me overcome this issue?
The code is retrieved from this repository: https://github.com/anilbatra2185/road_connectivity
I am discussing with the contributor, but we cannot solve this problem.


title: different between class name vs self calling in super()

body:
isn't it same meaning of class name NeuralNet and self keyword passing in super call  -
super(NeuralNet,self).__init__() # init super

here is the code snippet from example:

class NeuralNet(nn.Module):
    def __init__(self, use_batch_norm, input_size=784, hidden_dim=256, output_size=10):
        """
        Creates a PyTorch net using the given parameters.
        """
        super(NeuralNet, self).__init__() # init super
        # continues code


title: How much is the dimension of some bidirectional LSTM layers?

body:
I read a paper about machine translation, and it uses projection layer. Its encoder has 6 bidirectional LSTM layers. If input embedding dimension is 512, how much will be the dimension of the encoder output? 512*2**5?

The paper's link: https://www.aclweb.org/anthology/P18-1008.pdf


title: Efficient inference of 3D deep learning model (pytorch)

body:
I am trying to use a Pytorch 3D UNet for inference (from here: https://github.com/wolny/pytorch-3dunet) which receives images of size (96, 96, 96). I would like to use it on CPU instances, but I am getting very high memory usages (~18 GB). After researching on the subject I found out that this was due to the way convolutions are implemented on CPU (see https://discuss.pytorch.org/t/pytorch-high-memory-demand/2798/5). I thus have the following questions:

Is there a way to use a more memory-efficient implementation of the convolution in Pytorch?
How can I optimize my model for CPU inference? I saw that some tools like AWS Neo, Intel OpenVINO, etc. exist; could they solve my problem?
Does Tensorflow have a similar problem for using convolutions on CPU?
Any other tip, link on how to deploy such models in an efficient way is welcome!

Thanks!


title: CUDA out of memory during loss.backward(), torch.bmm works fine while doing forward computation

body:
I wrote the following codes to do doc-level pair-wise attention.
`

def bi_linear_attn(self, Q, K):
    """
    :param Q: 1*b*d
    :param K: n*b*d
    :return:  (for relative part and irrelative part)
    """
    v = K.permute(1, 2, 0)
    # b*1*n && b*s*n
    # here!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
    interaction = torch.bmm(Q.permute(1, 0, 2), torch.bmm(
        self.bi_linear_para.unsqueeze(0).expand(v.shape[0], *self.bi_linear_para.shape), v))

    print(interaction.shape)
    direct_score = F.softmax(interaction, dim=2)
    print(direct_score.shape)
    inverse_score = 1 - direct_score
    h_dir = torch.bmm(direct_score, K.permute(1, 0, 2))
    h_inv = torch.bmm(inverse_score, K.permute(1, 0, 2))
    # b * 1 * d
    return h_dir, h_inv

def forward(self, bid2emb):
    blk_num = len(bid2emb)
    tuple_lst = sorted([(bid, emb_tuple[0], emb_tuple[1]) for bid, emb_tuple in bid2emb.items()])
    id_zip, h_zip, c_zip = zip(*tuple_lst)
    h_stack = torch.cat([h.repeat(1, blk_num, 1) for h in list(h_zip)], dim=1)
    # -- pad context
    c_lst_padded = []
    stc_len = [c.shape[0] for c in list(c_zip)]
    max_stc_len = max(stc_len)
    for c in list(c_zip):
        if c.shape[0] < max_stc_len:
            c_pad = F.pad(c, (0, 0, 0, 0, 0, max_stc_len - c.shape[0]), 'constant', 0.)
        else:
            c_pad = c
        c_lst_padded.append(c_pad)
    c_stack = torch.cat(c_lst_padded * blk_num, dim=1)
    h_dir, h_inv = self.bi_linear_attn(h_stack, c_stack)
    # -- get ind
    AB_lst, BA_lst = [], []
    for bid_A, emb_tuple_A in bid2emb.items():
        for bid_B, emb_tuple_B in bid2emb.items():
            if bid_A == bid_B:
                continue
            AB_lst.append(bid_A * blk_num + bid_B)
            BA_lst.append(bid_B * blk_num + bid_A)
    AB_idx = torch.tensor(AB_lst).to(device)
    BA_idx = torch.tensor(BA_lst).to(device)
    # n * 1 * d
    h_dir = torch.mean(
        torch.cat((torch.index_select(h_dir, 0, AB_idx), torch.index_select(h_dir, 0, BA_idx)), dim=1), dim=1,
        keepdim=True)
    h_inv = torch.mean(
        torch.cat((torch.index_select(h_inv, 0, AB_idx), torch.index_select(h_inv, 0, BA_idx)), dim=1), dim=1,
        keepdim=True)
    print('hdir', h_dir.shape)
    return torch.cat((h_dir, h_inv), dim=2)`

In line 10, I tried to compute interaction score with x1 A x2, where x1 and x2 has the shape of 36000*1*512 and 36000*max_len*512.  Due to the large batch size, the matmul failed due to memory error, after changing matmul to bmm with expanded parameter matric, the forward computation works fine. However the memory error appears when doing backpropagation. Which is confusing to me.

I would really appreciate any help on this as I am completely perplexed


title: Can I use a PyTorch or Tensorflow project on a machine without GPU?

body:
I'm a noob when it comes to Python and machine learning. I'm trying to run two different projects that have to do with something called Deep Image Matting:

https://github.com/Joker316701882/Deep-Image-Matting with Tensorflow
https://github.com/huochaitiantang/pytorch-deep-image-matting with Pytorch

I'm just trying to run the tests in these projects but I run into various problems. Can I run these on a machine without GPU? I thought that GPU is only for speeding up processing, but I'm only interested in seeing these run before getting a machine with GPU.
I apologize in advance, as I know I'm a total noob in this

When I try the Tensorflow project:

I get an error with this line gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction = args.gpu_fraction) probably because I was tf2 and this requires tf1
After I downgraded to tf1 when I try to run the test I get W tensorflow/core/platform/cpu_feature_guard.cc:45] The TensorFlow library wasn't compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations.
 and InvalidArgumentError (see above for traceback): No OpKernel was registered to support Op 'MaxPoolWithArgmax' with these attrs.  Registered devices: [CPU], Registered kernels:
<no registered kernels> and now I'm stuck because I have no clue what this means

When I try the Pytorch project:

First I get this error: RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.
So I added map_location=torch.device('cpu') when the model is loaded, but now I get RuntimeError: Error(s) in loading state_dict for VGG16:
    size mismatch for conv6_1.weight: copying a param with shape torch.Size([512, 512, 1, 1]) from checkpoint, the shape in current model is torch.Size([512, 512, 3, 3]). And I'm stuck again

Can someone help?

thank you in advance!


title: Conv3D size doesn’t make sense with NIFTI data?

body:
So I am writing custom dataset for medical images, with .nii (NIFTI1 format), but there is a confusion.

My dataloader returns the shape torch.Size (1,1,256,256,51) . But NIFTI volumes use anatomical axes, different coordinate system, so it doesn’t make any sense to permute the axes, which I normally would with volume made of 2D images each  stored separately in local drive with 51 slice images (or depth), as Conv3D follows the convention (N,C,D,H,W).

so torch.Size (1,1,256,256,51) (ordinarily 51 would be the depth) doesn’t follow the convention (N,C,D,H,W) , but I should not permute the axes as the data uses entirely different coordinate system ?


I would like to split images using Open CV then feed to the by PyTorch model to count object in each image. I am getting the following error message when running the code: TypeError: pic should be PIL Image or ndarray. Got <class ‘bool’>
import os

import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms, utils
#from torchvision.transforms import Grayscalei
import pandas as pd
import pdb
import cv2

class CellsDataset(Dataset):
    # a very simple dataset

    def __init__(self, root_dir, transform=None, return_filenames=False):
        self.root = root_dir
        self.transform = transform
        self.return_filenames = return_filenames
        self.files = [os.path.join(self.root,filename) for filename in os.listdir(self.root)]
        self.files = [path for path in self.files
                      if os.path.isfile(path) and os.path.splitext(path)[1]=='.png']

    def __len__(self):
        return len(self.files)

    def __getitem__(self, idx):
        path = self.files[idx]
        sample = Image.open(path)

        sample = cv2.imread(path)
        b,g,r=cv2.split(sample)
        sample=cv2.imwrite('sample.png', g)

        #transform3 = Grayscale(num_output_channels=3)
        #sample = transform3(sample) # convert to a 3 channel grayscale, as it needs to be 3 channel.
        if self.transform:
            sample = self.transform(sample)

        if self.return_filenames:
            return sample, path
        else:
            return sample


title: Using SGD on MNIST dataset with Pytorch, loss not decreasing

body:
I tried to use SGD on MNIST dataset with batch size of 32, but the loss does not decrease at all.
I checked my model, loss function and read documentation but couldn't figure out what I've done wrong.
I defined my neural network as below
class classification(nn.Module):
def __init__(self):
    super(classification, self).__init__()

    # construct layers for a neural network
    self.classifier1 = nn.Sequential(
        nn.Linear(in_features=28*28, out_features=20*20),
        nn.Sigmoid(),
    )
    self.classifier2 = nn.Sequential(
        nn.Linear(in_features=20*20, out_features=10*10),
        nn.Sigmoid(),
    )
    self.classifier3 = nn.Sequential(
        nn.Linear(in_features=10*10, out_features=10),
        nn.LogSoftmax(dim=1),
    )

def forward(self, inputs):                 # [batchSize, 1, 28, 28]
    x = inputs.view(inputs.size(0), -1)    # [batchSize, 28*28]
    x = self.classifier1(x)                # [batchSize, 20*20]
    x = self.classifier2(x)                # [batchSize, 10*10]
    out = self.classifier3(x)              # [batchSize, 10]

    return out

And I defined my training process as below

classifier = classification().to("cuda")
#optimizer
optimizer = torch.optim.SGD(classifier.parameters(), lr=learning_rate_value)
#loss function
criterion = nn.NLLLoss()
batch_size=32
epoch = 30
#array to save loss history
loss_train_arr=np.zeros(epoch)

#used DataLoader to make split batch
batched_train = torch.utils.data.DataLoader(training_set, batch_size, shuffle=True)

for i in range(epoch):

    loss_train=0

    #train and compute loss, accuracy
    for img, label in batched_train:
        img=img.to(device)
        label=label.to(device)

        optimizer.zero_grad()
        predicted = classifier(img)

        label_predicted = torch.argmax(predicted,dim=1)
        loss = criterion(predicted, label)
        loss.backward
        optimizer.step()
        loss_train += loss.item()

    loss_train_arr[i]=loss_train/(len(batched_train.dataset)/batch_size)

I am using a model with LogSoftmax layer, so my loss function seems right. But the loss does not decrease at all.


title: Can YOLO pictures have a bounded box that covering the whole picture?

body:
I wonder why YOLO pictures need to have a bounding box.
Assume that we are using Darknet. Each image need to have a corresponding .txt file with the same name as the image file. Inside the .txt file it need to be. It's the same for all YOLO frameworks that are using bounded boxes for labeling.
<object-class> <x> <y> <width> <height>

Where x, y, width, and height are relative to the image's width and height.
For exampel. If we goto this page and press YOLO Darknet TXT button and download the .zip file and then go to train folder. Then we can see a these files
IMG_0074_jpg.rf.64efe06bcd723dc66b0d071bfb47948a.jpg
IMG_0074_jpg.rf.64efe06bcd723dc66b0d071bfb47948a.txt

Where the .txt file looks like this
0 0.7055288461538461 0.6538461538461539 0.11658653846153846 0.4110576923076923
1 0.5913461538461539 0.3545673076923077 0.17307692307692307 0.6538461538461539

Every image has the size 416x416. This image looks like this:

My idéa is that every image should have one class. Only one class. And the image should taked with a camera like this.

This camera snap should been taked as:

Take camera snap
Cut the camera snap into desired size
Upscale it to square 416x416

Like this:

And then every .txt file that correspons for every image should look like this:
<object-class> 0 0 1 1

Question
Is this possible for e.g Darknet or other framework that are using bounded boxes to labeling the classes?
Instead of let the software e.g Darknet upscale the bounded boxes to 416x416 for every class object, then I should do it and change the .txt file to x = 0, y = 0, width = 1, height = 1 for every image that only having one class object.
Is that possible for me to create a traing set in that way and train with it?


title: truncated bptt pytorch implementation question

body:
i'm trying to implement tbptt in pytorch.
I've found an implementation below in a forum and I get the logic behind the code but I keep getting a "inplace operation" error.
class TBPTT():
    def __init__(self, one_step_module, loss_module, k1, k2, optimizer):
        self.one_step_module = one_step_module
        self.loss_module = loss_module
        self.k1 = k1
        self.k2 = k2
        self.retain_graph = k1 < k2
        # You can also remove all the optimizer code here, and the
        # train function will just accumulate all the gradients in
        # one_step_module parameters
        self.optimizer = optimizer

    def train(self, input_sequence, init_state):
        states = [(None, init_state)]
        for j, (inp, target) in enumerate(input_sequence):

            state = states[-1][1].detach()
            state.requires_grad=True
            output, new_state = self.one_step_module(inp, state)
            states.append((state, new_state))

            while len(states) > self.k2:
                # Delete stuff that is too old
                del states[0]

            if (j+1)%self.k1 == 0:
                loss = self.loss_module(output, target)

                optimizer.zero_grad()
                # backprop last module (keep graph only if they ever overlap)
                start = time.time()
                loss.backward(retain_graph=self.retain_graph)
                for i in range(self.k2-1):
                    # if we get all the way back to the "init_state", stop
                    if states[-i-2][0] is None:
                        break
                    curr_grad = states[-i-1][0].grad
                    states[-i-2][1].backward(curr_grad, retain_graph=self.retain_graph)
                print("bw: {}".format(time.time()-start))
                optimizer.step()

seq_len = 20
layer_size = 50

idx = 0

class MyMod(nn.Module):
    def __init__(self):
        super(MyMod, self).__init__()
        self.lin = nn.Linear(2*layer_size, 2*layer_size)

    def forward(self, inp, state):
        global idx
        full_out = self.lin(torch.cat([inp, state], 1))
        # out, new_state = full_out.chunk(2, dim=1)
        out = full_out.narrow(1, 0, layer_size)
        new_state = full_out.narrow(1, layer_size, layer_size)
        def get_pr(idx_val):
            def pr(*args):
                print("doing backward {}".format(idx_val))
            return pr
        new_state.register_hook(get_pr(idx))
        out.register_hook(get_pr(idx))
        print("doing fw {}".format(idx))
        idx += 1
        return out, new_state

one_step_module = MyMod()
loss_module = nn.MSELoss()
input_sequence = [(torch.rand(200, layer_size), torch.rand(200, layer_size))] * seq_len

optimizer = torch.optim.SGD(one_step_module.parameters(), lr=1e-3)

runner = TBPTT(one_step_module, loss_module, 5, 7, optimizer)

runner.train(input_sequence, torch.zeros(200, layer_size))
print("done")

Here is the weird thing. When I tried to run the code the first time, I kept getting another error and after a thorough speculation I found that in some of the variables such as "one_step_module", "input_sequence" where shadowing other variables outer scope. so after renaming those variables the code ran just fine.
And then, I tried to revise the code a bit further for my own project, I started getting the "inplace operation" error. So, in order to see what went wrong, I fixed the code back to the original code above but I kept getting the error.. I even tried open a new file and copy paste the implementation right from the beginning, and I still can't get the code to run. This is driving me CRAZY.
Here's the "inplace operation" error I started getting from the implementation above.
C:\Users\bboyj\anaconda3\envs\jinkyu\python.exe C:/Users/bboyj/PycharmProjects/pythonProject/test1.py
doing fw 0
doing fw 1
doing fw 2
doing fw 3
doing fw 4
doing backward 4
doing backward 3
doing backward 2
doing backward 1
doing backward 0
bw: 0.17385029792785645
doing fw 5
doing fw 6
doing fw 7
doing fw 8
doing fw 9
doing backward 9
doing backward 8
doing backward 7
doing backward 6
doing backward 5
doing backward 4
C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\autograd\__init__.py:130: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  ..\c10\cuda\CUDAFunctions.cpp:100.)
  Variable._execution_engine.run_backward(
Traceback (most recent call last):
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 100, in <module>
    runner.train(input_sequence, torch.zeros(200, layer_size))
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 59, in train
    states[-i-2][1].backward(curr_grad, retain_graph=self.retain_graph)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\tensor.py", line 221, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\autograd\__init__.py", line 130, in backward
    Variable._execution_engine.run_backward(
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [100, 100]], which is output 0 of TBackward, is at version 2; expected version 1 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).

Process finished with exit code 1

just in case you want to see the specific code triggering the error. Here's the error log with torch anomaly detection.
C:\Users\bboyj\anaconda3\envs\jinkyu\python.exe C:/Users/bboyj/PycharmProjects/pythonProject/test1.py
doing fw 0
doing fw 1
doing fw 2
doing fw 3
doing fw 4
doing backward 4
doing backward 3
doing backward 2
doing backward 1
doing backward 0
bw: 0.17083358764648438
doing fw 5
doing fw 6
doing fw 7
doing fw 8
doing fw 9
doing backward 9
doing backward 8
doing backward 7
doing backward 6
doing backward 5
doing backward 4
C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\autograd\__init__.py:130: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  ..\c10\cuda\CUDAFunctions.cpp:100.)
  Variable._execution_engine.run_backward(
C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\autograd\__init__.py:130: UserWarning: Error detected in AddmmBackward. Traceback of forward call that caused the error:
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 101, in <module>
    runner.train(input_sequence, torch.zeros(200, layer_size))
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 41, in train
    output, new_state = self.one_step_module(inp, state)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 78, in forward
    full_out = self.lin(torch.cat([inp111, state111], 1))
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\nn\modules\linear.py", line 93, in forward
    return F.linear(input, self.weight, self.bias)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\nn\functional.py", line 1690, in linear
    ret = torch.addmm(bias, input, weight.t())
 (Triggered internally at  ..\torch\csrc\autograd\python_anomaly_mode.cpp:104.)
  Variable._execution_engine.run_backward(
Traceback (most recent call last):
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 101, in <module>
    runner.train(input_sequence, torch.zeros(200, layer_size))
  File "C:/Users/bboyj/PycharmProjects/pythonProject/test1.py", line 60, in train
    states[-i-2][1].backward(curr_grad, retain_graph=self.retain_graph)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\tensor.py", line 221, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "C:\Users\bboyj\anaconda3\envs\jinkyu\lib\site-packages\torch\autograd\__init__.py", line 130, in backward
    Variable._execution_engine.run_backward(
RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [100, 100]], which is output 0 of TBackward, is at version 2; expected version 1 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!

The main problem is that the first iteration is fine because the loss is calculated only with the new hidden state and the "detached and required_grad = True" state, but the second iteration when it tries to backward on the previous set of hidden states which already have "backwarded" it raises error. So in this case, after forwarding and backward on t =0,1,2,3,4 and forwarding on t = 5,6,7,8,9, when it tries to backward on t=9,8,7,6,5,4,3 (because k2 is 7), backward works fine for t=9,8,7,6,5 but fails on t = 4.
Can anyone please shed some light on this??


title: Batch processing delaunay triangulation algorithm for GPU?

body:
I have 500 groups of 2D points, and there is no connection between them.
Now I want to triangulate them at the same time. Is there anything I can do?
Now I need to convert it to numpy and then use scipy.spatial.Delaunay() to calculate each group separately.
import scipy.spatial
import torch
points = torch.rand(500,996,2)
corner = torch.tensor([[[0,0],[0,1],[1,0],[1,1]]],
                      dtype=torch.float32).expand(500,-1,-1)
points = torch.cat([points, corner], dim=1).numpy()
tri = [scipy.spatial.Delaunay(p).simplices for p in points]
tri = torch.from_numpy(np.concatenate(tri, axis=0)).view(500, -1, 3)

But it's too slow. Is it possible to run parallel computing on gpu?
I found one that might be useful, but it doesn't seem to support batch processing.
https://www.comp.nus.edu.sg/~tants/delaunay2DDownload.html


title: The size of tensor a (64) must match the size of tensor b (16) at non-singleton dimension 1

body:
It progresses during the training, then the loss_funstion (CrossEntropyLoss) runs, then the loss values go backwards and x is calculated again, it works in a loop, but then I get an error. While a.shape (64,64) - b.shape (64,64) becomes a.shape (16,16) b (64,64) after a while, what could be the reason, you can find the relevant code on this site, details.
https://github.com/nurkbts/error
 # Compute Q, K and V, concatenate heads on batch dimension
        queries = torch.cat(self._W_q(query).chunk(self._h, dim=-1), dim=0)
        keys = torch.cat(self._W_k(key).chunk(self._h, dim=-1), dim=0)
        values = torch.cat(self._W_v(value).chunk(self._h, dim=-1), dim=0)
        # Scaled Dot Product
        self._scores = (queries@keys.T) / np.sqrt(K)
        print(self._scores.shape)

        # Compute local map mask
        if self._attention_size is not None:
            attention_mask = generate_local_map_mask(K, self._attention_size, mask_future=False, device=self._scores.device)
            self._scores = self._scores.masked_fill(attention_mask, float('-inf'))


title: Normalize values of parameters in pytorch

body:
I was following this tutorial: https://towardsdatascience.com/conditional-random-field-tutorial-in-pytorch-ca0d04499463 to understand CRF better. Tutorial also has associated code.
pytorch model has a 2x3 matrix as parameters (in self.transition). The values are log of these probabilities:
p(Yt = Fair | Yt-1 = Fair),   p(Yt = Fair | Yt-1 = Biased),   p(first dice is fair)
p(Yt = Biased | Yt-1 = Fair), p(Yt = Biased | Yt-1 = Biased), p(first dice is Biased)

Clearly it has lot more degree of freedom and instead of 6 we only need 3 parameters.
What I need is, after each step of opt.step(), I want to normalize these parameters so that they reflect true probabilities.
I would like to update self.transition as:
self.transition = torch.log(
                    torch.exp(self.transition
                   )/torch.sum(
                        torch.exp(self.transition), 0
                            )
                           )

But I can't update it, so I tried instead as:
norm_val = torch.log(
        torch.exp(self.transition
                 )/torch.sum(
                    torch.exp(self.transition), 0
                            )
                        )
    self.transition.requires_grad = False
    self.transition -= self.transition
    self.transition += norm_val
    self.transition.requires_grad = True

Still I can't do it. If I initialize self.transition by creating nn.Parameter, then it does not update these parameters anymore. How can I normalize the model parameter after each opt step?


I am trying to build a simple CNN model using Pytorch C++ API ( libtorch ), but when I try to run CMake it gives me this error:
CMake Error at CMakeLists.txt:4 (add_executable):
Cannot find source file:

training_VGG.cpp

Tried extensions .c .C .c++ .cc .cpp .cxx .m .M .mm .h .hh .h++ .hm .hpp
  .hxx .in .txx

CMake Error: CMake can not determine linker language for target: example
CMake Error: Cannot determine link language for target "example".

----------------------------------------------------------------------------

Here is my CMakeLists.txt:
cmake_minimum_required(VERSION 3.0 FATAL_ERROR)

project(Digit-Recognition-MNIST)

find_package(Torch REQUIRED)

add_executable(example training_VGG.cpp)

target_link_libraries(example "${TORCH_LIBRARIES}")

set_property(TARGET example PROPERTY CXX_STANDARD 11)

I hope anyone can help me.


title: Hyperparameter Optimization of net structure with Pytorch

body:
I have built a neural net with a custom architecture, as designed below.
No I wanted to perform a hyperparameter optimization in order to find the optimal learning rate, batch size, but also the number of neurons per hidden layer and the number of layers.
I have three sperate NN in the beginning which concatenated and then processed again through some dense layers.
Until now I haven't found a way to tune the number of layers in Pytorch hyperparameter optimization.
Ideally, I would also like to test whether the three nets need a different number of layers.
class Individual(nn.Module):
    def __init__(self):
        super(Individual, self).__init__()
        self.fc1_left = nn.Linear(30, 24)
        self.fc1_right = nn.Linear(30, 24)
        self.fc1_cor = nn.Linear(2, 4)
        self.fc2_cor = nn.Linear(4, 4)
        self.fc2_left = nn.Linear(24,4)
        self.fc2_right = nn.Linear(24,4)
        self.fc3 = nn.Linear(12, 36)
        self.fc4 = nn.Linear(36,num_classes)
        self.Sigmoid_left = nn.Sigmoid()
        self.Sigmoid_right = nn.Sigmoid()

    def forward(self, x_left, x_right, x_cor, x_con):

        x_con_l = x_con[:,0].view(-1, 1)
        x_con_r = x_con[:,1].view(-1, 1)

        x_left = F.relu(self.fc1_left(x_left))
        x_left = F.relu(self.fc2_left(x_left))
        x_left = torch.mul(x_left, x_con_l)

        x_right = F.relu(self.fc1_right(x_right))
        x_right = F.relu(self.fc2_right(x_right))
        x_right = torch.mul(x_right, x_con_r)

        x_cor = F.relu(self.fc1_cor(x_cor))
        x_cor = F.relu(self.fc2_cor(x_cor))

        combined = torch.cat((x_left.view(x_left.size(0), -1),
                          x_right.view(x_right.size(0), -1)), dim=1)

        combined = torch.cat((combined.view(combined.size(0), -1),
                          x_cor.view(x_cor.size(0), -1)), dim=1)

        x1 = F.relu(self.fc3(combined))
        x2 = F.relu(self.fc4(x1))

        return x2


title: Overfitting on a small dataset while finetuning Layoutlm. Which regularization techniques are suggested?

body:
I am currently using huggingface package to train my layoutlm model. However, I am experiencing overfitting for a token classification task. My dataset contains only 400 documents. I know it is very small dataset but I don't have any other chance to collect more data.
My results are in the table below. I have tried weight_decay=0.1 which is a high number in my opinion and also tried early stopping based on f1 score and loss seperately, but they didn't work.

Which regularisation techniques should I try extra? Do you have any solution to overfitting to a small dataset with bert-like models?


title: Extracting features of the hidden layer of an autoencoder using Pytorch

body:
I am following this tutorial to train an autoencoder.
The training has gone well. Next, I am interested to extract features from the hidden layer (between the encoder and decoder).
How should I do that?


title: How to fine tune BERT model?

body:
class BertClassifier(nn.Module):
    #Bert Model for Classification Tasks.

    def __init__(self, freeze_bert=False):
        """
        @param    bert: a BertModel object
        @param    classifier: a torch.nn.Module classifier
        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model
        """
        super(BertClassifier, self).__init__()
        # Specify hidden size of BERT, hidden size of our classifier, and number of labels
        D_in, H, D_out = 768, 50, 2

        # Instantiate BERT model
        self.bert = BertModel.from_pretrained('bert-base-uncased')

        # Instantiate an one-layer feed-forward classifier
        self.classifier = nn.Sequential(
            nn.Linear(D_in, H),
            nn.ReLU(),
            #nn.Dropout(0.5),
            nn.Linear(H, D_out)
        )

        # Freeze the BERT model
        if freeze_bert:
            for param in self.bert.parameters():
                param.requires_grad = False

I want to know that if we call bert_classifier = BertClassifier(freeze_bert=False) which means we are not freezing the weights, so does it implies that we are fine tuning ? Kindly correct me if I am wrong.


title: OpenCV crash one time in two with the same ONNX model converted from PyTorch

body:
my config :

Windows 10
Python 3.9
Pytorch 1.8.1
Torchvision 0.9.1
Onnx 1.9.0
OpenCV 4.5.2 (C++)

I convert a PyTorch model to ONNX and load it with OpenCV.
file_onnx = './r2p1d18.onnx'

my_model=torchvision.models.video.r2plus1d_18(pretrained=True,
                                                  progress=False)
my_model.to(torch.device('cpu'))
my_model.eval()

dummy_input = torch.randn(1, 3, 16, 112,112,requires_grad=False)

torch.onnx._export(my_model, dummy_input,
                  file_onnx,
                  training=False,
                  export_params=True,
                  verbose=True
              )

The export seems OK (no error log, model visually checked with Netron) and then I use it with OpenCV :
void test_so()
{
    std::string f_classes("../models/action_recognition.txt");
    std::string f_video("e:/divx/snowboard.mkv");

    //std::string f_onnx("../models/resnet-34_kinetics.onnx"); // OK
    //std::string f_onnx("../models/r3d18.onnx"); // NOT OK
    std::string f_onnx("../models/r2p1d18.onnx"); // NOT OK

    vector<string>v_classes;
    std::ifstream ifs(f_classes);
    std::string line;
    while (std::getline(ifs, line))
    {
        v_classes.push_back(line);
    }

    Net net = readNetFromONNX(std::string(f_onnx));
    //net.setPreferableTarget(DNN_TARGET_CUDA);
    //net.setPreferableBackend(DNN_BACKEND_CUDA);
    net.setPreferableTarget(DNN_TARGET_CPU);
    net.setPreferableBackend(DNN_BACKEND_OPENCV);

    VideoCapture cap(f_video);

    int sample_duration = 16;
    int sample_size = 112;

    vector<Mat>v_frames;

    // only the first sample used for the example
    for (int i = 0; i < sample_duration; i++)
    {
        Mat frame, frame_resized, frame_f;
        cap >> frame;
        resize(frame, frame_resized, Size(sample_size, sample_size));
        frame_resized.convertTo(frame_f, CV_32FC3);
        v_frames.push_back(frame_f);
    }

    //Mat blob = blobFromImages(v_frames, 1.0/255.0, Size(112,112),(0.485, 0.456, 0.406), true, false, CV_32F);
    Mat blob = blobFromImages(v_frames, 1.0, Size(112, 112), (114.7748, 107.7354, 99.4750), true, false, CV_32F);

    // rearrange matrix to fit in the model
    int sz[] = { 1,blob.size[1], blob.size[0], blob.size[2], blob.size[3] };
    Mat newblob = Mat(5, sz, CV_32F, blob.ptr<float>(0));

    net.setInput(newblob);
    cout << "Fwd... " << endl;
    Mat score = net.forward();
    cout << "Done." << endl;

    Point pmin, pmax;
    double vmin, vmax;
    minMaxLoc(score, &vmin, &vmax, &pmin, &pmax);
    cout << "Action : " << v_classes[pmax.x] << endl;
}

Then, I launch several times the same exe, with the same onnx file, and the same video file, and I get randomly different results :
Fwd...
OpenCV: terminate handler is called! The last OpenCV error is:
OpenCV(4.5.2) Error: Assertion failed (srcMat.dims == 2 && srcMat.cols == weights.cols &&
dstMat.rows == srcMat.rows && dstMat.cols == weights.rows && srcMat.type() == weights.type() &&
weights.type() == dstMat.type() && srcMat.type() == CV_32F && (biasMat.empty() || (biasMat.type()
== srcMat.type() && biasMat.isContinuous() && (int)biasMat.total() == dstMat.cols))) in
cv::dnn::FullyConnectedLayerImpl::FullyConnected::run, file E:\Develop\opencv-4.5.2\modules
\dnn\src\layers\fully_connected_layer.cpp, line 180

or :
Fwd...
OpenCV: terminate handler is called! The last OpenCV error is:
OpenCV(4.5.2) Error: Assertion failed (total(os[i]) > 0) in
cv::dnn::dnn4_v20210301::Net::Impl::getLayerShapesRecursively,
file E:\Develop\opencv-4.5.2\modules\dnn\src\dnn.cpp, line 3534

and sometimes, everything is OK (no matter if the class is not the good one) :
Fwd...
Done
Score : 1 lines , 400 cols
Action : drawing

I have tested the OpenCV code with another model (resnet34-kinetics) already provided in ONNX format and it works perfectly, so I suspect the conversion part.
I have also tested to convert a more simple model (superresolution) and used it with openCV (not with the same code, of course), it works fine.
During the conversion to ONNX, I have tried some options (requires_grad=True), and in OpenCV, tried to specify input and output layers names, but that does not changes anything to the behavior.
I am a beginner in this area, so did I forget something, is there anything wrong in the code, or is it something special to do with video models ?
Above all, what I don't understand is : why does it works sometimes, and when it does not, why the error is not always the same ?
Thanks in advance for your help


title: How do I load a local model with torch.hub.load?

body:
I need to avoid downloading the model from the web (due to restrictions on the machine installed).
This works, but downloads the model from the net
model = torch.hub.load('pytorch/vision:v0.9.0', 'deeplabv3_resnet101', pretrained=True)

I have placed the .pth file and the hubconf.py file in the /tmp/ folder and changed my code to
model = torch.hub.load('/tmp/', 'deeplabv3_resnet101', pretrained=True,s ource='local')

but to my surprise it still downloads the model from the internet. What am I doing wrong? How can I load the model locally.
Just to give you a bit more details, I'm doing all this in a Docker container which has a read-only volume at runtime, so that's why the download of new files fails.
thanks,
John


I am running a pytorch NLP model in python and I keep encountering the following strange error:
RuntimeError: [enforce fail at ..\c10\core\CPUAllocator.cpp:75] data. DefaultCPUAllocator: not enough
memory: you tried to allocate 364742208 bytes. Buy new RAM!

This is strange considering I have 16GB of RAM and the allocation is only 0.3GB. So I fail to understand the memory error.
Any help on the above will be appreciated.


title: Is the torchvision MNIST loader not working properly or am I doing something wrong?

body:
I wanted to see that the images I used in my network were OK, so I saved a bunch of them using the following code:

train_set = dset.MNIST(root=root, train=True, transform=transforms.ToTensor(), download=download)

for it, (img, target) in enumerate(train_loader):
    X = Variable(img)
    tar = Variable(target)
    X = X.view(batch_size, -1)
    cur_img_batch = X.data.numpy()
    cur_tar_batch = tar.data.numpy()
    for i in range(batch_size):
        cur_img = cur_img_batch[i]
        im = Image.fromarray(cur_img.reshape((28, 28)).astype('uint8') * 255)
        if cur_tar_batch[i] == 8:
            im.save(test_img_dir + 'iter_' + str(it) + '_sample_' + str(i) + '.png')

This isn't the cleanest code, but it just saves a bunch of images that are all labeled as '8'. Upon opening them, I see that most of them look like  this, even though a small minority of them are perfectly fine.

Am I doing something wrong?


title: Fine Tuning dimensionality error

body:
I am trying to use Resnet50 for image classification problem. However it shows error and I could not fix it.

RuntimeError: inconsistent tensor size, expected tensor [120 x 2048] and src [1000 x 2048] to have the same number of elements, but got 245760 and 2048000 elements respectively at /Users/soumith/code/builder/wheel/pytorch-src/torch/lib/TH/generic/THTensorCopy.c:86

and error happens below.

self.resnet = models.resnet50(num_classes=num_breeds, pretrained='imagenet')

Model is below

class Resnet(nn.Module):
    def __init__(self):
        super(Resnet,self).__init__()
        self.resnet = models.resnet50(num_classes=num_breeds, pretrained='imagenet')
        #self.resnet = nn.Sequential(*list(resnet.children())[:-2])
        #self.fc = nn.Linear(2048,num_breeds)

    def forward(self,x):
        x = self.resnet(x)
        return x


title: PyTorch LSTM States

body:
Consider the following code snipped:

lstm = nn.LSTM(10, 5, batch_first=True)
states = (torch.rand(1, 1, 5), torch.rand(1, 1, 5))
h, states = lstm(torch.rand(1, 1, 10), states)
print('h:')
print(h)
print('states[0]:')
print(states[0])

Output:

h:
tensor([[[0.2808, 0.3357, 0.1290, 0.1413, 0.2648]]], grad_fn=<TransposeBackward0>)
states[0]:
tensor([[[0.2808, 0.3357, 0.1290, 0.1413, 0.2648]]], grad_fn=<ViewBackward>)

Because I have to hand over states as parameter for the forward() anyway I'd prefer using states[0] over h.

I've just noticed that the grad_fn is different, therefore I'm wondering if it makes any difference for backpropagation if using h or states for further computation of the outputs.

I can hardly imagine that there is a difference, so I'd probably just continue with states[0] but I also would like to understand why it is different.

Thanks in advance!


title: Save and restore autograd enabled state

body:
Is there a way of saving and restoring the current state of autograd?

eg:

saved_state = torch.autograd.get_grad_enabled()  # How do I do this?
torch.autograd.set_grad_enabled(temporary_state)
# do things for great justice
torch.autograd.set_grad_enabled(saved_state)


title: Should I run a separate thread to save the model in Pytorch

body:
Consider the following snippet from PyTorch Imagenet Example (Please notice the "#AREA OF INTEREST"):

try:
for epoch in range(1, args.epochs+1):
    epoch_start_time = time.time()
    train()
    val_loss = evaluate(val_data)
    # Save the model if the validation loss is the best we've seen so far.
    if not best_val_loss or val_loss < best_val_loss:
        # AREA OF INTEREST ##########
        with open(args.save, 'wb') as f:
            torch.save(model, f)
        #############################
        best_val_loss = val_loss
    else:
        # Anneal the learning rate if no improvement has been seen in the validation dataset.
        lr /= 4.0

The question is: should I run a separate thread that saves the model in order to save time, or am I overkilling the process? i.e., it does not worth running a separate thread? I've checked the doc of torch.save but I did not find what I want.


title: Why listing model components in pyTorch is not useful?

body:
I am trying to create Feed forward neural networks with N layers
So idea is suppose If I want 2 inputs 3 hidden and 2 outputs than I will just pass [2,3,2] to neural network class and neural network model will get created so if I want [100,1000,1000,2]
where in this case 100 is inputs, two hidden layers contains 1000 neuron each and 2 outputs so I want fully connected neural network where I just wanted to pass list which contains number of neuron in each layer.
So for that I have written following code

class FeedforwardNeuralNetModel(nn.Module):
    def __init__(self, layers):
        super(FeedforwardNeuralNetModel, self).__init__()
        self.fc=[]
        self.sigmoid=[]
        self.activationValue = []
        self.layers = layers
        for i in range(len(layers)-1):
            self.fc.append(nn.Linear(layers[i],layers[i+1]))
            self.sigmoid.append(nn.Sigmoid())

    def forward(self, x):
        out=x
        for i in range(len(self.fc)):
            out=self.fc[i](out)
            out = self.sigmoid[i](out)
        return out

when I tried to use it I found it kind of empty model

model=FeedforwardNeuralNetModel([3,5,10,2])

print(model)

>>FeedforwardNeuralNetModel()

and when I used following code

class FeedforwardNeuralNetModel(nn.Module):
    def __init__(self, input_dim, hidden_dim, output_dim):
        super(FeedforwardNeuralNetModel, self).__init__()
        # Linear function
        self.fc1 = nn.Linear(input_dim, hidden_dim)
        # Non-linearity
        self.tanh = nn.Tanh()
        # Linear function (readout)
        self.fc2 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        # Linear function
        out = self.fc1(x)
        # Non-linearity
        out = self.tanh(out)
        # Linear function (readout)
        out = self.fc2(out)
        return out

and when I tried to print this model I found following result

print(model)

>>FeedforwardNeuralNetModel(
(fc1): Linear(in_features=3, out_features=5, bias=True)
(sigmoid): Sigmoid()
(fc2): Linear(in_features=5, out_features=10, bias=True)
)

in my code I am just creating lists that is what difference
I just wanted to understand why in torch listing model components is not useful?


title: fastai tabular model trained but can not find categorical mapping

body:
After training my dataset which has a number of categorical data using fastai's tabular model, I wish to read out the entity embedding and use it to map to my original data values.
I can see the embedding weights. The number of input don't seem to match anything, but maybe it is based on the unique categorical values in the train_ds.

To get that map, I would like to get the self.categories dictionary from the Categorify transform class. Is there anyway to get that from the data variable obtained by calling TabularList.from_df?
Or maybe someone can tell me a better way to get this map. I know the input df into the TabularList.from_df() is not it, because the number of rows are wrong. Most likely because df is splitted into train and valid subsets. But there is no easy way to obtain the train part of the TabularList to check just the train part.

It's strange I can't find any code example that shows this. Doesn't anyone else care to map the entity embedding value back to its original categorical value?


title: Advance indexing in Pytorch to get rid of nested for-loops

body:
I have a situation for which I am using nested for-loops, but I want to know if there's a faster way of doing this using some advanced indexing in Pytorch.

I have a tensor named t:

t = torch.randn(3,8)
print(t)
tensor([[-1.1258, -1.1524, -0.2506, -0.4339,  0.8487,  0.6920, -0.3160, -2.1152],
        [ 0.4681, -0.1577,  1.4437,  0.2660,  0.1665,  0.8744, -0.1435, -0.1116],
        [ 0.9318,  1.2590,  2.0050,  0.0537,  0.6181, -0.4128, -0.8411, -2.3160]])

I want to create a new tensor which indexes values from t.
Let's say these indexes are stored in variable indexes

indexes = [[(0, 1, 4, 5), (0, 1, 6, 7), (4, 5, 6, 7)],
           [(2, 3, 4, 5)],
           [(4, 5, 6, 7), (2, 3, 6, 7)]]

Each inner tuple in indexes represents four indexes that are to be taken from a row.

As an example, based on these indexes my output would be a 6x4 dimension tensor (6 is the total number of tuples in indexes, and 4 corresponds to one value in a tuple)

For instance, this is what I want to do:

#counting the number of tuples in indexes
count_instances = sum([1 for lst in indexes for tupl in lst])

#creating a zero output matrix
final_tensor = torch.zeros(count_instances,4)

final_tensor[0] = t[0,indexes[0][0]]
final_tensor[1] = t[0,indexes[0][1]]
final_tensor[2] = t[0,indexes[0][2]]
final_tensor[3] = t[1,indexes[1][0]]
final_tensor[4] = t[2,indexes[2][0]]
final_tensor[5] = t[2,indexes[2][1]]

The final output looks like this:
print(final_tensor)

tensor([[-1.1258, -1.1524,  0.8487,  0.6920],
        [-1.1258, -1.1524, -0.3160, -2.1152],
        [ 0.8487,  0.6920, -0.3160, -2.1152],
        [ 1.4437,  0.2660,  0.1665,  0.8744],
        [ 0.6181, -0.4128, -0.8411, -2.3160],
        [ 2.0050,  0.0537, -0.8411, -2.3160]])

I created a function build_tensor (shown below) to achieve this with nested for-loops, but I want to know if there's a faster way of doing it with simple indexing in Pytorch. I want a faster way of doing it because I'm doing this operation hundreds of times with bigger index and t sizes.

Any help?

def build_tensor(indexes, t):
    #count tuples
    count_instances = sum([1 for lst in indexes for tupl in lst])
    #create a zero tensor
    final_tensor = torch.zeros(count_instances,4)
    final_tensor_idx = 0

    for curr_idx, lst in enumerate(indexes):
        for tupl in lst:
            final_tensor[final_tensor_idx] = t[curr_idx,tupl]
            final_tensor_idx+=1
    return final_tensor


title: Tensor size mismatch autoencoder pytorch

body:
I'm using stacked Autoencoder, which is a bunch of Conv layers.
However, I'm having a tensor mismatch error, and I'm not sure about the reason. Everything done in the Encoder is reversed in the Decoder!

This is for time-series data. Input shape is (bactch_size, 1, 3000)

Here's the code

class CDAutoEncoder(nn.Module):
    def __init__(self, input_size, output_size, kernel, stride):
        super(CDAutoEncoder, self).__init__()

        self.forward_pass = nn.Sequential(
            nn.Conv1d(input_size, output_size, kernel_size=kernel, stride=stride, padding=0),
            nn.PReLU(),
        )
        self.backward_pass = nn.Sequential(
            nn.ConvTranspose1d(output_size, input_size, kernel_size=kernel, stride=stride, padding=0),
            nn.PReLU(),
        )

    def forward(self, x):
        y = self.forward_pass(x)
        return y

    def reconstruct(self, x):
        return self.backward_pass(x)

class StackedAutoEncoder(nn.Module):
    def __init__(self):
        super(StackedAutoEncoder, self).__init__()

        self.ae1 = CDAutoEncoder(1, 32, 50, 10)
        self.ae2 = CDAutoEncoder(32, 64, 10, 3)
        self.ae3 = CDAutoEncoder(64, 64, 5, 1)

    def forward(self, x):
        a1 = self.ae1(x)
        a2 = self.ae2(a1)
        a3 = self.ae3(a2)

        return self.reconstruct(a3)

    def reconstruct(self, x):
        a2_reconstruct = self.ae3.reconstruct(x)
        a1_reconstruct = self.ae2.reconstruct(a2_reconstruct)
        x_reconstruct = self.ae1.reconstruct(a1_reconstruct)
        return x_reconstruct

The error:

RuntimeError: The size of tensor a (2990) must match the size of tensor b (3000) at non-singleton dimension 2

I've tried adding padding and it worked, but when I changed the kernel size I get different tensor-size-mismatch-error.

Apparently, there's nothing like 'same' padding, so is there automated solution for this?


title: "Exception from Layer: 149" Problem with CoreML model after converting from PyTorch

body:
I trained a YOLOv3-SPP model using PyTorch. I then saved it as an ONNX model and then converted it to CoreML using onnx-coreml. When I try to make a prediction using the model I get this error:

    YOLOv3-CoreML[13481:1004975] [espresso] [Espresso::handle_ex_plan] exception=Espresso exception: "Invalid state": reshape mismatching size: 13 13 24 1 1 -> 6 10 8 3 1 [Exception from Layer: 149: 300]
2020-03-16 13:46:05.248612-0500 YOLOv3-CoreML[13481:1004975] [coreml] Error computing NN outputs -1

This is the code I am using to make the prediction:

 if let prediction = try? model.prediction(input_1: image) {
      print("Output: \(prediction)")
    }

I did some digging to find layer #149. I used this script to find its name:

import coremltools
import numpy as np

mlmodel = coremltools.models.MLModel("model.mlmodel")
spec = mlmodel._spec

print(spec.neuralNetwork.layers[149])

I found its name to be "308". So I opened the model up in Netron and got this:

The layer in question is circled in red. How can I get my CoreML model to work properly?


title: Cannot create CustomDataset LibTorch/PyTorch

body:
I have a folder with a set of images that I want to load as a CustomDataset in LibTorch.

int main()
{
    ...
    std::string file_location{"dataset/img_align_celeba/*.jpg"};
    auto train_set = CustomDataset(file_location).map(data::transforms::Stack<>());
}

CustomDataset.h

using namespace torch;

torch::Tensor read_data(const std::string& loc);

std::vector<std::string> obtain_data(std::string &address);

class CustomDataset : public data::Dataset<CustomDataset> {
private:
    std::vector<std::string> m_filenames;
public:
    // Constructor
    explicit CustomDataset(std::string &files): m_filenames(obtain_data(files)) {
    };

    // Override get() function to return tensor at location index
    torch::data::Example<> get(size_t index) override {
        torch::Tensor sample_img = read_data(m_filenames[index]);
        torch::Tensor sample_label = torch::full({1}, 1);
        return {sample_img, sample_label};
    };

    // Return the length of data
    torch::optional<size_t> size() const override {
        return m_filenames.size();
    };
};

CustomDataset.cpp

#include "CustomDataset.h"

torch::Tensor read_data(const std::string& loc)
{
    // Read Data here
    // Return tensor form of the image
    cv::Mat img = cv::imread(loc);
    std::vector<cv::Mat> channels(3);
    cv::split(img, channels);

    auto R = torch::from_blob(
            channels[2].ptr(),
            {64, 64},
            torch::kUInt8);
    auto G = torch::from_blob(
            channels[1].ptr(),
            {64, 64},
            torch::kUInt8);
    auto B = torch::from_blob(
            channels[0].ptr(),
            {64, 64},
            torch::kUInt8);

    auto tdata = torch::cat({R, G, B})
            .view({3, 64, 64})
            .to(torch::kFloat);

    return tdata;
}

std::vector<std::string> obtain_data(std::string &address)
{
    std::vector<std::string> filenames;
    cv::glob(address, filenames);
    return filenames;
}

Even though when I am inside the CustomDataset class I see that the member variable m_filenames is not empty and that get() provides a Tensor with the image and label, once I get back to main.cpp the CustomDataset variable called train_set has size 0.


title: Libtorch operator/syntax errors in Visual Studio

body:
Hi recently I’ve installed Libtorch and I was able to use it in my new Visual Studio project without a problem. Currently I am trying to use Libtorch in an existing CUDA project. But I am having these strange errors when I include torch header and I couldn’t find any solution in the internet regarding to my problem. Does anyone have any idea what might be the cause of these errors?

Severity    Code    Description Project File    Line    Suppression State
Error   C2833   'operator {' is not a recognized operator or type   DepthSensing    e:\research\libtorch\include\c10\util\flat_hash_map.h   1433
Error   C2059   syntax error: 'newline' DepthSensing    e:\research\libtorch\include\c10\util\flat_hash_map.h   1433
Error   C2238   unexpected token(s) preceding ';'   DepthSensing    e:\research\libtorch\include\c10\util\flat_hash_map.h   1433
Error   C2143   syntax error: missing ';' before 'const'    DepthSensing    e:\research\libtorch\include\c10\util\flat_hash_map.h   1433
Error   C2833   'operator {' is not a recognized operator or type   DepthSensing    e:\research\libtorch\include\c10\util\order_preserving_flat_hash_map.h  1552
Error   C2059   syntax error: 'newline' DepthSensing    e:\research\libtorch\include\c10\util\order_preserving_flat_hash_map.h  1552
Error   C2238   unexpected token(s) preceding ';'   DepthSensing    e:\research\libtorch\include\c10\util\order_preserving_flat_hash_map.h  1552
Error   C2143   syntax error: missing ';' before 'const'    DepthSensing    e:\research\libtorch\include\c10\util\order_preserving_flat_hash_map.h  1552

Environment

Windows 10
CUDA 10.1
Visual Studio 2017
C++14


title: Pytorch neural networks, understanding fully connected layers

body:
def __init__(self):
        super(Net, self).__init__()
        # 1 input image channel, 6 output channels, 3x3 square convolution
        # kernel
        self.conv1 = nn.Conv2d(1, 6, 3)
        self.conv2 = nn.Conv2d(6, 16, 3)
        # an affine operation: y = Wx + b
        self.fc1 = nn.Linear(16 * 6 * 6, 120)  # 6*6 from image dimension
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

From the above image and code from the PyTorch neural network tutorial, I can understand the dimensions of the convolution. How is the output dimension of 'nn.Linear' determined? Also, why do we require three fully connected layers?
Any help will be highly appreciated. TIA


title: How to use pytorch in flask and run it on wsgi_mod for apache2

body:
I'm trying to run a flask app with apache in wsgi_mod
As Describe in following link
https://pytorch.org/tutorials/recipes/deployment_with_flask.html
i configure my app for working with pytorch
when my app tries to
import torchvision
it hangs with no error log.
Is there any way to fix this issue?
the main python code is:
import torchvision.models as models
import torchvision.transforms as transforms
from flask import Flask
app = Flask(__name__)
@app.route('/')
def hello():
    return ("Hello World qwertyuiop!!")
if __name__ == "main":
    app.run()

and the apache2 config is:
User daemon
Group daemon

</IfModule>

LoadFile "G:/Python3764/python37.dll"
LoadModule wsgi_module "G:/Python3764/lib/site-packages/mod_wsgi/server/mod_wsgi.cp37-win_amd64.pyd"
WSGIPythonHome "G:/Python3764"

<VirtualHost *:80>
    ServerName localhost:80
    WSGIScriptAlias / "D:/WSGIAppDir/web.wsgi"
    DocumentRoot "D:/WSGIAppDir"
    <Directory "D:/WSGIAppDir">
        Require all granted
    </Directory>
</VirtualHost>

import sys
sys.path.insert(0, 'D:/WSGIAppDir')
from hello import app as application


title: How do we convert more than one .pth files to one .pb file?

body:
If someone could help me I would be very very appreciate!
I would like to ask how about the conversion from .pth to .pb file.
Especially, I use MADDPG algorithm with pytorch and I have 4 .pth files.
How I can convert all of them and have one .pb file?
Firstly, I tried to convert one of my 4 .pth files to .pb file with these instructions
How can we convert a .pth model into .pb file?
but I receive this error.
C:\Python36\lib\site-packages\onnx_tf\common\__init__.py:89: UserWarning:
onnx_tf.common.get_outputs_names is deprecated. It will be removed in future release. Use
TensorflowGraph.get_outputs_names instead.
  warnings.warn(message)
Traceback (most recent call last):
  File "myscript.py", line 2, in <module>
    from onnx_tf.backend import prepare
  File "C:\Python36\lib\site-packages\onnx_tf\__init__.py", line 1, in <module>
    from . import backend
  File "C:\Python36\lib\site-packages\onnx_tf\backend.py", line 29, in <module>
   from onnx_tf.common.handler_helper import get_all_backend_handlers
  File "C:\Python36\lib\site-packages\onnx_tf\common\handler_helper.py", line 5, in <module>
    from onnx_tf.handlers.backend import *  # noqa
  File "C:\Python36\lib\site-packages\onnx_tf\handlers\backend\average_pool.py", line 7, in <module>
from .pool_mixin import PoolMixin
  File "C:\Python36\lib\site-packages\onnx_tf\handlers\backend\pool_mixin.py", line 12, in <module>
from .dilated_pooling import DilatedPooling
  File "C:\Python36\lib\site-packages\onnx_tf\handlers\backend\dilated_pooling.py", line 6, in
 <module>
from onnx_tf.common import pooling_helper
  File "C:\Python36\lib\site-packages\onnx_tf\common\pooling_helper.py", line 16, in <module>
pad_tf_ops = pad_ops(tf.maximum, tf.math.ceil, tf.math.floor,

AttributeError: module 'tensorflow' has no attribute 'math'


title: Resnet implementation: forward() takes 1 positional argument but 2 were given

body:
I wrote this code and when I run it I get the following error: forward() takes 1 positional argument but 2 were given. As far as I know, I am passing only one argument to forward().

ResNet is a basic residual block

class ResNet(nn.Module):
    def __init__(self, in_channels, mid_channels, mid2_channels ,out_channels):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels,mid_channels,kernel_size = 3, stride = 1, padding = 1)
        self.conv1_bn = nn.BatchNorm2d(mid_channels)
        self.conv2 = nn.Conv2d(mid_channels,mid2_channels,kernel_size = 3, stride = 1, padding = 1)
        self.conv2_bn = nn.BatchNorm2d(mid2_channels)
        self.conv3 = nn.Conv2d(mid2_channels,out_channels,kernel_size = 3, stride = 1, padding = 1)
        self.conv3_bn = nn.BatchNorm2d(out_channels)
        if (in_channels != out_channels):
            self.conv_shortcut = nn.Conv2d(in_channels, out_channels, kernel_size = 1, stride = 1, padding = 0 )

    def forward(self, X):
         X_shortcut = X
        X = F.relu(self.conv1(X))
        X =  self.conv1_bn(X)
        X = F.relu(self.conv2(X))
        X =  self.conv2_bn(X)
        X = F.relu(self.conv2(X))
        X =  self.conv2_bn(X)
        if (in_channels == out_channels):
            X = self.conv3(X) + X_shortcut
        else:
            X = self.conv3(X) + self.conv_shortcut(X_shortcut)
        X = self.conv3_bn(F.relu(x))
        return X

This the method for generating a model using the given layers.

class TotalNet(nn.Module):
    def __init__(self, Layers):
        super().__init__()
        self.hidden = nn.ModuleList()
        self.hidden.append(nn.BatchNorm2d(1))
        for i in range(0,len(Layers)-1,3):
            in_channels, mid_channels, mid2_channels, out_channels = Layers[i:(i+4)]
            self.hidden.append(ResNet(in_channels, mid_channels, mid2_channels, out_channels))
            self.hidden.append(nn.Flatten())

    def forward(self, X):
        X = self.hidden(X)
        return X

the following is how I am calling the function:

test = TotalNet([9,2,9,9,9,9,9,9,9,9])
a = torch.rand((1,9,9), dtype = torch.float32)
test(a)


title: How do I mutate the input using gradient descent in PyTorch?

body:
I'm new to PyTorch. I learned it uses autograd to automatically calculate the gradients for the gradient descent function.
Instead of adjusting the weights, I would like to mutate the input to achieve a desired output, using gradient descent. So, instead of the weights of neurons changing, I want to keep all of the weights the same and just change the input to minimize the loss.
For example. The network is a trained image classifier with the numbers 0-9. I input random noise, and I want to morph it so that the network considers it a 3 with 60% confidence. I would like to utilize gradient descent to adjust the values of the input (originally noise) until the network considers the input to be a 3, with 60% confidence.
Is there a way to do this?