title: cnn wrong prediction even though model shows good accuracy in training and validation data

body:
I have used the skin cancer classification competition data in Kaggle. There are 4 labels and the entire data is imbalanced. I ran the resnet 18 model on a 10 fold cross validation split to train the data and each fold was given around 2 epochs. The code has been attached below.
Basically the model gave 98.2% accuracy with 0.07 loss value in the train data and 98.1% accuracy and 0.06 loss value in the validation data. So this seemed pretty good.
However the problem is...prediction.py(code attached below). When I tried to predict, the model keeps giving the result as [0]. Even if it's a train image data.
Is there something wrong with my code?
Expected result:
if the image is the input, the output should be either 0,1,2 or 3
model.py(where the training happens)
import pandas as pd
import numpy as np

import torch
import torch.nn as nn

import os

import cv2

import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split

import albumentations as A

from torch.utils.data import TensorDataset, DataLoader,Dataset
from torchvision import models
from collections import defaultdict
from torch.utils.data.sampler import RandomSampler
import torch.optim as optim
from torch.optim import lr_scheduler
from sklearn import model_selection
from tqdm import tqdm
import gc

# generate data from csv file
class Build_dataset(Dataset):
    def __init__(self, csv, split, mode, transform=None):
        self.csv = csv.reset_index(drop=True)
        self.split = split
        self.mode = mode
        self.transform = transform

    def __len__(self):
        return self.csv.shape[0]

    def __getitem__(self, index):
        row = self.csv.iloc[index]

        image = cv2.imread(row.filepath)
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

        if self.transform is not None:
            res = self.transform(image=image)
            image = res['image'].astype(np.float32)
        else:
            image = image.astype(np.float32)

        image = image.transpose(2, 0, 1)
        data = torch.tensor(image).float()

        if self.mode == 'test':
            return data
        else:
            return data, torch.tensor(self.csv.iloc[index].target).long()

# training data
def train_epoch(model, loader, optimizer,loss_fn,device, scheduler,n_examples):

    model = model.train()

losses = []
correct_predictions = 0

for inputs, labels in tqdm(loader):
    inputs = inputs.to(device)
    labels = labels.to(device)

    outputs = model(inputs)

    _, preds = torch.max(outputs, dim=1)
    loss = loss_fn(outputs, labels)

    correct_predictions += torch.sum(preds == labels)
    losses.append(loss.item())

    loss.backward()
    optimizer.step()
    optimizer.zero_grad()
# here you delete inputs and labels and then use gc.collect
    del inputs, labels
    gc.collect()

return correct_predictions.double() / n_examples, np.mean(losses)

# validation data
def val_epoch(model, loader,loss_fn, device,n_examples):

    model = model.eval()

    losses = []
    correct_predictions = 0

    with torch.no_grad():
        for inputs, labels in tqdm(loader):
            inputs = inputs.to(device)
            labels = labels.to(device)
            outputs = model(inputs)
            _, preds = torch.max(outputs, dim=1)
            loss = loss_fn(outputs, labels)
            correct_predictions += torch.sum(preds == labels)
            losses.append(loss.item())
            # here you delete inputs and labels and then use gc.collect
            del inputs, labels
            gc.collect()

    return correct_predictions.double() / n_examples, np.mean(losses)

 def train(fold, model,device, num_epochs):

    df_train = df[df.kfold != fold].reset_index(drop=True)
    df_valid = df[df.kfold == fold].reset_index(drop=True)
    # generate data
    dataset_train = Build_dataset(df_train,  'train', 'train', transform=transforms_train)
    dataset_valid = Build_dataset(df_valid, 'train', 'val', transform=transforms_val)

    #load data
    train_loader = DataLoader(dataset_train, batch_size = 64,sampler=RandomSampler(dataset_train),
num_workers=4)
    valid_loader = DataLoader(dataset_valid, batch_size = 32,shuffle = True, num_workers= 4 )

    dataset_train_size = len(dataset_train)

    dataset_valid_size = len(dataset_valid)

    optimizer = optim.Adam(model.parameters(), lr = 1e-4)

    model = model.to(device)

    scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, patience = 3,threshold = 0.001, mode =
'max')

    loss_fn = nn.CrossEntropyLoss().to(device)

    history = defaultdict(list)

    best_accuracy = 0.0

    for epoch in range(num_epochs):
        print(f'Epoch {epoch+1} / {num_epochs}')
        print ('-'*30)

        train_acc, train_loss = train_epoch(model, train_loader, optimizer, loss_fn, device,
scheduler, dataset_train_size)
        print(f'Train loss {train_loss} accuracy {train_acc}')
        valid_acc, valid_loss = val_epoch(model, valid_loader, loss_fn, device,dataset_valid_size)
        print(f'Val   loss {valid_loss} accuracy {valid_acc}')
        print()

        history['train_acc'].append(train_acc)
        history['train_loss'].append(train_loss)
        history['val_acc'].append(valid_acc)
        history['val_loss'].append(valid_loss)

        if valid_acc > best_accuracy:
            print('saving model')
            torch.save(model.state_dict(), f'best_model_{fold}.bin')
            best_accuracy = valid_acc

    print(f'Best Accuracy: {best_accuracy}')

    model.load_state_dict(torch.load(f'best_model_{fold}.bin'))

    return model, history

 if __name__ == '__main__':
    #competition data -2020
    data_dir = "../input/jpeg-melanoma-384x384"
    #competition data - 2019
    data_dir2 = "../input/jpeg-isic2019-384x384"
    # device
    device = torch.device("cuda")

    # augmenting images

    image_size = 384
    transforms_train = A.Compose([
        A.Transpose(p=0.5),
        A.VerticalFlip(p=0.5),
        A.HorizontalFlip(p=0.5),
        A.RandomBrightness(limit=0.2, p=0.75),
        A.RandomContrast(limit=0.2, p=0.75),
        A.OneOf([
            A.MedianBlur(blur_limit=5),
            A.GaussianBlur(blur_limit=5),
            A.GaussNoise(var_limit=(5.0, 30.0)),
        ], p=0.7),

        A.OneOf([
            A.OpticalDistortion(distort_limit=1.0),
            A.GridDistortion(num_steps=5, distort_limit=1.),
            A.ElasticTransform(alpha=3),
        ], p=0.7),

        A.CLAHE(clip_limit=4.0, p=0.7),
        A.HueSaturationValue(hue_shift_limit=10, sat_shift_limit=20, val_shift_limit=10, p=0.5),
        A.ShiftScaleRotate(shift_limit=0.1, scale_limit=0.1, rotate_limit=15, border_mode=0, p=0.85),
        A.Resize(image_size, image_size),
        A.Cutout(max_h_size=int(image_size * 0.375), max_w_size=int(image_size * 0.375), num_holes=1,
p=0.7),
        A.Normalize()
    ])

    transforms_val = A.Compose([
        A.Resize(image_size, image_size),
        A.Normalize()
    ])
    # create data
    df_train = pd.read_csv(os.path.join(data_dir, "train.csv"))  #/kaggle/input/siim-isic-melanoma-classification/train.csv
    df_train.head()

    df_train['is_ext'] = 0
    df_train['filepath'] = df_train['image_name'].apply(lambda x: os.path.join(data_dir, 'train', f'{x}.jpg'))

    # dataset from 2020 data
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('seborrheic keratosis', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lichenoid keratosis', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('solar lentigo', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('lentigo NOS', 'BKL'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('cafe-au-lait macule', 'unknown'))
    df_train['diagnosis'] = df_train['diagnosis'].apply(lambda x: x.replace('atypical melanocytic proliferation', 'unknown'))

    # shuffle data
    df = df_train.sample(frac=1).reset_index(drop=True)

    # creating 8 different target values
    new_target = {d: idx for idx, d in enumerate(sorted(df.diagnosis.unique()))}
    df['target'] = df['diagnosis'].map(new_target)
    mel_idx = new_target['melanoma']

    # creating 10 fold cross validation data
    df = df_train.sample(frac=1).reset_index(drop=True)
    df['kfold'] = -1
    y = df_train.target.values
    kf = model_selection.StratifiedKFold(n_splits=10,shuffle=True)
    idx = kf.get_n_splits(X=df,y=y)
    print(idx)
    for fold,(x,y) in enumerate(kf.split(X=df,y=y)):
        df.loc[y,'kfold'] = fold

    df = df[['filepath','diagnosis', 'target', 'is_ext', 'kfold']]

    class_names = list(df['diagnosis'].unique())

    # create model

    def create_model(n_classes):
        model = models.resnet18(pretrained=True)

        n_features = model.fc.in_features
        model.fc = nn.Linear(n_features, n_classes)
        return model.to(device)

    base_model = create_model(len(class_names)) # model ready

    # run the model
    for i in range(10):
        #train
        base_model, history = train(i, base_model, device, num_epochs = 2) # train data

prediction.py
from torchvision import models
import torch
import torch.nn as nn
import albumentations as A
import cv2
import os
import numpy as np

device = torch.device("cuda")
MODEL = None
MODEL_PATH = "../input/prediction/best_model_4.bin"

def create_model(n_classes):
    model = models.resnet18(pretrained=True)

    n_features = model.fc.in_features
    model.fc = nn.Linear(n_features, n_classes)
    return model.to(device)
# generate the data to tensor with transform application

# converting the image to tensor by using the transforms function

class get_image:
    def __init__(self, image_path, targets, transform = None):
        self.image_path = image_path
        self.targets = targets
        self.transform = transform

    def __len__(self):
        return len(self.image_path)
    def __getitem__(self, item):
        targets = self.targets[item]
        resize = 384
        image = cv2.imread(self.image_path[item])
        image = cv2.resize(image, (resize, resize))
        image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)

        if self.transform is not None:
            res = self.transform(image = image)
            image = res['image'].astype(np.float32)
        image = image.transpose(2, 0, 1)
        data = torch.tensor(image).float()
        targets = torch.tensor(targets)

        return data, targets

# load the data by using torch data
# predict values

# predict function
def predict(image_path, model, model_path):
    image_size = 384

    transforms_val = A.Compose([
        A.Resize(image_size, image_size),
        A.Normalize()
    ])

    test_images = [image_path]
    test_targets = [0]

    test_data = get_image(
        image_path = test_images,
        targets = test_targets,
        transform=transforms_val)
    # loading the data
    test_dataloader = torch.utils.data.DataLoader(test_data, batch_size=1, shuffle = False,
num_workers=0)
    model = create_model(n_classes = 4)
    model.load_state_dict(torch.load(model_path))
    model.to(device)
    model.eval()
    prediction = []

    with torch.no_grad():
        for test_data, test_target in test_dataloader:
            test_data = test_data.to(device)
            test_target = test_target.to(device)

            outputs = model(test_data)
            _,preds = torch.max(outputs.cpu(), 1)

            #prediction.extend(preds)

            prediction = np.vstack((preds)).ravel()

            return prediction

def upload_predict():
    image_file = "../input/whatever/ISIC_0075663.jpg"

    if image_file:
        pred = predict(image_file, MODEL, MODEL_PATH)
        print(pred)

    return pred

the label and it's count is given right here
3    27126
2     5193
1      584
0      223

Here 0 is considered malignant type cancer and the other labels are of different types.
Here is the link to the data: https://www.kaggle.com/cdeotte/jpeg-melanoma-384x384


title: Computing mean of list of generator objects

body:
// Initialize 2-D array, each entry is some neural network
phi = [[None] * n for _ in range(m)]
for i in range(m):
    for j in range(n):
        phi[i][j] = NeuralNetwork()

// Let k, i be arbitrary indices
p1 = torch.nn.utils.parameters_to_vector(phi[k][i - 1].parameters())
p2 = torch.nn.utils.parameters_to_vector(mean of phi[:][i-1])

I want to basically compute the mean squared error between the parameters phi[k][i-1] and average of the entire column phi[:][i-1] i.e. ((p1 - p2)**2).sum() I tried in the following way:
tmp = [x.parameters() for x in self.phi[:][i - 1]]
mean_params = torch.mean(torch.stack(tmp), dim=0)
p2 = torch.nn.utils.parameters_to_vector(mean_params)

But this doesn't work out because tmp is a list of generator objects. More specifically, I guess my problem is to compute the mean from that generator object.


title: How to limit the performance overhead caused by Tensor.to() in Torch?

body:
I'm working on a real-time object tracking network based on Tracktor and the mmdetection framework. I'm simulating a constant stream of input data by running it with a batch size of 1. I noticed a very large slowdown caused by the constant usage of the .to() Torch method, over which I have no control, to my knowledge.
What techniques can be used to limit the overhead caused by the transfer of data from the CPU to the GPU?
Here's Pycharm's profiling with the important links highlighted:

And the code corresponding to the last box before .to():
def grid_anchors(self, featmap_sizes, device='cuda'):
    """Generate grid anchors in multiple feature levels.

    Args:
        featmap_sizes (list[tuple]): List of feature map sizes in
            multiple feature levels.
        device (str): Device where the anchors will be put on.

    Return:
        list[torch.Tensor]: Anchors in multiple feature levels. \
            The sizes of each tensor should be [N, 4], where \
            N = width * height * num_base_anchors, width and height \
            are the sizes of the corresponding feature lavel, \
            num_base_anchors is the number of anchors for that level.
    """
    assert self.num_levels == len(featmap_sizes)
    multi_level_anchors = []
    for i in range(self.num_levels):
        anchors = self.single_level_grid_anchors(
            self.base_anchors[i].to(device),
            featmap_sizes[i],
            self.strides[i],
            device=device)
        multi_level_anchors.append(anchors)
    return multi_level_anchors

And the main loop that loads the images and runs the tracking:
from torch.utils.data import DataLoader
from tqdm import tqdm

data_loader = DataLoader(seq, batch_size=1, shuffle=False)
for i, frame in enumerate(tqdm(data_loader)):
    if len(seq) * tracktor['frame_split'][0] <= i <= len(seq) * tracktor['frame_split'][1]:
        with torch.no_grad():
            tracker.step(frame)
        num_frames += 1


title: Docker full with URL path

body:
Good day all,
Anyone knows if it's possible to just pull a single container from github? I do have this link https://github.com/aws/sagemaker-pytorch-training-toolkit and I will like to pull the container in this link https://github.com/aws/sagemaker-pytorch-training-toolkit/tree/master/src/sagemaker_pytorch_container.
I did try using build docker build -t https://github.com/abc/sagemaker-pytorch-training-toolkit.git to just build an image of one file but there's an init.py file which i'm not sure if its necessary.
Thanks


title: Difference between Keras and Pytorch code with same architecture

body:
I am building an Autoencoder model and I have found two snippets of code with the same architecture model which are in Keras and PyTorch. But when I run it, a large difference in time even though they are using the same architecture. Could you please explain why I am encountering such a huge difference in time as well as performance?
PyTorch Code
class Encoder(nn.Module):

    def __init__(self, seq_len, n_features, embedding_dim=64):
        super(Encoder, self).__init__()

        self.seq_len, self.n_features = seq_len, n_features
        self.embedding_dim, self.hidden_dim = embedding_dim, 2 * embedding_dim

        self.rnn1 = nn.LSTM(
            input_size=n_features,
            hidden_size=self.hidden_dim,
            num_layers=1,
            batch_first=True
        )

        self.rnn2 = nn.LSTM(
            input_size=self.hidden_dim,
            hidden_size=embedding_dim,
            num_layers=1,
            batch_first=True
        )

    def forward(self, x):
        x = x.reshape((1, self.seq_len, self.n_features))

        x, (_, _) = self.rnn1(x)
        x, (hidden_n, _) = self.rnn2(x)

        return hidden_n.reshape((self.n_features, self.embedding_dim))

class Decoder(nn.Module):

    def __init__(self, seq_len, input_dim=64, n_features=1):
        super(Decoder, self).__init__()

        self.seq_len, self.input_dim = seq_len, input_dim
        self.hidden_dim, self.n_features = 2 * input_dim, n_features

        self.rnn1 = nn.LSTM(
            input_size=input_dim,
            hidden_size=input_dim,
            num_layers=1,
            batch_first=True
        )

        self.rnn2 = nn.LSTM(
            input_size=input_dim,
            hidden_size=self.hidden_dim,
            num_layers=1,
            batch_first=True
        )

        self.output_layer = nn.Linear(self.hidden_dim, n_features)

    def forward(self, x):
        x = x.repeat(self.seq_len, self.n_features)
        x = x.reshape((self.n_features, self.seq_len, self.input_dim))

        x, (hidden_n, cell_n) = self.rnn1(x)
        x, (hidden_n, cell_n) = self.rnn2(x)
        x = x.reshape((self.seq_len, self.hidden_dim))

        return self.output_layer(x)

class RecurrentAutoencoder(nn.Module):

    def __init__(self, seq_len, n_features, embedding_dim=64):
        super(RecurrentAutoencoder, self).__init__()

        self.encoder = Encoder(seq_len, n_features, embedding_dim).to(device)
        self.decoder = Decoder(seq_len, embedding_dim, n_features).to(device)

    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)

        return x

model = RecurrentAutoencoder(seq_len, n_features, 128)

import time
time_dict = {}
def train_model(model, train_dataset, val_dataset, n_epochs):

    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
    criterion = nn.L1Loss(reduction='sum').to(device)
    history = dict(train=[], val=[])

    best_model_wts = copy.deepcopy(model.state_dict())
    best_loss = 10000.0
    time_dict[0] = time.time()
    for epoch in range(1, n_epochs + 1):

        model = model.train()

        train_losses = []
        for seq_true in train_dataset:
            optimizer.zero_grad()

            seq_true = seq_true.to(device)
            seq_pred = model(seq_true)

            loss = criterion(seq_pred, seq_true)

            loss.backward()
            optimizer.step()

            train_losses.append(loss.item())

        val_losses = []
        model = model.eval()
        with torch.no_grad():
            for seq_true in val_dataset:

                seq_true = seq_true.to(device)
                seq_pred = model(seq_true)

                loss = criterion(seq_pred, seq_true)
                val_losses.append(loss.item())

        train_loss = np.mean(train_losses)
        val_loss = np.mean(val_losses)

        history['train'].append(train_loss)
        history['val'].append(val_loss)

        if val_loss < best_loss:
            best_loss = val_loss
            best_model_wts = copy.deepcopy(model.state_dict())

        print(f'Epoch {epoch}: train loss {train_loss} val loss {val_loss}')
        stop = time.time()
        time_dict[epoch] = stop

    model.load_state_dict(best_model_wts)

    return model.eval(), history

model, history = train_model(
  model,
  train_dataset,
  val_dataset,
  n_epochs=10)

Output of print(model)

RecurrentAutoencoder(     (encoder): Encoder(
(rnn1): LSTM(1, 256, batch_first=True)
(rnn2): LSTM(256, 128, batch_first=True)   )
(decoder): Decoder(
(rnn1): LSTM(128, 128, batch_first=True)
(rnn2): LSTM(128, 256, batch_first=True)
(output_layer): Linear(in_features=256, out_features=1, bias=True)   ) )

Keras Code
model = keras.Sequential()
model.add(keras.layers.LSTM(
    units=256,
    input_shape=(X_train.shape[1], X_train.shape[2]), return_sequences=True))
model.add(keras.layers.LSTM(units=128, return_sequences=True))

model.add(keras.layers.LSTM(units=256, return_sequences=True))
model.add(keras.layers.TimeDistributed(keras.layers.Dense(units=X_train.shape[2])))
model.compile(loss='mae', optimizer='adam')
nb_epoch = 10

autoencoder.compile(optimizer='adam',
                    loss='mean_squared_error')

tensorboard = TensorBoard(log_dir='/tmp/logs',
                          histogram_freq=0,
                          write_graph=True,       #to visualize
                          write_images=True)

history = autoencoder.fit(X_train, X_train,
                    epochs=nb_epoch,
                    shuffle=True,
                    validation_data=(X_validate, X_validate),
                    verbose=1,
                    callbacks=[tensorboard]).history

Output of print(model.summary())

Layer (type)                 Output Shape              Param #
lstm_6 (LSTM)                (None, 1, 256)            266240
lstm_7 (LSTM)                (None, 1, 128)            197120
lstm_8 (LSTM)                (None, 1, 256)            394240
time_distributed (TimeDistri (None, 1, 3)              771

The architecture of both these models look same, but there is a difference in performance and training time.


title: Passing the output of a CNN encoder to convLSTM layers. (PyTorch)

body:
I'm interested in implementing a LinkNet based encoder-decoder structure for semantic segmentation on a custom dataset. I'm trying to introduce convLSTM layers between the encoder and decoder. Typically, as expected, the output of the encoder is a 4-dim output (batch_size, channels, height, width). The convLSTM layers expect a 5-dim input (batch_size, sequence_length, channels, height, width). How do I convert this 4-dim tensor to a 5-dim tensor, without any loss of information? I initially thought of splitting the batch_size to accommodate the sequence_length as well, but that might be a problem since I'm dealing with video frames.
Maybe I'm looking at using sequences of four/five frames for training i.e. the semantic segmentation map of frame t is determined by means of the info of the last three to four frames, and hence, a sequence_length of 4 or 5 would do.
How do I introduce the sequence length?  Is it during pre-processing or right after the encoder structure?
Most importantly, HOW TO DO IT?


title: Find module corresponding to the method in python

body:
I am learning to work with fastai. And I come across this:
xt = tensor(3.).requires_grad_()
xt
[out1]: tensor(3., requires_grad=True)

I want to know where requires_grad_() is from i.e., is it from fastai or is it pytorch related. So I do:
requires_grad_.__module__
[out2]: NameError: name 'require_grad_' is not defined

What am I doing wrong? What am I missing?


title: Creating an array of successive matrix powers in python: what is optimal?

body:
I often have to create an array of matrices containing successive powers of an operator
(e.g. when integrating an ordinary differential equation, or computing a polynomial function of an operator).
However I can't figure what is the best way to do this. It seems like using a for-loop is rather wise, so that one may incrementally fill in the array by multiplying x**(k-1) with x to get x**k, and avoid repeating too many costly matrix multiplications. Is there any better construct to be aware of?
For instance, say I have represented a rank-k polynomial p as a 1D-array of shape (k),
and  want to write a function evaluate that applies a polynomial to a square matrix x of shape (n, n):
def evaluate1 (p, x):
    return np.sum(p * (x ** np.arange(p.shape[0])))

def evaluate2 (p, x):
    powers = [1]
    for i in range(1, p.shape[0]):
        powers.append(np.matmul(powers[-1], x))
    return np.sum(p * np.array(powers))

I would naively say that evaluate1 outperforms evaluate2 by making use of vectorisation for small n...
Is there any evaluate3 to follow best practices?


title: How to accomplish the multi-dimensional fancy indexing in PyTorch?

body:
I have a 3D tensor ([batch_size, seq_length, hidden_dim]) and a 2D list ([batch_size, seq_length]).
I want to using list to accomplish the selection of this tensor.
For example: the shape of 3D tensor t [2, 5, 3] and the shape of 2D list l  [2, 5].
Let t0 = t[0, :, :] and l0 = l[0]. I would like to select "t0[l0]".
Same for t1[l1], and so on.
I can only think of writng a for loop for achieve this:
new_ts = []
for i in range(t.shape[0]):
    new_t = t[i][l[i]]
    new_ts.append(new_t)
new_t = torch.cat(new_ts, dim=2)

There must be a more simple way to accomplish this. I have also tried multi-dimensional fancy indexing t[l], but the syntax is not valid and it doesn't work.
Looking forward to your suggestions.


title: Custom dataset Loader pytorch

body:
i am doing  covid-19 classification.i took dataset from kaggle. it has folder named dataset which contain 3 folders normal pnuemonia and covid-19 each contaning images for these classes i am stucked in writting getitem in pytorch custom dataloader ?
Dataset has 189 covid images but by this get item i get 920 images of covid kindly help
class_names = ['normal', 'viral', 'covid']
root_dir = 'COVID-19 Radiography Database'
source_dirs = ['NORMAL', 'Viral Pneumonia', 'COVID-19']

 if os.path.isdir(os.path.join(root_dir, source_dirs[1])):
   os.mkdir(os.path.join(root_dir, 'test'))

for i, d in enumerate(source_dirs):
    os.rename(os.path.join(root_dir, d), os.path.join(root_dir, class_names[i]))

for c in class_names:
    os.mkdir(os.path.join(root_dir, 'test', c))

for c in class_names:
    images = [x for x in os.listdir(os.path.join(root_dir, c)) if x.lower().endswith('png')]
    selected_images = random.sample(images, 30)
    for image in selected_images:
        source_path = os.path.join(root_dir, c, image)
        target_path = os.path.join(root_dir, 'test', c, image)
        shutil.move(source_path, target_path)

Above code is used to create test dataset which has 30 images of each class
 class ChestXRayDataset(torch.utils.data.Dataset):
   def __init__(self, image_dirs, transform):
      def get_images(class_name):
        images = [x for x in os.listdir(image_dirs[class_name]) if
        x[-3:].lower().endswith('png')]
        print(f'Found {len(images)} {class_name} examples')
        return images

    self.images = {}
    self.class_names = ['normal', 'viral', 'covid']

    for class_name in self.class_names:
        self.images[class_name] = get_images(class_name)

    self.image_dirs = image_dirs
    self.transform = transform

def __len__(self):
    return sum([len(self.images[class_name]) for class_name in self.class_names])

def __getitem__(self, index):
    class_name = random.choice(self.class_names)
    index = index % len(self.images[class_name])
    image_name = self.images[class_name][index]
    image_path = os.path.join(self.image_dirs[class_name], image_name)
    image = Image.open(image_path).convert('RGB')
    return self.transform(image), self.class_names.index(class_name)

**Stucked in get item of this **
images in folder are arranged as follows
Dataset is as follows
**Code for confusion matrix is **
nb_classes = 3

 confusion_matrix = torch.zeros(nb_classes, nb_classes)
 with torch.no_grad():
 for data in tqdm_notebook(dl_train,total=len(dl_train),unit='batch'):
    img,lab = data
    print(lab)
    img,lab = img.to(device),lab.to(device)
    _,output = torch.max(model(img),1)
    print(output)

    for t, p in zip(lab.view(-1), output.view(-1)):
            confusion_matrix[t.long(), p.long()] += 1

output for confusion matrix only one class is getting trained
confusio matrix image


title: What is the right input / shape for training a pretrained RoBERTa?

body:
Right now I am trying to train/finetune a pretrained RoBERTa model with a multichoice head, but I am having difficulty finding the right input so my model is able to train/finetune.
The dataframe I have right now looks like this:

With the 3 options being tokenized sentences, using:
tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
for i in range(0, len(train_data)):
  train_data["OptionA"][i] = tokenizer.encode(train_data["OptionA"][i])
  train_data["OptionB"][i] = tokenizer.encode(train_data["OptionB"][i])
  train_data["OptionC"][i] = tokenizer.encode(train_data["OptionC"][i])

My evaluation set looks like this as well, with the test set having 6500 rows and the evaluation set having 1500 rows.
I am trying to implement this with:
from transformers import RobertaForMultipleChoice, Trainer, TrainingArguments
model = RobertaForMultipleChoice.from_pretrained('roberta-base')

training_args = TrainingArguments(
    output_dir='./results',          # output directory
    num_train_epochs=1,              # total # of training epochs
    per_device_train_batch_size=32,  # batch size per device during training
    per_device_eval_batch_size=32,   # batch size for evaluation
    warmup_steps=500,                # number of warmup steps for learning rate scheduler
    weight_decay=0.01,               # strength of weight decay
    logging_dir='./logs',            # directory for storing logs
)

trainer = Trainer(
    model=model,                     # the instantiated  Transformers model to be trained
    args=training_args,              # training arguments, defined above
    train_dataset = train_split,     # training dataset
    eval_dataset = eval_split        # evaluation dataset
)

trainer.train()

But I keep getting different keyerrors, an example:
KeyError: 2526
If anyone knows what I am doing wrong, I would be very grateful as I am stuck trying to train this model for the past 3 days.


title: Torch model forward with a diferent image size

body:
I am testing some well known models for computer vision: UNet, FC-DenseNet103, this implementation
I train them with 224x224 randomly cropped patches and do the same on the validation set.
Now when I run inference on some videos, I pass it the frames directly (1280x640) and it works. It runs the same operations on different image sizes and never gives an error. It actually gives a nice output, but the quality of the output depends on the image size...
Now it's been a long time since I've worked with neural nets but when I was using tensorflow I remember I had to crop the input images to the train crop size.
Why don't I need to do this anymore? What's happening under the hood?


title: In Pytorch, when transferring to GPU, I get an error "is on CPU, but expected to be on GPU"

body:
Error example: "Tensor for 'out' is on CPU, Tensor for argument #1 'self' is on CPU, but expected them to be on GPU". I was stuck on the tutorial for classification:
https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html
Note: The code is for regression.
Code is below:
class Net(nn.Module):
    def __init__(self, num_features, size_hidden_layer, n_hidden_layer):
        super(Net, self).__init__()
        self.size_hidden_layer = size_hidden_layer
        self.n_hidden_layer = n_hidden_layer
        self.hidden_layers = list()
        self.hidden_layers.append(nn.Linear(num_features, size_hidden_layer))
        for _ in range(n_hidden_layer-1):
            self.hidden_layers.append(nn.Linear(size_hidden_layer, size_hidden_layer))
        self.last_layer = nn.Linear(size_hidden_layer, 1)

    def forward(self, x):
        for i in range(self.n_hidden_layer):
            x = torch.relu(self.hidden_layers[i](x))
        return self.last_layer(x)


title: How to actually apply a Conv2d filter in Pytorch

body:
I’m new to Python and trying to do some manipulations with filters in PyTorch.
I’m struggling re how to apply a Conv2d. I’ve got the following code which creates a 3x3 moving average filter:
resized_image4D = np.reshape(image_noisy, (1, 1, image_noisy.shape[0], image_noisy.shape[1]))
t = torch.from_numpy(resized_image4D)

conv = torch.nn.Conv2d(in_channels=1, out_channels=1, kernel_size=3, padding=1, bias=False)
conv.weight = torch.nn.Parameter(torch.ones((1,1,3, 3))/9.0)

Normally in NumPy I’d just call filtered_image = convolve2d(image, kernel), but I’ve not been able to figure out what the PyTorch equivalent is after days of searching.


title: What should be the input shape for 3D CNN on a sequence of images?

body:
https://pytorch.org/docs/stable/generated/torch.nn.Conv3d.html#conv3d Describes that the input to do convolution on 3D CNN is (N,Cin,D,H,W). Imagine if I have a sequence of images which I want to pass to 3D CNN. Am I right that:

N -> number of sequences (mini batch)
Cin -> number of channels (3 for rgb)
D -> Number of images in a sequence
H -> Height of one image in the sequence
W -> Width of one image in the sequence

The reason why I am asking is that when I stack image tensors: a = torch.stack([img1, img2, img3, img4, img5]) I get shape of a torch.Size([5, 3, 396, 247]), so is it compulsory to reshape my tensor to torch.Size([3, 5, 396, 247]) so that number of channels would go first or it does not matter inside the Dataloader?
Note that Dataloader would add one more dimension automatically which would correspond to N.


Why am I getting the error ValueError: Expected input batch_size (4) to match target batch_size (64)?
Is it something to do with an incorrect number of channels(?) in the first linear layer? In this example I have 128 *4 *4 as the channel.
I have tried looking online and on this site for the answer but I have not been able to find it. So, I asked here.
Here is the network:

class Net(nn.Module):
    """A representation of a convolutional neural network comprised of VGG blocks."""
    def __init__(self, n_channels):
        super(Net, self).__init__()
        # VGG block 1
        self.conv1 = nn.Conv2d(n_channels, 64, (3,3))
        self.act1 = nn.ReLU()
        self.pool1 = nn.MaxPool2d((2,2), stride=(2,2))
        # VGG block 2
        self.conv2 = nn.Conv2d(64, 64, (3,3))
        self.act2 = nn.ReLU()
        self.pool2 = nn.MaxPool2d((2,2), stride=(2,2))
        # VGG block 3
        self.conv3 = nn.Conv2d(64, 128, (3,3))
        self.act3 = nn.ReLU()
        self.pool3 = nn.MaxPool2d((2,2), stride=(2,2))
        # Fully connected layer
        self.f1 = nn.Linear(128 * 4 * 4, 1000)
        self.act4 = nn.ReLU()
        # Output layer
        self.f2 = nn.Linear(1000, 10)
        self.act5 = nn.Softmax(dim=1)

    def forward(self, X):
        """This function forward propagates the input."""
        # VGG block 1
        X = self.conv1(X)
        X = self.act1(X)
        X = self.pool1(X)
        # VGG block 2
        X = self.conv2(X)
        X = self.act2(X)
        X = self.pool2(X)
        # VGG block 3
        X = self.conv3(X)
        X = self.act3(X)
        X = self.pool3(X)
        # Flatten
        X = X.view(-1, 128 * 4 * 4)
        # Fully connected layer
        X = self.f1(X)
        X = self.act4(X)
        # Output layer
        X = self.f2(X)
        X = self.act5(X)

        return X

Here is the training loop:

def training_loop(
        n_epochs,
        optimizer,
        model,
        loss_fn,
        train_loader):
    for epoch in range(1, n_epochs + 1):
        loss_train = 0.0
        for i, (imgs, labels) in enumerate(train_loader):

            outputs = model(imgs)

            loss = loss_fn(outputs, labels)

            optimizer.zero_grad()

            loss.backward()

            optimizer.step()

            loss_train += loss.item()

        if epoch == 1 or epoch % 10 == 0:
            print('{} Epoch {}, Training loss {}'.format(
                datetime.datetime.now(),
                epoch,
                loss_train / len(train_loader)))


title: Is there a way to set attribute of attribute?

body:
I am currently trying to assign an attribute, consider the following example:
net = timm.create_model(model_name='regnetx_002', pretrained=False)
net.head.fc = torch.nn.Linear(1111, 2)

This seems to work just fine, however, some models have different attributes, like the following
net.classifier = torch.nn.Linear(1111,2)

Therefore, I am trying to set attribute dynamically. I have already manage to retrieve the names of the attributes, like "head" and "fc", but it seems that you cannot do something like:
setattr(net, "head.fc", torch.nn.Linear(1111, 2))

as it will create one new attribute "head.fc" instead. I think there is a misunderstanding in my knowledge, as I noticed that when I print the model's last layer, it looks like:
  (head): ClassifierHead(
    (global_pool): SelectAdaptivePool2d (pool_type=avg, flatten=True)
    (fc): Linear(in_features=368, out_features=1000, bias=True)
  )
)

Is there any way to achieve what I want dynamically.


title: Translate Conv2D from PyTorch code to Tensorflow

body:
I have the following PyTorch layer definition:
self.e_conv_1 = nn.Sequential(
    nn.ZeroPad2d((1, 2, 1, 2)),
    nn.Conv2d(in_channels=3, out_channels=64, kernel_size=(5, 5), stride=(2, 2)),
    nn.LeakyReLU(),
)

I want to have the same exact layer declaration in Tensorflow. How can I get to that?
self.e_conv_1 = tf.keras.Sequential([
  layers.Conv2D(64, kernel_size=(5, 5), activation=partial(tf.nn.leaky_relu, alpha=0.01), padding='same', strides=(1, 2))
])

Should it be something like this code above? I think that at least strides and padding isn't the same.
Thanks in advance to anyone who helps.


title: Reward of Pong game - (OpenAI gym)

body:
I know that the Pong Game initializes to new game when one side scores 20 points.
By the way, the reward shows that it goes down below -20.
Why is that so?
One thing to expect is that after one side gets 20 points, the game is reset by playing one more time. Does the game need to get 21 points to initialize?
(Use 8 workers, A2C, PongNoFrameskip-v4)


title: Receiving coordinates from inference Pytorch

body:
I'm trying to get the coordinates of the pixels inside of a mask that is generated by Pytorches DefaultPredictor, to later on get the polygon corners and use this in my application.
However, DefaultPredictor produced a tensor of pred_masks, in the following format: [False, False ... False], ... [False, False, .. False]
Where the length of each individual list is length of the image, and the number of total lists is the height of the image.
Now, as I need to get the pixel coordinates that are inside of the mask, the simple solution seemed to be looping through the pred_masks, checking the value and if == "True" creating tuples of these and adding them to a list. However, as we are talking about images with width x height of about 3200 x 1600, this is a relatively slow process (~4 seconds to loop through a single 3200x1600, yet as there are quite some objects for which I need to get the inference in the end - this will end up being incredibly slow).
What would be the smarter way to get the the coordinates (mask) of the detected object using the pytorch (detectron2) model?
Please find my code below for reference:
from __future__ import print_function

from detectron2.engine import DefaultPredictor
from detectron2.config import get_cfg
from detectron2.data import MetadataCatalog
from detectron2.data.datasets import register_coco_instances

import cv2
import time

# get image
start = time.time()

im = cv2.imread("inputImage.jpg")

# Create config
cfg = get_cfg()
cfg.merge_from_file("detectron2_repo/configs/COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5 # Set threshold for this model
cfg.MODEL.WEIGHTS = "model_final.pth" # Set path model .pth
cfg.MODEL.ROI_HEADS.NUM_CLASSES = 1
cfg.MODEL.DEVICE='cpu'

register_coco_instances("dataset_test",{},"testval.json","Images_path")
test_metadata = MetadataCatalog.get("dataset_test")

# Create predictor
predictor = DefaultPredictor(cfg)

# Make prediction
outputs = predictor(im)

#Loop through the pred_masks and check which ones are equal to TRUE, if equal, add the pixel values to the true_cords_list
outputnump = outputs["instances"].pred_masks.numpy()

true_cords_list = []
x_length = range(len(outputnump[0][0]))
#y kordinaat on range number
for y_cord in range(len(outputnump[0])):
    #x cord
    for x_cord in x_length:
        if str(outputnump[0][y_cord][x_cord]) == "True":
            inputcoords = (x_cord,y_cord)
            true_cords_list.append(inputcoords)

print(str(true_cords_list))

end = time.time()

print(f"Runtime of the program is {end - start}") # 14.29468035697937

//

EDIT:
After changing the for loop partially to compress - I've managed to reduce the runtime of the for loop by ~3x - however, ideally I would like to receive this from the predictor itself if possible.
y_length = len(outputnump[0])
x_length = len(outputnump[0][0])
true_cords_list = []
for y_cord in range(y_length):
    x_cords = list(compress(range(x_length), outputnump[0][y_cord]))
    if x_cords:
        for x_cord in x_cords:
            inputcoords = (x_cord,y_cord)
            true_cords_list.append(inputcoords)


title: PyTorch Lightning: includes some Tensor objects in checkpoint file

body:
As Pytorch Lightning provides automatic saving for model checkpoints, I use it to save top-k best models. Specifically in Trainer setting,
    checkpoint_callback = ModelCheckpoint(
                            monitor='val_acc',
                            dirpath='checkpoints/',
                            filename='{epoch:02d}-{val_acc:.2f}',
                            save_top_k=5,
                            mode='max',
                            )

This is working well but it does not save some attribute of the model object. My model stores some Tensor at every training epoch end such that
class SampleNet(pl.LightningModule):
    def __init__(self):
        super().__init__()
        self.save_hyperparameters()
        self.layer = torch.nn.Linear(100, 1)
        self.loss = torch.nn.CrossEntropy()

        self.some_data = None # Initialize as None

    def training_step(self, batch):
        x, t = batch
        out = self.layer(x)
        loss = self.loss(out, t)
        results = {'loss': loss}

        return results

    def training_epoch_end(self, outputs):

        self.some_data = some_tensor_object

This is a simplified example but I want the checkpoint file made by above checkpoint_callback to remember the attribute self.some_data but when I load the model from checkpoint, it always reset to None. I confirmed that it is successfully updated during the training.
I tried not to initialize it as None in init but then the attribute will disappear when loading model.
Saving the attribute as a distinct pt file is something I want to avoid as it is associated with model configuration so I manually need to match the file with corresponding checkpoint file later.
Would it be possible to include such tensor attribute in checkpoint file?


title: model.train() and model.eval() causing nan values

body:
Hey so I am trying my hand at image classification/transfer learning using the monkey species dataset and the resnet50 with a modified final fc layer to predict just the 10 classes. Eveything is working until I use model.train() and model.eval() then after the first epoch it starts to return nans and the accuracy drops off as you'll see below. I'm curious why is this only when switching to train/eval....?
First I import the model and attach the classifier and freeze the parameters
%%capture
resnet = models.resnet50(pretrained=True)

for param in resnet.parameters():
  param.required_grad = False

in_features = resnet.fc.in_features

# Build custom classifier
classifier = nn.Sequential(OrderedDict([('fc1', nn.Linear(in_features, 512)),
                                        ('relu', nn.ReLU()),
                                        ('drop', nn.Dropout(0.05)),
                                        ('fc2', nn.Linear(512, 10)),
                                        ]))

# ('output', nn.LogSoftmax(dim=1))
resnet.classifier = classifier

resnet.to(device)

Then setting my loss func, optimizer, and shceduler
# Step : Define criterion and optimizer
criterion = nn.CrossEntropyLoss()
# pass the optimizer to the appended classifier layer
optimizer = torch.optim.SGD(resnet.parameters(), lr=0.01)
# Scheduler
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=[10], gamma=0.05)

Then setting the training and validation loops
epochs = 20

tr_losses = []
avg_epoch_tr_loss = []
tr_accuracy = []

val_losses = []
avg_epoch_val_loss = []
val_accuracy = []
val_loss_min = np.Inf

resnet.train()
for epoch in range(epochs):
  for i, batch in enumerate(train_loader):
    # Pull the data and labels from the batch
    data, label = batch
    # If available push data and label to GPU
    if train_on_gpu:
      data, label = data.to(device), label.to(device)
    # Compute the logit
    logit = resnet(data)
    # Compte loss
    loss = criterion(logit, label)
    # Clearing the gradient
    resnet.zero_grad()
    # Backpropagate the gradients (accumulte the partial derivatives of loss)
    loss.backward()
    # Apply the updates to the optimizer step in the opposite direction to the gradient
    optimizer.step()
    # Store the losses of each batch
    # loss.item() seperates the loss from comp graph
    tr_losses.append(loss.item())
    # Detach and store the average accuracy of each batch
    tr_accuracy.append(label.eq(logit.argmax(dim=1)).float().mean())
    # Print the rolling batch training loss every 20 batches
    if i % 40 == 0 and not i == 1:
      print(f'Batch No: {i} \tAverage Training Batch Loss: {torch.tensor(tr_losses).mean():.2f}')
  # Print the average loss for each epoch
  print(f'\nEpoch No: {epoch + 1},Training Loss: {torch.tensor(tr_losses).mean():.2f}')
  # Print the average accuracy for each epoch
  print(f'Epoch No: {epoch + 1}, Training Accuracy: {torch.tensor(tr_accuracy).mean():.2f}\n')
  # Store the avg epoch loss for plotting
  avg_epoch_tr_loss.append(torch.tensor(tr_losses).mean())

  resnet.eval()
  for i, batch in enumerate(val_loader):
    # Pull the data and labels from the batch
    data, label = batch
    # If available push data and label to GPU
    if train_on_gpu:
      data, label = data.to(device), label.to(device)
    # Compute the logits without computing the gradients
    with torch.no_grad():
      logit = resnet(data)
    # Compte loss
    loss = criterion(logit, label)
    # Store test loss
    val_losses.append(loss.item())
    # Store the accuracy for each batch
    val_accuracy.append(label.eq(logit.argmax(dim=1)).float().mean())
    if i % 20 == 0 and not i == 1:
      print(f'Batch No: {i+1} \tAverage Val Batch Loss: {torch.tensor(val_losses).mean():.2f}')
  # Print the average loss for each epoch
  print(f'\nEpoch No: {epoch + 1}, Epoch Val Loss: {torch.tensor(val_losses).mean():.2f}')
  # Print the average accuracy for each epoch
  print(f'Epoch No: {epoch + 1}, Epoch Val Accuracy: {torch.tensor(val_accuracy).mean():.2f}\n')
  # Store the avg epoch loss for plotting
  avg_epoch_val_loss.append(torch.tensor(val_losses).mean())

  # Checpoininting the model using val loss threshold
  if torch.tensor(val_losses).float().mean() <= val_loss_min:
    print("Epoch Val Loss Decreased... Saving model")
    # save current model
    torch.save(resnet.state_dict(), '/content/drive/MyDrive/1. Full Projects/Intel Image Classification/model_state.pt')
    val_loss_min = torch.tensor(val_losses).mean()
  # Step the scheduler for the next epoch
  scheduler.step()
  # Print the updated learning rate
  print('Learning Rate Set To: {:.5f}'.format(optimizer.state_dict()['param_groups'][0]['lr']),'\n')

The model starts to train but then slowly becomes nan values
Batch No: 0     Average Training Batch Loss: 9.51
Batch No: 40    Average Training Batch Loss: 1.71
Batch No: 80    Average Training Batch Loss: 1.15
Batch No: 120   Average Training Batch Loss: 0.94

Epoch No: 1,Training Loss: 0.83
Epoch No: 1, Training Accuracy: 0.78

Batch No: 1     Average Val Batch Loss: 0.39
Batch No: 21    Average Val Batch Loss: 0.56
Batch No: 41    Average Val Batch Loss: 0.54
Batch No: 61    Average Val Batch Loss: 0.54

Epoch No: 1, Epoch Val Loss: 0.55
Epoch No: 1, Epoch Val Accuracy: 0.81

Epoch Val Loss Decreased... Saving model
Learning Rate Set To: 0.01000

Batch No: 0     Average Training Batch Loss: 0.83
Batch No: 40    Average Training Batch Loss: nan
Batch No: 80    Average Training Batch Loss: nan


title: How to combine 2 learning rate schedulers in PyTorch?

body:
I'm trying to follow this pytorch's optim documentation on using optimizer.param_groups to group several learning rate schedulers, but it seems like it only works for the LambdaLR one.
If I have my model:
net = Model()

I naively tried to add the schedulers with:
learning_rate = 0.02
optimizer = optim.Adam(net.parameters(), lr = learning_rate)
net.optimizer = optimizer

scheduler1 = torch.optim.lr_scheduler.CyclicLR(net.optimizer, base_lr=1e-10, max_lr=1e-3, step_size_up=3000, mode="triangular2", cycle_momentum=False)
net.scheduler1 = scheduler1

scheduler2 = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e3, max_lr=1e-1, step_size_up=1000, mode="triangular2", cycle_momentum=False)
net.scheduler2 = scheduler2

I'm trying to use scheduler1 to train one of my neural networks, and scheduler2 is just used for its value. That is, at each epoch I'm using its value to plug in into some function:
for epoch in num_epochs:
    loss = 5 - optimizer.param_groups[0]["lr"]

Note that that's just an example function.
The issue is that I think the model just takes the scheduler2 as its scheduler. Since
optimizer.param_groups[1]["lr"] returned:
IndexError: list index out of range

Update 1
For clarification, I have a neural network inside my model net. I want this neural network to have 1 lr scheduler, scheduler1. I have a certain function inside my model net, that needs to take certain values at each epoch. The values that it needs to take resemble the values from scheduler2. Hence, I need to add both schedulers to my model, using the first for my neural network, and the second to my function (not relating to any model's parameters except for the epoch number).


I am working on SVHN dataset, and I got this error, only during training phase. During the instantiation of the model, it works.
RuntimeError: Given groups=1, weight of size [64, 3, 3, 3], expected input[64, 1, 32, 32] to have 3 channels, but got 1 channels instead

To be sure of having 3 channels, I wrote that Grayscale(3) transformation.
# Compose transformations
data_transform = transforms.Compose([
  transforms.Resize((32,32)),
  transforms.Grayscale(num_output_channels=3),
  transforms.RandomHorizontalFlip(),
  transforms.ToTensor(),
])

# Compose transformations
test_transform = transforms.Compose([
  transforms.Resize((32,32)),
  transforms.Grayscale(num_output_channels=3),
  transforms.ToTensor(),
])

# Load MNIST dataset with transforms
train_set_svhn = torchvision.datasets.SVHN(root=base_dir, split='train', download=True, transform=data_transform, target_transform=None)
test_set_svhn = torchvision.datasets.SVHN(root=base_dir, split='test', download=True, transform=test_transform)

class VGG16(nn.Module):

    def __init__(self, num_classes):
        super(VGG16, self).__init__()

        # calculate same padding:
        # (w - k + 2*p)/s + 1 = o
        # => p = (s(o-1) - w + k)/2

        self.block_1 = nn.Sequential(
            nn.Conv2d(in_channels=3,
                      out_channels=64,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      # (1(32-1)- 32 + 3)/2 = 1
                      padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.Conv2d(in_channels=64,
                      out_channels=64,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.block_2 = nn.Sequential(
            nn.Conv2d(in_channels=64,
                      out_channels=128,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.Conv2d(in_channels=128,
                      out_channels=128,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(128),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.block_3 = nn.Sequential(
            nn.Conv2d(in_channels=128,
                      out_channels=256,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.Conv2d(in_channels=256,
                      out_channels=256,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.Conv2d(in_channels=256,
                      out_channels=256,
                      kernel_size=(3, 3),
                      stride=(1, 1),
                      padding=1),
            nn.BatchNorm2d(256),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=(2, 2),
                         stride=(2, 2))
        )

        self.classifier = nn.Sequential(
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout(p=0.25),
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout(p=0.25),
            nn.Linear(4096, num_classes),
        )

        for m in self.modules():
            if isinstance(m, torch.nn.Conv2d) or isinstance(m, torch.nn.Linear):
                nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='leaky_relu')
#                 nn.init.xavier_normal_(m.weight)
                if m.bias is not None:
                    m.bias.detach().zero_()

        # self.avgpool = nn.AdaptiveAvgPool2d((7, 7))

    def forward(self, x):

        x = self.block_1(x)
        x = self.block_2(x)
        x = self.block_3(x)
        # x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        logits = self.classifier(x)
        probas = F.softmax(logits, dim=1)
        # probas = nn.Softmax(logits)
        return probas
        # return logits

I have no idea where that 1 comes from
Moreover, this is the shape of the output of the model without classifier (fully connected layers):
output = model1(test_x)
output.shape

torch.Size([1, 256, 4, 4])

And indeed I will pass 256x4x4 as input to the first FC.
Edit:


Being a new guy and a beginner to deep learning and pytorch I am not sure what all inputs should I give you guys to answer my question. But I will try my best to make you guys understand my problem. I have loaded a model in pytorch using 'model= torch.load('model/resnet18-5c106cde.pth')'. But it is showing an AttributeError: 'collections.OrderedDict' object has no attribute 'predict', when I used the command 'prediction = model.predict(test_image)'. Hope you guys understood my problem and Thanks in advance...


title: How to efficiently multiply by torch tensor with repeated rows without storing all the rows in memory or iterating?

body:
Given a torch tensor:
# example tensor size 2 x 4
a = torch.Tensor([[1, 2, 3, 4], [5, 6, 7, 8]])

and another where every n rows are repeated:
# example tensor size 4 x 3 where every 2 rows repeated
b = torch.Tensor([[1, 2, 3], [4, 5, 6], [1, 2, 3], [4, 5, 6]])

how can one perform matrix multiplication:
>>> torch.mm(a, b)
tensor([[ 28.,  38.,  48.],
        [ 68.,  94., 120.]])

without copying the whole repeated row tensor into memory or iterating?
i.e. only store the first 2 rows:
# example tensor size 2 x 3 where only the first two rows from b are actually stored in memory
b_abbreviated = torch.Tensor([[1, 2, 3], [4, 5, 6]])

since these rows will be repeated.
There is a function
torch.expand()

but this does work when repeating more than a single row, and also, as this question:
Repeating a pytorch tensor without copying memory
indicates and my own tests confirm often ends up copying the whole tensor into memory anyway when calling
.to(device)

It is also possible to do this iteratively, but this is relatively slow.
Is there some way to perform this operation efficiently without storing the whole repeated row tensor in memory?
Edit explanation:
Sorry, for not initially clarifying: One was used as the first dimension of the first tensor to keep the example simple, but I am actually looking for a solution to the general case for any two tensors a and b such that their dimensions are compatible for matrix multiplication and the rows of b repeat every n rows. I have updated the example to reflect this.


title: Pytorch batchwise matrix vector rowwise multiplication

body:
I am using pytorch.
If I have a matrix M of size (d1, d2) and a vector V of size d2, doing M*V gives me an output OUT of size (d1, d2), where each row of M has been multiplied by V.
I need to do the same thing batch-wise, where the matrix M is fixed and I have a batch of dB vectors.
In practice, given a tensor M of size (d1, d2) and a tensor V of size (dB, d2), I need to get as output OUT a tensor of size (dB, d1, d2), so that OUT[i] = M*V[i].
How can I efficiently get it with pytorch?


title: Loss is stuck at `input_size/2`

body:
The weirdest thing is happening in my model and I can't figure out why.
So, TLDR, if my input has 100 points (in a batch), the difference at every output of my network will be 50 (half of that):
y - y_predict =  tensor([50.1337, 49.6812, 50.0728, 50.3125, 49.8369, 50.1456, 50.0340, 50.0508,         50.1231, 50.2237, 49.4339, 49.7306, 49.8324, 49.6193, 49.6949, 49.8605,         50.0395, 50.1412, 50.3325, 50.4694, 50.2878, 50.2262, 50.1926, 50.1933,         50.2082, 50.2194, 50.1791, 50.1530, 50.1509, 50.1562, 50.1650, 50.1777,         50.1900, 50.2044, 50.2156, 50.2246, 50.2187, 50.2143, 50.2464, 50.2789,         50.3046, 50.3081, 50.3118, 50.3157, 50.3172, 50.3146, 50.3121, 50.3128,         50.3165, 50.3203, 50.3240, 50.3278, 50.3316, 50.3353, 50.3391, 50.3429,         50.3468, 50.3506, 50.3545, 50.3584, 50.3622, 50.3660, 50.3699, 50.3738,         50.3777, 50.3815, 50.3854, 50.3892, 50.3931, 50.3970, 50.4009, 50.4047,         50.4086, 50.4125, 50.4163, 50.4183, 50.4201, 50.4219, 50.4236, 50.4253,         50.4270, 50.4288, 50.4306, 50.4323, 50.4341, 50.4358, 50.4375, 50.4471,         50.4729, 50.4986, 50.5243, 50.5499, 50.5756, 50.6001, 50.6285, 50.6592,         50.6900, 50.7367, 50.8411, 51.0187], dtype=torch.float64,        grad_fn=<SubBackward0>)

If my input size is 50, the difference at every output will be 25. And if the input size is 25, the difference is 12.5.
I have no idea why it's happening, but I know it has something to do with the number of points I'm using (as the difference between the actual and prediction seems to always converge to half of that when I use MSE).
I'm trying to figure out if it has anything to do with the backwards pass/gradients, but so far I'm lost.
My input is in a CSV file and I'm changing it to a batch as follows:
x = torch.tensor(x, requires_grad=True)
x_float = x.float()
x_batch = torch.reshape(x_float, (len(x),1)) #reshape for batch
x_batch.shape #torch.Size([100, 1])

x_batch goes into the model, and the output is 100 values (see above).
The loss function I'm using:
loss = (torch.mean(torch.square(self.y_actual - y_predict)))

The model is a simple 8 FC layers with 20 neurons each and Relu after each one. Input is 1 and output is 1. Loss gets lower for like a second, and then it converges around 10k. It's like the model see that y_actual-y_predict = 0 when it's actually input_size/2 for some reason
My code is a very convoluted & long otherwise I would share it. If there will be no ideas/suggestions I'll work on cleaning & simplifying my code and sharing it.
I tried changing the loss function to be nn.MSE and got the same results. I also tried several other loss functions for regression (MAE, etc.) but they didn't converge
Adding a plot to show that it learns the function perfectly, except for the difference:

Adding a plot of the loss function:

Update 1:
Tried changing the mean in my loss function to max -- it converged to almost similar values and the difference remained around 50.


title: How to load .pth file?

body:
I am given a pytorch model from [this repository][1] and I have to convert it to tflite.
Here's the code:
def get_torch_model(model_path):
    """
    Loads state-dict into model and creates an instance.
    """
    model= torch.load(model_path)
    return model

# Conversion
import torch
from torchvision import transforms

import onnx

import cv2
import numpy as np
import onnx
import tensorflow as tf
import torch
from PIL import Image

import torch.onnx

image, tf_lite_image, sample_input = get_sample_input("crop.jpg")
torch_model = get_torch_model("pose_resnet_152_256x256.pth")

ONNX_FILE = "./m_model.onnx"

Up until here everything runs smoothly. But when I run the below cell:
torch.onnx.export(
        model=torch_model,
        args=sample_input,
        f=ONNX_FILE,
        verbose=False,
        export_params=True,
        do_constant_folding=False,  # fold constant values for optimization
        input_names=['input'],
        opset_version=10,
        output_names=['output']
)

onnx_model = onnx.load(ONNX_FILE)

onnx.checker.check_model(onnx_model)

The complete log of error:
---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-33-15df717ec276> in <module>
      8         input_names=['input'],
      9         opset_version=10,
---> 10         output_names=['output']
     11 )
     12

~\anaconda3\envs\py36\lib\site-packages\torch\onnx\__init__.py in export(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes, keep_initializers_as_inputs, custom_opsets, enable_onnx_checker, use_external_data_format)
    274                         do_constant_folding, example_outputs,
    275                         strip_doc_string, dynamic_axes, keep_initializers_as_inputs,
--> 276                         custom_opsets, enable_onnx_checker, use_external_data_format)
    277
    278

~\anaconda3\envs\py36\lib\site-packages\torch\onnx\utils.py in export(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes, keep_initializers_as_inputs, custom_opsets, enable_onnx_checker, use_external_data_format)
     92             dynamic_axes=dynamic_axes, keep_initializers_as_inputs=keep_initializers_as_inputs,
     93             custom_opsets=custom_opsets, enable_onnx_checker=enable_onnx_checker,
---> 94             use_external_data_format=use_external_data_format)
     95
     96

~\anaconda3\envs\py36\lib\site-packages\torch\onnx\utils.py in _export(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, export_type, example_outputs, opset_version, _retain_param_name, do_constant_folding, strip_doc_string, dynamic_axes, keep_initializers_as_inputs, fixed_batch_size, custom_opsets, add_node_names, enable_onnx_checker, use_external_data_format, onnx_shape_inference, use_new_jit_passes)
    677         _set_opset_version(opset_version)
    678         _set_operator_export_type(operator_export_type)
--> 679         with select_model_mode_for_export(model, training):
    680             val_keep_init_as_ip = _decide_keep_init_as_input(keep_initializers_as_inputs,
    681                                                              operator_export_type,

~\anaconda3\envs\py36\lib\contextlib.py in __enter__(self)
     79     def __enter__(self):
     80         try:
---> 81             return next(self.gen)
     82         except StopIteration:
     83             raise RuntimeError("generator didn't yield") from None

~\anaconda3\envs\py36\lib\site-packages\torch\onnx\utils.py in select_model_mode_for_export(model, mode)
     36 def select_model_mode_for_export(model, mode):
     37     if not isinstance(model, torch.jit.ScriptFunction):
---> 38         is_originally_training = model.training
     39
     40         if mode is None:

AttributeError: 'collections.OrderedDict' object has no attribute 'training'

This error occurs when I use torch.onnx.export() .
Please let me know whats going wrong here.
Am I not loading the weights properly? If not then how do I load the model? I don't know the class, or architecture details so how do I use model.load_state_dict() ??

  [1]: https://github.com/leoxiaobin/deep-high-resolution-net.pytorch


title: PyTorch model training is suddenly super slow

body:
I'm using PyTorch (version 1.8.1) to train a set of 40 LSTMs on a set of speech data using a TitanV GPU with Ubuntu 18.04. I setup the code so that each LSTM is trained until it converges, so each LSTM converges on a different epoch. I'm using DataLoaders with a batch size of 16 and num_workers set to 4. The first few epochs lasted ~50 minutes, while the most recent epochs lasted ~10 minutes since most of the models have converged (see below). However, the current epoch started at 06:29 and is still running, but I would have expected this epoch to finish long ago.
I know the code is still running since nvidia-smi shows python as using > 6MiB of memory (see below), which is normally the amount of memory that is used.
What might be causing this problem?
As a side note, one thing I noticed was the message "System program problem detected", although I am not sure whether this is related.
INFO:root:Time: 05/24/21 16:24:40, Epoch: 1
INFO:root:Time: 05/24/21 17:16:19, Epoch: 2
INFO:root:Time: 05/24/21 18:08:10, Epoch: 3
INFO:root:Time: 05/24/21 18:59:52, Epoch: 4
INFO:root:Time: 05/24/21 19:51:41, Epoch: 5
INFO:root:Time: 05/24/21 20:43:21, Epoch: 6
INFO:root:Time: 05/24/21 21:35:09, Epoch: 7
INFO:root:Time: 05/24/21 22:26:56, Epoch: 8
INFO:root:Time: 05/24/21 23:18:40, Epoch: 9
INFO:root:Time: 05/25/21 00:10:27, Epoch: 10
INFO:root:Time: 05/25/21 01:02:24, Epoch: 11
INFO:root:Time: 05/25/21 01:54:16, Epoch: 12
INFO:root:Time: 05/25/21 02:44:33, Epoch: 13
INFO:root:Time: 05/25/21 03:34:58, Epoch: 14
INFO:root:Time: 05/25/21 04:20:09, Epoch: 15
INFO:root:Time: 05/25/21 04:59:01, Epoch: 16
INFO:root:Time: 05/25/21 05:28:44, Epoch: 17
INFO:root:Time: 05/25/21 05:50:44, Epoch: 18
INFO:root:Time: 05/25/21 06:08:50, Epoch: 19
INFO:root:Time: 05/25/21 06:20:41, Epoch: 20
INFO:root:Time: 05/25/21 06:29:32, Epoch: 21

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.73.01    Driver Version: 460.73.01    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  TITAN V             Off  | 00000000:03:00.0 Off |                  N/A |
| 28%   39C    P8    24W / 250W |   6751MiB / 12065MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      1293      G   /usr/lib/xorg/Xorg                 18MiB |
|    0   N/A  N/A      1509      G   /usr/bin/gnome-shell               73MiB |
|    0   N/A  N/A      3340      G   /usr/lib/xorg/Xorg                111MiB |
|    0   N/A  N/A      3534      G   ...mviewer/tv_bin/TeamViewer       27MiB |
|    0   N/A  N/A      3572      G   /usr/bin/gnome-shell               34MiB |
|    0   N/A  N/A      4406      G   ...f_4086.log --shared-files        6MiB |
|    0   N/A  N/A     11837      C   python                           6471MiB |
+-----------------------------------------------------------------------------+

EDIT 1: This is how I am implementing the DataLoaders. The below code also shows a simplified version of my train function.
    training_set = Dataset(train_list, conf_dict, le)
    training_generator = torch.utils.data.DataLoader(training_set, batch_size=conf_dict["batch_size"],
                                                     num_workers=4, collate_fn=collate_fn, shuffle=True)
    valid_set = Dataset(valid_list, conf_dict, le)
    valid_generator = torch.utils.data.DataLoader(valid_set, batch_size=conf_dict["batch_size"],
                                                  num_workers=4, collate_fn=collate_fn, shuffle=True)

class Dataset(torch.utils.data.Dataset):
    def __init__(self, file_list, conf_dict, le=None):
        self.file_list = file_list
        self.conf_dict = conf_dict
        self.le = le

    def __len__(self):
        return len(self.file_list)

    def __getitem__(self, index):
        # Read single feature file
        X, y, p = read_feat_file(self.file_list[index], self.conf_dict)

        if bool(self.le):
            p = self.le.transform(p).astype('long')

        return X, y, p

def collate_fn(batch):
    """
    Used to collate multiple files for dataloader
    """
    X_batch, y_batch, p_batch = zip(*batch)
    X_batch = list(X_batch)
    y_batch = list(y_batch)
    p_batch = list(p_batch)

    # Get maximum sequence length
    seq_lens = np.array(list(map(lambda a: len(a), y_batch)), dtype='int')
    max_seq_len = np.max(seq_lens)

    # Apply -1 padding
    for file_idx in range(len(y_batch)):
        if np.shape(y_batch[file_idx])[0] < max_seq_len:
            pad_len = max_seq_len - np.shape(y_batch[file_idx])[0]
            X_batch[file_idx] = np.concatenate((X_batch[file_idx], -np.ones((pad_len, np.shape(X_batch[file_idx])[1]), dtype='float32')), axis=0)
            y_batch[file_idx] = np.concatenate((y_batch[file_idx], -np.ones((pad_len, np.shape(y_batch[file_idx])[1]), dtype='float32')), axis=0)
            p_batch[file_idx] = np.concatenate((p_batch[file_idx], np.zeros((pad_len,), dtype='long')), axis=0)

    # Convert to np.array
    X_batch = np.array(X_batch)
    y_batch = np.array(y_batch)
    p_batch = np.array(p_batch)

    return X_batch, y_batch, p_batch

def train(experts, optimizers, le, conf_dict, train_generator):
    device = get_device()

    for X_batch, y_batch, p_batch in train_generator:
        X_batch = (torch.from_numpy(X_batch)).to(device)
        y_batch = (torch.from_numpy(y_batch)).to(device)
        p_batch = (torch.from_numpy(p_batch)).to(device)

        for key in list(experts.keys()):
            experts[key].train()
            optimizers[key].zero_grad()

            mask = (p_batch == ((le.transform([key])).astype('long')[0]))
            mask = mask.reshape(mask.size()[0], mask.size()[1], 1)
            loss_mask = mask * (y_batch != -1)

            if torch.sum(loss_mask) > 0:
                train_outputs = experts[key](X_batch)

                # Simplified loss calculation
                loss = torch.sum(loss_mask * F.mse_loss(y_pred, y_true, reduction='none'))

                loss.backward()
                optimizers[key].step()


title: Getting diagonal elements over several dimensions

body:
I'd like to transform a tensor T of size (n x n x m x m) into a tensor U of size (n x m x m)  while only retreiving the diagonal elements of T over the (NxN) chunks (i.e. Uikl=Tiikl). torch.diag() only works with 2-D tensors and I really fail to see how to do this without looping on the indexes of the elements (which I'd like to avoid given that I think that it is inefficient computationnally). In clear, I'd like to vectorize the following code:

U = torch.zeros(n, m, m)
for i in range(n):
    for k in range(m):
        for l in range(m):
            U[i][k][l] = T[i][i][k][l]

I'm totally new to pytorch and I tried many combination of functions but none of them gives me a satisfying result. Has anyone an idea?


title: Training (DC)GAN, D(G(z)) goes to 0.5 while D(x) stays 0.9 and G(z) becomes corrupt

body:
I'm currently training a DCGAN for 1x32x32 (channel, height, width) images.
Quite soon in training G(z) becomes reasonably realistic apart from a problem with the 'chessboard' artifacts being visible, but this should go away after lots of training?
However, after a long training session D(G(z)) goes to 0.5000 (and no longer changes) while D(x) stays between 0.8 and 0.9. Whenever D(G(z)) goes to 0.5 it also starts outputting fully black & white images. Hence, the generator no longer produces anything that looks close to what's in the training dataset. G(z) just becomes a black or white square.

The network used is from the original DCGAN paper, adapter for 1x32x32 images. With relu already replaced to leaky relu.


title: Pytorch parameters() behaving differently when collected to list or when kept in generator

body:
TL;DR - using a generator fails, using a list succeeds. why?

I am trying to change my model's parameters manually like so:

(1st code, works)

       delta = r_t + gamma * expected_reward_from_t1.data - expected_reward_from_t.data

        negative_expected_reward_from_t = -expected_reward_from_t
        self.critic_optimizer.zero_grad()
        negative_expected_reward_from_t.backward()

        for i, p in enumerate(self.critic_nn.parameters()):
             if not p.requires_grad:
                 continue
             p.grad[:] = delta.squeeze() * discount * p.grad

        self.critic_optimizer.step()

and it  seems to converge on the correct result 100% of the time.

However,

When attempting to use a function like so:

(2nd code, fails)

def _update_grads(self,delta, discount):
    params = self.critic_nn.parameters()
    for i, p in enumerate(params):
        if not p.requires_grad:
            continue
        p.grad[:] = delta.squeeze() * discount * p.grad

and then

       delta = r_t + gamma * expected_reward_from_t1.data - expected_reward_from_t.data

        negative_expected_reward_from_t = -expected_reward_from_t
        self.critic_optimizer.zero_grad()
        negative_expected_reward_from_t.backward()
        self._update_grads(delta=delta,
                           discount=discount)

        self.critic_optimizer.step()

which seems like the only thing I've done was put self.critic_nn.parameters() into a temporary local variable params,

Now the network does not converge.

(3rd code, again, works)

When replacing in the method _update_grads: params = self.critic_nn.parameters() with params = list(self.critic_nn.parameters())

Now again, convergence is restored.

This seems like a referencing issue, which, in PyTorch, I do not completely understand. I don't seem to understand fully what returns from parameters()

The question:
Why do the 1st and 3rd codes work, but the 2nd one doesn't?


title: After Pytorch Upgrade , my model is giving almost random output

body:
I trained, tested and still using a model in "Pytorch 0.4.1". It was, and is still working fine (output is what it should be) if I use pitch 0.4.1.
But as i upgrade to version 1.0.1, every time, I try to evaluate same input image, I get different output (Its a regression).

I tried to see what has been changed in those versions, but since i am not getting any errors, warnings anything, i am not sure what should I look for specifically.

PS: I checked the weights, they are also same when I load the model


title: vector multiplication in pytorch

body:
In pytorch. I want to multiply each vector of Matrix A by each vector of Matrix B:

A      = M x N
B      = L x N
result = (M x L) x N


title: Sharing gradients between the related classes in pytorch

body:
I am implementing a classifier with multi-label predictions. For the moment, there are only 5 classes. Following is the snippet of the code. So far I have no problem.

But I would like to further extend it where gradients between the related tasks (constrained by a matrix W which encodes the relatedness between the tasks) is shared. For example: there are five different classes a, b,c, d, e . I computed gradient for the mis-classification of a and updated the parameters. From the matrix W I discovered that task e is also related. I would also like to update the parameters for the task e with the gradients computed while predicting a with the rate encoded by W. Here W is precomputed and fixed.

I have no idea how to do it in PyTorch. I would really appreciate if someone can help me.

class Classifier(nn.Module):
    def __init__(self, image_size=128, conv_dim=64, c_dim=5, repeat_num=6):
        super(Classifier, self).__init__()
        layers = []
        layers.append(nn.Conv2d(3, conv_dim, kernel_size=4, stride=2, padding=1))
        layers.append(nn.LeakyReLU(0.01))

        curr_dim = conv_dim
        for i in range(1, repeat_num):
            layers.append(nn.Conv2d(curr_dim, curr_dim*2, kernel_size=4, stride=2, padding=1))
            layers.append(nn.LeakyReLU(0.01))
            curr_dim = curr_dim * 2

        kernel_size = int(image_size / np.power(2, repeat_num))
        self.main = nn.Sequential(*layers)
        self.conv = nn.Conv2d(curr_dim, 5, kernel_size=kernel_size, bias=False)

    def forward(self, x):
        h = self.main(x)
        out_cls = self.conv2(h)
        return out_cls.view(out_cls.size(0), out_cls.size(1))


title: What is `gpytorch.settings.max_preconditioner_size`? And `gpytorch.beta_features.checkpoint_kernel(checkpoint_size)`?

body:
I am following the tutorial Simple_MultiGPU_GP_Regression and I noticed that during the training two options were established:

    with gpytorch.beta_features.checkpoint_kernel(checkpoint_size), \
         gpytorch.settings.max_preconditioner_size(preconditioner_size):

What are or refer to the variables checkpoint_kernel and preconditioner_size?

I have check the documentation preconditioner_size but it is not quite clear to me what it refers to.

I intuit that checkpoint_size refers to something related to the number of training points associated with each GPU. But it is just an intuition.

Help is appreciated.

Thanks


title: Exception is Expected object of scalar type Float but got scalar type Long for argument #2 ‘other’

body:
I am getting this error when running the below code,

  Expected object of scalar type Float but got scalar type Long for argument #2 ‘other’

    def encode(words):
        max_l = max([len(i) for i in words])
        result = torch.empty((1,max_l, 26)).int()
        for word in words:
            ints = (np.fromstring(word,dtype=np.uint8)-ord('a'))
            addition = np.zeros((max_l - ints.shape[0],)) -1
            tr = torch.Tensor(np.expand_dims(np.hstack((ints,addition)),-1))
            tr = (tr[:] == torch.arange(26)).int()
            tr = torch.unsqueeze(tr, 0)
            result = torch.cat((result,tr))
        result = result[1::]
        return result

and would like to know how to fix it? Unfortunately, I don't have the full log stack


title: python memory explodes because of loop

body:
def compute_linear_gradient_manually(args,model,data_tr,data_te,device):
    N_tr = len(data_tr)
    N_te = len(data_te)
    kernel_tr = torch.zeros(N_tr,N_tr).to(device)
    kernel_te = torch.zeros(N_te,N_tr).to(device)
    for i in range(N_tr):
        grad_tr = compute_layer_gradient(args,model,data_tr[i])
        for j in range(N_tr):
            grad_tr_ = compute_layer_gradient(args,model,data_tr[j])
            kernel_tr[i][j] = torch.mm(grad_tr,grad_tr_.t())
        for k in range(N_te):
            grad_tr_ = compute_layer_gradient(args,model,data_te[k])
            kernel_te[i][k] = torch.mm(grad_tr,grad_tr_.t())
    return kernel_tr,kernel_te

def compute_layer_gradient(args,model,data):
    p = ((torch.mm(data.view(-1,3072),model.weight1)+model.bias1)>0).float()
    weight2_grad = ((torch.mm(data.view(-1,3072),model.weight1)+model.bias1)*p).t()
    weight1_grad = torch.mm(data.view(-1,3072).t(),model.weight2.t()*p)
    bias1_grad = math.sqrt(1/width)*model.weight2.t()*p.view(-1,width)
    grad = torch.cat([weight1_grad.view(-1,3072*width),bias1_grad.view(-1,width),weight2_grad.view(-1,width)],dim=1)
    return grad

I am using CUDA to compute two matrices kernel_tr and kernel_te. The function compute_layer_gradient returns a 1*m vector and data_tr and data_te are two matrices.

But the memory keeps increasing and I have tried using del grad_tr_ or something like that but it doesn't work.

How to correctly release the memory in this loop?


title: why my pytorch image classification sampler spit out the strange length value?

body:
I'm training the Image classification
I made the train and valid cross-validation sets

EPOCHS = 5
SAVE_DIR = 'models'
MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'please.pt')
from torch.utils.data import DataLoader
best_valid_loss = float('inf')

if not os.path.isdir(f'{SAVE_DIR}'):
    os.makedirs(f'{SAVE_DIR}')
print("start")
for epoch in range(EPOCHS):
    print('================================',epoch ,'================================')
    for i , (train_idx, valid_idx) in enumerate(zip(train_indexes, valid_indexes)):
        print(i,train_idx,valid_idx,len(train_idx),len(valid_idx))

        trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler= SubsetRandomSampler(train_idx))
        valloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler=SubsetRandomSampler(valid_idx))

        print(len(trainloader.dataset),len(valloader.dataset))

        train_loss, train_acc ,model= train(model, device, trainloader, optimizer, criterion)
        valid_loss, valid_acc,model = evaluate(model, device, valloader, criterion)

        if valid_loss < best_valid_loss:
            best_valid_loss = valid_loss
            torch.save(model,MODEL_SAVE_PATH)

        print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:05.2f}% | Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:05.2f}% |')

  start
  ================================ 0 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] 18827 2690  21517 21517

but the output of sampling is very strange
the output of

  len(train_idx) and len(valid_idx) = 18827 2690

this is correct

But. when I print the

  print(len(trainloader.dataset),len(valloader.dataset)) = 21517 , 21517

is something wrong with sampler???


title: How to manipulate only weights of neural network when biases are intact using Pytorch?

body:
In the following for loop I get name's and parameter's of my neural network. Depending on the bias flag I will do some operations on the weight's and bias's using block after 'if' and 'elif'. However, when bias is False, I just do operations on weight's but if bias flag is True, I will do it for both weight's and bias's.

for name, param in model.named_parameters():
    if bias = Fasle and 'weight' in name:
        weight_np = param.data.cpu().numpy()
        alive = weight_np[np.nonzero(weight_np)] # extract nonzero values
        percentile_value = np.percentile(abs(alive), percent)

        # Convert Tensors to numpy and calculate
        weight_dev = param.device
        new_mask = np.where(abs(weight_np) < percentile_value, 0, mask[step])
    elif bias = True and 'weight' or 'bias' in name:
        weight_np = param.data.cpu().numpy()
        alive = weight_np[np.nonzero(weight_np)] # extract nonzero values
        percentile_value = np.percentile(abs(alive), percent)

        # Convert Tensors to numpy and calculate
        weight_dev = param.device
        new_mask = np.where(abs(weight_np) < percentile_value, 0, mask[step])

Question: since block is the same for if and elif, I want to avoid writing it and have it once. Can you help me to figure this out?


title: Slice 4d tensor into 4D tensor of smaller subtensors (slice in last 2 dimensions only)

body:
The question is analogous to Slice 2d array into smaller 2d arrays except for the fact that I use tensors (torch) & I have a 4D, not 2D, tensor of the shape eg. (3, 1, 32, 32) - in my case, it is 3 images of size 32x32.

I want to split each tensor of form [i, 0, :, :] into smaller subarrays, so the output would have a shape eg. (3, 16, 8, 8), where each [:, j, :, :] is a small square cut from the original image. I cannot find a way to modify the proposed solution for 4D tensor.

I also tried to just use

subx = x.reshape(3, 16, 8, 8)

but this does not reshape it as I want.


title: How to create a Kubeflow component from a PyTorch job?

body:
I've been ramping up on Kubeflow recently. My goal is to get PyTorch running in Kubeflow. I've gone through the documentation on creating a distributed PyTorch job here. I've also read through all the documentation on how to create pipelines / components in Kubeflow.

My question is how can I now take a PyTorch job, which is a Kubernetes resource, and run it as a component. The ultimate goal is to have my PyTorch code, which is a distributed training of some model, run within the component / pipeline framework of Kubeflow. How do multi-worker jobs fit into the component / pipeline framework of Kubeflow?

The documentation gives plenty of information on how to run components from python code, Docker containers, etc... but nothing on how to do it from a PyTorchJob or a Kubernetes job. This seems like an obvious use case to me, and I feel like I'm missing something obvious, but I've gone through all the documentation for Kubeflow that I could find, and did additional searches for anything on how to do this.

Would appreciate any help, thank you!


title: opt.step() is not updating the weights of model

body:
print(self.global_model.state_dict())
    print("total_loss",total_loss)
    total_loss.backward()
    self.opt.step()
    print(self.global_model.state_dict())

it's output is

('dense1.weight', tensor([[ 0.3997, -0.1907,  0.1120,  0.3016],
        [ 0.1156,  0.0646,  0.1802,  0.3558],
        [ 0.0321,  0.2537,  0.0879,  0.2441],
        [-0.2952, -0.0886, -0.3235,  0.3006]])), ('dense1.bias', tensor([ 0.1927,  0.3048, -0.3551, -0.0302])), ('dense2.weig

total_loss.backward() tensor(2.5806, dtype=torch.float64, grad_fn=<MeanBackward0>)

('dense1.weight', tensor([[ 0.3997, -0.1907,  0.1120,  0.3016],
        [ 0.1156,  0.0646,  0.1802,  0.3558],
        [ 0.0321,  0.2537,  0.0879,  0.2441],
        [-0.2952, -0.0886, -0.3235,  0.3006]])), ('dense1.bias', tensor([ 0.192

We can see total_loss have some value but it is not updating the weights

self.opt = torch.optim.SGD(self.global_model.parameters(),lr = 0.01)

Update

If i do

            print(self.local_model.state_dict())
            print("total_loss.backward()",total_loss)
            total_loss.backward()
            opt_2 = torch.optim.SGD(self.local_model.parameters(),lr = 0.01)
            opt_2.step()
            self.opt.step()
            print(self.local_model.state_dict())

It updates the weights of local model.
But i need to apply that gradient to other model. So what i will need to do ?


title: Having trouble with my Dockerfile and setting correct working directory

body:
This is my first Dockerfile and this is what I have so far.

FROM python:3.6-stretch

# install build utilities
RUN apt-get update && \
    apt-get install -y gcc make apt-transport-https ca-certificates build-essential

# check our python environment
RUN python3 --version
RUN pip3 --version

# set the working directory for containers
WORKDIR  /usr/src/toxic-content-monitoring

# Installing python dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy all the files from the project’s root to the working directory
COPY src/ /src/
RUN ls -la /src/*

# Running Python Application
CMD ["python3", "/src/main.py"]

But when I am trying to run the docker image the files inside the data folder cannot be found.
Here is a picture of the my project structure.

Changed it to what was suggested and still getting an error.

FROM python:3.6-stretch

# install build utilities
RUN apt-get update && \
    apt-get install -y gcc make apt-transport-https ca-certificates build-essential

# check our python environment
RUN python3 --version
RUN pip3 --version

# set the working directory for containers
WORKDIR  /usr/src/toxic-content-monitoring

# Installing python dependencies
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

# Copy all the files from the project’s root to the working directory
COPY src/ /src/
COPY data /data/
RUN ls -la /src/*

# Running Python Application
CMD ["python3", "/src/main.py"]

This is the error message.

docker run toxic-content-monitoring:0.1
wiki.en.vec: 6.60GB [10:05, 10.9MB/s]
  0%|          | 0/2519371 [00:00<?, ?it/s]Skipping token 2519370 with 1-dimensional vector ['300']; likely a header
100%|██████████| 2519371/2519371 [08:36<00:00, 4878.67it/s]
Traceback (most recent call last):
  File "/src/main.py", line 11, in <module>
    from preprocessor import normalize_comment, clean_text
  File "/src/preprocessor.py", line 42, in <module>
    word_file = open(link, "r")
FileNotFoundError: [Errno 2] No such file or directory: 'data/identity_hateWordFile.txt'


title: Please compare numpy arrays and pytorch tensors on different performance parameters,uses and give the speciality or unique features of each?

body: I am confused between numpy arrays and py-torch tensors please help me resolve the confusion.I also want to know how py-torch tensors are so fast when compared to numpy arrays.If the py-torch tensors are so quick why don't we use them in data-science for data cleaning purposes.


title: cannot import clean_up_tokenization , bert tokenizer

body:
I am trying to do machine translation with fairseq and used bert as tokenizer. I installed pytorch-transformers from source. But i am not able to import a function clean_up_tokeinization.

from pytorch_transformers.tokenization_utils import clean_up_tokenization

Although, i can import pytorch_transformers.tokenization_utils and tokenization_utils has a function clean_up_tokenization. I cannot import it.

Could anyone help me with importing this function? Thanks in advance.


title: Multi band images in land prediction using tensorflow/pytorch

body:
I have patch of land and this patch of land  I have 2 satellite images  for it . The 1st one is is tiff format and contain the the land elevation and the 2nd satellite image which is in rgb format is the picture of that patch of land from above. Furthermore, I learned about transfer learning etc using pytorch and tensorflow.
Now the goal is to predict the type of the land (hills, valley, ridge...etc.) using shapes, colors, surface elevation, from those images.
Now my steps are:

Merging the Elevation band with rbg 3 bands to make three bands and we can have and array of depth of 4.

The labels corresponding to those 2 satellite images are in vector shapes and those vectors can be transferred to raster format. But for deep learning they require labels in certain format with certain folder structure.

Is using the 3 bands of the rgb on transfer learning because from what I heard is only work on rgb images and then use normal model on the elevation band (array) to predict land type.

fuse everything in step 3 to predict land type.

My question is how to make this possible.


title: select sub elements from another batch

body:
I have first tensor with size  torch.Size([12, 64, 8, 8, 3]) ,  (8,8,3) is the image size, 64 is the patches, 12 is the batch size
There is another tensor with size torch.Size([12, 10]) which select 10 patches for each item in the batch (select 10 patches from total of 64). Thus it store the indexes. How to use this to query for the first tensor  with list comprehension?


title: seq2seq model transformer model - what's the best way to batchify my inputs?

body:
I'm trying to build a character-level model that matches diacritics for Hebrew characters (each character is decorated with a diacritic). Note that the correct diacritic is dependent on the word, the context and the part-of-speech (not trivial).
I built an LSTM based model which achieves 18% word-level accuracy (18% of the words were exactly right in all their characters, on an unseen test set)
Now I'm trying to beat that with a transformer model, following pytorch seq-2-seq tutorial, and I'm reaching far worse results (7% word-level accuracy).
My training dataset is 100K sentences, most with up to 30 characters, but some go all the way to 80 characters.
My question (finally) - what's the best way to batchify these inputs for the transformer? I prepared 30-characters chunks that cover each sentence (e.g. a 55 characters sentence => 30 + 25) and padded with zeros when a chunk is shorter than 30. I'm also trying to split the chunks between words (on spaces) and not in mid-word.
Is this the way to go? Am I missing some better (and better-known) technique?


title: Fastai implementation - fit_one_cicle() and intepretation loss, r squared and rmse

body:
The following is a regression problem, where I have a dataset with composed by more than 40k consumption ids with some variables such as DateTime, month, year, temperature, humidity. I built a Fastai Tabular Data format using Embedding Layers for categorical variables. The historical consumption values (my target) are in the range [0, 1.400.000]. I used the tabular_learner with two dense layers [1000, 500].
After running the lr.find(), I chose a learning rate equal to  1e-01.

learn.fit_one_cycle(3, max_lr=slice(1e-02), wd=0.2)
Unfortunately, I reached the following results with an RMSE which is fluctuating during epochs:

I think I have a multicollinearity problem, that's why R^2 score is -inf. Another consideration: why validation and train losses are so high? Do I need to change model or do you think the problem stands in the data preparation?