title: How to troubleshoot the dimensions for my layers in PyTorch

body:
I am working through my training loop errors where my dimensions are incorrect. Of course there is other code here and there, but I thought this the most important to show here. I feel everywhere I look online I am getting told different info or in different ways that I don't know how to get at least the first layer correct so I can troubleshoot from there.  I know what goes out of one layer goes into the next layer, those dimensions must match.  Looking at this link https://towardsdatascience.com/pytorch-layer-dimensions-what-sizes-should-they-be-and-why-4265a41e01fd

I think the first layer should be nn.Conv2d([batch_size, channels, height, width])
or  nn.Conv2d([62,3,800,600]).  This, however, gives an error thinking
RuntimeError: Given groups=1, weight of size 3 62 800 600, expected input[62, 3, 800, 600] to have 62 channels, but got 3 channels instead.  So, I think it is expecting the channels to be the first number, the number of features, then (H, W).  So need to do switch #s around.......
However, get error

in _max_pool2d
    input, kernel_size, stride, padding, dilation, ceil_mode)
RuntimeError: Given input size: (62x1x1). Calculated output size: (62x0x0). Output size is too small

How can I understand these needed dimensions?

Image info:

Dimensions = 1920x1080 (W x H)
Bit depth 24

Code:

---------------------Hyper Parameters-------------------------------------------
SIZE TRAINING SET = 320
SIZE TEST SET = 80

-----------------------------------------------------------------------------
LR = 1e-1
N_EPOCHS = 30
BATCH_SIZE = 62
DROPOUT = 0.25

transformations = transforms.Compose([
    transforms.Resize((800,600),interpolation=Image.BICUBIC),
    transforms.RandomHorizontalFlip(),
    transforms.RandomVerticalFlip(),
    transforms.ColorJitter(brightness=10,saturation=10,contrast=30,hue=0.1),
    #transforms.CenterCrop(224),
    transforms.ToTensor()])

#  -------------------------------------- CNN Class ----------------------------------------------
class CNN(nn.Module):
    def __init__(self):
        super(CNN, self).__init__()

        self.conv1 = nn.Conv2d(3, 7, (3, 3))
        self.pool1 = nn.MaxPool2d((2, 2))

        self.conv2 = nn.Conv2d(7, 14, (3, 3))
        self.convnorm2 = nn.BatchNorm2d(14)
        self.pool2 = nn.MaxPool2d((2, 2))

        self.conv3 = nn.Conv2d(14, 28, (3, 3))

        self.conv4= nn.Conv2d(28, 56, (3, 3))
        self.convnorm4 = nn.BatchNorm2d(56)
        self.pool4 = nn.MaxPool2d((2, 2))

        self.conv5= nn.Conv2d(56, 112, (3, 3))
        self.convnorm5 = nn.BatchNorm2d(112)
        self.pool5 = nn.MaxPool2d((2, 2))

        self.conv6 = nn.Conv2d(112, 224, (2, 2))
        self.convnorm6 = nn.BatchNorm2d(224)
        self.pool6 = nn.MaxPool2d((2, 2))

        self.linear1 = nn.Linear(87584, 100)
        self.linear1_bn = nn.BatchNorm1d(100)
        self.drop = nn.Dropout(DROPOUT)
        self.linear2 = nn.Linear(100, 7)
        self.act = torch.relu

    def forward(self, x):
        x = self.pool1(self.act(self.conv1(x)))
        x = self.pool2(self.convnorm2(self.act(self.conv2(x))))
        x = self.act(self.conv3(x))
        x = self.pool4(self.convnorm4(self.act(self.conv4(x))))
        x = self.pool5(self.convnorm5(self.act(self.conv5(x))))
        x = self.pool6(self.convnorm6(self.act(self.conv6(x))))
        x = self.drop((self.linear1_bn(self.act(self.linear1(x.view(len(x), -1))))))
        return self.linear2(x)

# -------------------------------------- Training Prep --------------------------------------------------
model = CNN().to(device)
optimizer = torch.optim.SGD(model.parameters(), lr=LR)
criterion = nn.BCEWithLogitsLoss()

# -------------------------------------- Training Loop -----------------------------------------------
print("Starting training loop...")
for epoch in range(N_EPOCHS):
    model.train()
    loss_train = 0
    for iter, traindata in enumerate(train_loader):
        train_inputs, train_labels = traindata
        train_inputs, train_labels = train_inputs.to(device), train_labels.to(device)
        optimizer.zero_grad()
        logits = model(train_inputs)
        loss = criterion(logits, train_labels)
        loss.backward()
        optimizer.step()
        loss_train += loss.item()
        torch.save(model.state_dict(), "cnnmodel.pt")
        print('Batch {} and Loss {:.5f}'.format(iter,loss_train/BATCH_SIZE))
    model.load_state_dict(torch.load("cnnmodel.pt"))
    model.eval()

    with torch.no_grad():
        for iter, valdata in enumerate(val_loader, 0):
            val_inputs, val_labels = valdata
            val_inputs1, val_labels1 = val_inputs.to(device), val_labels.to(device)
            y_test_pred = model(val_inputs1)
            tar_=val_labels.cpu().numpy()
            loss = criterion(y_test_pred, val_labels1)
            loss_test = loss.item()
            print('Validation Loss {:.5f}'.format(loss_test))
            validation_output=np.where(y_test_pred.cpu().numpy()>0.5,1,0)

    print("Epoch {} | Train Loss {:.5f}".format( epoch, loss_train/BATCH_SIZE))

            return self.linear2(x)

Error message:

Starting training loop...
Expected: %s to exist.
NoneType: None
Expected: %s to exist.
NoneType: None
Expected: %s to exist.
NoneType: None
Expected: %s to exist.
NoneType: None
Traceback (most recent call last):
  File "/home/ubuntu/.pycharm_helpers/pydev/pydevd.py", line 1434, in _exec
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "/home/ubuntu/.pycharm_helpers/pydev/_pydev_imps/_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "/home/ubuntu/Deep-Learning/FinalProject/train_finalproject.py", line 310, in <module>
    logits = model(train_inputs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 541, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/ubuntu/Deep-Learning/FinalProject/train_finalproject.py", line 287, in forward
    x = self.pool2(self.convnorm2(self.act(self.conv2(x))))
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py", line 541, in __call__
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 345, in forward
    return self.conv2d_forward(input, self.weight)
  File "/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py", line 342, in conv2d_forward
    self.padding, self.dilation, self.groups)
RuntimeError: Given groups=1, weight of size 14 7 3 3, expected input[62, 2, 399, 299] to have 7 channels, but got 2 channels instead


title: How to feeding hidden state vectors from one transformer directly into a layer of different transformer

body:
The transformer models take as input token Ids, which are converted into embeddings. I am wondering how to input embeddings directly.

I am asking for both the Pytorch and Keras versions of the models.


title: Reward not increasing while training a Bipedal System

body:
I am completely new to reinforcement learning and this is my first program in practice. I am trying to train the bipedal system in the OpenAI gym environment using the policy gradient algorithm.
However, the reward never changes, either at episode 0 or at episode 1000 and I cannot figure out what is going wrong. Could anyone please help me..
TIA.
The code is as follows:
import torch
import torch.nn as nn
import numpy
from torch.autograd import Variable
import gym

def train(model, name):
    model.train()
    env = gym.make(name)
    env.reset()
    num_episodes = 20000
    max_steps = 10000
    optim = torch.optim.Adam(model.parameters(), lr=1)
    numsteps = []
    rewards = []
    avg_numsteps = []

    min_reward = -1000

    for episode in range(num_episodes):
        state = env.reset()
        probs = []
        rewards = []

        for steps in range(max_steps):
            action, log_prob = model.action(state)
            env.render()
            state, reward, finished, _ = env.step(action.squeeze(0).detach().numpy())
            env.render()
            probs.append(log_prob)
            rewards.append(reward)
            if finished:
                break

        if finished:
            Rewards = []
            for i in range(len(rewards)):
                G = 0
                p = 0
                for reward in rewards[i:]:
                    G = G + 0.9 * p * reward
                    p = p + 1
                Rewards.append(G)

            Rewards = torch.tensor(Rewards)

            discounted_reward = (Rewards - Rewards.mean()) / (Rewards.std() + 1e-9)

            gradients = []
            for log_prob, G in zip(log_prob, discounted_reward):
                gradients.append(-log_prob * G)

            optim.zero_grad()
            policy_gradient = Variable(torch.stack(gradients).sum(), requires_grad=True)
            policy_gradient.backward()
            optim.step()
            numsteps.append(steps)
            avg_numsteps.append(numpy.mean(numsteps[-10:]))
            rewards.append(numpy.sum(rewards))

            print("episode: {}, total reward: {}, average_reward: {}, length: {}\n".format(episode,
                                                                                numpy.sum(rewards),
                                                                                numpy.round(numpy.mean(
                                                                                rewards[-10:]),
                                                                                decimals=3),
                                                                                steps))

        if numpy.sum(rewards) > min_reward:
            torch.save(model.state_dict(), '/home/atharva/policyNet.pth')
            min_reward = numpy.sum(rewards)

def test(model, name):
    env = gym.make(name)
    model.eval()
    state = env.reset()
    with torch.no_grad():
        while True:
            action, log_prob = model(state)
            state, reward, finished, _ = env.step(action.squeeze(0).numpy())
            env.render()
            if finished:
                break

class PolicyNet(nn.Module):
    def __init__(self, inputs, actions, hidden_size):
        super(PolicyNet, self).__init__()
        self.num_actions = actions
        self.layer1 = nn.Linear(inputs, hidden_size)
        self.layer2 = nn.Linear(hidden_size, hidden_size)
        self.layer3 = nn.Linear(hidden_size, 2*hidden_size)
        self.layer4 = nn.Linear(2*hidden_size, hidden_size)
        self.layer5 = nn.Linear(hidden_size, actions)

    def forward(self, x):
        x = self.layer1(x)
        x = nn.functional.relu(x)
        x = self.layer2(x)
        x = nn.functional.relu(x)
        x = self.layer3(x)
        x = nn.functional.relu(x)
        x = self.layer4(x)
        x = nn.functional.relu(x)
        x = self.layer5(x)
        return x

    def action(self, state):
        state = torch.from_numpy(state).float().unsqueeze(0)
        actions = self.forward(Variable(state))
        #prob = numpy.random.choice(self.num_actions, p=numpy.squeeze(actions.detach().numpy()))
        log_prob = torch.log(actions.squeeze(0))
        return actions, log_prob

and the calling is as follows
from REINFORCE import PolicyNet, train

model = PolicyNet(24, 4, 256)
train(model, 'BipedalWalker-v3')


title: ImportError: No module named nltk macos

body:
Starting on a Python chatbot and using anaconda followed a tutorial and could not import nltk, using pytorch as well.
import nltk
#nltk.download('punkt')
from nltk.stem.porter import PorterStemmer
stemmer = PorterStemmer()

def tokenize(sentence):
    return nltk.word_tokenize(sentence)

def stem(word):
    return stemmer.stem(word.lower())

def bag_of_words(tokenized_sentence, all_words):
    pass

a = "How long does shipping take?"
print(a)
a = tokenize(a)
print(a)

The error after attempting to import nltk
(pytorch) jack.s@McUnicorn-Pro-2 Chatbot % python nltk_utils.py
Traceback (most recent call last):
  File "nltk_utils.py", line 1, in <module>
    import nltk
ImportError: No module named nltk
(pytorch) jack.s@McUnicorn-Pro-2 Chatbot %


title: PyTorch Model Accuracy

body:
I have the model below that trains in images that count a number of objects in a picture. I would like to calculate the model accuracy. The code at the moment works out the training loss, error, and test loss and error. What do I need in order to find a final accuracy please in % ? Many Thanks in advance
# main training loop
global_step = 0
best_test_error = 10000
for epoch in range(15):
    print("Epoch %d" % epoch)
    model.train()
    for images, paths in tqdm(loader_train):
        images = images.to(device)
        targets = torch.tensor([metadata['count'][os.path.split(path)[-1]] for path in paths]) # B
        targets = targets.float().to(device)

        # forward pass:
        output = model(images) # B x 1 x 9 x 9 (analogous to a heatmap)
        preds = output.sum(dim=[1,2,3]) # predicted cell counts (vector of length B)

        # backward pass:
        loss = torch.mean((preds - targets)**2)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # logging:
        count_error = torch.abs(preds - targets).mean()
        writer.add_scalar('train_loss', loss.item(), global_step=global_step)
        writer.add_scalar('train_count_error', count_error.item(), global_step=global_step)

        print("Step %d, loss=%f, count error=%f" % (global_step,loss.item(),count_error.item()))

        global_step += 1

    mean_test_error = 0
    model.eval()
    for images, paths in tqdm(loader_test):
        images = images.to(device)
        targets = torch.tensor([metadata['count'][os.path.split(path)[-1]] for path in paths]) # B
        targets = targets.float().to(device)

        # forward pass:
        output = model(images) # B x 1 x 9 x 9 (analogous to a heatmap)
        preds = output.sum(dim=[1,2,3]) # predicted cell counts (vector of length B)

        # logging:
        loss = torch.mean((preds - targets)**2)
        count_error = torch.abs(preds - targets).mean()
        mean_test_error += count_error
        writer.add_scalar('test_loss', loss.item(), global_step=global_step)
        writer.add_scalar('test_count_error', count_error.item(), global_step=global_step)

        global_step += 1

    mean_test_error = mean_test_error / len(loader_test)
    print("Test count error: %f" % mean_test_error)
    if mean_test_error < best_test_error:
        best_test_error = mean_test_error
        torch.save({'state_dict':model.state_dict(),
                    'optimizer_state_dict':optimizer.state_dict(),
                    'globalStep':global_step,
                    'train_paths':dataset_train.files,
                    'test_paths':dataset_test.files},checkpoint_path)


title: will loop decrease the utilization of the GPU?

body:
In PyTorch, I have a loop in my DeepLearning Pipeline's forward part to normalize the intermediate result.
Will it run on CPU and decrease the utilization of the GPU?
some snippet as follow:
def forward(self):
    ...
    for b in range(batch_size):
        self.points[b] = self.unit_cube(self.points[b])
    ....


title: Python scalars convert one Arrary

body:
I am getting the error below when running the Pytorch model. How can I convert into one size-1 array, please? here my code below: I have had a look at other StackOverflow answers but couldn't figure out how to do it. Sorry if this question is very basic. I hope someone can help. Many Thanks in advance!
Traceback (most recent call last):
File "train3_Eval.py", line 177, in
print("MAE Total: %f" % MAE_for_all_epochs)
TypeError: must be real number, not list
# main training loop
global_step = 0
best_test_error = 10000
MAE_for_all_epochs = []
MSE_for_all_epochs = []
for epoch in range(4):
    print("Epoch %d" % epoch)
    model.train()
    for images, paths in tqdm(loader_train):
        images = images.to(device)
        targets = torch.tensor([metadata['count'][os.path.split(path)[-1]] for path in paths]) # B
        targets = targets.float().to(device)

        # forward pass:
        output = model(images) # B x 1 x 9 x 9 (analogous to a heatmap)
        preds = output.sum(dim=[1,2,3]) # predicted cell counts (vector of length B)

        # backward pass:
        loss = torch.mean((preds - targets)**2)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        # logging:
        count_error = torch.abs(preds - targets).mean()
        writer.add_scalar('train_loss', loss.item(), global_step=global_step)
        writer.add_scalar('train_count_error', count_error.item(), global_step=global_step)

        print("Step %d, loss=%f, count error=%f" % (global_step,loss.item(),count_error.item()))

        global_step += 1

    mean_test_error = 0
    model.eval()
    for images, paths in tqdm(loader_test):
        images = images.to(device)
        targets = torch.tensor([metadata['count'][os.path.split(path)[-1]] for path in paths]) # B
        targets = targets.float().to(device)

        # forward pass:
        output = model(images) # B x 1 x 9 x 9 (analogous to a heatmap)
        preds = output.sum(dim=[1,2,3]) # predicted cell counts (vector of length B)

        # logging:
        #error = torch.abs(preds - targets).sum().data
        #squared_error = ((preds - targets)*(preds - targets)).sum().data

        #runnning_mae += error
        #runnning_mse += squared_error

        loss = torch.mean((preds - targets)**2)
        count_error = torch.abs(preds - targets).mean()
        mean_test_error += count_error
        writer.add_scalar('test_loss', loss.item(), global_step=global_step)
        writer.add_scalar('test_count_error', count_error.item(), global_step=global_step)

        global_step += 1
        #store_MAE = 0
        #store_MSE = 0

        mean_test_error = mean_test_error.detach() / len(loader_test)
        #store_MAE += mean_test_error
        MAE_for_all_epochs.append(mean_test_error)

        mse = math.sqrt(loss / len(loader_test))
        #store_MSE +=mse
        MSE_for_all_epochs.append(mse)

        print("Test count error: %f" % mean_test_error)
        print("MSE: %f" % mse)

    if mean_test_error < best_test_error:
        best_test_error = mean_test_error
        torch.save({'state_dict':model.state_dict(),
                    'optimizer_state_dict':optimizer.state_dict(),
                    'globalStep':global_step,
                    'train_paths':dataset_train.files,
                    'test_paths':dataset_test.files},checkpoint_path)

print("MAE Total: %f" % MAE_for_all_epochs)
print("MSE Total: %f" % MSE_for_all_epochs)

model_mae= MAE_for_all_epochs / epoch
model_mse= MSE_for_all_epochs / epoch
print("Model MAE: %f" % model_mae)
print("Model MSE: %f" % model_mse)


title: something wrong in custom pytorch dataset class for .h5 files

body:
class HDF5Dataset(torch.utils.data.Dataset):

    def __setup_files(self):
        files = glob.glob(os.path.join(self.dir_path,'**/*.h5'))
        return files

    def __init__(self, dir_path, IMG_SIZE):
        self.dir_path = dir_path
        self.IMG_SIZE = IMG_SIZE
        self.files = self.__setup_files()
        self.length = len(self.files)
        self.transform = transforms.Compose([
        transforms.Resize((IMG_SIZE, IMG_SIZE)),
        transforms.RandomHorizontalFlip(p=0.5),
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
    ])

    def __getitem__(self, idx):
        record = self.files[idx]
        h5 = h5py.File(record , 'r')
        image = h5['data'].value
        label = h5['label'].value
        h5.close()
        image = np.asarray(image)
        image = Image.fromarray(image.astype('uint8'), 'RGB')

        return self.transform(image), label

    def __len__(self):
        return self.length

this is my custom dataset class and I'm trying to load every h5 files in a directory recursively.
I think something wrong at def getitem, but I'm not sure what it is.
when I try to load with this
dataloaders['train'] = torch.utils.data.DataLoader(datasets['train'],
                                              batch_size=batch_size, shuffle=True, pin_memory=True,
                                              num_workers=12)

and this code,
inputs, classes = next(iter(dataloaders['train']))

it gives Type error:
TypeError: Caught TypeError in DataLoader worker process 0.
Original Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/_utils/worker.py", line 178, in _worker_loop
    data = fetcher.fetch(index)
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/_utils/collate.py", line 79, in default_collate
    return [default_collate(samples) for samples in transposed]
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/_utils/collate.py", line 79, in <listcomp>
    return [default_collate(samples) for samples in transposed]
  File "/usr/local/lib/python3.5/dist-packages/torch/utils/data/_utils/collate.py", line 62, in default_collate
    raise TypeError(default_collate_err_msg_format.format(elem.dtype))
TypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found object

I'd like to know exactly how to build a custom dataset for h5 files in pytorch and how to load them.
Thanks!


title: Does the loss of a model reflect its accuracy?

body:
So these are my loss per 75 epochs:
Epoch: 75, loss: 47382825795584.000000
Epoch: 150, loss: 47382825795584.000000
Epoch: 225, loss: 47382825795584.000000
Epoch: 300, loss: 47382825795584.000000
Epoch: 375, loss: 47382825795584.000000
Epoch: 450, loss: 47382825795584.000000
Epoch: 525, loss: 47382825795584.000000
Epoch: 600, loss: 47382825795584.000000
Epoch: 675, loss: 47382825795584.000000
Epoch: 750, loss: 47382825795584.000000

And these are the values from predictions and targets respectively
Predictions: tensor([[ 8109436.0000,  7734814.0000,  8737677.0000, 11230861.0000,
          3795826.7500,  3125072.7500,  1699706.1250,  5337285.0000,
          3474238.5000]], grad_fn=<TBackward>)
----------------------------------------
Targets: tensor([[ 8111607.,  7580798.,  8749436., 11183578.,  3822811.,  3148031.,
          2343278.,  5360924.,  3536146.]])

And this is the accuracy of the first, and second elements inside predictions against the first, and second elements of targets
8109436.0000/8111607*100 #First element
Output: 99.9732358828528

print(7734814.0000/7580798*100) #Second element
Output: 102.03165946381898

So I'm really not sure what is going on. Because I have a large loss there is a 99% accuracy for the first element and 98% accuracy on the second element? I'm not the best at math, so I'm not sure about the last percentage.
Could someone explain if the loss reflects the accuracy?


title: How does Pytorch's `autograd` handle non-mathematical functions?

body: During the course of my training process, I tend to use a lot of calls to torch.cat() and copying tensors into new tensors. How are these operations handled by autograd? Is the gradient value affected by these operations?


title: Is any additional masking required after setting padding_idx=0 in the nn.Embedding()?

body:
I am passing a batch of sentences to the GRU. The sentences are of varying length and have been padded (padding_idx = 0) to form the batch. The sentences vary a lot in lenghts, from 32 words to 105 words. I have specified the padding_idx while creating the embedding layer. If I am not using nn.pack_padded_sequence, is any kind of masking information is still required or will it be taken care of internally?
Also, after obtaining the hidden_states from the GRU, I want to form sentence representations. For this, I am applying a soft-attention mechanism on the hidden_states obtained for a sentence. Basically, I obtain a score for each hidden state and take the weighted mean. This would also assign a weight to a hidden state corresponding padding_idx. Is it correct or am I required to use masking here? Like setting the weight for the hidden state corresponding to the padding_idx to a very small value.


title: Trade off between losses?

body:
I have been working on a super-resolution task. I have this question about determining loss function, So in the case of the task at hand I felt like going with SSIM as a loss function to train my model. I did get a good set of results. Recently I come across perceptual loss function where we compare how a pretrained model looks at the Ground truth(GT) Images and the Super Resolution(SR) Image(Image generated by the model). My question is, I am thinking of using both ((1-SSIM(SR,GT))+Perceptual loss(SR,GT)) loss for backpropagation, so should I use a trade-off parameter between these two losses? if so how can I set up these trade-off parameters? or should I add these losses with equal weights.
PS: the perceptual loss is calculated by finding SSIMs between the feature maps of GT and SR images from the pre-trained model


title: Weights&Biases Sweep - Why might runs be overwriting each other?

body:
I am new to ML and W&B, and I am trying to use W&B to do a hyperparameter sweep. I created a few sweeps and when I run them I get a bunch of new runs in my project (as I would expect):
Image: New runs being created
However, all of the new runs say "no metrics logged yet" (Image) and are instead all of their metrics are going into one run (the one with the green dot in the photo above). This makes it not useable, of course, since all the metrics and images and graph data for many different runs are all being crammed into one run.
Is there anyone that has some experience in W&B? I feel like this is an issue that should be relatively straightforward to solve - like something in the W&B config that I need to change.
Any help would be appreciated. I didn't give too many details because I am hoping this is relatively straightforward, but if there are any specific questions I'd be happy to provide more info. The basics:

Using Google Colab for training
Project is a PyTorch-YOLOv3 object detection model that is based on this: https://github.com/ultralytics/yolov3

Thanks!


title: Using an ellipsis for the middle dimensions of a PyTorch tensor

body:
Suppose I have a torch.Tensor t of shape (8, 3, 32, 32). I want to index along the first and last 2 dimensions only.
In my usecase, t is a batch of 8 images, of which I want to modify a patch. Suppose the patch is given by indices idx_last = torch.tensor([[0, 0], [0, 1], [1, 0], [1, 1]]).
I also have idx1 = torch.arange(4) : I want the patch for the first 4 images.
The following does not work:
t[idx1, ..., idx_last]
Is there any way to do this ?


title: How do I properly mask values in my Convolutional Neural Network

body:
I'm creating an encoder-decoder CNN for some images. Each image has a geometric shape around the center - circle, ellipse, etc.
I want my CNN to ignore all the values that are in this shape. All my input data values have been normalized to be around 0-1. I set all the shape values to be 0.
I thought that setting them to zero would mean that they will not be updated, however, the output of my encoder-decoder CNN changes the shape.
What can I do to ensure these values stay put and do not update?
Thank you!


title: apex: TypeError: Class advice impossible in Python3

body:
i installed apex:
pip uninstall apex

My configurations are as follows:

torch version 1.7.1
python version 3.7
cuda 11.0

I came across this error after running import pytorch:
TypeError: Class advice impossible in Python3. Use the @implementer class decorator instead.


title: Computing loss along batch dimension in PyTorch

body:
I have tensors of BxCxHxW where B is the batch. I am interested in how is the loss implemented in PyTorch when batch size is more than 1. The below is what we usually do in PyTorch,
l1_loss = torch.nn.L1Loss()
loss = l1_loss(pred,gt)

Is it that the loss is the average loss over the batch?
If this is indeed the case, then is the following codes equivalent to what is we usually do in PyTorch (the code above)?
l1_loss = torch.nn.L1Loss()

for idx in range(B):
  if idx==0:
    loss = l1_loss(pred[idx,:,:,:],gt[idx,:,:,:])
  else:
    loss = l1_loss(pred[idx,:,:,:],gt[idx,:,:,:]) + loss

loss = loss/B


title: Why doesn't this input pass through this simple PyTorch model?

body:
I have an input/tensor whose shape is:
torch.Size([256, 3, 28, 28])

(batch size here is 256, 3 channels, 28x28 image)
And a model like so:
class Model(nn.Module):

    def __init__(self):
        super().__init__()
        self.network = nn.Sequential(
            nn.Conv2d(3, 28, kernel_size=3, padding=1),
            nn.ReLU(),
            nn.Conv2d(28, 56, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),  # output: 56 x 16 x 16

            nn.Conv2d(56, 112, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(112, 112, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),  # output: 112 x 8 x 8

            nn.Conv2d(112, 224, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(224, 224, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),  # output: 224 x 4 x 4

            nn.Flatten(),
            nn.Linear(224 * 4 * 4, 896),
            nn.ReLU(),
            nn.Linear(896, 512),
            nn.ReLU(),
            nn.Linear(512, 2))

    def forward(self, xb):
        return self.network(xb)

When I attempt to pass data forward, it fails with:
    ...
    return self.network(xb)
  File "/home/stark/anaconda3/envs/torch-env/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/stark/anaconda3/envs/torch-env/lib/python3.8/site-packages/torch/nn/modules/container.py", line 117, in forward
    input = module(input)
  File "/home/stark/anaconda3/envs/torch-env/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/stark/anaconda3/envs/torch-env/lib/python3.8/site-packages/torch/nn/modules/linear.py", line 93, in forward
    return F.linear(input, self.weight, self.bias)
  File "/home/stark/anaconda3/envs/torch-env/lib/python3.8/site-packages/torch/nn/functional.py", line 1690, in linear
    ret = torch.addmm(bias, input, weight.t())
RuntimeError: mat1 dim 1 must match mat2 dim 0

What am I missing?
Thanks!


title: pytorch require grad according to mask

body:
I calculate the distance matrix between a set of vertors, and backward the mse loss between calculated matrix and another one. But it seems like the backwarded grads on vectors are both nan, because the denominator of the quadratic radical's derivative in diagonal elements is 0.
import torch
import torch.nn as nn
from einops import rearrange

loss_f = nn.MSELoss()
vectors = torch.randn(1, 10, 3, requires_grad=True) # 10 vertors with 3 elements
pred_dist = torch.pow(rearrange(vectors, 'b n d -> b n () d') - rearrange(vectors, 'b n d -> b () n d'), 2).sum(axis=-1)**.5
real_dist = torch.randn(1, 10,10)

loss = loss_f(pred_dist, real_dist)
loss.backward()
print(vectors.grad)

"""
out:
tensor([[[nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan],
         [nan, nan, nan]]])
"""

Is there a mask-based method to require gradients only on no-diagnoal elements?


title: Tensor creation and destruction within a while loop in libtorch C++

body:
I have just started with libtorch and I am having some trouble with while loops and tensors :roll_eyes:
So, my main func looks like so:
int main()
{

  auto tensor_create_options = torch::TensorOptions().dtype(torch::kFloat32).device(torch::kCPU).requires_grad(false);
  torch::Tensor randn_tensor = torch::randn({10}, tensor_create_options);

  int randn_tensor_size = randn_tensor.sizes()[0];

  while (randn_tensor_size > 5)
  {
    std::cout << "==> randn_tensor shape: " << randn_tensor.sizes() << std::endl;
    randn_tensor.reset(); //reset();
    torch::Tensor randn_tensor = torch::randn({3}, tensor_create_options);
    std::cout << "==> randn_tensor shape 3: " << randn_tensor.sizes() << std::endl;
randn_tensor_size--;
  }

  return 0;
}

and I get thrown this:
==> randn_tensor shape: [10]
==> randn_tensor shape 3: [3]
terminate called after throwing an instance of 'c10::Error'
  what():  sizes() called on undefined Tensor

Essentially, what I want to do is recreate the tensor within the while loop and ensure that I can access it again in the whileloop.
Interestingly, it seems to have cerated the tensor of reduced size but the while loop does not seem to recognise this.
Thank you!


title: Pytorch LSTM - How do I implement it without using a for loop?

body:
I implemented this very simple LSTM model:
class LSTM_Model(nn.Module):
    def __init__(self, vocab_size, embed_size, hidden_size, num_layers, rnn_size, vf_h, vf_w, batch_size, num_steps, minval, maxval):
        super(LSTM_Model, self).__init__()
        self.num_steps = num_steps
        self.num_layers = num_layers
        self.batch_size = batch_size
        self.hidden_size = hidden_size
        self.embed_size = embed_size
        self.rnn_size = rnn_size
        self.vf_h = vf_h
        self.vf_w = vf_w
        self.embed = nn.Embedding(vocab_size, embed_size, padding_idx = 0) #12112->1000
        self.embed.weight.data.uniform_(minval, maxval)
        self.lstm = nn.LSTM(embed_size, hidden_size, num_layers, batch_first=True)

    def get_first_nonpad_index(self,words):
        i = 0
        encoded_word_list = words.view(-1).tolist()
        for word in encoded_word_list:
          if word == 0:
            i+=1
          else:
            break
        return i

    def forward(self, words, hidden_state):
        first_nonpad_index = self.get_first_nonpad_index(words)
        number_of_words = 20 - first_nonpad_index
        sentence_no_padding = torch.narrow(words, 2, first_nonpad_index, number_of_words).to(device)
        embedded_seq = self.embed(sentence_no_padding)
        state = hidden_state
        for n in range(number_of_words):
            w_emb = embedded_seq[:, :, n]
            rnn_output, state = self.lstm(w_emb, state)

        lang_feat = rnn_output.view(self.batch_size, 1, 1, self.rnn_size)         #shape:(1, 1, 1000) -> (1, 1, 1, 1000)
        lang_feat = f.normalize(lang_feat, p=2, dim=3)                            #shape=(1, 1, 1, 1000)

        lang_feat = lang_feat.repeat(1, self.vf_h, self.vf_w, 1)                  #shape=(1, 40, 40, 1000)
        return lang_feat

I wish to reimplement the same model without using a for loop in the forward function. I wrote this new forward function:
    def forward(self, words, state):
        first_nonpad_index = self.get_first_nonpad_index(words)
        number_of_words = 20 - first_nonpad_index
        sentence_no_padding = torch.narrow(words, 2, first_nonpad_index, number_of_words).to(device)

        embedded_seq = [self.embed(sentence_no_padding[:, :, n]) for n in range(number_of_words)]

        embedded_seq = torch.cat(embedded_seq).view(1, number_of_words, -1)

        rnn_output, state = self.lstm(embedded_seq, state)
        last_output = rnn_output[:, -1, :]

        lang_feat = last_output.view(self.batch_size, 1, 1, self.rnn_size)         #shape=(1, 1, 1, 1000)
        lang_feat = f.normalize(lang_feat, p=2, dim=3)                            #shape=(1, 1, 1, 1000)
        lang_feat = lang_feat.repeat(1, self.vf_h, self.vf_w, 1)                  #shape=(1, 40, 40, 1000)

        return lang_feat

However, when I use the new forward function I am getting different results than before. Can you please spot what did I do wrong? I obviously want the 2nd implementation to be equivalent to the first one


title: How to quickly generate a random permutation moving each element with a distance less than K?

body:
I know the following PyTorch API can perform a global random shuffle for 1D array [0, ... , n-1]:
torch.randperm(n)

but I'm confused on how to quickly generate a random permutation, such that each element of the shuffled array satisfying:
K = 10  # should be positive
shuffled_array = rand_perm_func(n, K)  # a mysterious function
for i in range(n):
    print(abs(shuffled_array[i] - i) < K)  # should be True for each i

which means that each element is moved with a distance less than K. Does there exists fast implementations for 1D arrays and 2D arrays?

Thanks to @PM 2Ring, I wrote the following code:
import torch

# randomly produces a 1-D permutation index array,
# such that each element of the shuffled array has
# a distance less than K from its original location
def rand_perm(n, K):
    o = torch.arange(0, n)
    if K <= 1:
        return o
    while True:
        p = torch.randperm(n)
        d = abs(p - o) < K
        if bool(d.all()):
            return p

if __name__ == '__main__':
    for i in range(10):
        print(rand_perm(10, 2))

but it seems that when n is large, and K is small, the generation will take a very long time. Does there exists a more efficient implementation?


title: How to run a Pytorch code on several GPUs?

body:
I have multiple GPU devices and want to run a Pytorch on them. I have already tried MULTI-GPU EXAMPLES and DATA PARALLELISM in my code by
device = torch.device("cuda:0,1,2")
model = torch.nn.DataParallel(model, device_ids=[0, 1, 2])
model.to(device)

in my code. But the training is still performed on one GPU (cuda:0). In the shell, I have also selected GPUs using export CUDA_VISIBLE_DEVICES=0,1,2 before running my code.
Could anyone help me to solve this issue please?


title: Get Pytorch - tensor values as a integer in python

body:
I have my output of my torch tensor which looks like below
(coordinate of a bounding box in object detection)
[tensor(299., device='cuda:0'), tensor(272., device='cuda:0'), tensor(327., device='cuda:0'), tensor(350., device='cuda:0')]

I wanted to extract each of the tensor value as an int in the form of minx,miny,maxx,maxy
so that I can pass it to a shapely function in the below form
from shapely.geometry import box
minx,miny,maxx,maxy=1,2,3,4
b = box(minx,miny,maxx,maxy)

What's the best way to do it? by avoiding, Cuda enabled or not or other exceptions?


title: Pytorch NCCL DDP freezes but Gloo Works

body:
I am trying to figure out whether both Nvidia 2070S GPUs on the same Ubuntu 20.04 system can access each other via NCCL and Pytorch 1.8.
My test script is based on the Pytorch docs, but with the backend changed from "gloo" to "nccl".
When the backend is "gloo", the script finishes running in less than a minute.
$ time python test_ddp.py
Running basic DDP example on rank 0.
Running basic DDP example on rank 1.

real    0m4.839s
user    0m4.980s
sys     0m1.942s

However, when the backend is set to "nccl", the script gets stuck with the below output and never returns to the bash prompt.
$ python test_ddp.py
Running basic DDP example on rank 1.
Running basic DDP example on rank 0.

Same problem when disabling IB
$ NCCL_IB_DISABLE=1 python test_ddp.py
Running basic DDP example on rank 1.
Running basic DDP example on rank 0.

I'm using the packages:

pytorch 1.8.1
cudatoolkit 11.1.1
python 3.8.8

How can we fix the problem when using NCCL? Thank you!
Python code used for testing NCCL:
import os
import sys
import tempfile
import torch
import torch.distributed as dist
import torch.nn as nn
import torch.optim as optim
import torch.multiprocessing as mp

from torch.nn.parallel import DistributedDataParallel as DDP

def setup(rank, world_size):
    os.environ["MASTER_ADDR"] = "localhost"
    os.environ["MASTER_PORT"] = "12355"

    # gloo: works
    # dist.init_process_group("gloo", rank=rank, world_size=world_size)

    # nccl: hangs forever
    dist.init_process_group(
        "nccl", init_method="tcp://10.1.1.20:23456", rank=rank, world_size=world_size
    )

def cleanup():
    dist.destroy_process_group()

class ToyModel(nn.Module):
    def __init__(self):
        super(ToyModel, self).__init__()
        self.net1 = nn.Linear(10, 10)
        self.relu = nn.ReLU()
        self.net2 = nn.Linear(10, 5)

    def forward(self, x):
        return self.net2(self.relu(self.net1(x)))

def demo_basic(rank, world_size):
    print(f"Running basic DDP example on rank {rank}.")
    setup(rank, world_size)

    # create model and move it to GPU with id rank
    model = ToyModel().to(rank)
    ddp_model = DDP(model, device_ids=[rank])

    loss_fn = nn.MSELoss()
    optimizer = optim.SGD(ddp_model.parameters(), lr=0.001)

    optimizer.zero_grad()
    outputs = ddp_model(torch.randn(20, 10))
    labels = torch.randn(20, 5).to(rank)
    loss_fn(outputs, labels).backward()
    optimizer.step()

    cleanup()

def run_demo(demo_fn, world_size):
    mp.spawn(demo_fn, args=(world_size,), nprocs=world_size, join=True)

if __name__ == "__main__":
    run_demo(demo_basic, 2)


title: PyTorch GPU out of memory although enough memory is free

body:
I am trying to finetune a Transformers language model for text classification using PyTorch . However, I get the following memory error:
CUDA out of memory. Tried to allocate 578.00 MiB (GPU 2; 10.76 GiB total capacity; 2.79 GiB already allocated; 6.55 GiB free; 3.24 GiB reserved in total by PyTorch

I am confused since I definitely have enough memory according to the error message.
I also tried to train the model on a different GPU with a little less memory before as well as on the CPU with much less data to check my code and both versions worked fine (but cannot be used for the full training). It also already worked for a different Transformers model when setting the batch size to 16. I also want to avoid decreasing my batch size any further since I am already at 16. Does anyone have an idea why the error occurs? Thanks in advance!


title: Negative of MNIST Dataset

body:
I am trying to get the negative of the mnist dataset using pytorch. I download the datasets using pytorhc's dataloader, but am unsure of how to approach next. I'm not sure the datatype of the the set and am trying to subtract each pixel value from 255 to get the negative. Any help with how to adjust either the set or the loader would be greatly appreciated.
 train_set = tv.datasets.MNIST(root = "./data", train = True, download = True, transform = transforms.ToTensor())
test_set = tv.datasets.MNIST(root = "./data", train = False, download = True, transform = transforms.ToTensor())

train_dl = torch.utils.data.DataLoader(train_set, batch_size=64, shuffle=True)
val_dl = torch.utils.data.DataLoader(test_set, batch_size=1000, shuffle=False)


title: The loss value does not decrease

body:
I am implementing a simple feedforward neural network with Pytorch and the loss function does not seem to decrease. Because of some other tests I have done, the problem seems to be in the computations I do to compute pred, since if I slightly change the network so that it spits out a 2-dimensional vector for each entry and save it as pred, everything works perfectly.
Do you see the problem in defining pred here? Thanks
import torch
import numpy as np
from torch import nn

dt = 0.1

class Neural_Network(nn.Module):
    def __init__(self, ):
        super(Neural_Network, self).__init__()

        self.l1 = nn.Linear(2,300)
        self.nl = nn.Tanh()
        self.l2 = nn.Linear(300,1)

    def forward(self, X):
        z = self.l1(X)
        z = self.nl(z)
        o = self.l2(z)
        return o

N = 1000
X = torch.rand(N,2,requires_grad=True)
y = torch.rand(N,1)
NN = Neural_Network()

criterion = torch.nn.MSELoss(reduction='sum')
optimizer = torch.optim.Adam(NN.parameters(), lr=1e-5)

epochs = 200

for i in range(epochs):  # trains the NN 1,000 times

    HH = torch.mean(NN(X))
    gradH =  torch.autograd.grad(HH, X)[0]
    XH= torch.cat((gradH[:,1].unsqueeze(0),-gradH[:,0].unsqueeze(0)),dim=0).t()
    pred = X + dt*XH

    #Optimize and improve the weights
    loss = criterion(pred, y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    print (" Loss: ", loss.detach().numpy())  # mean sum squared loss

P.S. With these X and y the loss is not expected to go to zero, I have added them here like them just for simplicity. I will apply this architecture to data points which are expected to satisfy this model. However I am just interested in seeing the loss decreasing.

My aim is to approximate with a neural network the Hamiltonian of a vector field where only some trajectory is known. For example only the updates x(t)\rightarrow x(t+\Delta t) for some choice of points. So the vector X contains the points x(t), while y contains the $x(t+\Delta t)$. My network above approximates in a simple way the Hamiltonian function H(x) and in order to optimize it I need to find the trajectories associated to this Hamiltonian.
In particular XH aims to be the Hamiltonian vector field associated to the approximated Hamiltonian. The time update pred = X + dt*XH is simply one step of forward Euler.
However, my main issue here can be abstracted in: how can I involve the gradient of a network with respect to its inputs in the loss function?


title: [Pytorch]Seek for a vectorised implementation

body:
Here is my code, 'x' is a regular CNN weight with size: (out_c, in_c, k_h, k_w) where out_c is also the number of CNN kernel. lb(lower bound) and ub (upper bond) are two predefined 1D numpy array with length out_c.

The task is to make sure that the value of all CNN kernel weights are within the range defined by lb and ub (i.e. lb[i] < x[i, :, :, :] < ub[i], 0 <= i < out_c) by clipping all the outliers.
Besides, I need to keep the mask of outliers for other usage.

However, I cannot find a fully vectorised solution such that no for loop is involved. I'm using Pytorch 0.3.1, is there any method to make this code segment fully vectorised (get rid of for loop)? Thanks.

N = x.size(0) # number of cnn kernel
lower_mask = []
upper_mask = []
for i in range(N):
    lower_mask.append(x[i] < lb.tolist()[i])
    upper_mask.append(ub.tolist()[i] < x[i])
    x[i].clamp_(lb.tolist()[i], ub.tolist()[i])
lower_mask = torch.stack(lower_mask).cuda()
upper_mask = torch.stack(upper_mask).cuda()


title: Shape of tensor

body:
I came across this piece of code

(x_train, y_train), (x_test, y_test) = mnist.load_data()
print("Shape of x_train: " + str(x_train.shape))
print("Shape of y_train: " + str(y_train.shape))

And found that the output looks like this

(60000, 28, 28)
(60000,)

For the first line of output

So far my understanding, does it means that in 1st dimension it can hold 60k items, then in next dimension, it can hold 28 "array of 60k items"
and finally, in the last dimension, it can hold 28 "array of 28 "array of 60k items""

What I want to clarify is, Is this 60k samples of 28x28 data or something else?

For the second line of output, it seems like its just a 1d array of 60k items. So what does it actually represents? (i know that in x_train it was handwritten numbers and each number represents the intensity of grey in that cell)

Please note I have taken this code from some online example(i don't remember and won't mind if you want your credit to be added to this) and public dataset
tf.keras.datasets.mnist


title: How can I run concurrent trials using AX?

body:
According to the AX documentation (https://ax.dev/docs/api.html), under the Service API, it says that "trials might be evaluated in parallel." How can I achieve this?

I have 4 GPUs, and I'd like to evaluate 4 different sets of hyperparameters at a time (4 training at the same time).

from ax.service.ax_client import AxClient
from ax.utils.measurement.synthetic_functions import branin

ax = AxClient()
ax.create_experiment(
    name="branin_test_experiment",
    parameters=[
        {
            "name": "x1",
            "type": "range",
            "bounds": [-5.0, 10.0],
            "value_type": "float",
        },
        {
            "name": "x2",
            "type": "range",
            "bounds": [0.0, 10.0],
        },
    ],
    objective_name="branin",
    minimize=True,
)

for _ in range(15):
    parameters, trial_index = ax.get_next_trial()
    ax.complete_trial(trial_index=trial_index, raw_data=branin(parameters["x1"], parameters["x2"]))

best_parameters, metrics = ax.get_best_parameters()


title: Mat2cell matlab equivalent in tensorflow or pytorch

body:
I have a square matrix and like to break it into some smaller matrices. For example, assume we have a matrix with the shape of [4,4] and would like to convert it into 4 smaller matrices with size [2,2].

input:

[9, 9, 9, 9,
 8, 8, 8, 8,
 7, 7, 7, 7,
 6, 6, 6, 6]

output:

[[9, 9  | [9, 9,
 8, 8]  | 8, 8],
 ---------------
 [7, 7  | [7, 7,
 6, 6]  | 6, 6]]


title: Fastest way to split tensors in groups of two and for each group, randomly assign one to 0 and the other to 1

body:
I have a big tensor with size (128, 64, 1, 1). For every two small tensors of size (1, 1) in that big tensor, I want to randomly assign one to be 1 and the other to be 0.

My following code works but it's awfully slow.

# a is a big tensor of size (128, 64, 1, 1)
for i in range(a.size()[0]):
    j = 0
    while j < a.size()[1] - 1:
        r = int(torch.randint(0, 2, (1,)))
        a[i][j + r] = 1
        a[i][j + 1 - r] = 0
        j += 2

Is there any way to avoid for loops and do every assignment in parallel? Thank you.


title: how can I import "Dense3D" and use it in code?

body:
I tried to run a code from git but i found a problem with importing Dense3D.
here the part of code witch relate to the problem:

here the part of code that relate to the problem:

   import torch
   from torch.models import Dense3D
   import torch.nn as nn
   #some lines deleted
   print("Loading options...")
   with open(sys.argv[1], 'r') as optionsFile:
       options = toml.loads(optionsFile.read())
   #Create the model.
   model = Dense3D(options)

it make error that "can't import Dense3D" or something like this in another tries with some changes.

and i used colab .


title: This is my training and validation accuracy is there something wrong with code ? or data?

body:
This is my classification code with pytorch.
it classify the image like cifar10.

The problem is I trained the classification model
but when I submit the result it always has very low accuracy
so I'm thinking is there something wrong with my code or data

does my training accuracy and validation accuracy seem right?

EPOCHS = 40
SAVE_DIR = 'models'
MODEL_SAVE_PATH = os.path.join(SAVE_DIR, 'please.pt')
from torch.utils.data import DataLoader
best_valid_loss = float('inf')

if not os.path.isdir(f'{SAVE_DIR}'):
    os.makedirs(f'{SAVE_DIR}')
print("start")
for epoch in range(EPOCHS):
    print('================================',epoch ,'================================')
    for i , (train_idx, valid_idx) in enumerate(zip(train_indexes, valid_indexes)):
        print(i,train_idx,valid_idx)
        ## create iterator objects for train and valid datasets
        trainloader = DataLoader(train_dataset, batch_size=batch_size, sampler= SubsetRandomSampler(train_idx))
        valloader = DataLoader(train_dataset, batch_size=batch_size, sampler=SubsetRandomSampler(valid_idx))

        train_loss, train_acc = train(model, device, trainloader, optimizer, criterion)
        valid_loss, valid_acc = evaluate(model, device, valloader, criterion)

        if valid_loss < best_valid_loss:
            best_valid_loss = valid_loss
            torch.save(model.state_dict(), MODEL_SAVE_PATH)

        print(f'| Epoch: {epoch+1:02} | Train Loss: {train_loss:.3f} | Train Acc: {train_acc*100:05.2f}% | Val. Loss: {valid_loss:.3f} | Val. Acc: {valid_acc*100:05.2f}% |')

And here is my train and validation accuracy
with this log is there something wrong with data or code?

  start
  ================================ 0 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 01 | Train Loss: 0.111 | Train Acc: 96.11% |
  Val. Loss: 0.122 | Val. Acc: 95.92% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 01 |
  Train Loss: 0.102 | Train Acc: 96.54% | Val. Loss: 0.118 | Val. Acc:
  96.28% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 01 | Train Loss: 0.091 | Train
  Acc: 96.79% | Val. Loss: 0.122 | Val. Acc: 95.53% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 01 | Train Loss: 0.092 | Train Acc: 96.87% | Val. Loss: 0.117 |
  Val. Acc: 95.82% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 01 | Train Loss: 0.093 |
  Train Acc: 96.82% | Val. Loss: 0.129 | Val. Acc: 95.71% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 01 | Train Loss: 0.090 | Train Acc: 96.83% | Val.
  Loss: 0.136 | Val. Acc: 95.17% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 01 | Train
  Loss: 0.094 | Train Acc: 96.69% | Val. Loss: 0.112 | Val. Acc: 95.74%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 01 | Train Loss: 0.090 | Train Acc: 96.80%
  | Val. Loss: 0.134 | Val. Acc: 94.96% |
  ================================ 1 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 02 | Train Loss: 0.092 | Train Acc: 96.96% |
  Val. Loss: 0.078 | Val. Acc: 97.16% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 02 |
  Train Loss: 0.090 | Train Acc: 96.77% | Val. Loss: 0.115 | Val. Acc:
  95.53% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 02 | Train Loss: 0.083 | Train
  Acc: 97.17% | Val. Loss: 0.105 | Val. Acc: 96.21% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 02 | Train Loss: 0.091 | Train Acc: 96.71% | Val. Loss: 0.162 |
  Val. Acc: 94.15% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 02 | Train Loss: 0.077 |
  Train Acc: 97.30% | Val. Loss: 0.124 | Val. Acc: 95.10% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 02 | Train Loss: 0.082 | Train Acc: 97.13% | Val.
  Loss: 0.100 | Val. Acc: 96.45% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 02 | Train
  Loss: 0.086 | Train Acc: 96.99% | Val. Loss: 0.121 | Val. Acc: 95.35%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 02 | Train Loss: 0.078 | Train Acc: 97.26%
  | Val. Loss: 0.132 | Val. Acc: 95.49% |
  ================================ 2 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 03 | Train Loss: 0.081 | Train Acc: 97.10% |
  Val. Loss: 0.108 | Val. Acc: 96.70% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 03 |
  Train Loss: 0.085 | Train Acc: 97.04% | Val. Loss: 0.098 | Val. Acc:
  96.52% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 03 | Train Loss: 0.080 | Train
  Acc: 97.27% | Val. Loss: 0.121 | Val. Acc: 95.99% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 03 | Train Loss: 0.075 | Train Acc: 97.43% | Val. Loss: 0.107 |
  Val. Acc: 96.17% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 03 | Train Loss: 0.076 |
  Train Acc: 97.39% | Val. Loss: 0.096 | Val. Acc: 96.35% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 03 | Train Loss: 0.076 | Train Acc: 97.32% | Val.
  Loss: 0.113 | Val. Acc: 96.06% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 03 | Train
  Loss: 0.079 | Train Acc: 97.21% | Val. Loss: 0.076 | Val. Acc: 97.30%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 03 | Train Loss: 0.071 | Train Acc: 97.45%
  | Val. Loss: 0.080 | Val. Acc: 97.05% |
  ================================ 3 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 04 | Train Loss: 0.077 | Train Acc: 97.24% |
  Val. Loss: 0.091 | Val. Acc: 96.95% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 04 |
  Train Loss: 0.073 | Train Acc: 97.45% | Val. Loss: 0.122 | Val. Acc:
  95.78% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 04 | Train Loss: 0.073 | Train
  Acc: 97.29% | Val. Loss: 0.088 | Val. Acc: 97.05% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 04 | Train Loss: 0.071 | Train Acc: 97.56% | Val. Loss: 0.099 |
  Val. Acc: 96.74% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 04 | Train Loss: 0.090 |
  Train Acc: 96.85% | Val. Loss: 0.127 | Val. Acc: 95.71% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 04 | Train Loss: 0.073 | Train Acc: 97.30% | Val.
  Loss: 0.099 | Val. Acc: 96.56% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 04 | Train
  Loss: 0.065 | Train Acc: 97.66% | Val. Loss: 0.086 | Val. Acc: 97.05%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 04 | Train Loss: 0.073 | Train Acc: 97.41%
  | Val. Loss: 0.073 | Val. Acc: 97.48% |
  ================================ 4 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 05 | Train Loss: 0.062 | Train Acc: 97.85% |
  Val. Loss: 0.097 | Val. Acc: 96.85% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 05 |
  Train Loss: 0.065 | Train Acc: 97.77% | Val. Loss: 0.088 | Val. Acc:
  96.70% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 05 | Train Loss: 0.065 | Train
  Acc: 97.74% | Val. Loss: 0.100 | Val. Acc: 96.21% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 05 | Train Loss: 0.061 | Train Acc: 97.88% | Val. Loss: 0.081 |
  Val. Acc: 96.95% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 05 | Train Loss: 0.067 |
  Train Acc: 97.64% | Val. Loss: 0.076 | Val. Acc: 97.34% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 05 | Train Loss: 0.063 | Train Acc: 97.79% | Val.
  Loss: 0.086 | Val. Acc: 97.02% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 05 | Train
  Loss: 0.066 | Train Acc: 97.69% | Val. Loss: 0.097 | Val. Acc: 96.73%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 05 | Train Loss: 0.061 | Train Acc: 97.79%
  | Val. Loss: 0.072 | Val. Acc: 97.69% |
  ================================ 5 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 06 | Train Loss: 0.057 | Train Acc: 98.07% |
  Val. Loss: 0.078 | Val. Acc: 97.52% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 06 |
  Train Loss: 0.057 | Train Acc: 97.97% | Val. Loss: 0.101 | Val. Acc:
  96.42% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 06 | Train Loss: 0.062 | Train
  Acc: 97.76% | Val. Loss: 0.067 | Val. Acc: 97.55% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 06 | Train Loss: 0.062 | Train Acc: 97.76% | Val. Loss: 0.062 |
  Val. Acc: 98.01% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 06 | Train Loss: 0.062 |
  Train Acc: 97.87% | Val. Loss: 0.055 | Val. Acc: 98.12% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 06 | Train Loss: 0.056 | Train Acc: 98.10% | Val.
  Loss: 0.086 | Val. Acc: 97.20% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 06 | Train
  Loss: 0.056 | Train Acc: 98.00% | Val. Loss: 0.070 | Val. Acc: 97.69%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 06 | Train Loss: 0.062 | Train Acc: 97.88%
  | Val. Loss: 0.058 | Val. Acc: 98.12% |
  ================================ 6 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 07 | Train Loss: 0.061 | Train Acc: 97.86% |
  Val. Loss: 0.052 | Val. Acc: 98.23% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 07 |
  Train Loss: 0.060 | Train Acc: 97.91% | Val. Loss: 0.056 | Val. Acc:
  97.80% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 07 | Train Loss: 0.053 | Train
  Acc: 98.11% | Val. Loss: 0.065 | Val. Acc: 97.48% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 07 | Train Loss: 0.056 | Train Acc: 98.01% | Val. Loss: 0.095 |
  Val. Acc: 96.56% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 07 | Train Loss: 0.057 |
  Train Acc: 97.96% | Val. Loss: 0.071 | Val. Acc: 97.20% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 07 | Train Loss: 0.057 | Train Acc: 98.16% | Val.
  Loss: 0.067 | Val. Acc: 97.87% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 07 | Train
  Loss: 0.055 | Train Acc: 98.05% | Val. Loss: 0.062 | Val. Acc: 97.83%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 07 | Train Loss: 0.053 | Train Acc: 98.17%
  | Val. Loss: 0.068 | Val. Acc: 97.76% |
  ================================ 7 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 08 | Train Loss: 0.053 | Train Acc: 98.17% |
  Val. Loss: 0.061 | Val. Acc: 97.98% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 08 |
  Train Loss: 0.053 | Train Acc: 98.15% | Val. Loss: 0.054 | Val. Acc:
  98.33% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 08 | Train Loss: 0.051 | Train
  Acc: 98.31% | Val. Loss: 0.067 | Val. Acc: 97.48% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 08 | Train Loss: 0.049 | Train Acc: 98.26% | Val. Loss: 0.053 |
  Val. Acc: 98.33% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 08 | Train Loss: 0.058 |
  Train Acc: 97.91% | Val. Loss: 0.066 | Val. Acc: 97.62% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 08 | Train Loss: 0.054 | Train Acc: 98.10% | Val.
  Loss: 0.069 | Val. Acc: 97.48% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 08 | Train
  Loss: 0.063 | Train Acc: 97.80% | Val. Loss: 0.058 | Val. Acc: 97.83%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 08 | Train Loss: 0.050 | Train Acc: 98.27%
  | Val. Loss: 0.053 | Val. Acc: 98.15% |
  ================================ 8 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 09 | Train Loss: 0.050 | Train Acc: 98.35% |
  Val. Loss: 0.059 | Val. Acc: 98.22% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 09 |
  Train Loss: 0.045 | Train Acc: 98.35% | Val. Loss: 0.047 | Val. Acc:
  98.09% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 09 | Train Loss: 0.047 | Train
  Acc: 98.42% | Val. Loss: 0.052 | Val. Acc: 98.37% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 09 | Train Loss: 0.050 | Train Acc: 98.27% | Val. Loss: 0.077 |
  Val. Acc: 97.87% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 09 | Train Loss: 0.050 |
  Train Acc: 98.27% | Val. Loss: 0.053 | Val. Acc: 98.05% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 09 | Train Loss: 0.046 | Train Acc: 98.42% | Val.
  Loss: 0.072 | Val. Acc: 97.62% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 09 | Train
  Loss: 0.050 | Train Acc: 98.21% | Val. Loss: 0.078 | Val. Acc: 97.23%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 09 | Train Loss: 0.053 | Train Acc: 98.11%
  | Val. Loss: 0.062 | Val. Acc: 97.90% |
  ================================ 9 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 10 | Train Loss: 0.051 | Train Acc: 98.18% |
  Val. Loss: 0.065 | Val. Acc: 98.01% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 10 |
  Train Loss: 0.049 | Train Acc: 98.33% | Val. Loss: 0.096 | Val. Acc:
  96.81% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 10 | Train Loss: 0.050 | Train
  Acc: 98.37% | Val. Loss: 0.056 | Val. Acc: 98.23% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 10 | Train Loss: 0.044 | Train Acc: 98.50% | Val. Loss: 0.053 |
  Val. Acc: 98.37% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 10 | Train Loss: 0.041 |
  Train Acc: 98.51% | Val. Loss: 0.045 | Val. Acc: 98.51% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 10 | Train Loss: 0.043 | Train Acc: 98.46% | Val.
  Loss: 0.060 | Val. Acc: 98.01% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 10 | Train
  Loss: 0.046 | Train Acc: 98.39% | Val. Loss: 0.051 | Val. Acc: 98.08%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 10 | Train Loss: 0.045 | Train Acc: 98.40%
  | Val. Loss: 0.059 | Val. Acc: 98.26% |
  ================================ 10 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 11 | Train Loss: 0.041 | Train Acc: 98.56% |
  Val. Loss: 0.052 | Val. Acc: 98.47% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 11 |
  Train Loss: 0.047 | Train Acc: 98.35% | Val. Loss: 0.061 | Val. Acc:
  97.91% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 11 | Train Loss: 0.047 | Train
  Acc: 98.42% | Val. Loss: 0.053 | Val. Acc: 98.33% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 11 | Train Loss: 0.039 | Train Acc: 98.65% | Val. Loss: 0.046 |
  Val. Acc: 98.47% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 11 | Train Loss: 0.034 |
  Train Acc: 98.83% | Val. Loss: 0.056 | Val. Acc: 98.19% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 11 | Train Loss: 0.047 | Train Acc: 98.44% | Val.
  Loss: 0.062 | Val. Acc: 97.70% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 11 | Train
  Loss: 0.040 | Train Acc: 98.63% | Val. Loss: 0.039 | Val. Acc: 98.69%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 11 | Train Loss: 0.038 | Train Acc: 98.70%
  | Val. Loss: 0.043 | Val. Acc: 98.40% |
  ================================ 11 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 12 | Train Loss: 0.039 | Train Acc: 98.65% |
  Val. Loss: 0.060 | Val. Acc: 97.87% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 12 |
  Train Loss: 0.043 | Train Acc: 98.54% | Val. Loss: 0.049 | Val. Acc:
  98.23% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 12 | Train Loss: 0.045 | Train
  Acc: 98.39% | Val. Loss: 0.069 | Val. Acc: 97.63% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 12 | Train Loss: 0.044 | Train Acc: 98.43% | Val. Loss: 0.052 |
  Val. Acc: 97.80% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 12 | Train Loss: 0.040 |
  Train Acc: 98.54% | Val. Loss: 0.061 | Val. Acc: 98.01% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 12 | Train Loss: 0.038 | Train Acc: 98.70% | Val.
  Loss: 0.052 | Val. Acc: 98.26% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 12 | Train
  Loss: 0.043 | Train Acc: 98.56% | Val. Loss: 0.056 | Val. Acc: 98.01%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 12 | Train Loss: 0.045 | Train Acc: 98.43%
  | Val. Loss: 0.043 | Val. Acc: 98.44% |
  ================================ 12 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 13 | Train Loss: 0.045 | Train Acc: 98.40% |
  Val. Loss: 0.060 | Val. Acc: 97.94% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 13 |
  Train Loss: 0.042 | Train Acc: 98.52% | Val. Loss: 0.062 | Val. Acc:
  97.73% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 13 | Train Loss: 0.038 | Train
  Acc: 98.66% | Val. Loss: 0.047 | Val. Acc: 98.23% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 13 | Train Loss: 0.040 | Train Acc: 98.61% | Val. Loss: 0.045 |
  Val. Acc: 98.16% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 13 | Train Loss: 0.033 |
  Train Acc: 98.82% | Val. Loss: 0.050 | Val. Acc: 98.37% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 13 | Train Loss: 0.042 | Train Acc: 98.60% | Val.
  Loss: 0.045 | Val. Acc: 98.12% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 13 | Train
  Loss: 0.040 | Train Acc: 98.70% | Val. Loss: 0.043 | Val. Acc: 98.65%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 13 | Train Loss: 0.038 | Train Acc: 98.63%
  | Val. Loss: 0.065 | Val. Acc: 97.80% |
  ================================ 13 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 14 | Train Loss: 0.041 | Train Acc: 98.62% |
  Val. Loss: 0.038 | Val. Acc: 98.76% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 14 |
  Train Loss: 0.039 | Train Acc: 98.74% | Val. Loss: 0.056 | Val. Acc:
  98.40% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 14 | Train Loss: 0.037 | Train
  Acc: 98.68% | Val. Loss: 0.043 | Val. Acc: 98.30% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 14 | Train Loss: 0.039 | Train Acc: 98.69% | Val. Loss: 0.041 |
  Val. Acc: 98.58% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 14 | Train Loss: 0.039 |
  Train Acc: 98.70% | Val. Loss: 0.038 | Val. Acc: 98.65% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 14 | Train Loss: 0.035 | Train Acc: 98.77% | Val.
  Loss: 0.046 | Val. Acc: 98.40% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 14 | Train
  Loss: 0.039 | Train Acc: 98.70% | Val. Loss: 0.043 | Val. Acc: 98.37%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 14 | Train Loss: 0.034 | Train Acc: 98.87%
  | Val. Loss: 0.045 | Val. Acc: 98.37% |
  ================================ 14 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 15 | Train Loss: 0.039 | Train Acc: 98.68% |
  Val. Loss: 0.046 | Val. Acc: 98.23% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 15 |
  Train Loss: 0.038 | Train Acc: 98.70% | Val. Loss: 0.046 | Val. Acc:
  98.54% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 15 | Train Loss: 0.035 | Train
  Acc: 98.80% | Val. Loss: 0.072 | Val. Acc: 97.73% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 15 | Train Loss: 0.041 | Train Acc: 98.69% | Val. Loss: 0.044 |
  Val. Acc: 98.44% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 15 | Train Loss: 0.031 |
  Train Acc: 98.87% | Val. Loss: 0.047 | Val. Acc: 98.58% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 15 | Train Loss: 0.036 | Train Acc: 98.73% | Val.
  Loss: 0.029 | Val. Acc: 98.97% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 15 | Train
  Loss: 0.038 | Train Acc: 98.65% | Val. Loss: 0.046 | Val. Acc: 98.44%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 15 | Train Loss: 0.035 | Train Acc: 98.78%
  | Val. Loss: 0.041 | Val. Acc: 98.76% |
  ================================ 15 ================================ 0 [  451   452   453 ... 21514 21515 21516] [    0     1     2 ... 18402
  18403 18404] | Epoch: 16 | Train Loss: 0.040 | Train Acc: 98.66% |
  Val. Loss: 0.043 | Val. Acc: 98.55% | 1 [    0     1     2 ... 21514
  21515 21516] [  451   452   453 ... 18847 18848 18849] | Epoch: 16 |
  Train Loss: 0.033 | Train Acc: 98.86% | Val. Loss: 0.037 | Val. Acc:
  98.72% | 2 [    0     1     2 ... 21514 21515 21516] [  902   903   904 ... 19292 19293 19294] | Epoch: 16 | Train Loss: 0.033 | Train
  Acc: 98.82% | Val. Loss: 0.034 | Val. Acc: 98.58% | 3 [    0     1
  2 ... 21514 21515 21516] [ 1353  1354  1355 ... 19737 19738 19739] |
  Epoch: 16 | Train Loss: 0.033 | Train Acc: 98.77% | Val. Loss: 0.046 |
  Val. Acc: 98.26% | 4 [    0     1     2 ... 21514 21515 21516] [ 1803
  1804  1805 ... 20182 20183 20184] | Epoch: 16 | Train Loss: 0.036 |
  Train Acc: 98.66% | Val. Loss: 0.031 | Val. Acc: 99.11% | 5 [    0
  1     2 ... 21514 21515 21516] [ 2253  2254  2255 ... 20626 20627
  20628] | Epoch: 16 | Train Loss: 0.040 | Train Acc: 98.65% | Val.
  Loss: 0.040 | Val. Acc: 98.90% | 6 [    0     1     2 ... 21514 21515
  21516] [ 2703  2704  2705 ... 21070 21071 21072] | Epoch: 16 | Train
  Loss: 0.033 | Train Acc: 98.91% | Val. Loss: 0.037 | Val. Acc: 98.76%
  | 7 [    0     1     2 ... 21070 21071 21072] [ 3153  3154  3155 ...
  21514 21515 21516] | Epoch: 16 | Train Loss: 0.032 | Train Acc: 98.90%
  | Val. Loss: 0.040 | Val. Acc: 98.65% |


title: Incorrect predictions on extracted images from text

body:
I trained a model in PyTorch on the EMNIST data set - and got about 85% accuracy on the test set. Now, I have an image of handwritten text from which I have extracted individual letters, but I'm getting very poor accuracy on the images that I have extracted.

One hot mappings that I'm using -

letters_EMNIST = {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9',
         10: 'A', 11: 'B', 12: 'C', 13: 'D', 14: 'E', 15: 'F', 16: 'G', 17: 'H', 18: 'I', 19: 'J',
         20: 'K', 21: 'L', 22: 'M', 23: 'N', 24: 'O', 25: 'P', 26: 'Q', 27: 'R', 28: 'S', 29: 'T',
         30: 'U', 31: 'V', 32: 'W', 33: 'X', 34: 'Y', 35: 'Z', 36: 'a', 37: 'b', 38: 'd', 39: 'e',
         40: 'f', 41: 'g', 42: 'h', 43: 'n', 44: 'q', 45: 'r', 46: 't'}

For reference, this is an example of the image used for testing data -

And this is an example of the image I extracted -

How can I debug this?


title: How can I efficiently modify/make pairwise distance matrix?

body:
x_norm = (x**2).sum(1).view(-1, 1)
    if y is not None:
        y_norm = (y**2).sum(1).view(1, -1)
    else:
        y = x
        y_norm = x_norm.view(1, -1)
    dist = (x_norm + y_norm - 2.0 * torch.mm(x, torch.transpose(y, 0, 1)))
    return dist

Above is a code used to calculate pairwise distance matrix(M*N) between x (M points) and y (N points).

I hope to make pairwise distance matrix that has 0 element when distance between two points is larger than specific value 'T'.

In this case, what should I do?

Thanks


title: Pytorch unsqueeze function and concatenating

body:
I have initialized a tensor t2 and applied unsqueeze function

t2=torch.tensor([[1,2],[3,4]])
t2.unsqueeze(dim=0)
tensor([[[1, 2], [3, 4]]])
t2.unsqueeze(dim=1)
tensor([[[1, 2]],[[3, 4]]])

Where I have done dim=1, there is a vacant space between the first row and second row, what does it signify?

[


title: Is it possible to add a trainable filter after an autoencoder?

body:
So I’m building a denoiser with an autoencoder. The idea is that before computing my loss (after the autoencoder), I apply an empirical wiener filter to a texture map of the image and add it back to my autoencoder output (adding back ‘lost detail’). I’ve coded this filter with PyTorch.

My first attempt worked by adding the filter to the end of my autoencoder’s forward function. I can train this network and it backpropagates through my filter in training. However, if I print my network, the filter is not listed, and torchsummary doesn’t include it when calculating parameters.

This has me thinking that I am only training the autoencoder and my filter is filtering the same way every time and not learning.

Is what I’m trying to do possible?

Below is my Autoencoder:

class AutoEncoder(nn.Module):
"""Autoencoder simple implementation """
def __init__(self):
    super(AutoEncoder, self).__init__()
    # Encoder
    # conv layer
    self.block1 = nn.Sequential(
        nn.Conv2d(1, 48, 3, padding=1),
        nn.Conv2d(48, 48, 3, padding=1),
        nn.MaxPool2d(2),
        nn.BatchNorm2d(48),
        nn.LeakyReLU(0.1)

    )
    self.block2 = nn.Sequential(
        nn.Conv2d(48, 48, 3, padding=1),
        nn.MaxPool2d(2),
        nn.BatchNorm2d(48),
        nn.LeakyReLU(0.1)
    )
    self.block3 = nn.Sequential(
        nn.Conv2d(48, 48, 3, padding=1),
        nn.ConvTranspose2d(48, 48, 2, 2, output_padding=1),
        nn.BatchNorm2d(48),
        nn.LeakyReLU(0.1)
    )
    self.block4 = nn.Sequential(
        nn.Conv2d(96, 96, 3, padding=1),
        nn.Conv2d(96, 96, 3, padding=1),
        nn.ConvTranspose2d(96, 96, 2, 2),
        nn.BatchNorm2d(96),
        nn.LeakyReLU(0.1)
    )
    self.block5 = nn.Sequential(
        nn.Conv2d(144, 96, 3, padding=1),
        nn.Conv2d(96, 96, 3, padding=1),
        nn.ConvTranspose2d(96, 96, 2, 2),
        nn.BatchNorm2d(96),
        nn.LeakyReLU(0.1)
    )
    self.block6 = nn.Sequential(
        nn.Conv2d(97, 64, 3, padding=1),
        nn.BatchNorm2d(64),
        nn.Conv2d(64, 32, 3, padding=1),
        nn.BatchNorm2d(32),
        nn.Conv2d(32, 1, 3, padding=1),
        nn.LeakyReLU(0.1)
    )

    # self.blockNorm = nn.Sequential(
    #     nn.BatchNorm2d(1),
    #     nn.LeakyReLU(0.1)
    # )

def forward(self, x):
    # torch.autograd.set_detect_anomaly(True)
    # print("input: ", x.shape)
    pool1 = self.block1(x)
    # print("pool1: ", pool1.shape)
    pool2 = self.block2(pool1)
    # print("pool2: ", pool2.shape)
    pool3 = self.block2(pool2)
    # print("pool3: ", pool3.shape)
    pool4 = self.block2(pool3)
    # print("pool4: ", pool4.shape)
    pool5 = self.block2(pool4)
    # print("pool5: ", pool5.shape)
    upsample5 = self.block3(pool5)
    # print("upsample5: ", upsample5.shape)
    concat5 = torch.cat((upsample5, pool4), 1)
    # print("concat5: ", concat5.shape)
    upsample4 = self.block4(concat5)
    # print("upsample4: ", upsample4.shape)
    concat4 = torch.cat((upsample4, pool3), 1)
    # print("concat4: ", concat4.shape)
    upsample3 = self.block5(concat4)
    # print("upsample3: ", upsample3.shape)
    concat3 = torch.cat((upsample3, pool2), 1)
    # print("concat3: ", concat3.shape)
    upsample2 = self.block5(concat3)
    # print("upsample2: ", upsample2.shape)
    concat2 = torch.cat((upsample2, pool1), 1)
    # print("concat2: ", concat2.shape)
    upsample1 = self.block5(concat2)
    # print("upsample1: ", upsample1.shape)
    concat1 = torch.cat((upsample1, x), 1)
    # print("concat1: ", concat1.shape)
    output = self.block6(concat1)

    t_map = x - output

    for i in range(4):
        tensor = t_map[i, :, :, :]                 # Take each item in batch separately. Could account for this in Wiener instead

        tensor = torch.squeeze(tensor)              # Squeeze for Wiener input format

        tensor = wiener_3d(tensor, 0.05, 10)        # Apply Wiener with specified std and block size
        tensor = torch.unsqueeze(tensor, 0)         # unsqueeze to put back into block
        t_map[i, :, :, :] = tensor                  # put back into block

    filtered_output = output + t_map
    return filtered_output

The for loop at the end is to apply the filter to each image in the batch. I get that this isn’t parallelisable so if anyone has ideas for this, I’d appreciate it. I can post the ‘wiener 3d()’ filter function if that helps, just want to keep the post short.

I’ve tried to define a custom layer class with the filter inside it but I got lost very quickly.

Any help would be greatly appreciated!


title: Dealing with imbalance dataset for multi-label classification

body: In my case, I’ve 33 labels per samples. The input label tensors for a corresponding image are like [0,0,1,0,1,1,1,0,0,0,0,0…...33]. And the samples for some labels are quite low and some are high. I'm looking for predict the regression values. So what will be the best approach to improve the prediction? I would like to apply data balancing technique. But so far I found the balancing technique available only for multi-class. I’m grateful to you if you share your best knowledge about regarding my problem or any other idea to improve the performance. Thanks in Advance.


title: The features from CNN and R-CNN for a region

body:
I have a dataset of images with bounding boxes around the regions in the image. I do not need R-CNN to detect the regions as they are given in the dataset, but I need to extract the features of these regions. So, my question if I extract each region from the image and treat it as standalone image, then pass it to CNN to extract the features of that region is the same as using R-CNN to extract the features of those regions? In short, which is better the features extracted from the CNN for each regions or the features extracted from R-CNN?
If using R-CNN is better, how can I identify the regions from my dataset and the regions from the R-CNN?


title: PyTorch LSTM data dimension

body:
I'm using every past 7 days' data to predict today's value (price).

For each day, there are 6 features (Let's call them feature1 - feature5, and price).

Suppose I have 1000 rows of data. Therefore, what should be the shape of my data to be used in LSTM Pytorch?

Is it (1000, 7, 6)?


title: torch stack producing wrong dimensions

body:
I am quite new to pytorch, and I am trying to create data within a dataloader and my code looks like so:

a=[]
.. within a for loop
self.a.append(torch.stack([b[ith_idx][j], \
b[ith_idx][rnd_dist], \
b[rnd_cls_idx][rnd_dist_rnd_cls]]\
))
self.c.append([1,0])

where, b is a python list of tensors. For example, the first element of b has shape torch.Size([46, 3, 512, 512]) (3 channels 512 x 512).

        self.a = torch.stack(self.a)
        self.c = torch.tensor(self.c)

and I notice I have shapes of [500,3,3,512,512] and [500,2] for a and b, while I was expecting 500,3,3,512,512 and 500 as tensor shapes.

Any pointers as to why this is happening would be helpful.


title: Pytorch Cityscapes Dataset, train_distribute problem - "Typeerror: path should be string, bytes, pathlike or integer, not NoneType"

body:
I'm very unfamiliar with Machine Learning, python, and such, so forgive my oblivious errors. I'm trying to use machine learning systems on a dataset of streetscapes I have. I found a lot or resources, and I'm working off of this package which has a lot of examples and seems straightforward.
When I attempted to run the train_distribute.py file, I received this error:
(base) corey@corona:~/Desktop/pycity/GALD-Net-master$ python train_distribute.py

Traceback (most recent call last):
  File "train_distribute.py", line 261, in <module>
    main()
  File "train_distribute.py", line 136, in main
    if not os.path.exists(args.save_dir):
  File "/home/corey/anaconda3/lib/python3.7/genericpath.py", line 19, in exists
    os.stat(path)
TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType

Looking in the code, it's coming from these lines:
def main():

    # make save dir
    if args.local_rank == 0:
        if not os.path.exists(args.save_dir):
            os.makedirs(args.save_dir)
    # launch the logger
    Log.init(
        log_level=args.log_level,

I'm guessing this means I need a more exact file structure, and to point the code at the right location. I am in no way a computer scientist and have close to zero understanding of what does what and how things like this work. Any advice for what I'm doing wrong and how I can approach fixing things?


title: What is the correct way to fetch weights and biases in a pytorch model and copy those to a similar layer in another model?

body:
I am trying to copy weights from a pretrained model layer by layer into another model of exactly similar structure. The original model gives an accuracy of 94% on a binary image classification problem but the target model is unable to predict, results in predicting only one class for entire test set.
For example, I used this piece of code to manually copy weights from the stem of the pretrained model to the stem of the target:
modelmix.stem[0].weight = modelSep.stem[0].weight
modelmix.stem[1].weight = modelSep.stem[1].weight
modelmix.stem[1].bias = modelSep.stem[1].bias

where modelmix is the target and modelSep is the pretrained model. Used a similar snippet for all the other layers. The target model is not working even though I can see the weights are similar for all layers. I am Using pytorch 1.1. Thank you


title: autograd.grad always returns none while .grad has a correct value

body:
This is a simple program of polynomial regression, I try to get gradient of the polynomial by autograd.grad but it always returns none. The major problem might be caused by the funtion Train
Here is my code:
import torch
import torch.autograd
import numpy as np

def target_fun(x):
    return x*x + 2*x + 1

class polynomial:
    def __init__(self,d):
        self.dim = d
        self.weight = torch.rand(d).float()
    def Train(self,x,y):
        self.weight.requires_grad_(True)
        x = x.float()
        y = y.float()
        ty = torch.zeros(x.size(0))
        for i in range(self.dim):
            ty += self.weight[i]*x.pow(i)
        E = (y - ty).norm(2).float()
        E.backward()
        grad_I = (torch.autograd.grad(E,self.weight.double(),create_graph = True, allow_unused = True))
        print(self.weight.grad,"\n",grad_I)

x = np.linspace(-10,10,10)
x = torch.tensor(x).double()
y = target_fun(x)

po = polynomial(6)
po.Train(x,y)


title: How does one train a list of models in Torch?

body:
I am trying to train a list of torch neural network models. I am using a list because I want to have any number of models, and be able to iterate through the list. I am currently trying to
for i in range(len(model_list)):

    old_model = model_list[i]

    new_model = train_model(old_model, data) # train_model take a model, trains it, and returns it

    model_list[i] = new_model

However I am getting some kind of scope problem, since the models in the list do not update their parameters. I assume it has something to do with the model I am updating being some kind of clone, though I do not understand why returning the model has no effect. My training code looks like this:
def train_model(model, data):
    model_optimizer = optim.Adam(model.parameters())
    model_output = model(data)
    model_loss = criterion(model_output, target) # lets just say target we get from data
    model_loss.backward()
    model_optimizer.step()

    return model

I don't see why code like this would not work, however the models in the list are not updating (after checking their coeffficients), and the loss does not change. Is this some kind of scope problem with regard to the models or their parameters? Or is there some other problem? Thanks.


title: How to avoid recalculating a function when we need to backpropagate through it twice?

body:
In PyTorch, I want to do the following calculation:
l1 = f(x.detach(), y)
l1.backward(retain_graph=True)
l2 = -1*f(x, y.detach())
l2.backward()

where f is some function, and x and y are tensors that require gradient. Notice that x and y may both be the results of previous calculations which utilize shared parameters (for example, maybe x=g(z) and y=g(w) where g is an nn.Module).
The issue is that l1 and l2 are both numerically identical, up to the minus sign, and it seems wasteful to repeat the calculation f(x,y) twice. It would be nicer to be able to calculate it once, and apply backward twice on the result. Is there any way of doing this?
One possibility is to manually call autograd.grad and update the w.grad field of each nn.Parameter w. But I'm wondering if there is a more direct and clean way to do this, using the backward function.


title: Retraining model starts with higher loss than it finished with last session

body:
I am training a model for a denoising task.
Every time i train it the first loss in the new training session is much higher than the loss i finished with before. for example after first training:

Then when i wanted to train it for 100 more epochs:

20x higher loss.
Why does it happen?
Edit:
Here is the loss graphed. You can clearly see that after epoch 2500 there is a massive increase in the loss:

Added the training loop here:
num_epochs = 500

criterion = torch.nn.MSELoss()

sgd_params = {
    "lr": 0.30,
    "momentum": 0.9
}

lin_optimizer = torch.optim.SGD(lin_model.parameters(), lr=0.30, momentum=0.9)
sta_optimizer = torch.optim.SGD(sta_model.parameters(), lr=0.30, momentum=0.9)
ref_optimizer = torch.optim.SGD(ref_model.parameters(), lr=0.30, momentum=0.9)

_, val_clean, val_noisy = util.lincomb_generate_data(batch_size*10, B, K, functions, sample_offset=sample_offset, noise_type="gaussian", noise_mean=noise_mean, noise_std=noise_std)

print("STARTED TRAINING")

for epoch in range(num_epochs):
    print("Current epoch {}\r".format(epoch), end="")
    # generate data returns the x-axis used for plotting as well as the clean and noisy data
    _, t_clean, t_noisy = util.lincomb_generate_data(batch_size, B, K, functions, sample_offset=sample_offset, noise_type="gaussian", noise_mean=noise_mean, noise_std=noise_std)

    # ===================forward=====================
    lin_output = lin_model(t_noisy.float())
    sta_output = sta_model(t_noisy.float())
    ref_output = ref_model(t_noisy.float())

    lin_loss = criterion(lin_output.float(), t_clean.float())
    sta_loss = criterion(sta_output.float(), t_clean.float())
    ref_loss = criterion(ref_output.float(), t_clean.float())

    lin_loss_list.append(lin_loss.data)
    sta_loss_list.append(sta_loss.data)
    ref_loss_list.append(ref_loss.data)
    # ===================backward====================
    lin_optimizer.zero_grad()
    sta_optimizer.zero_grad()
    ref_optimizer.zero_grad()

    lin_loss.backward()
    sta_loss.backward()
    ref_loss.backward()

    lin_optimizer.step()
    sta_optimizer.step()
    ref_optimizer.step()

    val_lin_loss = F.mse_loss(lin_model(val_noisy.float()), val_clean.float())
    val_sta_loss = F.mse_loss(sta_model(val_noisy.float()), val_clean.float())
    val_ref_loss = F.mse_loss(ref_model(val_noisy.float()), val_clean.float())

print("DONE TRAINING")

edit 2: bump


title: Non-MNIST Digit Recognition Pytorch

body: I'm looking for examples of pytorch being used to classify non-MNIST digits. After hours of searching, it appears the algorithms are against me. Does anyone have a good example? Thanks.


title: Get encoder from trained UNet

body:
I have trained a UNet model on some images but now, I want to extract the encoder part of the model. My UNet has the following architecture:
UNet(
  (conv_final): Conv2d(8, 1, kernel_size=(1, 1), stride=(1, 1))
  (down_convs): ModuleList(
    (0): DownConv(
      (conv1): Conv2d(1, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    )
    (1): DownConv(
      (conv1): Conv2d(8, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    )
    (2): DownConv(
      (conv1): Conv2d(16, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    )
    (3): DownConv(
      (conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (pool): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
    )
    (4): DownConv(
      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    )
  )
  (up_convs): ModuleList(
    (0): UpConv(
      (upconv): ConvTranspose2d(128, 64, kernel_size=(2, 2), stride=(2, 2))
      (conv1): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    )
    (1): UpConv(
      (upconv): ConvTranspose2d(64, 32, kernel_size=(2, 2), stride=(2, 2))
      (conv1): Conv2d(64, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    )
    (2): UpConv(
      (upconv): ConvTranspose2d(32, 16, kernel_size=(2, 2), stride=(2, 2))
      (conv1): Conv2d(32, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    )
    (3): UpConv(
      (upconv): ConvTranspose2d(16, 8, kernel_size=(2, 2), stride=(2, 2))
      (conv1): Conv2d(16, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (conv2): Conv2d(8, 8, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
    )
  )
)

I have tried to load the encoder layers through model.down_convs but I get the following error:

TypeError Traceback (most recent call last) in
----> 1 res = encoder(train_img)
~/anaconda3/envs/work/lib/python3.8/site-packages/torch/nn/modules/module.py
in call(self, *input, **kwargs) 548 result =
self._slow_forward(*input, **kwargs) 549 else: –> 550 result =
self.forward(*input, **kwargs) 551 for hook in
self._forward_hooks.values(): 552 hook_result = hook(self, input,
result)
TypeError: forward() takes 1 positional argument but 2 were given

I have attached the model so you can try it out. And the weights from here
Please let me know.