title: PyTorch demands image-like dimensionality in the DataLoader even though no images are used

body:
I cannot get a simple neural network to run with a custom dummy dataset. You can find the error message at the very bottom of this question.

I would like to deeply understand how PyTorch processes data inputs and therefore built a simple dataset that gets two series of boolean values (encoded as 0s and 1s) as inputs, with OR, AND, and XOR series as targets. Everything should work with a custom Dataset and a DataLoader (for learning purposes). The input data looks as follows (naturally, only one target column is used at a time):

      column_1  column_2  or  and  xor
0            1         1   1    1    0
1            0         1   1    0    1
2            0         1   1    0    1
3            0         1   1    0    1
4            1         0   1    0    1
...        ...       ...  ..  ...  ...
9995         1         1   1    1    0
9996         1         0   1    0    1
9997         1         0   1    0    1
9998         0         1   1    0    1
9999         0         1   1    0    1

So I would like to build a neural network that represents either an OR gate, an AND gate, or an XOR gate.

Can someone shed some light on why one-dimensional data does not seem to be accepted in the iterator? It seems like image-like data is assumed? Is it even possible to solve this problem with a custom Dataset and with a DataLoader or will I have to compromise to not use a DataLoader (like in this intro to logic gates in PyTorch)?

Minimum working example with XOR as the target column:

# Imports
from pandas import read_csv, DataFrame
import torch
from torch.utils.data import Dataset, DataLoader
import torch.nn as torch_nn
import torch.nn.functional as torch_functional
import torchvision
import random

# Classes
class CustomDataset(Dataset):
    def __init__(self, data, target_column_name, transform=None):
        self.dataframe = data
        self.x = data[['column_1', 'column_2']].values
        self.y = data[[target_column_name]].values
        self.n_samples = len(data)
        self.transform = transform

    def __len__(self):
        return self.n_samples

    def __getitem__(self, index):
        x = self.x[index]
        y = self.y[index]

        if not self.transform == None:
            return (self.transform(x), self.transform(y))
        return (x, y)

class CustomNet(torch_nn.Module):
    def __init__(self):
        super().__init__()
        self.fully_connected_input_layer = torch_nn.Linear(2, 5)
        self.fully_connected_hidden_layer_1 = torch_nn.Linear(5, 5)
        self.fully_connected_hidden_layer_2 = torch_nn.Linear(5, 5)
        self.fully_connected_output_layer = torch_nn.Linear(5, 2)

    def forward(self, data):
        data = torch_functional.relu(self.fully_connected_input_layer(data))
        data = torch_functional.relu(self.fully_connected_hidden_layer_1(data))
        data = torch_functional.relu(self.fully_connected_hidden_layer_2(data))
        data = torch_functional.log_softmax(
            self.fully_connected_output_layer(data),
            dim=1
        )
        # data = data.squeeze(1)

        return data

# Global variables
NUMBER_OF_OBSERVATIONS = 10000
N_TRAIN_OBSERVATIONS = 7000
BATCH_SIZE = 4
N_EPOCHS = 3
random.seed(42)

# Generating logical gate data
## Generating two columns with random 0s and 1s.
df_data = DataFrame({
    'column_1': random.choices([0, 1], k=NUMBER_OF_OBSERVATIONS),
    'column_2': random.choices([0, 1], k=NUMBER_OF_OBSERVATIONS)
})
## Adding the logic gate results of the previously generated two columns.
df_data.loc[:,'or'] = (df_data['column_1'] == 1) | (df_data['column_2'] == 1)
df_data.loc[:,'or'] = df_data.loc[:,'or'].astype(int)
df_data.loc[:,'and'] = (df_data['column_1'] == 1) & (df_data['column_2'] == 1).astype(int)
df_data.loc[:,'and'] = df_data.loc[:,'and'].astype(int)
df_data.loc[:,'xor'] = ((df_data['column_1'] == 1) & (df_data['column_2'] != 1)) | ((df_data['column_1'] != 1) & (df_data['column_2'] == 1)).astype(int)
df_data.loc[:,'xor'] = df_data.loc[:,'xor'].astype(int)
print(df_data.info())
print(df_data)

# Instantiating a CustomDataSet object and a DataLoader object.
dataset_data = CustomDataset(df_data, target_column_name='xor', transform=torchvision.transforms.ToTensor())
dataset_data_train, dataset_data_test = torch.utils.data.random_split(
    dataset_data,
    [N_TRAIN_OBSERVATIONS, len(dataset_data)-N_TRAIN_OBSERVATIONS]
)
dataloader_data_train = DataLoader(
    dataset=dataset_data_train,
    batch_size=BATCH_SIZE,
    shuffle=True
)
dataloader_data_test = DataLoader(
    dataset=dataset_data_test,
    batch_size=BATCH_SIZE,
    shuffle=True
)

# Instantiating the neural network
custom_net = CustomNet()

# Running one epoch
for epoch in range(N_EPOCHS):
    for data in dataloader_data_train:
        X, y = data[0].float(), data[1].float()
        net.zero_grad()
        output = net(X)
        print(output)
        print(y)
        loss = torch_functional.nll_loss(output, y)
        loss.backward()
        optimizer.step()
        break

Error message:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-2-f2833deb4b11> in <module>
     95 # Running one epoch
     96 for epoch in range(N_EPOCHS):
---> 97     for data in dataloader_data_train:
     98         X, y = data[0].float(), data[1].float()
     99         net.zero_grad()

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in __next__(self)
    343
    344     def __next__(self):
--> 345         data = self._next_data()
    346         self._num_yielded += 1
    347         if self._dataset_kind == _DatasetKind.Iterable and \

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataloader.py in _next_data(self)
    383     def _next_data(self):
    384         index = self._next_index()  # may raise StopIteration
--> 385         data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
    386         if self._pin_memory:
    387             data = _utils.pin_memory.pin_memory(data)

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py in fetch(self, possibly_batched_index)
     42     def fetch(self, possibly_batched_index):
     43         if self.auto_collation:
---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
     45         else:
     46             data = self.dataset[possibly_batched_index]

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py in <listcomp>(.0)
     42     def fetch(self, possibly_batched_index):
     43         if self.auto_collation:
---> 44             data = [self.dataset[idx] for idx in possibly_batched_index]
     45         else:
     46             data = self.dataset[possibly_batched_index]

~/anaconda3/lib/python3.7/site-packages/torch/utils/data/dataset.py in __getitem__(self, idx)
    255
    256     def __getitem__(self, idx):
--> 257         return self.dataset[self.indices[idx]]
    258
    259     def __len__(self):

<ipython-input-2-f2833deb4b11> in __getitem__(self, index)
     26
     27         if not self.transform == None:
---> 28             return (self.transform(x), self.transform(y))
     29         return (x, y)
     30

~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/transforms.py in __call__(self, pic)
     90             Tensor: Converted image.
     91         """
---> 92         return F.to_tensor(pic)
     93
     94     def __repr__(self):

~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/functional.py in to_tensor(pic)
     43
     44     if _is_numpy(pic) and not _is_numpy_image(pic):
---> 45         raise ValueError('pic should be 2/3 dimensional. Got {} dimensions.'.format(pic.ndim))
     46
     47     if isinstance(pic, np.ndarray):

ValueError: pic should be 2/3 dimensional. Got 1 dimensions.


title: Mask for neural network training

body:
I am training a neural network and I found a dataset of pictures taken by drones. It says that it has 20 classes that should be detected. However the masks all looks like this (I'm sorry the image is really dark!):
This is my mask
When I try to train the network it always says that the image only has 3 channels. (I figured it was probably RGB). The problem is that my network expects a 20 channels input for the mask (one for each category to detect like tree, car, human, etc.) Is there a way for me to transform the image in a 20 channels image? I looked if the mask only had 20 different values for the pixel colors but it has 254 so I do not think there is something to do with that...
Thank you! (It is my first question on StackOverflow so if there is a problem in the question just tell me! :-) )


title: Meaning of the outlined code written in C?

body:
I am currently building an understanding around PyTorch and I have come across few codes written in C language. Since I am not that familiar with C conventions, could anyone elaborate what does this code mean?
typedef float float4 __attribute__((ext_vector_type(4)));
typedef float float8 __attribute__((ext_vector_type(8)));
#define LoadFloat8(PTR) *((const float8 *)(PTR))

The code is taken from here


title: pytorch: Random classifier: ValueError: optimizer got an empty parameter list

body:
Is there any best practice or efficient way to have a random classifier in pytorch? My random classifier basically looks like this:
def forward(self, inputs):
    # get a random tensor
    logits = torch.rand(batch_size, num_targets, num_classes)
    return logits

This should be fine in principle, but the optimizer raises a ValueError because the classifier - in contrast to all other classifiers / models in the system - does not have any parameters that can be optimized, obviously. Is there a torch built-in solution to this or must I change the system's architecture (to not perform optimization)?
Edit: If adding some arbitrary parameters to the model as shown below, the loss will raise an RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
def __init__(self, transformer_models: Dict, opt: Namespace):
    super(RandomMulti, self).__init__()
    self.num_classes = opt.polarities_dim
    # add some parameters so that the optimizer doesn't raise an exception
    self.some_params = nn.Linear(2, 2)

My assumption really would be that there is a simpler solution, since having a random baseline classifier is a rather common thing in machine learning.


title: PyTorch: Running unseen text through generated model

body:
I am trying to implement a PyTorch project, found here.
import os
from process_file import process_doc
import random
import torch
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
import numpy as np
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import classification_report, confusion_matrix
from full_model import Classifier
import torch.nn as nn
import torch.optim as optim
import time

def get_batch(doc, ref_type='headline'):
    sent, ls, out, sids = [], [], [], []
    sent.append(doc.headline)
    ls.append(len(doc.headline))
    for sid in doc.sentences:
        if SPEECH:
            out.append(out_map[doc.sent_to_speech.get(sid, 'NA')])
        else:
            out.append(out_map[doc.sent_to_event.get(sid)])
        sent.append(doc.sentences[sid])
        ls.append(len(doc.sentences[sid]))
        sids.append(sid)
    ls = torch.LongTensor(ls)
    out = torch.LongTensor(out)
    return sent, ls, out, sids

def train(epoch, data):
    start_time = time.time()
    total_loss = 0
    global prev_best_macro

    for ind, doc in enumerate(data):
        model.train()
        optimizer.zero_grad()
        sent, ls, out, _ = get_batch(doc)
        if has_cuda:
            ls = ls.cuda()
            out = out.cuda()

        _output, _, _, _ = model.forward(sent, ls)
        loss = criterion(_output, out)
        total_loss += loss.item()
        loss.backward()
        optimizer.step()
        del sent, ls, out
        if has_cuda:
            torch.cuda.empty_cache()

    print("--Training--\nEpoch: ", epoch, "Loss: ", total_loss, "Time Elapsed: ", time.time()-start_time)
    perf = evaluate(validate_data)
    # print(perf)
    if prev_best_macro < perf:
        prev_best_macro = perf
        print ("-------------------Test start-----------------------")
        _ = evaluate(test_data, True)
        print ("-------------------Test end-----------------------")
        torch.save(model.state_dict(), 'discourse_lstm_model.pt')

def evaluate(data, is_test=False):
    y_true, y_pred = [], []
    model.eval()
    for doc in data:
        sent, ls, out, sids = get_batch(doc)
        if has_cuda:
            ls = ls.cuda()
            #out = out.cuda()

        _output, _, _, _ = model.forward(sent, ls)
        _output = _output.squeeze()
        _, predict = torch.max(_output, 1)
        y_pred += list(predict.cpu().numpy() if has_cuda else predict.numpy())
        temp_true = list(out.numpy())
        y_true += temp_true

    print("MACRO: ", precision_recall_fscore_support(y_true, y_pred, average='macro'))
    print("MICRO: ", precision_recall_fscore_support(y_true, y_pred, average='micro'))
    if is_test:
        print("Classification Report \n", classification_report(y_true, y_pred))
    print("Confusion Matrix \n", confusion_matrix(y_true, y_pred))
    return precision_recall_fscore_support(y_true, y_pred, average='macro')[2]

if __name__ == '__main__':

    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
    # parser.add_argument('--drop', help='DROP', default=6, type=float)
    # parser.add_argument('--learn_rate', help='LEARNING RATE', default=0, type=float)
    # parser.add_argument('--loss_wt', help='LOSS WEIGHTS', default=0, type=str)
    parser.add_argument('--seed', help='SEED', default=0, type=int)

    args = parser.parse_args()

    has_cuda = torch.cuda.is_available()

    SPEECH = 0
    if SPEECH:
        out_map = {'NA':0, 'Speech':1}
    else:
        out_map = {'NA':0,'Main':1,'Main_Consequence':2, 'Cause_Specific':3, 'Cause_General':4, 'Distant_Historical':5,
        'Distant_Anecdotal':6, 'Distant_Evaluation':7, 'Distant_Expectations_Consequences':8}

    train_data = []
    validate_data = []
    test_data = []
    for domain in ["Business", "Politics", "Crime", "Disaster", "kbp"]:
        subdir = "../data/train/"+domain
        files = os.listdir(subdir)
        for file in files:
            if '.txt' in file:
                doc = process_doc(os.path.join(subdir, file), domain) #'../data/Business/nyt_corpus_data_2007_04_27_1843240.txt'
                #print(doc.sent_to_event)
                train_data.append(doc)
        subdir = "../data/test/"+domain
        files = os.listdir(subdir)
        for file in files:
            if '.txt' in file:
                doc = process_doc(os.path.join(subdir, file), domain) #'../data/Business/nyt_corpus_data_2007_04_27_1843240.txt'
                #print(doc.sent_to_event)
                test_data.append(doc)

    subdir = "../data/validation"
    files = os.listdir(subdir)
    for file in files:
        if '.txt' in file:
            doc = process_doc(os.path.join(subdir, file), 'VAL') #'../data/Business/nyt_corpus_data_2007_04_27_1843240.txt'
            #print(doc.sent_to_event)
            validate_data.append(doc)
    print(len(train_data), len(validate_data), len(test_data))

    seed = args.seed
    np.random.seed(seed)
    torch.manual_seed(seed)
    if has_cuda:
        torch.cuda.manual_seed(seed)
    random.seed(seed)
    np.random.seed(seed)

    prev_best_macro = 0.

    model = Classifier({'num_layers': 1, 'hidden_dim': 512, 'bidirectional': True, 'embedding_dim': 1024,
                        'dropout': 0.5, 'out_dim': len(out_map)})

    if has_cuda:
        model = model.cuda()
    model.init_weights()

    criterion = nn.CrossEntropyLoss()

    print("Model Created")

    params = filter(lambda p: p.requires_grad, model.parameters())
    optimizer = optim.Adam(params, lr=5e-5, betas=[0.9, 0.999], eps=1e-8, weight_decay=0)

    try:
        for epoch in range(15):
            print("---------------------------Started Training Epoch = {0}--------------------------".format(epoch+1))
            train(epoch, train_data)

    except KeyboardInterrupt:
        print ("----------------- INTERRUPTED -----------------")
        evaluate(validate_data)
        evaluate(test_data)

Running this code, I have successfully outputted a .pt model trained on a corpus of ~400 articles, each article annotated according to its sectional content (data from the Github repo).
Now, I want to annotate a new, unseen article using this model, but I can't quite figure out how to do so. I have a feeling the classification code is already implemented in the snippet above and I'd really appreciate any help/guidance as to how I could go about classifying an unseen article using this code. Thanks so much in advance!


title: Is there an equivalent of _keras_shape in pytorch?

body:
I am trying to convert some code from a keras implementation to a pytorch equivalent. I don't understand this assignment in particular f_real._keras_shape = self.kernel_shape. As I can see the _keras_shape seems to be an auto-generated attribute of 'self.kernel' - which is being assigned the kernel_shape value. In keras, I think the kernel is initialized as a tensor placeholder of sorts. Here's the code:
from keras.layers import Layer
from keras import backend as K
class cconv(Layer):
.......
  self.kernel = self.add_weight(
        self.kernel_shape,
        initializer=kern_init,
        name='kernel',
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint
    )
  real = self.kernel[:, :, :, :self.filters]
  imag = self.kernel[:, :, :, self.filters:]

I am struggling with these 2 lines:
  real._keras_shape = self.kernel_shape
  imag._keras_shape = self.kernel_shape

so far, I got:
  self.kernel = nn.Parameter(
        self.kernel_shape,
        initializer=self.kernel_initializer,
        name='kernel',
        regularizer=self.kernel_regularizer,
        constraint=self.kernel_constraint
    )
  real = self.kernel[:, :, :, :self.filters]
  imag = self.kernel[:, :, :, self.filters:]

Is there a torch.nn.Parameter equivalent of '_keras_shape' or any workaround to this?
edit: I have done some digging but I can't seem to find the exact file where this '_keras_shape' attribute originates! There is a Variable class which seems relevant (can't reach the bottom), i.e. some code in keras.backend.py -
from tensorflow.python.ops import variables as variables_module

def variable(value, dtype=None, name=None, constraint=None):

    v = variables_module.Variable(
        value,
        dtype=dtypes_module.as_dtype(dtype),
        name=name,
        constraint=constraint)
    if isinstance(value, np.ndarray):
        v._keras_shape = value.shape
    elif hasattr(value, 'shape'):
        v._keras_shape = int_shape(value)
    track_variable(v)
    return v

The variables.py file does not make sense - can't find how far this _keras_shape goes back to...


title: How to deploy trained VGG-16 model, using python what could the best possible and free options

body: I have trained a vgg16 model using transferlearning, now I want to deploy my trained model which is of size 520 Mbs, I want to opt for an easy method, either a web app or an android one, I have no prior development experience is their any possible way for deployment, also I don't want to go for a paid deployment option. For training purposes I have used Python and PyTorch framework. How may I deploy the model?


title: Creating a Pseudo-Cyclic signal using a cyclic signal

body:
I am looking for a way using numpy or pytorch to skew a tensor.
For example given an array of samples of sin(x), i hope to get a skewed version of it (preferably the same size) such that the cycle of the function is either stretched or shrinks or even both (if it can interpolate it randomly), so in some parts the frequency is higher and in some parts lower.
I need to create "pseudo-cyclic" signals which means that they are almost cyclic but not perfectly.


title: Is there a training/validation split happening internally, or is there just one training set and testing set?

body:
so recently I've been following the tutorial in https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html and I came up with the following question: is there a training/validation split happening internally?
The thing is, in this tutorial, the main dataset is spliced into training and testing. Here, the training set is used for training and the testing in the evaluate() function.
To my knowledge, when dealing with neural networks usually the data is split into 3 sets: training, validation and testing. In this tutorial though, it is only split into training and testing. For what I know, usually the model is trained and then evaluated, and the weights are then updated according to what was learnt in the evaluation step. However, I can't seem to find any connection between the evaluate function and training. Therefore, in this example the model is being evaluated AND tested using the same dataset.
Is there something here that I might be missing? Is there an internal split of the training dataset happening during training (into training and validation) and the function evaluate() is simply used for testing the performance of the model?
for epoch in range(num_epochs):
        # train for one epoch, printing every 10 iterations
        train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq=10)
        # update the learning rate
        lr_scheduler.step()
        # evaluate on the test dataset
        evaluate(model, data_loader_test, device=device)```


title: How to define neural network neuron-by-neuron?

body:
So we want to have a random brain-like neuron mess. Meaning:

We have AxB inputs and CxD outputs.
We want to have K (where K >= CxD) neurons that are connected to each other at random.
Yet so that all K neurons are connected to at least one of AxB inputs,
And all K neurons are connected to at least one CxD output.

Something like ths (here AxB=5, K=4, CxD=2):

Operations that neurons shall do are weighted summ + some reduction like LeakyReLu.
So one can imagine that when connection randomnes is controlled so that connections are localised over image patches alike layers of CNN it can produce intresting results.
How one could do such thing (handle neurons one-by-one) in PyTorch?


title: PPO2 reinforcement learning 'catastrophic forgetting'?

body:
I'm implementing PPO2 reinforcement learning on my self-build tasks and always encounter such situations where the agent seems to be nearly matured then suddenly catstrophically loses its performance and couldn't hold its stable performance. I don't know what's the right word for it.
I'm just wondering what could be the cause for such catastrophic drop in performance? Any hints or tips?
Many thanks
learningprocess1        learningprocess2


title: Tensor power and multiplication in pytorch

body:
I have a matrix A and a tensor b of size (1,3) - so a vector of size 3.
I want to compute
C = b1 * A + b2 * A^2 + b3 * A^3 where ^n is the n-th power of A.
At the end, C should have the same shape as A. How can I do this efficiently?


title: multiplying each element of a matrix by a vector (or array)

body:
Say I have a an input array of size (64,100)
t = torch.randn((64,100))

Now say I want to multiply each of the 6400 elements of t with 6400 separate vectors each of size 256 to produce a tensor of size [64, 100, 256]. This is what I am doing currently -
import copy
def clones(module, N):
    "Produce N identical layers."
    return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])

linears = clones(nn.Linear(1,256, bias=False), 6400)

idx = 0
t_final = []
for i in range(64):
    t_bs = []
    for j in range(100):
        t1 = t[i, j] * linears[idx].weight.view(-1)
        idx += 1
        t_bs.append(t1)
    t_bs = torch.cat(t_bs).view(1, 100, 256)
    t_final.append(t_bs)
t_final = torch.cat(t_final)
print(t_final.shape)
Output: torch.Size([64, 100, 256])

Is there a faster and cleaner way of doing the same thing? I tried torch.matmul and torch.dot but couldn't do any better.


title: How to enforce Pyro to only infer posterior for a subset of the sample sites?

body:
Recently I learned out how to use MCMC to infer input parameters of a neural network. After running the code I noticed that it takes a very long time for MCMC to finish running. By very long, I mean it might take 1-3 hours (or even more) to run MCMC with 40 warmup steps and 100 samples depending on the kernel parameters. This is very unacceptable to me specially since the neural network model I'm using can process thousands of images per minute. Then I learned that this is probably because MCMC is in fact estimating a distribution for all of my sample sites which have very large dimensions (see the dimensions below in the code). However, I am only interested in inferring a posterior distribution for one of the variable v_q after conditioning the model's output on the observation x_q (an image). So I wonder, is there a way to enforce MCMC to only infer posterior distribution for v_q and somehow ignores the other variables?
Here's my attempt: I set z to z=torch.distributions.Normal(mu_q, std_q).rsample() and MCMC inference runs very fast but acc. prob is very close to 0 (e.g. 0.01) and step_size keeps going down (it got down to 1e-140 after 200 warmup steps). So I assume this is not the right thing to do.
Also, inference is much faster if I set adapt_step_size to False but the results are really bad (acc. prob is around 0.1) when I manually set step_size to a value that a previous inference run with adap_step_size found.
Here's the code
class model(nn.Module):
    def __init__(self):
        super(model, self).__init__()
        .
        . # define model layers
        .

    def infer_v_q(self, x, v, x_q, dataset):
        .
        . # do some other operations here to get c_e, h_e, h_g, u and some other variables
        .

        # x.shape = (batch_dim, 3, 64, 64), batch_dim=36
        with pyro.plate("data", x.shape[0]):
                # v_q.shape = (batch_dim, 7), batch_dim=36
                v_q = pyro.sample('v_q', pyro.distributions.Uniform(v_q_min, v_q_max).to_event(1))

                for l in range(self.L):
                    # note that the followings are done L times so I have L number of z sample sites
                    c_e, h_e = self.inference_network(x_q, v_q, r, c_e, h_e, h_g, u)

                    mu_q, logvar_q = torch.split(self.hidden_state_network(h_e), 1, dim=1)
                    std_q = torch.exp(0.5*logvar_q)
                    # z.shape = (batch_dim, 1, 16, 16)
                    z = pyro.sample("z"+str(l), pyro.distributions.Normal(mu_q, std_q).to_event(3))

                    c_g, h_g, u = self.generation_network(v_q, r, c_g, h_g, u, z)
                # x.shape = (batch_dim, 3, 64, 64)
                return pyro.sample("x", pyro.distributions.Normal(self.image_generator_network(u), 0.001).to_event(3), obs=x_q)

# Here's how I run MCMC inference
nuts_kernel = pyro.infer.NUTS(model.infer_v_q, adapt_step_size=True, step_size=1e-9, jit_compile=True, ignore_jit_warnings=True)
mcmc = pyro.infer.MCMC(nuts_kernel, num_samples=200, warmup_steps=100, num_chains=1)
mcmc.run(x, v, x_q)

# get samples
mcmc.get_samples()["v_q"] # I only need this

# I don't want MCMC to get a posterior for the "z" variables:
mcmc.get_samples()["z1"]
.
.
.
mcmc.get_samples()["z7"]

And here are my sample sites:
Sample Sites:
data dist |
value 36 |
v_q dist 36 | 7
value 36 | 7
z0 dist 36 | 1 16 16
value 36 | 1 16 16
z1 dist 36 | 1 16 16
value 36 | 1 16 16
z2 dist 36 | 1 16 16
value 36 | 1 16 16
z3 dist 36 | 1 16 16
value 36 | 1 16 16
z4 dist 36 | 1 16 16
value 36 | 1 16 16
z5 dist 36 | 1 16 16
value 36 | 1 16 16
z6 dist 36 | 1 16 16
value 36 | 1 16 16
z7 dist 36 | 1 16 16
value 36 | 1 16 16


title: Optuna on Pytorch CNN

body:
class ConvolutionalNetwork(nn.Module):
    def __init__(self, in_features, trial):
        super().__init__()
        self.in_features = in_features
        self.trial = trial
        # this computes no of features outputted by 2 conv layers
        c1 = int(((self.in_features - 2)) / 64)  # this is to account for the loss due to conversion to int type
        c2 = int((c1 - 2) / 64)
        self.n_conv = int(c2 * 16)
        # self.n_conv = int((( ( (self.in_features - 2)/4 ) - 2 )/4 ) * 16)
        num_filters1 = trial.suggest_int("num_filters1",16,64,step=16)
        num_filters2 = trial.suggest_int("num_filters2",16,64,step=16)
        #num_filters = 16
        kernel_size = trial.suggest_int('kernel_size', 2, 7)
        self.conv1 = nn.Conv1d(1, num_filters1, kernel_size, 1)
        self.conv1_bn = nn.BatchNorm1d(num_filters1)
        self.conv2 = nn.Conv1d(num_filters1, num_filters2, kernel_size, 1)
        self.conv2_bn = nn.BatchNorm1d(num_filters2)
        #Add in trial range for dropout to determine optimal dropout value
        self.dp = nn.Dropout(trial.suggest_uniform('dropout_rate',0,1.0))
        self.fc3 = nn.Linear(self.n_conv, 2)

I tried to add in the Optuna parameter tuning trial for number of filters 1&2 as per below, but the following error occurred. Trial 0 managed to go through but Trial 1 no. I made adjustments to the batch size but the same error format occurred with a different recommended batch size.
num_filters1 = trial.suggest_int("num_filters1",16,64,step=16)
            num_filters2 = trial.suggest_int("num_filters2",16,64,step=16)

[I 2020-12-07 09:22:53,512] Trial 0 finished with value: 0.6597743630409241 and parameters: {'num_filters1': 16, 'num_filters2': 16, 'kernel_size': 7, 'dropout_rate': 0.5225509182876455, 'optimizer': 'SGD', 'lr': 0.00020958259416674875, 'Weight_decay': 3.8111213220887506e-08}. Best is trial 0 with value: 0.6597743630409241.
[W 2020-12-07 09:22:55,502] Trial 1 failed because of the following error: ValueError('Expected input batch_size (8000) to match target batch_size (2000).')


title: Multiple values for argument

body:
I am trying to convert this code  passing it with pysyft refference
like this :
class SyNet(sy.Module):
  def __init__(self,embedding_size, num_numerical_cols, output_size, layers, p ,torch_ref):
    super(SyNet, self ).__init__(  embedding_size, num_numerical_cols , output_size , layers , p=0.4  ,torch_ref=torch_ref  )
    self.all_embeddings=self.torch_ref.nn.ModuleList([nn.Embedding(ni, nf) for ni, nf in embedding_size])
    self.embedding_dropout=self.torch_ref.nn.Dropout(p)
    self.batch_norm_num=self.torch_ref.nn.BatchNorm1d(num_numerical_cols)

    all_layers= []
    num_categorical_cols = sum((nf for ni, nf in embedding_size))
    input_size = num_categorical_cols + num_numerical_cols

    for i in layers:
      all_layers.append(self.torch_ref.nn.Linear(input_size,i))
      all_layers.append(self.torch_ref.nn.ReLU(inplace=True))
      all_layers.append(self.torch_ref.nn.BatchNorm1d(i))
      all_layers.append(self.torch_ref.nn.Dropout(p))
      input_size = i

    all_layers.append(self.torch_ref.nn.Linear(layers[-1], output_size))

    self.layers = self.torch_ref.nn.Sequential(*all_layers)

  def forward(self, x_categorical, x_numerical):
    embeddings= []
    for i,e in enumerate(self.all_embeddings):
      embeddings.append(e(x_categorical[:,i]))

    x_numerical = self.batch_norm_num(x_numerical)
    x = self.torch_ref.cat([x, x_numerical], 1)
    x = self.layers(x)
    return x

But when I try to create a instance of the model
model = SyNet( categorical_embedding_sizes, numerical_data.shape[1], 2, [200,100,50], p=0.4 ,torch_ref= th)

I got a TypeError
TypeError: multiple values for argument 'torch_ref'
I tried to change the order of the arguments but i got an error about positional arguments .
Can you help me , I am not very experienced in classes and functions (oop)
Thank you in advance !


title: predict sequence of tuples using Transformer model

body:
I am fairly new to seq2seq models and transformers.
Basically I am working on a sequence generation problem. I want to use the transformer. I am using python and pyTorch.
I know how the transformer model works for a sequence generation like given [1,2,3] it can generate [1,2,3,4,5].
But the problem I am facing is that, in my dataset each point/element in the sequence has 2 attributes. So, the sequence looks like following:
[(2,4), (1,8), (1,9)]

and the generated sequence will be:
[(2,4), (1,8), (1,9), (3,1), (2,9)]

The first element of each tuple can be between 1-5 and the second element can be between 1-10.
I want to follow the general approach of Transformers like creating embedding for each element, pass it through a decoder block of multihead attention and pointwise feed forward network and finally use softmax to sample out the output.
My question is, as each data point contains 2 values, how do I change the transformer model to work with this kind of sequences? Any direction will be appreciated. Thanks.


title: I am Calculating Average RMSE Loss in Pytorch Neural Network regression model. Is it correct?

body:
train_losses=[]
val_losses=[]

for epoch in range(EPOCH):
  train_loss = 0
  valid_loss = 0

  # train steps
  net.train()
  for batch_index, (data, target) in enumerate(train_loader):
    target = target.reshape(-1,1)

    # clears gradients
    optimizer.zero_grad()

    # forward pass
    output, dense2_output = net(data)
    eps = 1e-6
    #loss in batch
    loss = torch.sqrt(criterion(output, target.double())+eps)

    # backward pass for loss gradient
    loss.backward()

    # update paremeters/weights
    optimizer.step()

    # update training loss
    #train_loss += math.pow(loss.item(),2)*data.size(0)
    train_loss += loss.item()*data.size(0)

  # average loss calculations
  #train_loss = math.sqrt(train_loss/len(train_loader.sampler))
  train_loss = train_loss/len(train_loader.sampler)

  # Display loss statistics
  print(f'Current Epoch: {epoch}\n Training Loss: {round(train_loss, 6)}')

  train_losses.append(train_loss)

I am Calculating Average RMSE Loss in Pytorch Neural Network regression model. Is it the correct way ?
Please help me understand if there is any error with my approach.
Actually i want to know if there is any bug in my code


title: How to use k channels in CNN for k FC Layers

body:
I have an encoder, which outputs a tensor with shape (bn, c * k, 32, 32). I now want produce k means with shape (bn, k, 1, 2). So the means are 2-dim coordinates. To do so, I want to use k FC Layers, while for each mean k_i I only want to use c channels.
So my idea is, that I reshape the encoder output out to a 5d tensor with shape (bn, k, c, 32, 32). Then I can use the flattened out[:, 0] ... out[:, k] as input for the k linear layers.
The trivial solution would be to define the linear layers manually:
self.fc0 = nn.Linear(c * 32 * 32, 2)
...
self.fck = nn.Linear(c * 32 * 32, 2)

Then I could define the forward pass for each mean as follows:
mean_0 = self.fc0(out[:, 0].reshape(bn, -1))
...
mean_k = self.fck(out[:, k].reshape(bn, -1))

Is there a more efficient way to do that?


title: How to squeeze all but one torch dims?

body:
torch.squeeze can convert the shape of a tensor to not have dimensions of size 1.
I want to squeeze my tensor in all dimensions but one (in this example, not squeeze dim=0).
All I can see in the doc is

dim (int, optional) – if given, the input will be squeezed only in
this dimension

I want the opposite:
t = torch.zeros(5, 1, 6, 1, 7, 1)

squeezed = torch.magic_squeeze(keep_dim=3)

assert squeezed == (5, 6, 1, 7)

Can this be done?


title: Pytorch: Reshape 1d to 2d target

body:
Apologies if this has been answered already but I couldn't find it.
I have a binary classification problem which I have been using CrossEntropyLoss for which expects the following input and target tensor dimensions:

I have switched to using BCEWithLogitsLoss with pre-calculated class weights to address a label imbalance problem. The issue is BCEWithLogitsLoss expects the following dimensions:

How would I go about shaping my 1D tensor, ie. tensor([[0, 0, 0,  ..., 1, 0, 0]]) to the shape of my inputs which is (No. of X samples, 2)? I have tried .unsqueeze(1) but this gives me (#X, 1). To clarify, my input shape in the prior problem was [32,2] with target shape [32] as per above documentation and I am looking for [32,2] for both input and target dims.


title: Why does my pytorch rpc workers deadlock, is it because I am using main as my master?

body:
I wanted to run the simplest example I could for computing a distributed autograd and print the gradients in my main script. For that I do the following:

start the main process as the master/worker0
start a helper as worker1
send from worker1 to do my computation through an rpc
get gradients from remote and print them

This doesn't work for some reason. It seems my workers deadlock each other. I am unsure why. At first I was getting an error complaining that it seemed the workers were trying to use the same port (thats what I assume because it got fixed once the workers/processes used a different port).
Now that I fixed that I get that they deadlock each other and they don't do my distributed autograd computation. I am usure why. I assume it has something to do with init_rpc since they stop right there it seems. The docs the the following for init_rpc:

torch.distributed.rpc.init_rpc(name, backend=None, rank=-1, world_size=None, rpc_backend_options=None) Initializes RPC primitives such as the local RPC agent and distributed autograd, which immediately makes the current process ready to send and receive RPCs.

or it might be due to shutdown

torch.distributed.rpc.shutdown(graceful=True) Perform a shutdown of the RPC agent, and then destroy the RPC agent. This stops the local agent from accepting outstanding requests, and shuts down the RPC framework by terminating all RPC threads. If graceful=True, this will block until all local and remote RPC processes reach this method and wait for all outstanding work to complete. Otherwise, if graceful=False, this is a local shutdown, and it does not wait for other RPC processes to reach this method.

I tried removing/adding the shutdowns from my code in different combinations but it still got deadlocked...which is why I ended up making a question instead since I am unsure why the issue might arise.
My code is super simple, I just multiple two tensors remotely and then try to get the gradients. Nothing more. It is insipired from this a simple end to end example.
I ran out of ideas of what to do. Anyone know what's going on?
import os

import torch
from torch.distributed import rpc
import torch.multiprocessing as mp
from torch.multiprocessing import Process

def start_worker(rank, world_size):
    # initialize worker proc
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = f'2950{rank}'
    print(f'initializing worker{rank}')
    rpc.init_rpc(f'worker{rank}', rank=rank, world_size=world_size)
    print(f'finished initializing worker{rank}')
    # block until all rpcs finish, and shutdown the RPC instance
    # rpc.shutdown()

def test1_rpc():
    print('running test1_rpc')
    world_size = 2

    # initialize worker
    worker_rank = 1
    worker_proc = Process(target=start_worker, args=(worker_rank, world_size))
    worker_proc.start()

    # initialize main as master
    rank = 0  # master rank
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = f'2950{rank}'
    # Initializes RPC primitives such as the local RPC agent and distributed autograd, which immediately makes the current process ready to send and receive RPCs.
    print(f'initializing worker{rank}')
    rpc.init_rpc(f'worker{rank}', rank=rank, world_size=world_size)
    print(f'finished initializing worker{rank}')

    # # initialize worker
    # worker_rank = 1
    # worker_proc = Process(target=start_worker, args=(worker_rank, world_size))
    # worker_proc.start()
    # print('initialized and started worker proc')

    # do distributed autograd backward pass
    def loss(a, b):
        return a.mm(b).sum()

    with torch.distributed.autograd.context() as context_id:
        print('in distributed autograd')
        a = torch.ones([2, 5], requires_grad=True)
        b = torch.ones([5, 3], requires_grad=True)
        loss = rpc.rpc_sync(to='worker1', func=loss, args=(a, b))

        torch.distributed.autograd(context_id, loss)
        print(a.grad)
        print(b.grad)

    rpc.shutdown()

def test1():
    a = torch.ones([2, 5], requires_grad=True)
    b = torch.ones([5, 3], requires_grad=True)
    loss = a.mm(b).sum()
    loss.backward()
    print(a.grad)
    print(b.grad)
    print()

if __name__ == '__main__':
    test1()
    test1_rpc()

ouput:
tensor([[3., 3., 3., 3., 3.],
        [3., 3., 3., 3., 3.]])
tensor([[2., 2., 2.],
        [2., 2., 2.],
        [2., 2., 2.],
        [2., 2., 2.],
        [2., 2., 2.]])
running test1_rpc
initializing worker0
initializing worker1

crossposted: https://discuss.pytorch.org/t/why-does-my-pytorch-rpc-workers-deadlock-is-it-because-i-am-using-main-as-my-master/112648
reddit: https://www.reddit.com/r/pytorch/comments/lq0gu7/why_does_my_pytorch_rpc_workers_deadlock_is_it/
Quora: https://www.quora.com/unanswered/Why-does-my-PyTorch-RPC-workers-deadlock-Is-it-because-I-am-using-main-as-my-master


title: Why DataLoader return list that has a different length with batch_size

body:
I am writting a customed dataloader, while the returned value makes me confused.
import torch
import torch.nn as nn
import numpy as np
import torch.utils.data as data_utils

class TestDataset:
    def __init__(self):
        self.db = np.random.randn(20, 3, 60, 60)

    def __getitem__(self, idx):
        img = self.db[idx]
        return img, img.shape[1:]

    def __len__(self):
        return self.db.shape[0]

if __name__ == '__main__':
    test_dataset = TestDataset()
    test_dataloader = data_utils.DataLoader(test_dataset,
                                       batch_size=1,
                                       num_workers=4,
                                       shuffle=False, \
                                       pin_memory=True
                                       )
    for i, (imgs, sizes) in enumerate(test_dataloader):
        print(imgs.size())  # torch.Size([1, 3, 60, 60])
        print(sizes)  # [tensor([60]), tensor([60])]
        break

Why "sizes" returns a list of length 2? I think it should be "torch.Size([1, 2])" which indicates height and width of a image(1 batch_size).
Further more, should the length of the returned list be the same to batch_size? If I want to get the size, I have to write "sizes = [sizes[0][0].item(), sizes[1][0].item()]". And this makes me very confused.
Thank you for your time.


title: Why is this nerual network performing poorly on MNIST?

body:
Hi I am building a nerual network in pytorch to classify MNIST and for the life of me, I can't seem to find out why this network won't perform above 7% accuracy. Any guidance would be nice.
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from keras.datasets import mnist
from keras.utils.np_utils import to_categorical
import numpy as np
from sklearn.metrics import precision_score, recall_score, f1_score, accuracy_score, confusion_matrix

(X_train, Y_train), (X_test, Y_test) = mnist.load_data()

X_train = X_train.astype("float32")/255
X_test = X_test.astype("float32")/255

X_train = X_train.reshape(X_train.shape[0],(X_train.shape[1] * X_train.shape[2]));
X_test = X_test.reshape(X_test.shape[0],(X_test.shape[1] * X_test.shape[2]));

class Net(torch.nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.lin_1 = nn.Linear(784, 128)
    self.lin_2 = nn.Linear(128, 64)
    self.lin_3 = nn.Linear(64, 10)

  def forward(self,x) :
    x = self.lin_1(x)
    x = torch.relu(x)
    x = self.lin_2(x)
    x = torch.relu(x)
    x = self.lin_3(x)
    x = torch.softmax(x, dim=0)
    return x

net = Net();
loss = torch.nn.CrossEntropyLoss();
optimizer = torch.optim.SGD(net.parameters(),lr = 0.01);

X_train = torch.from_numpy(X_train);
X_test = torch.from_numpy(X_test);
y_train = torch.from_numpy(Y_train);
y_test = torch.from_numpy(Y_test)

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu");
X_train.to(device);
X_test.to(device);
y_train.to(device);
y_test.to(device);
net.to(device);
loss.to(device);

y_train = y_train.type(torch.long)
y_test = y_test.type(torch.long)

net.train()
for epoch in range(10):
  #pred = torch.max(net(X_train),1);
  pred = net(X_train.to(device));

  train_loss = loss(pred,y_train.to(device));
  optimizer.zero_grad()
  train_loss.backward()
  optimizer.step()

net.eval()

pred = torch.max(net(X_test.to(device)),1)[1];
print('The accuracy for pytorch is ' , accuracy_score(y_test.cpu().numpy(),pred.cpu().numpy()));

I feel like I have to transform the data somehow. That's why I am dividing the the training and testing data by 255, and the network excepts a float for the input and a long for the output.
Here is a numpy version I made without pytorch
from keras.datasets import mnist
from keras.utils.np_utils import to_categorical
(X_train, Y_train), (X_test, Y_test) = mnist.load_data()

X_train = X_train.astype("float32")/255
X_test = X_test.astype("float32")/255

X_train = X_train.reshape(X_train.shape[0],(X_train.shape[1] * X_train.shape[2]));
X_test = X_test.reshape(X_test.shape[0],(X_test.shape[1] * X_test.shape[2]));

Y_train = to_categorical(Y_train);
Y_test = to_categorical(Y_test)
import numpy as np

print(Y_test.shape)

class DNN():
  def __init__(self, sizes, epochs=10, lr = 0.01):
    self.sizes = sizes
    self.epochs = epochs
    self.lr = lr
    self.params = self.initialization();

  def ReLu(self, x, derivative=False):
    if derivative:
      return 1. * (x > 0)
    else:
      return x * (x > 0)

  def softmax(self, x, derivative=False):
        # Numerically stable with large exponentials
        exps = np.exp(x - x.max())
        if derivative:
            return exps / np.sum(exps, axis=0) * (1 - exps / np.sum(exps, axis=0))
        return exps / np.sum(exps, axis=0)

  def initialization(self):
        # number of nodes in each layer
        input_layer=self.sizes[0]
        hidden_1=self.sizes[1]
        hidden_2=self.sizes[2]
        output_layer=self.sizes[3]

        params = {
            "W1":np.random.randn(hidden_1, input_layer) * np.sqrt(1. / hidden_1),
            "W2":np.random.randn(hidden_2, hidden_1) * np.sqrt(1. / hidden_2),
            "W3":np.random.randn(output_layer, hidden_2) * np.sqrt(1. / output_layer)
        }

        return params
  def forward (self,X_train):

    self.params["X0"] = X_train;

    self.params["Z1"] = np.dot(self.params["W1"], self.params["X0"])
    self.params['X1'] = self.ReLu(self.params["Z1"])

    self.params['Z2'] = np.dot(self.params["W2"], self.params["X1"])
    self.params["X2"] = self.ReLu(self.params["Z2"])

    self.params["Z3"] = np.dot(self.params["W3"], self.params["X2"])
    self.params["X3"] = self.softmax(self.params["Z3"])

    return self.params["X3"]

  def backpropagation (self, Y_train, output):

    update = {};

    error = 2 * (output - Y_train) / output.shape[0] * self.softmax(self.params["Z3"], derivative=True)
    update["W3"] = np.outer(error, self.params["X2"])

    error = np.dot(self.params["W3"].T, error) * self.ReLu(self.params["Z2"], derivative=True)
    update["W2"] = np.outer(error, self.params["X1"])

    error = np.dot(self.params["W2"].T, error) * self.ReLu(self.params["Z1"], derivative=True)
    update["W1"] = np.outer(error, self.params["X0"])

    return update

  def updateParams (self,update):
    for key, value in update.items():
      #print(key)
      self.params[key] -= self.lr * value

  def test_accuracy(self, X_test, Y_train):
    predictions = []
    for i in range(len(X_test)):
      output = self.forward(X_test[i])
      pred = np.argmax(output)
      predictions.append(pred == np.argmax(Y_train[i]))

    return np.mean(predictions)

  def train(self, X_train, Y_train):
        for epoch in range(self.epochs):
            print("epoch ", epoch)
            for i in range(len(X_train)):
                output = self.forward(X_train[i])
                update = self.backpropagation(Y_train[i], output)
                self.updateParams(update)

dnn = DNN(sizes=[784, 200, 50, 10],epochs=10)
dnn.train(X_train, Y_train)

print("The accuracy of the numpy network on the test dataset is ", dnn.test_accuracy(X_test,Y_test))


I am new to Pytorch and I've been working on training the MLP model using the MNIST dataset. Basically, I am feeding the model with images and labels as an input and training the dataset on it. I am using CrossEntropyLoss() as a loss function, however I am getting the dimension error whenever I run my model.
IndexError                                Traceback (most recent call last)
<ipython-input-37-04f8cfc1d3b6> in <module>()
     47
     48         # Forward
---> 49         outputs = model(images)
     50

5 frames
/usr/local/lib/python3.7/dist-packages/torch/nn/modules/flatten.py in forward(self, input)
     38
     39     def forward(self, input: Tensor) -> Tensor:
---> 40         return input.flatten(self.start_dim, self.end_dim)
     41
     42     def extra_repr(self) -> str:

IndexError: Dimension out of range (expected to be in range of [-4, 3], but got 64)

Here is the MLP class that I've created
class MLP(nn.Module):
    def __init__(self, device, input_size = 1*28*28, output_size = 10):
        super().__init__()

        self.seq = nn.Sequential(nn.Flatten(BATCH=64, input_size),
                                                       nn.Linear(input_size, 32),
                                                       nn.ReLU(),
                                                       nn.Linear(32, output_size))

        self.to(device)

    def forward(self, x):
        return self.seq(x)

And rest of the training model is
from tqdm.notebook import tqdm
from datetime import datetime

from torch.utils.tensorboard import SummaryWriter
import torch.optim as optim

exp_name = "MLP version 1"

# log_name = "logs/" + exp_name + f" {datetime.now()}"
# print("Tensorboard logs will be written to:", log_name)
# writer = SummaryWriter(log_name)

criterion = nn.CrossEntropyLoss()
model = MLP(device)

optimizer = torch.optim.Adam(model.parameters(), lr = 0.0001)
num_epochs = 10

for epoch in tqdm(range(num_epochs)):
    epoch_train_loss = 0.0
    epoch_accuracy = 0.0

    for data in train_loader:
        images, labels = data
        images, labels = images.to(device), labels.to(device)
        images = images.permute(0, 3, 1, 2)

        optimizer.zero_grad()
        print("hello")

        outputs = model(images)

        loss = criterion(outputs, labels)
        epoch_train_loss += loss.item()

        loss.backward()
        optimizer.step()

        accuracy = compute_accuracy(outputs, labels)
        epoch_accuracy += accuracy
    writer.add_scalar("Loss/training", epoch_train_loss, epoch)
    writer.add_scalar("Accuracy/training", epoch_accuracy / len(train_loader), epoch)

    print('epoch: %d loss: %.3f' % (epoch + 1, epoch_train_loss / len(train_loader)))
    print('epoch: %d accuracy: %.3f' % (epoch + 1, epoch_accuracy / len(train_loader)))

    epoch_accuracy = 0.0
    # The code below computes the validation results
    for data in val_loader:
        images, labels = data
        images, labels = images.to(device), labels.to(device)
        images = images.permute(0, 3, 1, 2)

        model.eval()
        with torch.no_grad():
            outputs = model(images)

        accuracy = compute_accuracy(outputs, labels)
        epoch_accuracy += accuracy
    writer.add_scalar("Accuracy/validation", epoch_accuracy / len(val_loader), epoch)
print("finished training")

Any help would be appreciated. Thank you.


title: Confusion when displaying an image from matplotlib.pyplot to tensorflow

body:
I have this error: TypeError: Invalid shape (28, 28, 1) for image data
Here is my code:
import torch
import torchvision
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor
from torchvision.utils import make_grid
from torch.utils.data.dataloader import DataLoader
from torch.utils.data import random_split
%matplotlib inline

# Load dataset

!wget www.di.ens.fr/~lelarge/MNIST.tar.gz
!tar -zxvf MNIST.tar.gz

from torchvision.datasets import MNIST

dataset = MNIST(root = './', train=True, download=True, transform=ToTensor())
#val_data = MNIST(root = './', train=False, download=True, transform=transform)

image, label = dataset[0]
print('image.shape:', image.shape)
plt.imshow(image.permute(1, 2, 0), cmap='gray') # HELP WITH THIS LINE
print('Label:', label)

I know that the pytorch does processing via this way: C x H x W,
and that matplotlib does it this way: H x W x C, yet when I change it to  matplotlib's way, it gives me an error here. Am I missing something? Why does this happen?


title: Get the length of every sentence before padding in torchtext bucketiterator

body:
Is it possible to get the length of every sentence before padding in torchtext bucketiterator :
train_loader = torchtext.legacy.data.BucketIterator(train_data, batch_size = 64, repeat=True, shuffle=True, sort_key = lambda x: len(x.text), sort=False, sort_within_batch=True, device = device)

bucketiterator dataloader :
inputs: tensor([[  34,   87,    2,  ...,  227,  239,  263],
        [ 138,    7, 1006,  ...,  840,  142,  665],
        [ 549,    4, 1028,  ...,   11,   14,    4],
        ...,
        [   1,    1,    5,  ...,   66,   23,   13],
        [   1,    1, 1062,  ...,  177,  252, 1587],
        [   1,    1,   66,  ...,  553,   52,   73]]), shape: torch.Size([64, 91])

Like when using pytorch dataloader:
train_loader = data.DataLoader(train_data, batch_size = 64, shuffle=True, collate_fn=padding)

def padding(batch):
 doc = [doc['input'] for doc in batch]
 len_doc = [len(doc['input']) for doc in batch]
 doc_pad = pad_sequence(doc, batch_first=True, padding_value=0)
 return doc_pad, len_doc

pytorch dataloader :
inputs: tensor([[   2, 1396, 2686,  ...,    0,    0,    0],
        [   2, 1391, 1396,  ...,    0,    0,    0],
        [   2, 2018, 2597,  ...,    0,    0,    0],
        ...,
        [   2, 1546, 1623,  ...,    0,    0,    0],
        [   2, 1435, 1396,  ...,    0,    0,    0],
        [   2, 1391, 1396,  ...,    0,    0,    0]]), shape: torch.Size([64, 40])

inputs_len_before_padding: tensor([18,  8, 21, 16, 16, 12, 40, 12,  9, 12, 17, 12, 17, 15, 16, 12,  8, 24,
        25, 10, 22,  8,  8, 13, 12, 22, 17, 14, 21, 14, 19, 13, 21,  8, 28, 16,
        31, 24, 23, 19, 10,  7, 16, 12, 16, 12, 17, 12, 18, 11,  8, 13, 17, 14,
        11, 13, 13, 20,  8, 12, 22,  7,  9, 11]), shape: torch.Size([64])


title: Dataset size is smaller than memory, What's wrong with my code?

body:
The following is part of the code, epoch=300, each npz file is 2.73M, but the batch size of my dataloader gives 64, a total of 8 gpuss, so a mini batch should be 64×8×2.73M≈1.1G, my actual memory is 128G. Even if it becomes larger after decompression, it will not reach the size of 128G. The following figure link shows that all 128G of memory is occupied. How should I change my code?
class VimeoDataset(Dataset):
def __init__(self, dataset_name, batch_size=64):
    def __init__(self, dataset_name, batch_size=32):
    self.batch_size = batch_size
    self.path = '/data/train_sample/dataset/'
    self.dataset_name = dataset_name
    #self.load_data()
    self.h = 256
    self.w = 448
    xx = np.arange(0, self.w).reshape(1,-1).repeat(self.h,0) #xx shape is(256,448)
    yy = np.arange(0, self.h).reshape(-1,1).repeat(self.w,1) #yy shape is(448,256)
    self.grid = np.stack((xx,yy),2).copy()

def __len__(self):
    return len(self.meta_data)

def getimg(self, index):
    f = np.load('/data/train_sample/dataset/'+ str(index) + '.npz')
    if index < 8000:
        train_data = f['i0i1gt']
        flow_data = f['ft0ft1']
    elif 8000 <= index < 10000:
        val_data = f['i0i1gt']
    else:
        pass

    if self.dataset_name == 'train':
        meta_data = train_data
    else:
        meta_data = val_data

    data = meta_data
    img0 = data[0:3].transpose(1, 2, 0)
    img1 = data[3:6].transpose(1, 2, 0)
    gt = data[6:9].transpose(1, 2, 0)
    flow_gt = flow_data.transpose(1, 2, 0)
    return img0, gt, img1, flow_gt

    dataset = VimeoDataset('train')
def __getitem__(self, index):
    img0, gt, img1, flow_gt = self.getimg(index)
    ...

sampler = DistributedSampler(dataset)
train_data = DataLoader(dataset, batch_size=args.batch_size, num_workers=8, pin_memory=True,
drop_last=True, sampler=sampler)

system usage figure


title: How to check accuracy on BCELoss Pytorch?

body:
I'm trying to use Pytorch to take a HeartDisease.csv and predict whether the patient has heart disease or not... the .csv provides 13 inputs and 1 target
I'm using BCELoss and I'm having trouble understanding how to write an accuracy check function.
My num_samples is correct but not my num_correct. I think this is a result of not understanding the predictions tensor. Right now my num_correct is usually over 8000 while my num_samples is 303...
Any insight on how to write this check accuracy function is much appreciated
I wrote this on a google co lab
#imports
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
import pandas as pd

#create fully connected network
class NN(nn.Module):
  def __init__(self, input_size, num_classes):
    super(NN, self).__init__()
    self.outputs = nn.Linear(input_size, 1)

  def forward(self, x):
     x = self.outputs(x)
     return torch.sigmoid(x)

#set device
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

#hyperparameters
input_size = 13 # 13 inputs
num_classes = 1 # heartdisease or not
learning_rate = 0.001
batch_size = 64
num_epochs = 1

#load data
class MyDataset(Dataset):
    def __init__(self, root, n_inp):
        self.df = pd.read_csv(root)
        self.data = self.df.to_numpy()
        self.x , self.y = (torch.from_numpy(self.data[:,:n_inp]),
                           torch.from_numpy(self.data[:,n_inp:]))
    def __getitem__(self, idx):
        return self.x[idx, :], self.y[idx,:]
    def __len__(self):
        return len(self.data)

train_dataset = MyDataset("heart.csv", input_size)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle =True)
test_dataset = MyDataset("heart.csv", input_size)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle =True)

#initialize network
model = NN(input_size=input_size, num_classes=num_classes).to(device)

#loss and optimizer
criterion = nn.BCELoss()
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

#train network
for epoch in range(num_epochs):
  for batch_idx, (data, targets) in enumerate(train_loader):
    #get data to cuda if possible
    data = data.to(device=device)
    targets = targets.to(device=device)

    #forward
    scores = model(data.float())
    targets = targets.float()
    loss = criterion(scores, targets)

    #backward
    optimizer.zero_grad()
    loss.backward()

    #grad descent or adam step
    optimizer.step()

#check accuracy of model
def check_accuracy(loader, model):
  num_correct = 0
  num_samples = 0
  model.eval()

  with torch.no_grad():
    for x, y in loader:
      x = x.to(device=device)
      y = y.to(device=device)

      scores = model(x.float())
      _, predictions = scores.max(1)
      num_correct += (predictions == y).sum()
      num_samples += predictions.size(0)

    print("Got {} / {} with accuracy {}".format(num_correct, num_samples, float(num_correct)/float(num_samples)*100))

  model.train()

print("checking accuracy on training data")
check_accuracy(train_loader, model)

print("checking accuracy on test data")
check_accuracy(test_loader, model)


title: Elegant Numpy Tensor product

body:
I need to take the product over two tensors in numpy (or pytorch):
I have
A = np.arange(1024).reshape(8,1,128)
B = np.arange(9216).reshape(8, 128, 9)

And want to obtain C, with dot products summing over the last dim of A (axis=2) and the middle dim of B (axis=1). This should have dimensions 8x9. Currently, I am doing:
C = np.zeros([8, 9])
for i in range(8):
    C[i,:] = np.matmul(A[i,:,:], B[i,:,:])

How to do this elegantly?
I tried:
np.tensordot(weights, features, axes=(2,1)).

but it returns 8x1x8x9.


title: How to implement Flatten layer with batch size > 1 in Pytorch (Pytorch_Geometric)

body:
I am new to Pytorch and am trying to transfer my previous code from Tensorflow to Pytorch due to memory issues.  However, when trying to reproduce Flatten layer, some issues kept coming out.
In my DataLoader object, batch_size is mixed with the first dimension of input (in my GNN, the input unpacked from DataLoader object is of size [batch_size*node_num, attribute_num], e.g. [4*896, 32] after the GCNConv layers). Basically, if I implement torch.flatten() after GCNConv, samples are mixed together (to [4*896*32]) and there would be only 1 output from this network, while I expect #batch_size outputs. And if I use nn.Flatten() instead, nothing seems to happen (still [4*896, 32]).  Should I set batch_size as the first dim of the input at the very beginning, or should I directly use view() function?  I tried directly using view() and it (seemed to have) worked, although I am not sure if this is the same as Flatten.  Please refer to my code below.  I am currently using global_max_pool because it works (it can separate batch_size directly).
By the way, I am not sure why training is so slow in Pytorch...  When node_num is raised to 13000, I need an hour to go through an epoch, and I have 100 epoch per test fold and 10 test folds.  In tensorflow the whole training process only takes several hours.  Same network architecture and raw input data, as shown here in another post of mine, which also described the memory issues I met when using TF.
Have been quite frustrated for a while.  I checked this and this post, but it seems their problems somewhat differ from mine.  Would greatly appreciate any help!
Code:
# Generate dataset
class STDataset(InMemoryDataset):
    def __init__(self, root, transform=None, pre_transform=None):
        super(STDataset, self).__init__(root, transform, pre_transform)
        self.data, self.slices = torch.load(self.processed_paths[0])

    @property
    def raw_file_names(self):
        return []

    @property
    def processed_file_names(self):
        return ['pygdata.pt']

    def download(self):
        pass

    def process(self):
        data_list= []
        for i in range(sample_size):
            data = Data(x=torch.tensor(X_all[i],dtype=torch.float),edge_index=edge_index,y=torch.FloatTensor(y_all[i]))
            data_list.append(data)

        data, slices = self.collate(data_list)
        torch.save((data, slices), self.processed_paths[0])

dataset = STDataset(root=save_dir)
train_dataset = dataset[:len(X_train)]
val_dataset = dataset[len(X_train):(len(X_train)+len(X_val))]
test_dataset = dataset[(len(X_train)+len(X_val)):]

# Build network

from torch_geometric.nn import GCNConv, GATConv, TopKPooling, global_max_pool, global_mean_pool
from torch.nn import Flatten, Linear, ELU
import torch.nn.functional as F

class GCN(torch.nn.Module):
    def __init__(self):
        super(GCN, self).__init__()
        self.conv1 = GCNConv(in_channels = feature_num, out_channels = 32)
        self.conv2 = GCNConv(in_channels = 32, out_channels = 32)
        self.fc1 = Flatten()
#         self.ln1 = Linear(in_features = batch_size*N*32, out_features = 512)
        self.ln1 = Linear(in_features = 32, out_features = 32)
        self.ln2 = Linear(in_features = 32, out_features = 1)

    def forward(self,x,edge_index,batch):
#         x, edge_index, batch = data.x, data.edge_index, data.batch
#         print(np.shape(x),np.shape(edge_index),np.shape(batch))
        x = F.elu(self.conv1(x,edge_index))
#         x = x.squeeze(1)
        x = F.elu(self.conv2(x,edge_index))
        print(np.shape(x))
        x = self.fc1(x)
#         x = torch.flatten(x,0)
#         x = torch.cat([global_max_pool(x,batch),global_mean_pool(x,batch)],dim=1)
        print(np.shape(x))
        x = self.ln1(x)
        x = F.relu(x)
        ## Dropout?
        print("o")
        x = torch.sigmoid(self.ln2(x))
        return x

# training
def train():
    model.train()
    loss_all=0
    correct = 0
    for i, data in enumerate(train_loader, 0):
        data = data.to(device)
        optimizer.zero_grad()
        output = model(data.x, data.edge_index,data.batch)
        label = data.y.to(device)
        loss = loss_func(output, label)
        loss.backward()
        loss_all += loss.item()

        output = output.detach().cpu().numpy().squeeze()
        label = label.detach().cpu().numpy().squeeze()
        correct += (abs(output-label)<0.5).sum()

        optimizer.step()

    return loss_all / len(train_dataset), correct / len(train_dataset)

device = torch.device('cuda')
model = GCN().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
loss_func = torch.nn.BCELoss()  # binary cross-entropy
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle = True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle = True)
for epoch in range(num_epochs):
    gc.collect()
    train_loss, train_acc = train()

Error message for using torch.nn.Flatten(start_dim = 1) (code above):
ValueError                                Traceback (most recent call last)
<ipython-input-42-c96e8b058742> in <module>
     65 for epoch in range(num_epochs):
     66     gc.collect()
---> 67     train_loss, train_acc = train()

<ipython-input-42-c96e8b058742> in train()
     10         output = model(data.x, data.edge_index,data.batch)
     11         label = data.y.to(device)
---> 12         loss = loss_func(output, label)
     13         loss.backward()
     14         loss_all += loss.item()

~/miniconda3/envs/ST-Torch/lib/python3.7/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    530             result = self._slow_forward(*input, **kwargs)
    531         else:
--> 532             result = self.forward(*input, **kwargs)
    533         for hook in self._forward_hooks.values():
    534             hook_result = hook(self, input, result)

~/miniconda3/envs/ST-Torch/lib/python3.7/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
    496
    497     def forward(self, input, target):
--> 498         return F.binary_cross_entropy(input, target, weight=self.weight, reduction=self.reduction)
    499
    500

~/miniconda3/envs/ST-Torch/lib/python3.7/site-packages/torch/nn/functional.py in binary_cross_entropy(input, target, weight, size_average, reduce, reduction)
   2068     if input.numel() != target.numel():
   2069         raise ValueError("Target and input must have the same number of elements. target nelement ({}) "
-> 2070                          "!= input nelement ({})".format(target.numel(), input.numel()))
   2071
   2072     if weight is not None:

ValueError: Target and input must have the same number of elements. target nelement (4) != input nelement (3584)


title: No such file or directory

body:
import os
from PIL import Image
path='D:/SomeExperiments/KITTRawData/2011_09_26/2011_09_26_drive_0091_sync/image_03/data/0000000100.jpg'
with open(path,'rb') as f:
    pass

error as below
[Errno 2] No such file or directory: 'D:/SomeExperiments/KITTRawData/2011_09_26/2011_09_26_drive_0091_sync/image_03/data/0000000224.jpg'

but I can find this path on my computer:


title: How to take shape [1,1,256] from [1,4,256] cuda.FloatTensor?

body:
I have an batch of output hidden vector from GRU. It's shape is [1,4,256]

( 0 ,.,.) =
-0.9944  1.0000  0.0000  ...  -1.0000  0.0000 -1.0000
-1.0000  1.0000  0.0000  ...  -1.0000  0.0000 -1.0000
-1.0000  1.0000  0.0000  ...  -1.0000  0.0000 -1.0000
-1.0000  1.0000  0.0000  ...  -1.0000  0.0000 -1.0000
[torch.cuda.FloatTensor of size (1,4,256) (GPU 0)]

I need a shape of [1,1,256] to pass to another model. How can I take it? Through this line I can only have a shape of [1,256]

decoder_hidden = encoder_hidden[:, index]

resize and wrap a new FloatTensor didn't work.


title: trouble installing a protocol-buffer known as SentencePiece too, to alleviate the open vocabulary problems in neural machine translation

body:
im trying to install a protocol-buffer known as SentencePiece too, to alleviate the open vocabulary problems in neural machine translation .

I used this command as suggested on github documentation :

sudo apt-get install cmake build-essential pkg-config libgoogle-perftools-dev

it gives an error :

eventhough I have cuda10.1 installed .

any ideas ?


title: Would backpropagation work as expect when using this code as swish in a CNN?

body:
I would like to use swish (as a layer) in a CNN.

I am not sure if this is the correct way to implement a such activation function.

Will back propagation work properly with this code?

class Swish(nn.Module):
    def forward(self,x):
        return x * F.sigmoid(x)

The relevant information on what programming language I am using can be found in the tags.


title: Mask a 4d tensor with a 2d mask

body: I have a tensor which size is (batch_size, seqLength, seqLength, label_number), and a mask tensor with size (batch_size, seqLength), when I calculate loss, I need to mask out the word that I padding in. My question is how to use this 2d mask to masking out the padding samples in the 4d tensor so that I can calculate the right loss?


title: How to convert PyTorch recurrent_block to Keras equvalet

body:
I'm testing several architectures for semantic segmentation and came across an implementation in PyTorch I want to try. My problem is that I have no experience with PyTorch, and therefore struggle to convert this the following code snippet to Keras.

class Recurrent_block(nn.Module):
    def __init__(self,ch_out,t=2):
        super(Recurrent_block,self).__init__()
        self.t = t
        self.ch_out = ch_out
        self.conv = nn.Sequential(
            nn.Conv2d(ch_out,ch_out,kernel_size=3,stride=1,padding=1,bias=True),
            nn.BatchNorm2d(ch_out),
            nn.ReLU(inplace=True)
        )

    def forward(self,x):
        for i in range(self.t):

            if i==0:
                x1 = self.conv(x)

            x1 = self.conv(x+x1)
        return x1


title: Pytorch slows down while training

body:
I am training an Actor-Critic Reinforcement Learning algorithm like DDPG for many episodes. However, the time per step is increasing at first and then remaining on a stable level, like in the following picture.

I first thought I forgot to .detach()/.item() elements which I put in the replay buffer (a deque element of size 1e6) as it is listed in some other stackoverflow posts, but this is not the case. Has anybody encountered a similar situation before and can pinpoint me where to look for the cause?


title: PIL Image.open display image reversely rotated

body:
I'm working on predicting the number picture as  below with MNIST dataset and LeNet Model . Firstly, I show test images with Image.open, it displays test images in the way of reversely rotated.

from PIL import Image
import matplotlib.cm as cm
import pylab as pl
import numpy as np

img = Image.open('./test/2.png').convert('L')
img = np.invert(img) # convert to white on black
pl.imshow(np.asarray(img), origin='lower', cmap=cm.Greys_r)
pl.show()

Another issue is  the accuracy ratio of prediction seem very low. For example, the 2 here has been predicted as 4.

Someone could help with that or explain?  In my point of views, this number is much clear than handwriting MNIST. Thanks a lot.


title: How to concatenate two pytorch models by observation?

body:
I have two image classification models that have multiple images per observation.  The target variable is binary.  I would like to predict by observation.  How can I concat these two models when there are different numbers of images per observation, at the observation level?

Would I have to average out the predictions?  Or is there a way to program pytorch to consider the observation id?

I have my data class returning the image, label and observation ID.  See data class code below:

class image_Dataset(Dataset):
    '''
    roof data class

    '''
    def __init__(self, csv_file, transform = None):
        '''
        Args:
            csv_file (string): Path to the csv file with annotations.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        '''
        self.roof_frame = pd.read_csv(csv_file)
        self.transform = transform

    def __len__(self):
        return len(self.roof_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        root_dir = self.roof_frame.iloc[idx,4]
        pic = self.roof_frame.iloc[idx, 1]
        label = self.roof_frame.iloc[idx, 3]
        img_name = os.path.join(root_dir, pic)
        image = Image.open(img_name)
        id= self.roof_frame.iloc[idx, 0]
        sample = {'image': image, 'id': id, 'label':label}

        if self.transform:
            image = self.transform(image)

        return image, label, id

I have a model for my images of roofs and a model for my images of houses.  The same IDs are in both data sets.  An ID in roofs may have 4 pictures or less of a roof.  The same IDs in the house images may have 4 to 8 images in that data set.  I need to figure out how to predict on each image, average the results by for both the roof and house models for each ID, then concatenation them together for a final model at the ID-observation level.  I’m not sure how to ensure the models concat with the right ID.


title: when I train my network, The same number appears periodically on my my training acc ， I don't konow the error come from

body:
this is my train.py
I use dice loss for my network ,outputs size is [4,2,224,224] where 4 means batchsize, 2 means channels, 224 means h and w .output_c1 size is [4,224,224] ,labels size is [4,224,224] too.

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
from model import U_net
import visdom
from dataset import driveDateset
from torch import optim
from Dice_loss import DiceLoss
from Dice_loss import MulticlassDiceLoss
import matplotlib.pylab as plt
import numpy as np
import time

if __name__ == '__main__':
    DATA_DIRECTORY = "F:\\experiment_code\\U-net\\DRIVE\\training"
    DATA_LIST_PATH = "F:\\experiment_code\\U-net\DRIVE\\training\\images_id.txt"
    Batch_size = 4
    epochs = 100
    #MEAN = (104.008, 116.669, 122.675)
    dst = driveDateset(DATA_DIRECTORY, DATA_LIST_PATH)

    # Initialize model
    device = torch.device("cuda")
    model = U_net()
    model.to(device)

    #Initial optimizer
    optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9)
    best_acc, best_epoch =0, 0
    global_step = 0
    start_time = time.time()
    viz = visdom.Visdom()

    for epoch in range(epochs):
        running_corrects = 0
        since_epoch = time.time()
        trainloader = torch.utils.data.DataLoader(dst, batch_size=Batch_size) #,shuffle =True
        for step, data in enumerate(trainloader):
            imgs, labels, _, _ = data

            imgs, labels = imgs.to(device), labels.to(device)
            labels = labels.float()
            # labels = labels / 255.0
            # labels[labels > 0.5] = 1.
            # labels[labels < 0.5] = 0.
            # imgs = imgs / 255.0
            model.train()
            outputs = model(imgs)   # output  B * C * H *W
            output_c1 = outputs[:,1,:,:] # C are 2 channels

            # Dice loss
            binary_output_c1 = torch.round(output_c1)
            # Network output visualization
            binary_output_c1_show = binary_output_c1.cpu().detach().numpy().astype(np.uint8)
            plt.figure()
            plt.suptitle('Network visualization output')
            plt.subplot(1, 4, 1)
            plt.imshow(binary_output_c1_show[0,:,:] ),plt.axis('off')
            plt.subplot(1, 4, 2)
            plt.imshow(binary_output_c1_show[1, :, :] ),plt.axis('off')
            plt.subplot(1, 4, 3)
            plt.imshow(binary_output_c1_show[2, :, :] ),plt.axis('off')
            plt.subplot(1, 4, 4)
            plt.imshow(binary_output_c1_show[3, :, :] ),plt.axis('off')
            plt.show()
            plt.pause(0.5)

            #dice = MulticlassDiceLoss()
            dice = DiceLoss()
            loss = dice(binary_output_c1,labels)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            viz.line([loss.item()],[global_step], win='loss', update='append',opts=dict(title='train_loss'))
            running_corrects = torch.sum(binary_output_c1 == labels).float()
            labels_size = labels.size(1) * labels.size(2) * 4
            training_acc = running_corrects / labels_size
            time_elapsed_epoch = time.time() - since_epoch
            print('epoch :', epoch, '\t', 'loss:', loss.item(),'\t','training_acc',training_acc,'\t','{:.0f}m {:.0f}s'.format(time_elapsed_epoch // 60, time_elapsed_epoch % 60))
            global_step += 1

    time_elapsed = time.time() - start_time
    print('Training complate in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))

my Dice_loss.py is:

import torch
import torch.nn as nn

class DiceLoss(nn.Module):
    def __init__(self):
        super(DiceLoss, self).__init__()

    def forward(self, input, target):
        N = target.size(0)
        smooth = 1

        input_flat = input.view(N, -1)
        target_flat = target.view(N, -1)
        #target_flat = torch.t(target_flat)

        intersection = input_flat * target_flat

        loss = 2 * (intersection.sum(1) + smooth) / (input_flat.sum(1) + target_flat.sum(1) + smooth)
        loss = 1 - loss.sum() / N

        return loss

class MulticlassDiceLoss(nn.Module):
    """
    requires one hot encoded target. Applies DiceLoss on each class iteratively.
    requires input.shape[0:1] and target.shape[0:1] to be (N, C) where N is
      batch size and C is number of classes
    """

    def __init__(self):
        super(MulticlassDiceLoss, self).__init__()

    def forward(self, input, target, weights=None):

        C = target.shape[1]

        if weights is None:
         weights = torch.ones(C) #uniform weights for all classes

        dice = DiceLoss()
        totalLoss = 0

        for i in range(C):
            diceLoss = dice(input[:, i], target[:, i])
            if weights is not None:
                diceLoss *= weights[i]
            totalLoss += diceLoss

        return totalLoss

I test Data incoming has no problem ,every training data has Traversed. training data and label like this

but when I train my network  ,output picture like this ,

and some result like this :

epoch : 0    loss: -0.8808882236480713   training_acc tensor(0.3331, device='cuda:0')    0m 2s
epoch : 0    loss: -0.478731632232666    training_acc tensor(0.4376, device='cuda:0')    0m 3s
epoch : 0    loss: -0.42925024032592773      training_acc tensor(0.4485, device='cuda:0')    0m 4s
epoch : 0    loss: -0.60923171043396     training_acc tensor(0.4187, device='cuda:0')    0m 5s
epoch : 0    loss: -0.4442058801651001   training_acc tensor(0.4383, device='cuda:0')    0m 6s
epoch : 1    loss: -0.8808882236480713   training_acc tensor(0.3331, device='cuda:0')    0m 1s
epoch : 1    loss: -0.478731632232666    training_acc tensor(0.4376, device='cuda:0')    0m 2s
epoch : 1    loss: -0.42925024032592773      training_acc tensor(0.4485, device='cuda:0')    0m 2s
epoch : 1    loss: -0.60923171043396     training_acc tensor(0.4187, device='cuda:0')    0m 3s
epoch : 1    loss: -0.4442058801651001   training_acc tensor(0.4383, device='cuda:0')    0m 4s
epoch : 2    loss: -0.8808882236480713   training_acc tensor(0.3331, device='cuda:0')    0m 1s

I dont't know how to solve this problem
I try to use one-hot encoding to transform outputs and crossentropyloss

imgs, labels = imgs.to(device), labels.to(device) # imgs size [4,224,224]
criteon = nn.CrossEntropyLoss() #reduce=False
model.train()
outputs = model(imgs)   # output  B * C * H *W
output_c1 = outputs[:,1,:,:] # outputs have 2 channels ,I choose the second channels
Rounding_output_c1 = torch.round(output_c1)
Rounding_output_c11 = torch.stack([(Rounding_output_c1 == i).float() for i in range(256)]) # [4,224,224]-> [256,4,224,224] ,256 means the number of classes
Rounding_output_c11 = Rounding_output_c11.permute(1,0,2,3)   #[256,4,224,224]->[4,256,224,224]
loss = criteon(Rounding_output_c11,labels)
loss.requires_grad = True
optimizer.zero_grad()
loss.backward()
optimizer.step()
labels_float = labels.float()
running_corrects = torch.sum(Rounding_output_c1 == labels_float).float()
labels_size = labels.size(1) * labels.size(2) * 4
training_acc = running_corrects / labels_size

and get the same phenomenon

epoch : 0    loss: 5.2415571212768555    training_acc tensor(0.3103, device='cuda:0')    0m 2s
epoch : 0    loss: 5.228370666503906     training_acc tensor(0.3235, device='cuda:0')    0m 2s
epoch : 0    loss: 5.224219799041748     training_acc tensor(0.3276, device='cuda:0')    0m 2s
epoch : 0    loss: 5.222436428070068     training_acc tensor(0.3294, device='cuda:0')    0m 2s
epoch : 0    loss: 5.2202887535095215    training_acc tensor(0.3316, device='cuda:0')    0m 2s
epoch : 1    loss: 5.2415571212768555    training_acc tensor(0.3103, device='cuda:0')    0m 0s
epoch : 1    loss: 5.22836971282959      training_acc tensor(0.3235, device='cuda:0')    0m 0s
epoch : 1    loss: 5.224219799041748     training_acc tensor(0.3276, device='cuda:0')    0m 0s
epoch : 1    loss: 5.222436428070068     training_acc tensor(0.3294, device='cuda:0')    0m 1s
epoch : 1    loss: 5.2202887535095215    training_acc tensor(0.3316, device='cuda:0')    0m 1s
epoch : 2    loss: 5.2415571212768555    training_acc tensor(0.3103, device='cuda:0')    0m 0s
epoch : 2    loss: 5.22836971282959      training_acc tensor(0.3235, device='cuda:0')    0m 0s
epoch : 2    loss: 5.224219799041748     training_acc tensor(0.3276, device='cuda:0')    0m 0s
epoch : 2    loss: 5.222436428070068     training_acc tensor(0.3294, device='cuda:0')    0m 1s
epoch : 2    loss: 5.2202887535095215    training_acc tensor(0.3316, device='cuda:0')    0m 1s
epoch : 3    loss: 5.2415571212768555    training_acc tensor(0.3103, device='cuda:0')    0m 0s
epoch : 3    loss: 5.22836971282959      training_acc tensor(0.3235, device='cuda:0')    0m 0s
epoch : 3    loss: 5.224219799041748     training_acc tensor(0.3276, device='cuda:0')    0m 0s
epoch : 3    loss: 5.222436428070068     training_acc tensor(0.3294, device='cuda:0')    0m 1s
epoch : 3    loss: 5.2202887535095215    training_acc tensor(0.3316, device='cuda:0')    0m 1s

I don't konw hw to solve this problem


title: DCGAN for RGB dataset strange results

body:
I am implementing a DCGAN network in LibTorch/Pytorch. I am following the official example in https://github.com/pytorch/examples/blob/master/cpp/dcgan/dcgan.cpp .
The only differences between my problem and the example are:

My dataset is composed by RGB pictures (CelebA dataset) while the one from the example is black and white (MNIST)
The dimensions of my pictures are 64x64 while MNIST pictures are 28x28

That said here is my code:

#include <torch/torch.h>

#include <cmath>
#include <cstdio>
#include <iostream>
#include "CustomDataset.h"
#include "parameters.h"

// The size of the noise vector fed to the generator.
const int64_t kNoiseSize = 100;

// The batch size for training.
const int64_t kBatchSize = 64;

// The number of epochs to train.
const int64_t kNumberOfEpochs = 30;

// Where to find the MNIST dataset.
const char* kDataFolder = "./data";

// After how many batches to create a new checkpoint periodically.
const int64_t kCheckpointEvery = 20;

// How many images to sample at every checkpoint.
const int64_t kNumberOfSamplesPerCheckpoint = 10;

// After how many batches to log a new update with the loss value.
const int64_t kLogInterval = 10;

using namespace torch;

struct DCGANGeneratorImpl : nn::Module {
    DCGANGeneratorImpl(int kNoiseSize)
            : conv1(nn::ConvTranspose2dOptions(kNoiseSize, 256, 4)
                            .bias(false)),
              batch_norm1(256),
              conv2(nn::ConvTranspose2dOptions(256, 128, 4)
                            .stride(2)
                            .padding(1)
                            .bias(false)),
              batch_norm2(128),
              conv3(nn::ConvTranspose2dOptions(128, 64, 4)
                            .stride(2)
                            .padding(1)
                            .bias(false)),
              batch_norm3(64),
              conv4(nn::ConvTranspose2dOptions(64, 32, 4)
                            .stride(2)
                            .padding(1)
                            .bias(false)),
              batch_norm4(32),
              conv5(nn::ConvTranspose2dOptions(32, 3, 4)
                            .stride(2)
                            .padding(1)
                            .bias(false))

    {
        register_module("conv1", conv1);
        register_module("conv2", conv2);
        register_module("conv3", conv3);
        register_module("conv4", conv4);
        register_module("conv5", conv5);
        register_module("batch_norm1", batch_norm1);
        register_module("batch_norm2", batch_norm2);
        register_module("batch_norm3", batch_norm3);
        register_module("batch_norm4", batch_norm4);

    }

    torch::Tensor forward(torch::Tensor x)
    {
        x = torch::relu(batch_norm1(conv1(x)));
        x = torch::relu(batch_norm2(conv2(x)));
        x = torch::relu(batch_norm3(conv3(x)));
        x = torch::relu(batch_norm4(conv4(x)));
        x = torch::tanh(conv5(x));
        return x;
    }

    nn::ConvTranspose2d conv1, conv2, conv3, conv4, conv5;
    nn::BatchNorm2d batch_norm1, batch_norm2, batch_norm3, batch_norm4;
};

TORCH_MODULE(DCGANGenerator);

int main(int argc, const char* argv[]) {
    torch::manual_seed(1);

    // Create the device we pass around based on whether CUDA is available.
    torch::Device device(torch::kCPU);
    if (torch::cuda::is_available()) {
        std::cout << "CUDA is available! Training on GPU." << std::endl;
        device = torch::Device(torch::kCUDA);
    }

    DCGANGenerator generator(kNoiseSize);
    generator->to(device);

    nn::Sequential discriminator(
            // Layer 1
            nn::Conv2d(
                    nn::Conv2dOptions(3, 64, 4).stride(2).padding(1).bias(false)),
            nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)),
            //output is 32x32
            // Layer 2
            nn::Conv2d(
                    nn::Conv2dOptions(64, 128, 4).stride(2).padding(1).bias(false)),
            nn::BatchNorm2d(128),
            nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)),
            //output is 16x16
            // Layer 3
            nn::Conv2d(
                    nn::Conv2dOptions(128, 64, 4).stride(2).padding(1).bias(false)),
            nn::BatchNorm2d(64),
            nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)),
            //output is 8x8
            // Layer 4
            nn::Conv2d(
                    nn::Conv2dOptions(64, 32, 5).stride(1).padding(0).bias(false)),
            nn::LeakyReLU(nn::LeakyReLUOptions().negative_slope(0.2)),
            // output is 4x4
            // Layer 5
            nn::Conv2d(
                    nn::Conv2dOptions(32, 1, 4).stride(1).padding(0).bias(false)),
            nn::Sigmoid());
    discriminator->to(device);

    // Where all my pictures are;
    std::string file_location{"dataset/img_align_celeba/*.jpg"};
    auto dataset = CustomDataset(file_location).map(data::transforms::Stack<>());

    const int64_t batches_per_epoch =
            std::ceil(dataset.size().value() / static_cast<double>(kBatchSize));

    auto data_loader = torch::data::make_data_loader(
            std::move(dataset),
            torch::data::DataLoaderOptions().batch_size(kBatchSize).workers(2));

    torch::optim::Adam generator_optimizer(
            generator->parameters(), torch::optim::AdamOptions(2e-4).beta1(0.5));
    torch::optim::Adam discriminator_optimizer(
            discriminator->parameters(), torch::optim::AdamOptions(2e-4).beta1(0.5));

    int64_t checkpoint_counter = 1;
    for (int64_t epoch = 1; epoch <= kNumberOfEpochs; ++epoch) {
        int64_t batch_index = 0;
        for (torch::data::Example<>& batch : *data_loader) {
            // Train discriminator with real images.
            discriminator->zero_grad();
            torch::Tensor real_images = batch.data.to(device);
            torch::Tensor real_labels =
                    torch::empty(batch.data.size(0), device).uniform_(0.8, 1.0);
            torch::Tensor real_output = discriminator->forward(real_images);
            torch::Tensor d_loss_real =
                    torch::binary_cross_entropy(real_output, real_labels);
            d_loss_real.backward();

            // Train discriminator with fake images.
            torch::Tensor noise =
                    torch::randn({batch.data.size(0), kNoiseSize, 1, 1}, device);
            torch::Tensor fake_images = generator->forward(noise);
            torch::Tensor fake_labels = torch::zeros(batch.data.size(0), device);
            torch::Tensor fake_output = discriminator->forward(fake_images.detach());
            torch::Tensor d_loss_fake =
                    torch::binary_cross_entropy(fake_output, fake_labels);
            d_loss_fake.backward();

            torch::Tensor d_loss = d_loss_real + d_loss_fake;
            discriminator_optimizer.step();

            // Train generator.
            generator->zero_grad();
            fake_labels.fill_(1);
            fake_output = discriminator->forward(fake_images);
            torch::Tensor g_loss =
                    torch::binary_cross_entropy(fake_output, fake_labels);
            g_loss.backward();
            generator_optimizer.step();
            batch_index++;

            if (batch_index % kCheckpointEvery == 0) {
                // Checkpoint the model and optimizer state.
                torch::save(generator, "generator-checkpoint.pt");
                torch::save(generator_optimizer, "generator-optimizer-checkpoint.pt");
                torch::save(discriminator, "discriminator-checkpoint.pt");
                torch::save(
                        discriminator_optimizer, "discriminator-optimizer-checkpoint.pt");
                // Sample the generator and save the images.
                torch::Tensor samples = generator->forward(torch::randn(
                        {kNumberOfSamplesPerCheckpoint, kNoiseSize, 1, 1}, device));
                torch::save(
                        samples,
                        torch::str("dcgan-sample-", checkpoint_counter, ".pt"));
                std::cout << "\n-> checkpoint " << ++checkpoint_counter << '\n';
            }
        }
    }

    std::cout << "Training complete!" << std::endl;
}

I save the minibatches from time to time and plot the result of inputing noise over the Generator. The problem is that in the MNIST example results are correct but in my case for each output picture I see like 9 smaller pictures with faces instead of one (see the picture attached).

How is it possible that the generator is outputting a correct shape but with 9 almost identical faces instead of one?


title: Testing BERT for large sequences

body:
I'm a beginner with pytorch and I want just to test the pretrained model CamemBERT for Question Answering task with long input text sequences (over than 512 tokens).
So, I'm trying to implement it using the following code:

class CamemBERTQA(nn.Module):
  def __init__(self, hidden_size, num_labels):
      super(CamemBERTQA, self).__init__()
      self.hidden_size = hidden_size
      num_labels = num_labels
      self.camembert = CamembertForQuestionAnswering.from_pretrained('fmikaelian/camembert-base-fquad')
      self.tokenizer = CamembertTokenizer.from_pretrained('fmikaelian/camembert-base-fquad', do_lower_case=True)
#         self.qa_outputs = nn.Linear(self.hidden_size, self.num_labels)

  def forward(self, ids):
      input_ids = ids
      start_scores, end_scores = self.camembert(torch.tensor([input_ids]))
      start_logits = torch.argmax(start_scores)
      end_logits = torch.argmax(end_scores)+1
      outputs = (start_logits, end_logits,)
#       print(outputs)
      return outputs

      def text_representation(self, text):
          input_ids = torch.tensor([tokenizer.encode(text, add_special_tokens=True)])  # Add special tokens takes care of adding [CLS], [SEP], <s>... tokens in the right way for each model.
          with torch.no_grad():
              last_hidden_states = model(input_ids)[0]  # Models outputs are now tuples
#             print(last_hidden_states)
          return last_hidden_states

      def tokenize(self, text):
          return self.tokenizer.convert_tokens_to_ids(self.tokenizer.tokenize(text))

To deal with text length sequence limitation I want to split the input sequence into subsequences and generate the text representation of each one of them and then using a max pooling layer to generate the final text representation (510 tokens).

So, can any one help me in order to adapt this script to attend this objective ?

Thank you,