title: RuntimeError: Cost Function returns nan values in its 1th output

body:
I am working on a temporal model to predict future events. Here is the link to my colab notebook. I am facing an issue trying to train the model. I am getting NaN values for Train and valid loss. The loss function is a joint loss consisting of the cross entropy loss and the squared loss.  Link to the blog here.
Tried following solution which didn`t worked -
Smaller learning rate - 0.01, 0.001, 0.0001
class cost_function():
    def __init__(self, yhat, y, L_2=0.001, logEps=1e-8):
        # logEps : log epsilon, very small positive value greater that 0.0
        # CE = - [ ln(y*)(y) + ln(1-y*)(1-y) ],

        self.yhat = yhat
        self.y = y

        self.logEps = logEps
        self.L_2 = L_2

        self.W_out = nn.Parameter(torch.randn(hiddenDimSize, numClass)*0.01)

    def cross_entropy(self):
        ce = -(self.y * torch.log(self.yhat + self.logEps) + (1. - self.y) * torch.log(1. - self.yhat + self.logEps))
        print("Inside CrossEntrophy Loss fn : ", ce)
        return ce

    def prediction_loss(self):
        # return  (torch.sum(torch.sum(self.cross_entropy(), dim=0),dim=1)).float()/  lengths.float()

        tmp_tensor = torch.sum(self.cross_entropy(), dim=0)
        print("Inside PredictionLoss fn : Sum Dim 0", tmp_tensor)
        print("Inside PredictionLoss fn : Sum Dim 1", torch.sum(tmp_tensor,dim=1))
        print("Inside PredictionLoss fn : Final Result ", (torch.sum(tmp_tensor,dim=1)).float()/  lengths.float())
        return (torch.sum(tmp_tensor,dim=1)).float()/  lengths.float()

    def cost(self):
        print("Inside Cost fn :", torch.mean(self.prediction_loss()) + self.L_2 * (self.W_out ** 2).sum())
        return torch.mean(self.prediction_loss()) + self.L_2 * (self.W_out ** 2).sum() # regularize

build_EHRNN class -  I have made modification in the forward method params to resolve the 'h' is not defined error.
torch.manual_seed(1)

class build_EHRNN(nn.Module):
    def __init__(self, inputDimSize=4894, hiddenDimSize=[200,200], batchSize=100, embSize=200, numClass=4894, dropout=0.5, logEps=1e-8):
        super(build_EHRNN, self).__init__()

        self.inputDimSize = inputDimSize
        self.hiddenDimSize = hiddenDimSize
        self.numClass = numClass
        self.embSize = embSize
        self.batchSize = batchSize
        self.dropout = nn.Dropout(p=0.5)
        self.logEps = logEps

        # Embedding inputs
        self.W_emb = nn.Parameter(torch.randn(self.inputDimSize, self.embSize).cuda())
        self.b_emb = nn.Parameter(torch.zeros(self.embSize).cuda())

        self.W_out = nn.Parameter(torch.randn(self.hiddenDimSize, self.numClass).cuda())
        self.b_out = nn.Parameter(torch.zeros(self.numClass).cuda())

        self.params = [self.W_emb, self.W_out,
                       self.b_emb, self.b_out]

    # def forward(self,x, y, h, lengths, mask):
    def forward(self,x, y, lengths, mask):
        self.emb = torch.tanh(torch.matmul(x, self.W_emb) + self.b_emb)
        input_values = self.emb
        self.outputs = [input_values]
        for i, hiddenSize in enumerate([self.hiddenDimSize, self.hiddenDimSize]):  # iterate over layers
            rnn = EHRNN(self.inputDimSize,hiddenSize,self.embSize,self.batchSize,self.numClass) # calculate hidden states
            hidden_state = []
            h = self.init_hidden().cuda()
            for i,seq in enumerate(input_values): # loop over sequences in each batch
                h = rnn(seq, h)
                hidden_state.append(h)
            hidden_state = self.dropout(torch.stack(hidden_state))    # apply dropout between layers
            input_values = hidden_state

        y_linear = torch.matmul(hidden_state, self.W_out)  + self.b_out # fully connected layer
        yhat = F.softmax(y_linear, dim=1)  # yhat
        yhat = yhat*mask[:,:,None]   # apply mask

        # Loss calculation
        cross_entropy = -(y * torch.log(yhat + self.logEps) + (1. - y) * torch.log(1. - yhat + self.logEps))
        last_step = -torch.mean(y[-1] * torch.log(yhat[-1] + self.logEps) + (1. - y[-1]) * torch.log(1. - yhat[-1] + self.logEps))
        prediction_loss = torch.sum(torch.sum(cross_entropy, dim=0),dim=1)/ torch.cuda.FloatTensor(lengths)
        cost = torch.mean(prediction_loss) + 0.000001 * (self.W_out ** 2).sum() # regularize
        return (yhat, hidden_state, cost)

    def init_hidden(self):
        return torch.zeros(self.batchSize, self.hiddenDimSize)  # initial state

model training
artificalData_seqs = np.array(pickle.load(open(os.path.join(GOOGLE_DRV_PATH,BASE_DIR,'data.encodedDxs'),'rb')))
train, test, valid = load_data(artificalData_seqs, artificalData_seqs)

batchSize = 50     # decreased from 100 to 50
n_batches = int(np.ceil(float(len(train[0])) / float(batchSize)))-1
n_batches_valid = int(np.ceil(float(len(valid[0])) / float(batchSize)))-1
model = build_EHRNN(inputDimSize=4894, hiddenDimSize=200, batchSize=50, embSize=200, numClass=4894, dropout=0.5, logEps=1e-8)
model = model.to(device)

 import torch.nn.functional as F
import pdb

optimizer = torch.optim.Adadelta(model.parameters(), lr = 0.001, rho=0.95)
epochs = 10

counter = 0
# with torch.autograd.detect_anomaly():
for e in range(epochs):
    for x, y in train_dl:
        x, y , mask, lengths = padding(x, y, inputDimSize, numClass)
        output, h = model(x, mask)

        loss = cost_function(output, y).cost()
        # pdb.set_trace()
        loss.backward()
        print("loss ",loss)
        nn.utils.clip_grad_norm_(model.parameters(), 5) # Constraining the weight matrix directly == regularization.
        optimizer.step()
        optimizer.zero_grad()

    with torch.no_grad():
            model.eval()
            val_loss = []
            for x_valid, y_valid in valid_dl:
                    x_val, y_val, mask, lengths = padding(x_valid, y_valid, inputDimSize, numClass)
                    outputs_val, hidden_val = model(x_val,  mask)
                    loss = cost_function(outputs_val, y_val).cost()
                    val_loss.append(loss.item())
            model.train()

            print("Epoch: {}/{}...".format(e+1, epochs),
                                "Step: {}...".format(counter),
                                "Training Loss: {:.4f}...".format(loss.item()),
                                "Val Loss: {:.4f}".format(torch.mean(torch.tensor(val_loss))))

Error (Start getting NaNs in the loss)
Inside PredictionLoss fn : Sum Dim 0 tensor([[0.1008, 0.1539, 0.1211,  ..., 0.1533, 0.1218, 0.1418],
        [0.0253, 0.0449, 0.0249,  ..., 0.0439, 0.0134, 0.0332],
        [0.0306, 0.0799, 0.0570,  ..., 0.0790, 0.0484, 0.0678],
        ...,
        [0.0253, 0.0450, 0.0249,  ..., 0.0439, 0.0134, 0.0332],
        [0.0253, 0.0450, 0.0249,  ..., 0.0439, 0.0134, 0.0332],
        [0.0038, 0.0106, 0.0106,  ..., 0.0098, 0.0004, 0.0106]],
       grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([  372.4754,   133.2620,   219.1195,    37.5425,   141.3354,    37.5070,
          229.2947,     0.0000,   379.1829,   217.3962,    80.1226,    37.5074,
          138.4665,    82.1034,    89.7893,    81.8173,    92.8159,   141.8856,
           95.9898,   216.0511,   133.2535,   385.0391,   369.4958,   244.9362,
           37.5088,    37.5087,   141.6083,     0.0000,    95.3367,    37.5074,
          735.0569,   378.0407,    37.5135,    40.7778,    82.0872,   225.9998,
          216.6189,   379.0732,    81.4742,   144.4226,    93.3905,   214.0228,
           37.5078,   224.0793,    88.3753,    41.2919,   140.4855,    37.5086,
          226.6366,   148.7171,   137.9226, 13887.5811,    81.1428,    84.6804,
          226.6779,    37.5065,   223.8841,   220.5979,    83.2484,    37.5080,
           84.5247,   384.2115,    80.1173,     0.0000,   146.9714,    37.6982,
          134.6618,    84.1838,    37.5421,   730.5516,    37.5085,     0.0000,
          215.1523,   136.5673,    81.2887,    37.5080,    94.4181,   140.6268,
          133.9295,   136.2485,    80.1173,   386.2103,    39.0282,     0.0000,
           37.5055,    42.1506,    80.1662,   228.5819,    39.3403,   138.7672,
         1768.6033,   143.5350,    40.2060,   147.7809,    37.5085,   380.9214,
          750.6883,   141.0447,   136.9028,    37.5049],
       grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([5., 3., 4., 1., 3., 1., 4., 0., 5., 4., 2., 1., 3., 2., 2., 2., 2., 3.,
        2., 4., 3., 5., 5., 4., 1., 1., 3., 0., 2., 1., 6., 5., 1., 1., 2., 4.,
        4., 5., 2., 3., 2., 4., 1., 4., 2., 1., 3., 1., 4., 3., 3., 9., 2., 2.,
        4., 1., 4., 4., 2., 1., 2., 5., 2., 0., 3., 1., 3., 2., 1., 6., 1., 0.,
        4., 3., 2., 1., 2., 3., 3., 3., 2., 5., 1., 0., 1., 1., 2., 4., 1., 3.,
        7., 3., 1., 3., 1., 5., 6., 3., 3., 1.])
Inside PredictionLoss fn : Final Result  tensor([  74.4951,   44.4207,   54.7799,   37.5425,   47.1118,   37.5070,
          57.3237,       nan,   75.8366,   54.3491,   40.0613,   37.5074,
          46.1555,   41.0517,   44.8946,   40.9086,   46.4080,   47.2952,
          47.9949,   54.0128,   44.4178,   77.0078,   73.8992,   61.2340,
          37.5088,   37.5087,   47.2028,       nan,   47.6683,   37.5074,
         122.5095,   75.6081,   37.5135,   40.7778,   41.0436,   56.5000,
          54.1547,   75.8146,   40.7371,   48.1409,   46.6952,   53.5057,
          37.5078,   56.0198,   44.1876,   41.2919,   46.8285,   37.5086,
          56.6591,   49.5724,   45.9742, 1543.0646,   40.5714,   42.3402,
          56.6695,   37.5065,   55.9710,   55.1495,   41.6242,   37.5080,
          42.2623,   76.8423,   40.0586,       nan,   48.9905,   37.6982,
          44.8873,   42.0919,   37.5421,  121.7586,   37.5085,       nan,
          53.7881,   45.5224,   40.6443,   37.5080,   47.2090,   46.8756,
          44.6432,   45.4162,   40.0587,   77.2421,   39.0282,       nan,
          37.5055,   42.1506,   40.0831,   57.1455,   39.3403,   46.2557,
         252.6576,   47.8450,   40.2060,   49.2603,   37.5085,   76.1843,
         125.1147,   47.0149,   45.6343,   37.5049], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([2., 1., 1., 2., 3., 5., 3., 3., 1., 4., 2., 2., 2., 3., 2., 2., 5., 2.,
        4., 2., 1., 4., 2., 5., 3., 0., 4., 7., 6., 4., 4., 1., 7., 1., 3., 3.,
        5., 3., 5., 5., 4., 2., 2., 4., 1., 5., 6., 2., 5., 5., 2., 1., 3., 1.,
        4., 4., 3., 3., 0., 2., 2., 4., 2., 2., 1., 0., 3., 2., 3., 0., 1., 2.,
        4., 4., 5., 1., 1., 3., 3., 2., 5., 0., 2., 6., 5., 5., 5., 1., 2., 4.,
        0., 2., 6., 1., 2., 0., 1., 1., 2., 3.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([3., 2., 4., 4., 6., 1., 1., 4., 2., 2., 4., 2., 1., 1., 1., 1., 3., 1.,
        2., 3., 4., 2., 2., 3., 2., 0., 1., 3., 4., 1., 2., 3., 4., 3., 3., 1.,
        2., 3., 3., 2., 1., 4., 6., 3., 4., 2., 3., 0., 1., 1., 3., 7., 2., 3.,
        4., 2., 3., 0., 3., 3., 2., 1., 1., 3., 6., 2., 2., 3., 2., 3., 3., 2.,
        1., 2., 5., 3., 4., 2., 5., 3., 5., 5., 5., 3., 2., 1., 4., 3., 3., 3.,
        1., 4., 3., 2., 5., 3., 6., 4., 3., 2.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([4., 6., 2., 2., 4., 5., 0., 3., 2., 3., 4., 3., 3., 3., 4., 3., 0., 1.,
        2., 2., 4., 2., 4., 4., 1., 0., 3., 0., 5., 5., 1., 2., 3., 2., 2., 3.,
        1., 2., 1., 2., 3., 1., 3., 1., 3., 0., 4., 2., 1., 6., 2., 2., 0., 1.,
        4., 4., 2., 2., 0., 3., 5., 5., 5., 5., 3., 4., 4., 4., 5., 4., 2., 2.,
        3., 1., 5., 6., 3., 4., 1., 3., 5., 1., 3., 2., 3., 6., 2., 4., 3., 2.,
        3., 2., 4., 1., 2., 2., 1., 2., 3., 4.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([1., 5., 1., 5., 3., 0., 1., 4., 3., 3., 5., 5., 2., 0., 1., 2., 3., 2.,
        2., 2., 2., 2., 2., 2., 2., 3., 4., 1., 4., 2., 1., 3., 4., 0., 4., 2.,
        0., 4., 1., 1., 2., 3., 3., 5., 1., 3., 2., 3., 1., 1., 3., 6., 3., 0.,
        2., 3., 2., 3., 3., 4., 1., 5., 2., 6., 2., 1., 5., 3., 2., 1., 3., 5.,
        2., 0., 3., 0., 1., 3., 3., 2., 3., 1., 2., 4., 1., 2., 7., 2., 7., 2.,
        2., 2., 1., 1., 1., 2., 4., 2., 2., 3.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([2., 2., 1., 4., 3., 2., 2., 4., 5., 2., 3., 1., 1., 2., 3., 1., 3., 1.,
        3., 1., 6., 4., 2., 2., 0., 4., 1., 0., 2., 0., 1., 1., 1., 1., 2., 2.,
        3., 1., 3., 2., 0., 1., 3., 2., 4., 1., 2., 1., 6., 2., 1., 2., 1., 2.,
        2., 1., 1., 1., 4., 1., 0., 2., 2., 2., 3., 6., 5., 1., 1., 3., 3., 3.,
        1., 4., 4., 2., 4., 2., 3., 7., 1., 1., 2., 3., 0., 0., 1., 4., 2., 3.,
        1., 2., 3., 2., 7., 0., 2., 3., 4., 4.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([1., 2., 6., 7., 2., 1., 1., 3., 1., 2., 2., 4., 3., 3., 4., 3., 2., 2.,
        5., 2., 1., 3., 2., 2., 2., 1., 4., 3., 2., 2., 1., 4., 2., 0., 2., 2.,
        1., 3., 4., 4., 5., 1., 5., 6., 1., 3., 1., 3., 0., 1., 6., 1., 2., 0.,
        3., 2., 2., 3., 4., 3., 3., 2., 3., 2., 2., 3., 4., 3., 2., 0., 3., 1.,
        1., 3., 1., 4., 7., 2., 2., 1., 1., 1., 2., 3., 3., 2., 2., 3., 1., 3.,
        3., 1., 1., 3., 1., 2., 7., 3., 1., 6.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([2., 2., 4., 2., 5., 1., 2., 4., 2., 2., 2., 5., 1., 2., 1., 4., 1., 2.,
        4., 3., 4., 0., 2., 5., 2., 6., 2., 6., 3., 4., 3., 5., 7., 4., 3., 5.,
        1., 3., 2., 3., 2., 2., 2., 3., 3., 3., 2., 3., 1., 4., 3., 3., 2., 1.,
        6., 9., 2., 1., 6., 3., 1., 5., 1., 6., 2., 2., 6., 2., 4., 2., 4., 3.,
        2., 2., 4., 1., 2., 2., 2., 1., 3., 1., 3., 2., 3., 3., 2., 2., 4., 3.,
        3., 3., 5., 4., 3., 2., 4., 2., 1., 8.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : Sum Dim 1 tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<SumBackward1>)
Inside PredictionLoss fn : lengths tensor([1., 3., 2., 3., 3., 2., 7., 4., 3., 1., 1., 5., 1., 3., 4., 4., 1., 5.,
        1., 3., 3., 1., 4., 3., 0., 4., 4., 2., 4., 1., 2., 4., 3., 2., 3., 2.,
        2., 2., 2., 3., 3., 2., 5., 1., 2., 1., 2., 2., 3., 2., 2., 2., 3., 3.,
        3., 5., 1., 4., 8., 4., 0., 2., 4., 2., 0., 1., 4., 4., 1., 5., 0., 1.,
        3., 1., 2., 3., 3., 4., 3., 4., 2., 4., 2., 3., 4., 1., 1., 2., 4., 1.,
        1., 0., 0., 0., 4., 3., 1., 3., 4., 3.])
Inside PredictionLoss fn : Final Result  tensor([nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,
        nan, nan, nan, nan], grad_fn=<DivBackward0>)
Inside PredictionLoss fn : Sum Dim 0 tensor([[nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        ...,
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan],
        [nan, nan, nan,  ..., nan, nan, nan]], grad_fn=<SumBackward1>)


title: 'BCEWithLogitsLoss' object has no attribute 'backward' error PyTorch

body:
I'm trying to fine-tune the Bert model i.e., bert-base-uncased for a text classification task. I'm getting a strange error while calling the backward() on loss, which is as follows
torch.nn.modules.module.ModuleAttributeError: 'BCEWithLogitsLoss' object has no attribute 'backward'.
I can't find any syntax error and also checked the inputs(outputs and targets) to the loss function i.e, nn.BCEWithLogitsLoss(outputs, targets) and I found then to be in the correct format. I don't know what is causing this error. I would be great if any of you could help me with this......thanks in advance :)
Here is the code
'''def train_loop_fn(data_loader, model, optimizer, device, scheduler = None):
model.train()
for bi, d in enumerate(data_loader):
ids = d["ids"]
mask = d["mask"]
token_type_ids = d["token_type_ids"]
targets = d["targets"]
    ids = ids.to(device, dtype=torch.long)
    mask = mask.to(device, dtype=torch.long)
    token_type_ids = token_type_ids.to(device, dtype=torch.long)
    targets = targets.to(device, dtype=torch.float)

    optimizer.zero_grad()
    outputs = model(ids=ids, mask = mask, token_type_ids = token_type_ids)
    outputs = outputs.reshape(1)
    print("here is the outputs")
    print(outputs)
    print("here is the targets")
    print(targets)
    loss = nn.BCEWithLogitsLoss(outputs, targets)
    print("this is the loss")
    print(loss)
    loss.backward()
    optimizer.step()
    if scheduler is not None:
        scheduler.step()

'''


title: Skorch RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

body:
I'm trying to develop an image segmentation model. In the below code I keep hitting a RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same. I'm not sure why as I've tried to load both my data and my UNet model to the GPU using .cuda() (although not the skorch model-- not sure how to do that). I'm using a library for active learning, modAL, which wraps skorch.
from modAL.models import ActiveLearner
import numpy as np
import torch

from torch import nn
from torch import Tensor
from torch.utils.data import DataLoader
from torch.utils.data import Dataset

from skorch.net import NeuralNet

from modAL.models import ActiveLearner
from modAL.uncertainty import classifier_uncertainty, classifier_margin
from modAL.utils.combination import make_linear_combination, make_product
from modAL.utils.selection import multi_argmax
from modAL.uncertainty import uncertainty_sampling

from model import UNet
from skorch.net import NeuralNet
from skorch.helper import predefined_split
from torch.optim import SGD

import cv2

# Map style dataset,
class ImagesDataset(Dataset):
    """Constructs dataset of satellite images + masks"""
    def __init__(self, image_paths):
        super().__init__()
        self.image_paths = image_paths

    def __len__(self):
        return len(self.image_paths)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()
        print("idx:", idx)
        sample_dir = self.image_paths[idx]
        img_path = sample_dir +"/images/"+ Path(sample_dir).name +'.png'
        mask_path = sample_dir +'/mask.png'
        img, mask = cv2.imread(img_path), cv2.imread(mask_path)
        print("shape of img", img.shape)
        return img, mask

# turn data into dataset
train_ds = ImagesDataset(train_dirs)
val_ds = ImagesDataset(valid_dirs)

train_loader = torch.utils.data.DataLoader(train_ds, batch_size=3, shuffle=True, pin_memory=True)
val_loader = torch.utils.data.DataLoader(val_ds, batch_size=1, shuffle=True, pin_memory=True)

# make sure data loaded in cuda for train, validation
for i, (tr, val) in enumerate(train_loader):
    tr, val = tr.cuda(), val.cuda()

for i, (tr2, val2) in enumerate(val_loader):
    tr2, val2 = tr2.cuda(), val2.cuda()

X, y = next(iter(train_loader))
X_train = np.array(X.reshape(3,3,1024,1024))
y_train = np.array(y.reshape(3,3,1024,1024))

X2, y2 = next(iter(val_loader))
X_test = np.array(X2.reshape(1,3,1024,1024))
y_test = np.array(y2.reshape(1,3,1024,1024))

module = UNet(pretrained=True)
if torch.cuda.is_available():
    module = module.cuda()

# create the classifier

net = NeuralNet(
    module,
    criterion=torch.nn.NLLLoss,
    batch_size=32,
    max_epochs=20,
    optimizer=SGD,
    optimizer__momentum=0.9,
    iterator_train__shuffle=True,
    iterator_train__num_workers=4,
    iterator_valid__shuffle=False,
    iterator_valid__num_workers=4,
    train_split=predefined_split(val_ds),
    device='cuda',
)

# assemble initial data
n_initial = 1
initial_idx = np.random.choice(range(len(X_train)), size=n_initial, replace=False)
X_initial = X_train[initial_idx]
y_initial = y_train[initial_idx]

# generate the pool, remove the initial data from the training dataset
X_pool = np.delete(X_train, initial_idx, axis=0)
y_pool = np.delete(y_train, initial_idx, axis=0)

# train the activelearner
# shape of 4D matrix is 'batch', 'channel', 'width', 'height')
learner = ActiveLearner(
    estimator= net,
    X_training=X_initial, y_training=y_initial,
)

The full error trace is:
    ---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-83-0af6007b6b72> in <module>
      8 learner = ActiveLearner(
      9     estimator= net,
---> 10     X_training=X_initial, y_training=y_initial,
     11     # X_training=X_initial, y_training=y_initial,
     12 )

~/.local/lib/python3.7/site-packages/modAL/models/learners.py in __init__(self, estimator, query_strategy, X_training, y_training, bootstrap_init, on_transformed, **fit_kwargs)
     80                  ) -> None:
     81         super().__init__(estimator, query_strategy,
---> 82                          X_training, y_training, bootstrap_init, on_transformed, **fit_kwargs)
     83
     84     def teach(self, X: modALinput, y: modALinput, bootstrap: bool = False, only_new: bool = False, **fit_kwargs) -> None:

~/.local/lib/python3.7/site-packages/modAL/models/base.py in __init__(self, estimator, query_strategy, X_training, y_training, bootstrap_init, on_transformed, force_all_finite, **fit_kwargs)
     70         self.y_training = y_training
     71         if X_training is not None:
---> 72             self._fit_to_known(bootstrap=bootstrap_init, **fit_kwargs)
     73             self.Xt_training = self.transform_without_estimating(self.X_training) if self.on_transformed else None
     74

~/.local/lib/python3.7/site-packages/modAL/models/base.py in _fit_to_known(self, bootstrap, **fit_kwargs)
    160         """
    161         if not bootstrap:
--> 162             self.estimator.fit(self.X_training, self.y_training, **fit_kwargs)
    163         else:
    164             n_instances = self.X_training.shape[0]

~/.local/lib/python3.7/site-packages/skorch/net.py in fit(self, X, y, **fit_params)
    901             self.initialize()
    902
--> 903         self.partial_fit(X, y, **fit_params)
    904         return self
    905

~/.local/lib/python3.7/site-packages/skorch/net.py in partial_fit(self, X, y, classes, **fit_params)
    860         self.notify('on_train_begin', X=X, y=y)
    861         try:
--> 862             self.fit_loop(X, y, **fit_params)
    863         except KeyboardInterrupt:
    864             pass

~/.local/lib/python3.7/site-packages/skorch/net.py in fit_loop(self, X, y, epochs, **fit_params)
    774
    775             self.run_single_epoch(dataset_train, training=True, prefix="train",
--> 776                                   step_fn=self.train_step, **fit_params)
    777
    778             if dataset_valid is not None:

~/.local/lib/python3.7/site-packages/skorch/net.py in run_single_epoch(self, dataset, training, prefix, step_fn, **fit_params)
    810             yi_res = yi if not is_placeholder_y else None
    811             self.notify("on_batch_begin", X=Xi, y=yi_res, training=training)
--> 812             step = step_fn(Xi, yi, **fit_params)
    813             self.history.record_batch(prefix + "_loss", step["loss"].item())
    814             self.history.record_batch(prefix + "_batch_size", get_len(Xi))

~/.local/lib/python3.7/site-packages/skorch/net.py in train_step(self, Xi, yi, **fit_params)
    707             return step['loss']
    708
--> 709         self.optimizer_.step(step_fn)
    710         return step_accumulator.get_step()
    711

~/.local/lib/python3.7/site-packages/torch/autograd/grad_mode.py in decorate_context(*args, **kwargs)
     24         def decorate_context(*args, **kwargs):
     25             with self.__class__():
---> 26                 return func(*args, **kwargs)
     27         return cast(F, decorate_context)
     28

~/.local/lib/python3.7/site-packages/torch/optim/sgd.py in step(self, closure)
     84         if closure is not None:
     85             with torch.enable_grad():
---> 86                 loss = closure()
     87
     88         for group in self.param_groups:

~/.local/lib/python3.7/site-packages/skorch/net.py in step_fn()
    703         def step_fn():
    704             self.optimizer_.zero_grad()
--> 705             step = self.train_step_single(Xi, yi, **fit_params)
    706             step_accumulator.store_step(step)
    707             return step['loss']

~/.local/lib/python3.7/site-packages/skorch/net.py in train_step_single(self, Xi, yi, **fit_params)
    643         """
    644         self.module_.train()
--> 645         y_pred = self.infer(Xi, **fit_params)
    646         loss = self.get_loss(y_pred, yi, X=Xi, training=True)
    647         loss.backward()

~/.local/lib/python3.7/site-packages/skorch/net.py in infer(self, x, **fit_params)
   1046             x_dict = self._merge_x_and_fit_params(x, fit_params)
   1047             return self.module_(**x_dict)
-> 1048         return self.module_(x, **fit_params)
   1049
   1050     def _get_predict_nonlinearity(self):

~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/al/model.py in forward(self, x)
     51
     52     def forward(self, x):
---> 53         conv1 = self.conv1(x)
     54         conv2 = self.conv2(conv1)
     55         conv3 = self.conv3(conv2)

~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/.local/lib/python3.7/site-packages/torch/nn/modules/container.py in forward(self, input)
    115     def forward(self, input):
    116         for module in self:
--> 117             input = module(input)
    118         return input
    119

~/.local/lib/python3.7/site-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

~/.local/lib/python3.7/site-packages/torch/nn/modules/conv.py in forward(self, input)
    421
    422     def forward(self, input: Tensor) -> Tensor:
--> 423         return self._conv_forward(input, self.weight)
    424
    425 class Conv3d(_ConvNd):

~/.local/lib/python3.7/site-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight)
    418                             _pair(0), self.dilation, self.groups)
    419         return F.conv2d(input, weight, self.bias, self.stride,
--> 420                         self.padding, self.dilation, self.groups)
    421
    422     def forward(self, input: Tensor) -> Tensor:

RuntimeError: Input type (torch.cuda.ByteTensor) and weight type (torch.cuda.FloatTensor) should be the same

If anyone could help that would be so so so appreciated! I've been really stuck despite searching all over-- casting my UNet model to floats has not helped and I think I've called .cuda() where I'm supposed to.
Specific things I've tried:

RuntimeError: Input type (torch.FloatTensor) and weight type (torch.cuda.FloatTensor) should be the same loading entries of my DataLoader to CUDA
adding pin_memory to my DataLoader
loading my skorch NeuralNetwork to CUDA as stated here Pytorch, INPUT (normal tensor) and WEIGHT (cuda tensor) mismatch (which didn't work because it's not a function in skorch)
Casting my data to float (https://discuss.pytorch.org/t/input-type-torch-cuda-doubletensor-and-weight-type-torch-cuda-floattensor-should-be-the-same/22704)


title: Tensorflow finfo (numeric limits)

body:
How do I access something like numpy.finfo (Or torch.finfo) in the tensorflow Python API? I want to look up things like the smallest increment or largest finite value of the given type (for example tf.float32). Some attributes are accessible directly;

tf.float32.max
>> -3.4028235e+38
tf.float32.min
>> -3.4028235e+38

But how about epsilon, infinity or similar?

I would expect a straightforward interface to c++ std::numeric_limits, but I can't seem to find it.


title: Efficiency of .to(device) in pytorch

body:
For achieving maximum efficiency, which command would be more efficient?

x = torch.randn(100, 100).to(device)
x = torch.randn(100, 100, device = device)

Is there a benefit to using one versus the other when doing heavy tensor operations? I was told one of them is less efficient, but can't properly figure how to compare the two.

I'm assuming the second is better as it directly exports to device instead of first creating a tensor and then having to transfer it to device.


title: Compute gradient between a scalar and vector in PyTorch

body:
I am trying to replicate code which was written using Theano, to PyTorch. In the code, the author computes the gradient using

import theano.tensor as T
gparams = T.grad(cost, params)

and the shape of gparams is (256, 240)

I have tried using backward() but it doesn't seem to return anything. Is there an equivalent to grad within PyTorch?

Assume this is my input,

import torch
from torch.autograd import Variable
cost = torch.tensor(1.6019)
params = Variable(torch.rand(1, 73, 240))


title: PyTorch: Different training accuracies using same random seed

body:
I am trying to evaluate my model on the whole training set after each epoch.
This is what I did:

torch.manual_seed(1)
model = ConvNet(num_classes=num_classes)
cost_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

def compute_accuracy(model, data_loader):
    correct_pred, num_examples = 0, 0
    for features, targets in data_loader:
        logits = model(features)
        predicted_labels = torch.argmax(logits, 1)
        num_examples += targets.size(0)
        correct_pred += (predicted_labels == targets).sum()
    return correct_pred.float()/num_examples * 100

for epoch in range(num_epochs):
    model = model.train()
    for features, targets in train_loader:
        logits = model(features)
        cost = cost_fn(logits, targets)
        optimizer.zero_grad()
        cost.backward()
        optimizer.step()

    model = model.eval()
    print('Epoch: %03d/%03d training accuracy: %.2f%%' % (
          epoch+1, num_epochs,
          compute_accuracy(model, train_loader)))

the output was convincing:

Epoch: 001/005 training accuracy: 89.08%
Epoch: 002/005 training accuracy: 90.41%
Epoch: 003/005 training accuracy: 91.70%
Epoch: 004/005 training accuracy: 92.31%
Epoch: 005/005 training accuracy: 92.95%

But then I added another line at the end of the training loop, to also evaluate the model on the whole test set after each epoch:

for epoch in range(num_epochs):
    model = model.train()
    for features, targets in train_loader:
        logits = model(features)
        cost = cost_fn(logits, targets)
        optimizer.zero_grad()
        cost.backward()
        optimizer.step()

    model = model.eval()
    print('Epoch: %03d/%03d training accuracy: %.2f%%' % (
          epoch+1, num_epochs,
          compute_accuracy(model, train_loader)))
    print('\t\t testing accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))

But the training accuracies started to change:

Epoch: 001/005 training accuracy: 89.08%
         testing accuracy: 87.66%
Epoch: 002/005 training accuracy: 90.42%
         testing accuracy: 89.04%
Epoch: 003/005 training accuracy: 91.84%
         testing accuracy: 90.01%
Epoch: 004/005 training accuracy: 91.86%
         testing accuracy: 89.83%
Epoch: 005/005 training accuracy: 92.45%
         testing accuracy: 90.32%

Am I doing something wrong? I expected the training accuracies to remain the same because the manual seed is 1 in both cases.
Is this an expected output ?


title: Wrong value of standard deviation

body:
Hello I am trying to evaluate the standard deviation and the mean of the dataset MNIST and I get a wrong value for the standard deviation. Here is my code :

import torch
from torchvision import datasets, transforms
import torch.nn.functional as F

loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data', train=True, download=True, transform=transform1),
                     batch_size=32,
                     num_workers=0,
                     shuffle=False)

mean = 0.
std = 0.
for images, _ in loader:
    batch_samples = images.size(0)
    images = images.view(batch_samples, images.size(1), -1)
    mean += images.mean(2).sum(0)
    std += images.std(2).sum(0)

mean /= len(loader.dataset)
std /= len(loader.dataset)

print("The mean is ", mean)
print("The standard deviation is ", std)

My problem is the following, I get as mean the value 0.1307 and for the standard deviation the value 0.3015 instead of 0.3081. I suppose I have an error in my code but I don't see where.

Could you help me please ?

Thank you very much !


title: What is the purpose of the dim parameter in torch.nn.Softmax

body:
I don't understand to what does the dim parameter applies in torch.nn.Softmax. There is a warning that tells me to use it and I set it to 1, but I don't understand what I am setting. Where is it being used in the formula:

Softmax(xi)=exp(xi)/∑jexp(xj)

There is no dim here, so to what does it apply?


title: Pytorch Geometric Graph Classification : AttributeError: 'Batch' object has no attribute 'local_var'

body:
I am currently working on doing graph classification on the IMDB-Binary dataset using deep learning and specifically the pytorch geometric environment.

I have split my data into test/train samples that are list of tuples containing a graph and its label. One thing I've had to do is to treat the different graph as a "Batch", a large disconnected graph, using torch_geometric.data.Batch. To start, I am using a data loader with the following collate function

def collate(samples) :
  graphs,labels = map(list,zip(*samples))
  datalist = make_datalist(graphs)
  datalist = Batch.from_data_list(datalist)
  return datalist, torch.tensor(labels)

and my classifier is the following :

class Classifier(nn.Module):
    def __init__(self, in_dim, hidden_dim, n_classes):
        super(Classifier, self).__init__()
        self.conv1 = GraphConv(in_dim, hidden_dim)
        self.conv2 = GraphConv(hidden_dim, hidden_dim)
        self.classify = nn.Linear(hidden_dim, n_classes)

    def forward(self, g):
        # Use node degree as the initial node feature. For undirected graphs, the in-degree
        # is the same as the out_degree.
        h = g.in_degrees
        # Perform graph convolution and activation function.
        h = F.relu(self.conv1(g, h))
        h = F.relu(self.conv2(g, h))
        g.ndata['h'] = h
        # Calculate graph representation by averaging all the node representations.
        hg = dgl.mean_nodes(g, 'h')
        return self.classify(hg)

Which simply averages the nodes representations of each graph, and feeds it to a MLP

The problem I come up with is that during the prediction of our batch, I have the error
AttributeError: 'Batch' object has no attribute 'local_var'
and I can't find where it may come from, would anyone know ?

Thank you for taking the time to read !


title: How does torch.argmax work for 4-dimensions

body:
I am a newbie in Pytorch. Even though I read the documentation, it is unclear for me how does torch.argmax() applied to first dimension work when we have 4-dimensional input. Also, how does keepdims=True change the output?
Here is an example of each case:
k = torch.rand(2, 3, 4, 4)
print(k):

tensor([[[[0.2912, 0.4818, 0.1123, 0.3196],
          [0.6606, 0.1547, 0.0368, 0.9475],
          [0.4753, 0.7428, 0.5931, 0.3615],
          [0.6729, 0.7069, 0.1569, 0.3086]],

         [[0.6603, 0.7777, 0.3546, 0.2850],
          [0.3681, 0.5295, 0.8812, 0.6093],
          [0.9165, 0.2842, 0.0260, 0.1768],
          [0.9371, 0.9889, 0.6936, 0.7018]],

         [[0.5880, 0.0349, 0.0419, 0.3913],
          [0.5884, 0.9408, 0.1707, 0.1893],
          [0.3260, 0.4410, 0.6369, 0.7331],
          [0.9448, 0.7130, 0.3914, 0.2775]]],

        [[[0.9433, 0.8610, 0.9936, 0.1314],
          [0.8627, 0.3103, 0.3066, 0.3547],
          [0.3396, 0.1892, 0.0385, 0.5542],
          [0.4943, 0.0256, 0.7875, 0.5562]],

         [[0.2338, 0.2498, 0.4749, 0.2520],
          [0.4405, 0.1605, 0.6219, 0.8955],
          [0.2326, 0.1816, 0.5032, 0.8732],
          [0.2089, 0.6131, 0.1898, 0.0517]],

         [[0.1472, 0.8059, 0.6958, 0.9047],
          [0.6403, 0.2875, 0.5746, 0.5908],
          [0.8668, 0.4602, 0.8224, 0.9307],
          [0.2077, 0.5665, 0.8671, 0.4365]]]])

argmax = torch.argmax(k, axis=1)
print(argmax):
tensor([[[1, 1, 1, 2],
         [0, 2, 1, 0],
         [1, 0, 2, 2],
         [2, 1, 1, 1]],

        [[0, 0, 0, 2],
         [0, 0, 1, 1],
         [2, 2, 2, 2],
         [0, 1, 2, 0]]])

argmax = torch.argmax(k, axis=1, keepdims=True)
print(argmax):
tensor([[[[1, 1, 1, 2],
          [0, 2, 1, 0],
          [1, 0, 2, 2],
          [2, 1, 1, 1]]],

        [[[0, 0, 0, 2],
          [0, 0, 1, 1],
          [2, 2, 2, 2],
          [0, 1, 2, 0]]]])


title: PyTorch C++ extension: How to index tensor and update it?

body:
I'm creating a PyTorch C++ extension and after much research I can't figure out how to index a tensor and update its values. I found out how to iterate over a tensor's entries using the data_ptr() method, but that's not applicable to my use case.
Given is a matrix M, a list of lists (blocks) of index pairs P and a function f: dtype(M)^2 -> dtype(M)^2 that takes two values and spits out two new values.
I'm trying to implement the following pseudo code:
for each block B in P:
    for each row R in M:
        for each index-pair (i,j) in B:
            M[R,i], M[R,j] = f(M[R,i], M[R,j])

After all, this code is going to run on the GPU using CUDA, but since I don't have any experience with that, I wanted to first write a pure C++ program and then convert it.
Can anyone suggest how to do this or how to convert the algorithm to do something equivalent?


title: Installing YOLOv5 dependencies - torchvision?

body:
I want to use yolov5. According to https://pytorch.org/hub/ultralytics_yolov5/, you should have Python>=3.8 and PyTorch>=1.7 installed, as well as YOLOv5 dependencies.
Python and pytorch are up to date:
pip show torch
Version: 1.7.1

python --version
Python 3.9.1

But when I try to install the yolov5 dependencies, I get an error message:
pip install -qr https://raw.githubusercontent.com/ultralytics/yolov5/master/requirements.txt
ERROR: Could not find a version that satisfies the requirement torchvision>=0.8.1
ERROR: No matching distribution found for torchvision>=0.8.1

An updated version of torchvision is needed (???). So I run the update, but when I check the version it hasn't worked.
pip show torchvision
Version: 0.2.2.post3

-m pip install --upgrade torchvision

pip show torchvision
Version: 0.2.2.post3

Is torchvision needed for installing the yolov5 dependencies? How do I move forward? I'm on Windows 10.
Thanks!


title: Conda - ModuleNotFoundError: No module named 'torch'

body:
Steps to reproduce:
I am using Anaconda on Windows to set up environment for this repo.
conda create --name pytorch-yolo
Then I install all dependencies with conda install --file requirements.txt
Which returns
PackagesNotFoundError: The following packages are not available from current channels:
- torch[version='>=1.2']

So I install pytorch with conda install pytorch torchvision cudatoolkit=10.2 -c pytorch
When I am trying to evaluate the model from the following repo: https://github.com/eriklindernoren/PyTorch-YOLOv3 with python3 test.py --weights_path weights/yolov3.weights inside conda environment, it returns the following error ModuleNotFoundError: No module named 'torch'
If I run a Jupyter lab or notebook, or even go with python inside conda terminal, torch is easily imported.
What I've already tried:
Almost everything from this thread: No module named "Torch"
Specifically, creating a new conda environment with python set to 3.8.2, installing torch to base and then to the environment.
I'm also using pyenv to set global python to 3.8.2, but that doesn't help at all.
conda list shows I have pytorch installed
Can't wrap my head around this issue.


title: I have a trouble after trying to parallelize data using nn.Dataparallel

body:
I didn't have any probelm without data parallelization, but after I put JUST A ONE LINE "model = nn.DataParallel(model)" the error message "TypeError: 'list' object is not callable" comes. If I push out that damn line the source works clean. plz help me

I can't do anything but just mad. I search that google and there are some ways to solve that error message but I can't do anything. Because nn.Dataparallel is already used by other coders. Sorry English is not my mothertongue.

    if use_cuda:
            [[[model = nn.DataParallel(model)]]]
            model = model.cuda()
            criterion = criterion.cuda()
            print('cuda is used')

I just put model = nn.DataParallel(model) and error comes right after.

  Traceback (most recent call last):
    File "/home/scrcdeep2/YBJ/espnet_myself/Main.py", line 119, in
      train(loader, model_, criterion_, optimizer_, use_cuda_, pretrained=None)

  File "/home/scrcdeep2/YBJ/espnet_myself/Main.py", line 83, in train
      outputs = model(inputs)

  File "/home/scrcdeep2/YBJ/lib/python3.5/site-packages/torch/nn/modules/module.py", line 493, in call
      result = self.forward(*input, **kwargs)
   File "/home/scrcdeep2/YBJ/lib/python3.5/site-packages/torch/nn/parallel/data_parallel.py", line 151, in forward
      replicas = self.replicate(self.module, self.device_ids[:len(inputs)])

  File "/home/scrcdeep2/YBJ/lib/python3.5/site-packages/torch/nn/parallel/data_parallel.py", line 156, in replicate
      return replicate(module, device_ids)

  File "/home/scrcdeep2/YBJ/lib/python3.5/site-packages/torch/nn/parallel/replicate.py", line 114, in replicate
      modules = list(network.modules())

  TypeError: 'list' object is not callable


title: Why does pre-trained ResNet18 have a higher validation accuracy than training?

body: For PyTorch's tutorial on performing transfer learning for computer vision (https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html), we can see that there is a higher validation accuracy than training accuracy. Applying the same steps to my own dataset, I see similar results. Why is this the case? Does it have something to do with ResNet 18's architecture?


title: Why can't I import PyTorch? AttributeError: 'module' object has no attribute 'inf'

body:
I have a code using PyTorch, but got the following error in the first line when I tried to import PyTorch. I am using Python3.4 and PyTorch-0.4

Traceback (most recent call last):
  File "my_test.py", line 1, in <module>
    import torch
  File "/my_path/lib/python3.4/site-packages/torch/__init__.py", line 17, in <module>
    from ._six import string_classes as _string_classes
  File "/my_path/lib/python3.4/site-packages/torch/_six.py", line 33, in <module>
    inf = math.inf
AttributeError: 'module' object has no attribute 'inf'

Any idea why I got this error? Thanks!


title: How can I show multiple predictions of the next word in a sentence?

body:
I am using the GPT-2 pre trained model. the code I am working on will get a sentence and generate the next word for that sentence. I want to print multiple predictions, like the three first predictions with best probabilities!
for example if I put in the sentence "I's an interesting ...."
predictions:  "Books"    "story"    "news"

is there a way I can modify this code to show these predictions instead of one?!

also there are two parts in the code, I do not understand, what is the meaning of the numbers in  (predictions[0, -1, :])?  and why do we put [0] after predictions = output[0]?

import torch
from pytorch_transformers import GPT2Tokenizer, GPT2LMHeadModel

# Load pre-trained model tokenizer (vocabulary)
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')

# Encode a text inputs
text = "The fastest car in the  "
indexed_tokens = tokenizer.encode(text)

# Convert indexed tokens in a PyTorch tensor
tokens_tensor = torch.tensor([indexed_tokens])

# Load pre-trained model (weights)
model = GPT2LMHeadModel.from_pretrained('gpt2')

# Set the model in evaluation mode to deactivate the DropOut modules
model.eval()

# If you have a GPU, put everything on cuda
#tokens_tensor = tokens_tensor.to('cuda')
#model.to('cuda')

# Predict all tokens
with torch.no_grad():
    outputs = model(tokens_tensor)
    predictions = outputs[0]
    #print(predictions)

# Get the predicted next sub-word
predicted_index = torch.argmax(predictions[0, -1, :]).item()
predicted_text = tokenizer.decode(indexed_tokens + [predicted_index])

# Print the predicted word
#print(predicted_index)
print(predicted_text)

The result for the above code will be :

The fastest car in the world.


title: How can I calculate accuracy for keypoints detection CNN model in pytorch?

body:
Can someone help me with this please,

def train_net(n_epochs):
    valid_loss_min = np.Inf
    history = {'train_loss': [], 'valid_loss': [], 'epoch': []}

    for epoch in range(n_epochs):
        train_loss = 0.0
        valid_loss = 0.0
        net.train()
        running_loss = 0.0
        for batch_i, data in enumerate(train_loader):
            images = data['image']
            key_pts = data['keypoints']
            key_pts = key_pts.view(key_pts.size(0), -1)
            key_pts = key_pts.type(torch.FloatTensor).to(device)
            images = images.type(torch.FloatTensor).to(device)
            output_pts = net(images)
            loss = criterion(output_pts, key_pts)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            train_loss += loss.item()*images.data.size(0)
        net.eval()

        with torch.no_grad():
            for batch_i, data in enumerate(test_loader):
                images = data['image']
                key_pts = data['keypoints']
                key_pts = key_pts.view(key_pts.size(0), -1)
                key_pts = key_pts.type(torch.FloatTensor).to(device)
                images = images.type(torch.FloatTensor).to(device)
                output_pts = net(images)
                loss = criterion(output_pts, key_pts)
                valid_loss += loss.item()*images.data.size(0)
        train_loss = train_loss/len(train_loader.dataset)
        valid_loss = valid_loss/len(test_loader.dataset)
        print('Epoch: {} \tTraining Loss: {:.6f}'.format(epoch+1,train_loss,valid_loss))

        if valid_loss <= valid_loss_min:
            print('Validation loss decreased ({:.6f} --> {:.6f}).  Saving model ...'.format(valid_loss_min,valid_loss))
            torch.save(net,f'X:\\xxxx\\xxx\\xxx\\epoch{epoch + 1}_loss{valid_loss}.pth')
            valid_loss_min = valid_loss
        history['epoch'].append(epoch + 1)
        history['train_loss'].append(train_loss)
        history['valid_loss'].append(valid_loss)
    print('Finished Training')
    return history
'''

Above is the training code for reference!


title: Embedding Error Index out of Range in self

body:
I followed this tutorial

However I get this error.

`This is a test of our preprocessing function. It took 46.7 seconds to load the data. Our German vocab has size 13354 and our English vocab has size 11560. Our training data has 7443 batches, each with 16 sentences, and our validation data has 570 batches.
Validating Epoch [0/50]  Average loss: 9.379     Perplexity: 11833.884
Epoch [1/50]     Batch [10/7443]     Loss: 8.345
Epoch [1/50]     Batch [1010/7443]   Loss: 4.692
Epoch [1/50]     Batch [2010/7443]   Loss: 4.192
Traceback (most recent call last):
  File "/home/hasara/python code/benchmark- IWSLT data/test.py", line 334, in <module>
    train(train_iter, val_iter, model, criterion, optimizer, 50)
  File "/home/hasara/python code/benchmark- IWSLT data/test.py", line 255, in train
    scores = model(src, tgt)
  File "/home/hasara/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/hasara/python code/benchmark- IWSLT data/test.py", line 159, in forward
    out_e, final_e = self.encoder(src)
  File "/home/hasara/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/hasara/python code/benchmark- IWSLT data/test.py", line 75, in forward
    x = self.dropout(self.embedding(x)) # embedding
  File "/home/hasara/.local/lib/python3.6/site-packages/torch/nn/modules/module.py", line 550, in __call__
    result = self.forward(*input, **kwargs)
  File "/home/hasara/.local/lib/python3.6/site-packages/torch/nn/modules/sparse.py", line 114, in forward
    self.norm_type, self.scale_grad_by_freq, self.sparse)
  File "/home/hasara/.local/lib/python3.6/site-packages/torch/nn/functional.py", line 1724, in embedding
    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
IndexError: index out of range in self`

Some resources mentioned that issue is with embedding. But I still couldn't figure out the error. I'm new to pyTorch. Any insight will be highly appreciated.


title: Pytorch for loop inefficience

body:
I have a efficient issue with some tensor for loop.

I’m extracting the features from the last layer of a CNN through a image data loader (I’m using batch size 8). Im getting the euclidean distance of the batch tensor and a table with previous features.

I want to add to the table a tensor every time all the tensors in the table are above a threshole. I have implemented a successful running code but the loop i use its not efficient and im wondering how i could do something similar using something more efficient rather than this secuential way.

for i, data in enumerate(dataloader, 0):
  input, label = data
  input, label = input.to(device), label.to(device)
  n,c h,w = input.size()
  outputs = model(input)
  if (i == 0):
    features_list = torch.cat( (features_list, outputs[0].view(1,-1)), 0)
  dist_tensores = torch.cdist(outputs, features_list, p=2.0)
  activation = torch.gt(dist_tensores, AVG, out=torch.cuda.FloatTensor(len(outputs), len(features_list)))
  counter = len(features_list)
  activation_list = torch.sum(activation, dim=0)
  for x in range(len(activation)):
    if (torch.sum(activation[x], dim=0) == counter):
      features_list = torch.cat( (features_list, outputs[x].view(1,-1)), 0)

The last loop is the part i want to change but i really don´t know how to assign and add the tensor i want if it's not by creating a loop where i can control the tensor to add.


title: save to csv - numpy.ndarray' object has no attribute 'append'

body:
I'm trying to append the filenames of the images during training into a csv file, but getting an error. I can print the values okay but cannot append to csv file for some reason. Here is the full code:
  File "train_filename.py", line 140, in <module>
    names_preds.append() + '\n'
AttributeError: 'numpy.ndarray' object has no attribute 'append'

        with open('preds_base_model_teste1016.csv','a') as fd:
            #dict_writer = csv.writer(fd)
            #dict_writer.writerow('Target' + '\n')
            #dict_writer.writerow( ','.join(map(str, preds.detach().tolist())) + '\n')
            #dict_writer.writerow('Prediction' + '\n')
            #dict_writer.writerow( ','.join(map(str, targets.detach().tolist())) + '\n')
            #fd.write('Target' + '\n')
            fd.write(','.join(map(str, preds.detach().tolist())) + '\n')
            #fd.write('Prediction' + '\n')
            fd.write( ','.join(map(str, targets.detach().tolist())) + '\n')
            #fd.write('Image_Name' + '\n')
            np.append(paths)
            #fd.write([filename])


title: Unable to import Pytorch in Jetson nano (ubuntu)

body:
I having trouble importing PyTorch in my jetson nano (jetpack 4.4, Cuda 10.2.89), I have successfully installed it from .whl file and it is in my pip3 lib. But when I import it, it shows this error. Please help.
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/parikshit/.local/lib/python3.6/site-packages/torch/__init__.py", line 188, in <module>
    _load_global_deps()
  File "/home/parikshit/.local/lib/python3.6/site-packages/torch/__init__.py", line 141, in _load_global_deps
    ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)
  File "/usr/lib/python3.6/ctypes/__init__.py", line 348, in __init__
    self._handle = _dlopen(self._name, mode)
OSError: libmpi_cxx.so.20: cannot open shared object file: No such file or directory```


title: complex functions that already support autograd - Pytorch

body:
I am using this customized function to reshape my tensors in the customized loss function.
def reshape_fortran(x, shape):
if len(x.shape) > 0:
    x = x.permute(*reversed(range(len(x.shape))))
return x.reshape(*reversed(shape)).permute(*reversed(range(len(shape))))

Though, I receive this error:
RuntimeError: _unsafe_view does not support automatic differentiation for outputs with complex dtype.
for reshape_fortran output.
Do you know what might be the problem? which function is not supported in Pytorch autograd for complex numbers?


title: BERT NER: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first

body:
I want to train my BERT NER model on colab. But following error occurs
Code:
tr_logits = tr_logits.detach().cpu().numpy()
tr_label_ids = torch.masked_select(b_labels, (preds_mask == 1))
tr_batch_preds = np.argmax(tr_logits[preds_mask.squeeze()], axis=1)
tr_batch_labels = tr_label_ids.to(device).numpy()
tr_preds.extend(tr_batch_preds)
tr_labels.extend(tr_batch_labels)

Error:
Using TensorFlow backend.
Saved standardized data to ./data/en/combined/train_combined.txt.
Saved standardized data to ./data/en/combined/dev_combined.txt.
Saved standardized data to ./data/en/combined/test_combined.txt.
Constructed SentenceGetter with 25650 examples.
Constructed SentenceGetter with 8934 examples.
Loaded training and validation data into DataLoaders.
Initialized model and moved it to cuda.
Initialized optimizer and set hyperparameters.
Epoch:   0% 0/5 [00:00<?, ?it/s]Starting training loop.
Epoch:   0% 0/5 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "/content/FYP_Presentation/python/main.py", line 102, in <module>
    valid_dataloader,
  File "/content/FYP_Presentation/python/utils/main_utils.py", line 431, in train_and_save_model
    tr_batch_preds = torch.max(tr_logits[preds_mask.squeeze()], axis=1)
  File "/usr/local/lib/python3.6/dist-packages/torch/tensor.py", line 412, in __array__
    return self.numpy()
TypeError: can't convert CUDA tensor to numpy. Use Tensor.cpu() to copy the tensor to host memory first.

How would I solve this issue?


title: Pytorch already installed using Conda but fails when called

body:
I am trying to install pytorch for using BERT but when following the installation instructions found here: https://pytorch.org/get-started/locally/ I am getting an error.
When I try and initalise the BERT model I get the following error:
ImportError:
    BertForSequenceClassification requires the PyTorch library but it was not found in your environment.
    Checkout the instructions on theinstallation page: https://pytorch.org/get-started/locally/
    and follow the ones that match your environment.

I have followed the instructions and run the following command line in my Conda prompt terminal AND in my current working directory:
conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch

It looks to complete but when I try and call the following line I get the same error as the start as if it hasn't installed at all.
Can anyone help me out please.
EDIT:
The code I am using to execute bert is:
model = BertForSequenceClassification.from_pretrained(r'C:\Users\441\bert\pytorch_model.bin', config = r'C:\Users\441\bert\config.json')


title: How can I compute the tensor in Pytorch efficiently?

body:
I have a tensor x and x.shape=(batch_size,10), now I want to take

 x[i][0] = x[i][0]*x[i][1]*...*x[i][9] for i in range(batch_size)

Here is my code:

for i in range(batch_size):
    for k in range(1, 10):
        x[i][0] = x[i][0] * x[i][k]

But when I implement this in forward() and call loss.backward(), the speed of back-propagation is very slow. Why is it slow and is there any way to implement it efficiently?


title: loading pretrained (CNN) model from .ckpt file using Pytorch

body:
I am using Pytorch for image classification. I am looking for CNN models pretrained on a dataset other than ImageNet, I have found a link to a ".ckpt" file. I also found tutorials on loading this file with Tenserflow, but not using pytorch.
How can I load pretrained model using Pytorch from ".ckpt" file ?


title: Array Assignment with Autograd ( not again :( )

body:
I have a fairly non trivial function I want to differentiate with autograd but I'm not quite enough of a numpy wizard to figure our how to do it without array assingment.

I also apologize that I had to make this example incredibly contrived and meaningless to be able to run standalone. The actual code I'm working with is for non linear finite elements and is trying to compute the jacobian for a complex non linear system.

import autograd.numpy as anp
from autograd import jacobian

def alpha(x):
    return anp.exp(-(x - 10) ** 2) / (x + 1)

def f(x):
    # Matrix getting constructed
    k = anp.zeros((x.shape[0], x.shape[0]))

    # loop over some random 3 dimensional vectors
    for element in anp.random.randint(0, x.shape[0], (x.shape[0], 3)):

        # select 3 values from x
        x_ijk = anp.array([[x[i] for i in element]])

        norm = anp.linalg.norm(
            x_ijk @ anp.vstack((element, element)).transpose()
        )

        # make some matrix from the element
        m = element.reshape(3, 1) @ element.reshape(1, 3)

        # alpha is an arbitrary differentiable function R -> R
        alpha_value = alpha(norm)

        # combine m matricies into k scaling by alpha_value
        n = m.shape[0]
        for i in range(n):
            for j in range(n):
                k[element[i], element[j]] += m[i, j] * alpha_value

    return k @ x

print(jacobian(f)(anp.random.rand(10)))
# And course we get an error
# k[element[i], element[j]] += m[i, j] * alpha_value
# ValueError: setting an array element with a sequence.

I don't really understand this message since no type error is happening. I assume it must be from assignment.

After writting the above I made a trivial switch to PyTorch and the code runs just fine. But I would still prefer to use autograd

#pytorch version
import torch
from torch.autograd.gradcheck import zero_gradients

def alpha(x):
    return torch.exp(x)

def f(x):
    # Matrix getting constructed
    k = torch.zeros((x.shape[0], x.shape[0]))

    # loop over some random 3 dimensional vectors
    for element in torch.randint(0, x.shape[0], (x.shape[0], 3)):

        # select 3 values from x
        x_ijk = torch.tensor([[1. if n == e else 0 for n in range(len(x))] for e in element]) @ x
        norm = torch.norm(
            x_ijk @ torch.stack((torch.tanh(element.float() + 4), element.float() - 4)).t()
        )

        m = torch.rand(3, 3)

        # alpha is an arbitrary differentiable function R -> R
        alpha_value = alpha(norm)

        n = m.shape[0]
        for i in range(n):
            for j in range(n):
                k[element[i], element[j]] += m[i, j] * alpha_value

    print(k)
    return k @ x

x = torch.rand(4, requires_grad=True)
print(x, '\n')
y = f(x)
print(y, '\n')
grads = []
for val in y:
    val.backward(retain_graph=True)
    grads.append(x.grad.clone())
    zero_gradients(x)
if __name__ == '__main__':
    print(torch.stack(grads))


title: Does calling forward() on a model in pytorch require extra gpu memory after already having loaded the model and data in gpu memory?

body:
I can load the model and a data sample in gpu memory, but when I call forward on the model with the sample, it gives a CUDA out of memory error.

I'm sure the model and data have been loaded, as my code is structured as follows (pseudocode):

model = Model()
sample = load_sample()
sleep(5) # to check memory usage with nvidia-smi
print('before forward')
model(sample)
print('after forward')

"before forward" gets printed, but "after forward" does not.

I assumed all the necessary memory for a forward pass gets allocated during construction of the model, but I don't know how else this error can happen. I also cannot find it on Google.

Python:  3.6.9
PyTorch: 1.2.0


title: Converting a pytorch model to nn.Module for exporting to onnx for lens studio

body:
I am trying to convert pix2pix to a pb or onnx that can run in Lens Studio. Lens studio has strict requirements for the models. I am trying to export this pytorch model to onnx using this guide provided by lens studio. The issue is the pytorch model found here uses its own base class, when in the example it uses Module.nn, and therefore doesnt have methods/variables that the torch.onnx.export function needs to run. So far Ive run into its missing a variable called training and a method called train
Would it be worth it to try to modify the base model, or should I try to build it from scratch using nn.Module? Is there a way to make the pix2pix model inherit from both the abstract base class and nn.module? Am I not understanding the situation? The reason why I want to do it using the lens studio tutorial is because I have gotten it to export onnx different ways but Lens Studio wont accept those for various reasons.
Also this is my first time asking a SO question (after 6 years of coding), let me know if I make any mistakes and I can correct them. Thank you.
This is the important code from the tutorial creating a pytorch model for Lens Studio:
import torch
import torch.nn as nn
class Model(nn.Module):
    def __init__(self):
        super().__init__()
        self.layer = nn.Conv2d(in_channels=3, out_channels=1,
                               kernel_size=3, stride=2, padding=1)

    def forward(self, x):
        out = self.layer(x)
        out = nn.functional.interpolate(out, scale_factor=2,
                                        mode='bilinear', align_corners=True)
        out = torch.nn.functional.softmax(out, dim=1)
        return out

I'm not going to include all the code from the pytorch model bc its large, but the beginning of the baseModel.py is
import os
import torch
from collections import OrderedDict
from abc import ABC, abstractmethod
from . import networks

class BaseModel(ABC):
    """This class is an abstract base class (ABC) for models.
    To create a subclass, you need to implement the following five functions:
        -- <__init__>:                      initialize the class; first call BaseModel.__init__(self, opt).
        -- <set_input>:                     unpack data from dataset and apply preprocessing.
        -- <forward>:                       produce intermediate results.
        -- <optimize_parameters>:           calculate losses, gradients, and update network weights.
        -- <modify_commandline_options>:    (optionally) add model-specific options and set default options.
    """

    def __init__(self, opt):
        """Initialize the BaseModel class.

        Parameters:
            opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions

        When creating your custom class, you need to implement your own initialization.
        In this function, you should first call <BaseModel.__init__(self, opt)>
        Then, you need to define four lists:
            -- self.loss_names (str list):          specify the training losses that you want to plot and save.
            -- self.model_names (str list):         define networks used in our training.
            -- self.visual_names (str list):        specify the images that you want to display and save.
            -- self.optimizers (optimizer list):    define and initialize optimizers. You can define one optimizer for each network. If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an example.
        """
        self.opt = opt
        self.gpu_ids = opt.gpu_ids
        self.isTrain = opt.isTrain
        self.device = torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu')  # get device name: CPU or GPU
        self.save_dir = os.path.join(opt.checkpoints_dir, opt.name)  # save all the checkpoints to save_dir
        if opt.preprocess != 'scale_width':  # with [scale_width], input images might have different sizes, which hurts the performance of cudnn.benchmark.
            torch.backends.cudnn.benchmark = True
        self.loss_names = []
        self.model_names = []
        self.visual_names = []
        self.optimizers = []
        self.image_paths = []
        self.metric = 0  # used for learning rate policy 'plateau'

and for pix2pix_model.py
import torch
from .base_model import BaseModel
from . import networks

class Pix2PixModel(BaseModel):
    """ This class implements the pix2pix model, for learning a mapping from input images to output images given paired data.

    The model training requires '--dataset_mode aligned' dataset.
    By default, it uses a '--netG unet256' U-Net generator,
    a '--netD basic' discriminator (PatchGAN),
    and a '--gan_mode' vanilla GAN loss (the cross-entropy objective used in the orignal GAN paper).

    pix2pix paper: https://arxiv.org/pdf/1611.07004.pdf
    """
    @staticmethod
    def modify_commandline_options(parser, is_train=True):
        """Add new dataset-specific options, and rewrite default values for existing options.

        Parameters:
            parser          -- original option parser
            is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options.

        Returns:
            the modified parser.

        For pix2pix, we do not use image buffer
        The training objective is: GAN Loss + lambda_L1 * ||G(A)-B||_1
        By default, we use vanilla GAN loss, UNet with batchnorm, and aligned datasets.
        """
        # changing the default values to match the pix2pix paper (https://phillipi.github.io/pix2pix/)
        parser.set_defaults(norm='batch', netG='unet_256', dataset_mode='aligned')
        if is_train:
            parser.set_defaults(pool_size=0, gan_mode='vanilla')
            parser.add_argument('--lambda_L1', type=float, default=100.0, help='weight for L1 loss')

        return parser

    def __init__(self, opt):
        """Initialize the pix2pix class.

        Parameters:
            opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions
        """

(Also sidenote if you see this and it looks like no easy way out let me know, I know what its like seeing someone getting started in something who goes in to deep too early on)


title: Gensim's word2vec has a loss of 0 from epoch 1?

body:
I am using the Word2vec module of Gensim library to train a word embedding, the dataset is 400k sentences with 100k unique words (its not english)
I'm using this code to monitor and calculate the loss :
class MonitorCallback(CallbackAny2Vec):
    def __init__(self, test_words):
        self._test_words = test_words

    def on_epoch_end(self, model):
        print("Model loss:", model.get_latest_training_loss())  # print loss
        for word in self._test_words:  # show wv logic changes
            print(model.wv.most_similar(word))

monitor = MonitorCallback(["MyWord"])  # monitor with demo words

w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE, window=W2V_WINDOW, min_count=W2V_MIN_COUNT  , callbacks=[monitor])

w2v_model.build_vocab(tokenized_corpus)

words = w2v_model.wv.vocab.keys()
vocab_size = len(words)
print("Vocab size", vocab_size)

print("[*] Training...")

# Train Word Embeddings
w2v_model.train(tokenized_corpus, total_examples=len(tokenized_corpus), epochs=W2V_EPOCH)

The problem is from epoch 1 the loss is 0 and the vector of the monitored words dont change at all!
[*] Training...
Model loss: 0.0
Model loss: 0.0
Model loss: 0.0
Model loss: 0.0

so what is the problem here? is this normal?  the tokenized corpus is a list of lists that are something like tokenized_corpus[0] = [ "word1" , "word2" , ...]
I googled and seems like some of the old versions of gensim had problem with calculating loss function, but they are from almost a year ago and it seems like it should be fixed right now?
I tried the code provided in the answer of this question as well but still the loss is 0 :
Loss does not decrease during training (Word2Vec, Gensim)
EDIT1 : after adding compute_loss=True, the loss shows up, but it keeps going higher and higher, and the top similar words and their similarity doesn't change at all :
Model loss: 2187903.5
Model loss: 3245492.0
Model loss: 4103624.5
Model loss: 4798541.0
Model loss: 5413940.0
Model loss: 5993822.5
Model loss: 6532631.0
Model loss: 7048384.5
Model loss: 7547147.0


title: How do I compute bootstrapped cross entropy loss in PyTorch?

body:
I have read some papers that use something called "Bootstrapped Cross Entropy Loss" to train their segmentation network. The idea is to focus only on the hardest k% (say 15%) of the pixels into account to improve learning performance, especially when easy pixels dominate.
Currently, I am using the standard cross entropy:
loss = F.binary_cross_entropy(mask, gt)

How do I convert this to the bootstrapped version efficiently in PyTorch?


title: Loading data from Custom Data-Loader in pytorch only if the data specifies a certain condition

body:
I have a CSV file with filename in the first column and a label for the filename in the second column. I also have a third column, which specifies something about the data (whether the data meets a specific condition). It will look something like,
+-----------------------------+
| Filepath 1   Label 1    'n' |
|                             |
+-----------------------------+
| Filepath 2   Label 2    'n' |
|                             |
|                             |
+-----------------------------+
| Filepath 3   Label 3     'n'|
|                             |
+-----------------------------+
| Filepath 4   Label 4     'y'|
+------------------------------+

I want to be able to load the custom dataset using getitem only when attribute column == 'y'. However, I get the following error:
TypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'NoneType'>
My code is as follows:
'''
class InterDataset(Dataset):
  def __init__(self, csv_file, mode, root_dir = None, transform = None, run = None):
    self.annotations = pd.read_csv(csv_file, header = None)
    self.root_dir = root_dir
    self.transform = transform
    self.mode = mode
    self.run = run

  def __len__(self):
    return len(self.annotations)

  def __getitem__(self, index):
    if self.mode == 'train':
        if (self.annotations.iloc[index, 2] == 'n'):
                    img_path = self.annotations.iloc[index,0]
                    image = cv2.imread(img_path,1)

                    y_label = self.annotations.iloc[index,1]

                    if self.transform:
                        image = self.transform(image)
                    if (index+1)%300 == 0:
                        print('Loop {0} done'.format(index))
                    return [image, y_label]

'''


title: How do I fix the UserWarning: Using a target size (torch.Size([1])) that is different to the input size (torch.Size([])) is deprecated

body:
When I try to run my model, I get this UserWarning. How do I go about fixing this? My model's input is expecting (torch.Size([])), a scalar, but my target is torch.Size([1])). Should I be changed my input size or my target size? Any help would be greatly appreciated. Thanks!
UserWarning Message:
/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:34: UserWarning: Using a target size (torch.Size([1])) that is different to the input size (torch.Size([])) is deprecated. Please ensure they have the same size.

Model:
TextClassifier(
  (dropout): Dropout(p=0.25, inplace=False)
  (embedding): Embedding(5001, 64, padding_idx=0)
  (conv_1): Conv1d(75, 32, kernel_size=(2,), stride=(2,))
  (conv_2): Conv1d(75, 32, kernel_size=(3,), stride=(2,))
  (conv_3): Conv1d(75, 32, kernel_size=(4,), stride=(2,))
  (conv_4): Conv1d(75, 32, kernel_size=(5,), stride=(2,))
  (pool_1): MaxPool1d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  (pool_2): MaxPool1d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)
  (pool_3): MaxPool1d(kernel_size=4, stride=2, padding=0, dilation=1, ceil_mode=False)
  (pool_4): MaxPool1d(kernel_size=5, stride=2, padding=0, dilation=1, ceil_mode=False)
  (fc): Linear(in_features=1856, out_features=1, bias=True)
)

Code for my model:
class TextClassifier(nn.ModuleList):

    def __init__(self, params):
        super(TextClassifier, self).__init__()

        # Parameters regarding text preprocessing
        self.seq_len = params.seq_len
        self.num_words = params.num_words
        self.embedding_size = params.embedding_size

        # Dropout definition
        self.dropout = nn.Dropout(0.25)

        # CNN parameters definition
        # Kernel sizes
        self.kernel_1 = 2
        self.kernel_2 = 3
        self.kernel_3 = 4
        self.kernel_4 = 5

      # Output size for each convolution
        self.out_size = params.out_size
      # Number of strides for each convolution
        self.stride = params.stride

      # Embedding layer definition
        self.embedding = nn.Embedding(self.num_words + 1, self.embedding_size, padding_idx=0)

      # Convolution layers definition
        self.conv_1 = nn.Conv1d(self.seq_len, self.out_size, self.kernel_1, self.stride)
        self.conv_2 = nn.Conv1d(self.seq_len, self.out_size, self.kernel_2, self.stride)
        self.conv_3 = nn.Conv1d(self.seq_len, self.out_size, self.kernel_3, self.stride)
        self.conv_4 = nn.Conv1d(self.seq_len, self.out_size, self.kernel_4, self.stride)

      # Max pooling layers definition
        self.pool_1 = nn.MaxPool1d(self.kernel_1, self.stride)
        self.pool_2 = nn.MaxPool1d(self.kernel_2, self.stride)
        self.pool_3 = nn.MaxPool1d(self.kernel_3, self.stride)
        self.pool_4 = nn.MaxPool1d(self.kernel_4, self.stride)

      # Fully connected layer definition
        self.fc = nn.Linear(self.in_features_fc(), 1)

    def in_features_fc(self):
        '''Calculates the number of output features after Convolution + Max pooling
        Convolved_Features = ((embedding_size + (2 * padding) - dilation * (kernel - 1) - 1) / stride) + 1
        Pooled_Features = ((embedding_size + (2 * padding) - dilation * (kernel - 1) - 1) / stride) + 1

        source: https://pytorch.org/docs/stable/generated/torch.nn.Conv1d.html
        '''
        # Calculate size of convolved/pooled features for convolution_1/max_pooling_1 features
        out_conv_1 = ((self.embedding_size - 1 * (self.kernel_1 - 1) - 1) / self.stride) + 1
        out_conv_1 = math.floor(out_conv_1)
        out_pool_1 = ((out_conv_1 - 1 * (self.kernel_1 - 1) - 1) / self.stride) + 1
        out_pool_1 = math.floor(out_pool_1)

        # Calculate size of convolved/pooled features for convolution_2/max_pooling_2 features
        out_conv_2 = ((self.embedding_size - 1 * (self.kernel_2 - 1) - 1) / self.stride) + 1
        out_conv_2 = math.floor(out_conv_2)
        out_pool_2 = ((out_conv_2 - 1 * (self.kernel_2 - 1) - 1) / self.stride) + 1
        out_pool_2 = math.floor(out_pool_2)

        # Calculate size of convolved/pooled features for convolution_3/max_pooling_3 features
        out_conv_3 = ((self.embedding_size - 1 * (self.kernel_3 - 1) - 1) / self.stride) + 1
        out_conv_3 = math.floor(out_conv_3)
        out_pool_3 = ((out_conv_3 - 1 * (self.kernel_3 - 1) - 1) / self.stride) + 1
        out_pool_3 = math.floor(out_pool_3)

        # Calculate size of convolved/pooled features for convolution_4/max_pooling_4 features
        out_conv_4 = ((self.embedding_size - 1 * (self.kernel_4 - 1) - 1) / self.stride) + 1
        out_conv_4 = math.floor(out_conv_4)
        out_pool_4 = ((out_conv_4 - 1 * (self.kernel_4 - 1) - 1) / self.stride) + 1
        out_pool_4 = math.floor(out_pool_4)

        # Returns "flattened" vector (input for fully connected layer)
        return (out_pool_1 + out_pool_2 + out_pool_3 + out_pool_4) * self.out_size

    def forward(self, x):

        # Sequence of tokes is filterd through an embedding layer
        x = self.embedding(x)

        # Convolution layer 1 is applied
        x1 = self.conv_1(x)
        x1 = torch.relu(x1)
        x1 = self.pool_1(x1)

        # Convolution layer 2 is applied
        x2 = self.conv_2(x)
        x2 = torch.relu((x2))
        x2 = self.pool_2(x2)

        # Convolution layer 3 is applied
        x3 = self.conv_3(x)
        x3 = torch.relu(x3)
        x3 = self.pool_3(x3)

        # Convolution layer 4 is applied
        x4 = self.conv_4(x)
        x4 = torch.relu(x4)
        x4 = self.pool_4(x4)

        # The output of each convolutional layer is concatenated into a unique vector
        union = torch.cat((x1, x2, x3, x4), 2)
        union = union.reshape(union.size(0), -1)

        # The "flattened" vector is passed through a fully connected layer
        out = self.fc(union)
        # Dropout is applied
        out = self.dropout(out)
        # Activation function is applied
        out = torch.sigmoid(out)

        return out.squeeze()

Where I call model/train it:
# Initialize dataset maper
train = DatasetMaper(df_train['text_padded'], df_train['Target'])
test = DatasetMaper(df_test['text_padded'], df_test['Target'])

# Initialize loaders
loader_train = DataLoader(train, batch_size=Parameters.batch_size)
loader_test = DataLoader(test, batch_size=Parameters.batch_size)
print('DataLoaders initialized.')

# Define optimizer
optimizer = optim.RMSprop(model.parameters(), lr=Parameters.learning_rate)
print('Done. Everything loaded. Preparing to train model.')

# Starts training phase
for epoch in range(Parameters.epochs):
    print('Epoch: ', epoch+1)
    # Set model in training model
    model.train()
    predictions = []
        # Starts batch training
    for x_batch, y_batch in loader_train:
        #print(x_batch)
        x_batch = torch.tensor(x_batch)
        #print(x_batch)

        y_batch = y_batch.type(torch.FloatTensor)
        #print(y_batch)

        # Feed the model
        y_pred = model(x_batch)
        #print(y_pred.detach().numpy())

        # Loss calculation
        loss = F.binary_cross_entropy(y_pred, y_batch)

        # Clean gradientes
        optimizer.zero_grad()

        # Gradients calculation
        loss.backward()

        # Gradients update
        optimizer.step()

        #if epoch % 1000==1:
            #print(y_pred.detach().numpy())
        # Save predictions
        predictions += y_pred.detach().numpy()

    # Evaluation phase
    test_predictions = Run.evaluation(model, loader_test)

    # Metrics calculation
    train_accuary = Run.calculate_accuray(df_train['Target'], predictions)
    test_accuracy = Run.calculate_accuray(df_test['Target'], test_predictions)
    print("Epoch: %d, loss: %.5f, Train accuracy: %.5f, Test accuracy: %.5f" % (epoch+1, loss.item(), train_accuary, test_accuracy))


title: ModuleNotFoundError: No module named 'torch.nn'; 'torch' is not a package on Mac OS

body:
I am trying to get pytorch to work but I keep getting this error.
ModuleNotFoundError: No module named 'torch.nn'; 'torch' is not a package
I am using a Macbook, i've tried looking at the other answers on here but nothing is working.
import torch
import torchvision
import torchvision.transforms as transforms

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
                                        download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                          shuffle=True, num_workers=2)

testset = torchvision.datasets.CIFAR10(root='./data', train=False,
                                       download=True, transform=transform)
testloader = torch.utils.data.DataLoader(testset, batch_size=4,
                                         shuffle=False, num_workers=2)

classes = ('plane', 'car', 'bird', 'cat',
           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')


title: What is the correct way to use OHE lookup table for a pytorch RNN?

body:
I am currently trying to build a LSTM RNN using pytorch. One input vector is represented as an array of 50 integers corresponding to a sequence of  at most 50 tokens with padding where each integer corresponds to an element from my vocabulary and the index of the 1 in the OHE vector. I want to have an embedding layer that just uses a lookup table to One-hot encode the integer-- kind of like tensorflow's OHE layer.

Something like this "kind of" works

import torch
import numpy as np
import torch.nn as nn

# vocab_size is the number of words in your train, val and test set
# vector_size is the dimension of the word vectors you are using
vocab_size, vector_size = 5, 5
embed = nn.Embedding(vocab_size, vector_size)

# intialize the word vectors, pretrained_weights is a
# numpy array of size (vocab_size, vector_size) and
# pretrained_weights[i] retrieves the word vector of
# i-th word in the vocabulary
pretrained_weights = np.zeros((vocab_size, vector_size))
np.fill_diagonal(pretrained_weights, 1)
tmp =torch.from_numpy(pretrained_weights)
embed.weight = nn.Parameter(tmp,requires_grad=False )

# Then turn the word index into actual word vector
vocab = {"some": 0, "words": 1}
word_indexes = torch.from_numpy(np.array([vocab[w] for w in ["some", "words"]]))
word_vectors = embed(word_indexes)
word_vectors.data.numpy()

>>>output
array([[1., 0., 0., 0., 0.],
       [0., 1., 0., 0., 0.]])

but it is very hacky, and doesn't play nicely with batches of input vectors.

What is the correct way to declare a OHE embedding layer at the begining of an RNN?


title: Extracting the encoded representations of an image in PyTorch?

body:
I have a template autoencoder neural net model made in PyTorch that I'm using on the Omniglot dataset. I'd like to extract the encoded representations of the image, but I'm unsure how.

# Load data
mean = 0.5
std = 0.5
batch_size = 128
img_transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize((mean,), (std,))
])
dataset = Omniglot('.', download=True, transform=img_transform)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)

# Define Autoencoder
class Autoencoder(nn.Module):

    def __init__(self, n=64):
        super(Autoencoder, self).__init__()
        self.encoder = nn.Sequential(
            nn.Linear(105*105, 256, bias=True),
            nn.ReLU(True),
            nn.Linear(256, 64, bias=True),
            nn.ReLU(True),
            nn.Linear(64, n, bias=True),
            nn.ReLU(True)
        )
        self.decoder = nn.Sequential(
            nn.Linear(n, 64, bias=True),
            nn.ReLU(True),
            nn.Linear(64, 256, bias=True),
            nn.ReLU(True),
            nn.Linear(256, 105*105, bias=True),
            nn.Tanh()
        )

    def forward(self, x):
        x = self.encoder(x)
        x = self.decoder(x)
        return x

def train(num_epochs, dataloader, model, criterion, optimizer):
    for epoch in range(num_epochs):
        for data in dataloader:
            img, label = data
            img = img.view(img.size(0), -1)
            img = Variable(img).cuda()
            output = model(img)
            loss = criterion(output, img)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
    return model

# Train model
num_epochs = 25
learning_rate = 1e-3
model = Autoencoder().cuda()
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
trained_model = train(num_epochs, dataloader, model, criterion, optimizer)