title: No update of bias in linear Regression via Pytorch

body:
I want to use PyTorch to build linear regression to solve a simple task. The 1-D output is just the average of 10-D input. What is strange is that during training, weights would converge to 0.1 while the bias keep the same as the default value. The expected bias should be 0. The code and result are shown as below.
from torch import nn
from torch.autograd import Variable
from torch.optim.lr_scheduler import LambdaLR, ReduceLROnPlateau
from torch.utils.data import DataLoader, Dataset
from matplotlib import pyplot
import torch as th
import torch.nn.functional as F
import pandas as pd
import numpy as np
import sys

# Linear Regression Model
class linearRegression(nn.Module):
    def __init__(self):
        super(linearRegression, self).__init__()
        self.linear = nn.Linear(10, 1)  # input and output is 1 dimension

    def forward(self, x):
        out = self.linear(x)
        return out

#th.manual_seed(2020)
data = pd.read_csv('./train.csv',dtype=np.float64).values[: , 1:]
train_data_label = data[:,0].reshape(-1,1)
train_data = np.delete(data, 0, axis=1)

x_train_tensor = th.tensor(train_data[0:9500, :]).float().to(device)
y_train_tensor = th.tensor(train_data_label[0:9500, :]).float().to(device)
x_valid_tensor = th.tensor(train_data[9500:, :]).float().to(device)
y_valid_tensor = th.tensor(train_data_label[9500:, :]).float().to(device)

model = linearRegression().float() # Transform the precision of model
model = model.to(device)

criterion = nn.MSELoss()
optimizer = th.optim.SGD(model.parameters(), lr=1e-8)

train_loss_history = []
valid_loss_history = []

print("initial weights: ", model.linear.weight.data, " bias: ", model.linear.bias.data)
# Start training
num_epochs = 500
for epoch in range(num_epochs):
    inputs = x_train_tensor
    target = y_train_tensor

    # Forward
    out = model(inputs)
    loss = criterion(out, target)
    # Backward
    optimizer.zero_grad() # Set gradients of all parameters to zero
    loss.backward()
    optimizer.step()
    train_loss_history.append(loss.item())

    # For validation
    model.eval() # Use pretrained weights and bias
    with th.no_grad(): # No calculation of gradient during validation
      predict = model(x_valid_tensor)
      valid_loss = criterion(predict, y_valid_tensor)
      valid_loss_history.append(valid_loss.item())

    if (epoch+1) % 1 == 0:
      print(f'Epoch[{epoch+1}/{num_epochs}], bias: {model.linear.bias.data}, weight: {model.linear.weight.data}')

print("weight: ", model.linear.weight.data)
print("bias: ", model.linear.bias.data)

initial weights:  tensor([[ 0.2636, -0.0230, -0.2399,  0.3074,  0.0735,  0.2014, -0.2753,  0.1922,
         -0.1379,  0.2794]], device='cuda:0')  bias:  tensor([0.2830], device='cuda:0')
Epoch[1/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2604, -0.0206, -0.2334,  0.3034,  0.0741,  0.1993, -0.2677,  0.1907,
         -0.1335,  0.2760]], device='cuda:0')
Epoch[2/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2572, -0.0183, -0.2269,  0.2995,  0.0747,  0.1973, -0.2603,  0.1891,
         -0.1292,  0.2726]], device='cuda:0')
Epoch[3/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2540, -0.0160, -0.2206,  0.2956,  0.0753,  0.1953, -0.2530,  0.1876,
         -0.1250,  0.2693]], device='cuda:0')
Epoch[4/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2510, -0.0138, -0.2144,  0.2918,  0.0759,  0.1934, -0.2458,  0.1861,
         -0.1208,  0.2660]], device='cuda:0')
Epoch[5/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2480, -0.0116, -0.2084,  0.2881,  0.0765,  0.1915, -0.2389,  0.1846,
         -0.1167,  0.2628]], device='cuda:0')
Epoch[6/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2450, -0.0094, -0.2024,  0.2845,  0.0770,  0.1896, -0.2320,  0.1832,
         -0.1128,  0.2597]], device='cuda:0')
Epoch[7/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2421, -0.0073, -0.1966,  0.2809,  0.0776,  0.1878, -0.2253,  0.1818,
         -0.1088,  0.2566]], device='cuda:0')
Epoch[8/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2393, -0.0052, -0.1908,  0.2774,  0.0781,  0.1860, -0.2187,  0.1804,
         -0.1050,  0.2536]], device='cuda:0')
Epoch[9/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2365, -0.0032, -0.1852,  0.2740,  0.0786,  0.1842, -0.2123,  0.1790,
         -0.1012,  0.2506]], device='cuda:0')
Epoch[10/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2338, -0.0012, -0.1797,  0.2706,  0.0791,  0.1825, -0.2060,  0.1777,
         -0.0975,  0.2477]], device='cuda:0')
Epoch[11/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2311,  0.0008, -0.1743,  0.2673,  0.0796,  0.1808, -0.1998,  0.1763,
         -0.0939,  0.2449]], device='cuda:0')
Epoch[12/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2285,  0.0027, -0.1690,  0.2641,  0.0801,  0.1792, -0.1937,  0.1750,
         -0.0903,  0.2421]], device='cuda:0')
Epoch[13/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2260,  0.0046, -0.1638,  0.2609,  0.0806,  0.1776, -0.1878,  0.1737,
         -0.0868,  0.2393]], device='cuda:0')
Epoch[14/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2235,  0.0064, -0.1587,  0.2578,  0.0810,  0.1760, -0.1820,  0.1725,
         -0.0833,  0.2367]], device='cuda:0')
Epoch[15/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2210,  0.0082, -0.1537,  0.2547,  0.0815,  0.1744, -0.1763,  0.1713,
         -0.0800,  0.2340]], device='cuda:0')
Epoch[16/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2186,  0.0100, -0.1488,  0.2517,  0.0819,  0.1729, -0.1707,  0.1700,
         -0.0767,  0.2314]], device='cuda:0')
Epoch[17/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2162,  0.0117, -0.1440,  0.2488,  0.0824,  0.1714, -0.1652,  0.1688,
         -0.0734,  0.2289]], device='cuda:0')
Epoch[18/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2139,  0.0134, -0.1393,  0.2459,  0.0828,  0.1700, -0.1599,  0.1677,
         -0.0702,  0.2264]], device='cuda:0')
Epoch[19/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2116,  0.0151, -0.1347,  0.2431,  0.0832,  0.1685, -0.1546,  0.1665,
         -0.0671,  0.2240]], device='cuda:0')
Epoch[20/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2094,  0.0168, -0.1301,  0.2403,  0.0836,  0.1671, -0.1495,  0.1654,
         -0.0640,  0.2216]], device='cuda:0')
Epoch[21/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2072,  0.0184, -0.1257,  0.2376,  0.0840,  0.1658, -0.1444,  0.1642,
         -0.0610,  0.2193]], device='cuda:0')
Epoch[22/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2051,  0.0199, -0.1213,  0.2349,  0.0844,  0.1644, -0.1395,  0.1631,
         -0.0580,  0.2170]], device='cuda:0')
Epoch[23/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2030,  0.0215, -0.1171,  0.2323,  0.0847,  0.1631, -0.1346,  0.1621,
         -0.0551,  0.2147]], device='cuda:0')
Epoch[24/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.2010,  0.0230, -0.1129,  0.2297,  0.0851,  0.1618, -0.1299,  0.1610,
         -0.0522,  0.2125]], device='cuda:0')
Epoch[25/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1989,  0.0245, -0.1088,  0.2272,  0.0854,  0.1606, -0.1253,  0.1599,
         -0.0494,  0.2103]], device='cuda:0')
Epoch[26/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1970,  0.0260, -0.1047,  0.2248,  0.0858,  0.1593, -0.1207,  0.1589,
         -0.0466,  0.2082]], device='cuda:0')
Epoch[27/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1950,  0.0274, -0.1008,  0.2223,  0.0861,  0.1581, -0.1162,  0.1579,
         -0.0439,  0.2061]], device='cuda:0')
Epoch[28/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1932,  0.0288, -0.0969,  0.2200,  0.0865,  0.1569, -0.1119,  0.1569,
         -0.0413,  0.2041]], device='cuda:0')
Epoch[29/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1913,  0.0302, -0.0931,  0.2177,  0.0868,  0.1558, -0.1076,  0.1559,
         -0.0387,  0.2021]], device='cuda:0')
Epoch[30/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1895,  0.0315, -0.0894,  0.2154,  0.0871,  0.1546, -0.1034,  0.1550,
         -0.0361,  0.2001]], device='cuda:0')
Epoch[31/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1877,  0.0328, -0.0857,  0.2131,  0.0874,  0.1535, -0.0993,  0.1540,
         -0.0336,  0.1982]], device='cuda:0')
Epoch[32/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1860,  0.0341, -0.0821,  0.2110,  0.0877,  0.1524, -0.0953,  0.1531,
         -0.0311,  0.1963]], device='cuda:0')
Epoch[33/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1842,  0.0354, -0.0786,  0.2088,  0.0880,  0.1514, -0.0913,  0.1522,
         -0.0287,  0.1944]], device='cuda:0')
Epoch[34/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1826,  0.0367, -0.0752,  0.2067,  0.0883,  0.1503, -0.0875,  0.1513,
         -0.0263,  0.1926]], device='cuda:0')
Epoch[35/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1809,  0.0379, -0.0718,  0.2046,  0.0886,  0.1493, -0.0837,  0.1504,
         -0.0240,  0.1908]], device='cuda:0')
Epoch[36/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1793,  0.0391, -0.0685,  0.2026,  0.0889,  0.1483, -0.0800,  0.1495,
         -0.0217,  0.1891]], device='cuda:0')
Epoch[37/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1777,  0.0403, -0.0652,  0.2006,  0.0891,  0.1473, -0.0763,  0.1487,
         -0.0195,  0.1874]], device='cuda:0')
Epoch[38/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1762,  0.0414, -0.0620,  0.1987,  0.0894,  0.1463, -0.0728,  0.1479,
         -0.0173,  0.1857]], device='cuda:0')
Epoch[39/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1747,  0.0425, -0.0589,  0.1968,  0.0896,  0.1454, -0.0693,  0.1470,
         -0.0151,  0.1840]], device='cuda:0')
Epoch[40/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1732,  0.0437, -0.0558,  0.1949,  0.0899,  0.1445, -0.0659,  0.1462,
         -0.0130,  0.1824]], device='cuda:0')
Epoch[41/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1717,  0.0447, -0.0528,  0.1930,  0.0901,  0.1436, -0.0625,  0.1454,
         -0.0109,  0.1808]], device='cuda:0')
Epoch[42/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1703,  0.0458, -0.0499,  0.1912,  0.0904,  0.1427, -0.0592,  0.1446,
         -0.0088,  0.1793]], device='cuda:0')
Epoch[43/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1689,  0.0469, -0.0470,  0.1895,  0.0906,  0.1418, -0.0560,  0.1439,
         -0.0068,  0.1778]], device='cuda:0')
Epoch[44/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1675,  0.0479, -0.0441,  0.1877,  0.0908,  0.1409, -0.0529,  0.1431,
         -0.0049,  0.1763]], device='cuda:0')
Epoch[45/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1662,  0.0489, -0.0414,  0.1861,  0.0911,  0.1401, -0.0498,  0.1424,
         -0.0029,  0.1748]], device='cuda:0')
Epoch[46/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1648,  0.0499, -0.0386,  0.1844,  0.0913,  0.1393, -0.0468,  0.1416,
         -0.0010,  0.1733]], device='cuda:0')
Epoch[47/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1636,  0.0508, -0.0360,  0.1828,  0.0915,  0.1385, -0.0438,  0.1409,
          0.0008,  0.1719]], device='cuda:0')
Epoch[48/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1623,  0.0518, -0.0333,  0.1811,  0.0917,  0.1377, -0.0409,  0.1402,
          0.0027,  0.1706]], device='cuda:0')
Epoch[49/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1611,  0.0527, -0.0308,  0.1796,  0.0919,  0.1369, -0.0380,  0.1395,
          0.0045,  0.1692]], device='cuda:0')
Epoch[50/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1598,  0.0536, -0.0282,  0.1780,  0.0921,  0.1362, -0.0353,  0.1388,
          0.0062,  0.1679]], device='cuda:0')
Epoch[51/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1586,  0.0545, -0.0258,  0.1765,  0.0923,  0.1354, -0.0325,  0.1382,
          0.0080,  0.1666]], device='cuda:0')
Epoch[52/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1575,  0.0554, -0.0233,  0.1750,  0.0925,  0.1347, -0.0299,  0.1375,
          0.0097,  0.1653]], device='cuda:0')
Epoch[53/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1563,  0.0563, -0.0210,  0.1736,  0.0927,  0.1340, -0.0272,  0.1369,
          0.0113,  0.1640]], device='cuda:0')
Epoch[54/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1552,  0.0571, -0.0186,  0.1722,  0.0928,  0.1333, -0.0247,  0.1362,
          0.0130,  0.1628]], device='cuda:0')
Epoch[55/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1541,  0.0579, -0.0163,  0.1708,  0.0930,  0.1326, -0.0221,  0.1356,
          0.0146,  0.1616]], device='cuda:0')
Epoch[56/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1530,  0.0587, -0.0141,  0.1694,  0.0932,  0.1320, -0.0197,  0.1350,
          0.0162,  0.1604]], device='cuda:0')
Epoch[57/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1520,  0.0595, -0.0119,  0.1681,  0.0934,  0.1313, -0.0173,  0.1344,
          0.0177,  0.1592]], device='cuda:0')
Epoch[58/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1509,  0.0603, -0.0097,  0.1667,  0.0935,  0.1307, -0.0149,  0.1338,
          0.0192,  0.1581]], device='cuda:0')
Epoch[59/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1499,  0.0611, -0.0076,  0.1654,  0.0937,  0.1301, -0.0126,  0.1332,
          0.0207,  0.1570]], device='cuda:0')
Epoch[60/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1489,  0.0618, -0.0055,  0.1642,  0.0938,  0.1294, -0.0103,  0.1326,
          0.0222,  0.1559]], device='cuda:0')
Epoch[61/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1480,  0.0626, -0.0035,  0.1629,  0.0940,  0.1288, -0.0081,  0.1320,
          0.0236,  0.1548]], device='cuda:0')
Epoch[62/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1470,  0.0633, -0.0015,  0.1617,  0.0941,  0.1283, -0.0059,  0.1315,
          0.0251,  0.1537]], device='cuda:0')
Epoch[63/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1461,  0.0640,  0.0005,  0.1605,  0.0943,  0.1277, -0.0038,  0.1309,
          0.0264,  0.1527]], device='cuda:0')
Epoch[64/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[ 0.1451,  0.0647,  0.0024,  0.1594,  0.0944,  0.1271, -0.0017,  0.1304,
          0.0278,  0.1517]], device='cuda:0')
Epoch[65/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1442, 0.0654, 0.0043, 0.1582, 0.0946, 0.1266, 0.0004, 0.1299, 0.0291,
         0.1507]], device='cuda:0')
Epoch[66/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1434, 0.0661, 0.0061, 0.1571, 0.0947, 0.1260, 0.0024, 0.1294, 0.0304,
         0.1497]], device='cuda:0')
Epoch[67/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1425, 0.0667, 0.0079, 0.1560, 0.0948, 0.1255, 0.0044, 0.1289, 0.0317,
         0.1488]], device='cuda:0')
Epoch[68/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1417, 0.0673, 0.0097, 0.1549, 0.0949, 0.1250, 0.0063, 0.1284, 0.0330,
         0.1478]], device='cuda:0')
Epoch[69/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1408, 0.0680, 0.0114, 0.1538, 0.0951, 0.1245, 0.0082, 0.1279, 0.0342,
         0.1469]], device='cuda:0')
Epoch[70/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1400, 0.0686, 0.0132, 0.1528, 0.0952, 0.1240, 0.0100, 0.1274, 0.0355,
         0.1460]], device='cuda:0')
Epoch[71/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1392, 0.0692, 0.0148, 0.1518, 0.0953, 0.1235, 0.0118, 0.1269, 0.0367,
         0.1451]], device='cuda:0')
Epoch[72/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1384, 0.0698, 0.0165, 0.1508, 0.0954, 0.1230, 0.0136, 0.1264, 0.0378,
         0.1442]], device='cuda:0')
Epoch[73/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1377, 0.0704, 0.0181, 0.1498, 0.0955, 0.1225, 0.0154, 0.1260, 0.0390,
         0.1434]], device='cuda:0')
Epoch[74/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1369, 0.0709, 0.0197, 0.1488, 0.0957, 0.1221, 0.0171, 0.1255, 0.0401,
         0.1426]], device='cuda:0')
Epoch[75/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1362, 0.0715, 0.0212, 0.1479, 0.0958, 0.1216, 0.0188, 0.1251, 0.0412,
         0.1417]], device='cuda:0')
Epoch[76/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1355, 0.0721, 0.0227, 0.1469, 0.0959, 0.1212, 0.0204, 0.1246, 0.0423,
         0.1409]], device='cuda:0')
Epoch[77/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1348, 0.0726, 0.0242, 0.1460, 0.0960, 0.1207, 0.0220, 0.1242, 0.0434,
         0.1402]], device='cuda:0')
Epoch[78/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1341, 0.0731, 0.0257, 0.1451, 0.0961, 0.1203, 0.0236, 0.1238, 0.0444,
         0.1394]], device='cuda:0')
Epoch[79/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1334, 0.0736, 0.0271, 0.1443, 0.0962, 0.1199, 0.0251, 0.1234, 0.0454,
         0.1386]], device='cuda:0')
Epoch[80/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1327, 0.0742, 0.0285, 0.1434, 0.0963, 0.1195, 0.0266, 0.1230, 0.0465,
         0.1379]], device='cuda:0')
Epoch[81/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1321, 0.0746, 0.0299, 0.1426, 0.0964, 0.1191, 0.0281, 0.1226, 0.0475,
         0.1371]], device='cuda:0')
Epoch[82/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1314, 0.0751, 0.0313, 0.1418, 0.0965, 0.1187, 0.0296, 0.1222, 0.0484,
         0.1364]], device='cuda:0')
Epoch[83/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1308, 0.0756, 0.0326, 0.1409, 0.0965, 0.1183, 0.0310, 0.1218, 0.0494,
         0.1357]], device='cuda:0')
Epoch[84/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1302, 0.0761, 0.0339, 0.1402, 0.0966, 0.1180, 0.0324, 0.1214, 0.0503,
         0.1350]], device='cuda:0')
Epoch[85/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1296, 0.0765, 0.0352, 0.1394, 0.0967, 0.1176, 0.0337, 0.1210, 0.0512,
         0.1344]], device='cuda:0')
Epoch[86/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1290, 0.0770, 0.0364, 0.1386, 0.0968, 0.1172, 0.0351, 0.1207, 0.0521,
         0.1337]], device='cuda:0')
Epoch[87/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1284, 0.0774, 0.0376, 0.1379, 0.0969, 0.1169, 0.0364, 0.1203, 0.0530,
         0.1331]], device='cuda:0')
Epoch[88/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1279, 0.0779, 0.0388, 0.1371, 0.0970, 0.1165, 0.0377, 0.1200, 0.0539,
         0.1324]], device='cuda:0')
Epoch[89/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1273, 0.0783, 0.0400, 0.1364, 0.0970, 0.1162, 0.0389, 0.1196, 0.0548,
         0.1318]], device='cuda:0')
Epoch[90/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1268, 0.0787, 0.0412, 0.1357, 0.0971, 0.1159, 0.0401, 0.1193, 0.0556,
         0.1312]], device='cuda:0')
Epoch[91/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1262, 0.0791, 0.0423, 0.1350, 0.0972, 0.1155, 0.0414, 0.1189, 0.0564,
         0.1306]], device='cuda:0')
Epoch[92/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1257, 0.0795, 0.0434, 0.1343, 0.0973, 0.1152, 0.0425, 0.1186, 0.0572,
         0.1300]], device='cuda:0')
Epoch[93/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1252, 0.0799, 0.0445, 0.1337, 0.0973, 0.1149, 0.0437, 0.1183, 0.0580,
         0.1294]], device='cuda:0')
Epoch[94/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1247, 0.0803, 0.0456, 0.1330, 0.0974, 0.1146, 0.0448, 0.1180, 0.0588,
         0.1289]], device='cuda:0')
Epoch[95/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1242, 0.0807, 0.0466, 0.1324, 0.0975, 0.1143, 0.0459, 0.1176, 0.0596,
         0.1283]], device='cuda:0')
Epoch[96/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1237, 0.0811, 0.0477, 0.1318, 0.0975, 0.1140, 0.0470, 0.1173, 0.0603,
         0.1278]], device='cuda:0')
Epoch[97/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1232, 0.0814, 0.0487, 0.1311, 0.0976, 0.1137, 0.0481, 0.1170, 0.0610,
         0.1272]], device='cuda:0')
Epoch[98/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1228, 0.0818, 0.0497, 0.1305, 0.0977, 0.1135, 0.0491, 0.1167, 0.0618,
         0.1267]], device='cuda:0')
Epoch[99/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1223, 0.0821, 0.0506, 0.1300, 0.0977, 0.1132, 0.0502, 0.1164, 0.0625,
         0.1262]], device='cuda:0')
Epoch[100/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1219, 0.0825, 0.0516, 0.1294, 0.0978, 0.1129, 0.0512, 0.1161, 0.0632,
         0.1257]], device='cuda:0')
...

Epoch[480/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[481/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[482/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[483/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[484/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[485/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[486/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[487/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[488/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[489/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[490/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[491/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[492/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[493/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[494/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[495/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[496/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[497/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[498/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[499/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
Epoch[500/500], bias: tensor([0.2830], device='cuda:0'), weight: tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
weight:  tensor([[0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000, 0.1000,
         0.1000]], device='cuda:0')
bias:  tensor([0.2830], device='cuda:0')


title: Accuracy and loss does not change with RMSprop optimizer

body:
The dataset is CIFAR10. I've created a VGG-like network:
class FirstModel(nn.Module):
    def __init__(self):
        super(FirstModel, self).__init__()
        self.vgg1 = nn.Sequential(
            nn.Conv2d(3, 16, 3, padding=1),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.Conv2d(16, 16, 3, padding=1),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(2,2),
            nn.Dropout(0.2)
        )

        self.vgg2 = nn.Sequential(
            nn.Conv2d(16, 32, 3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.Conv2d(32, 32, 3, padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(2,2),
            nn.Dropout(0.2)
        )

        self.vgg3 = nn.Sequential(
            nn.Conv2d(32, 64, 3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.Conv2d(64, 64, 3, padding=1),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2,2),
            nn.Dropout(0.2)
        )

        self.fc1 = nn.Linear(4 * 4 * 64, 4096)
        self.relu = nn.ReLU()
        self.fc2 = nn.Linear(4096, 4096)
        self.fc3 = nn.Linear(4096, 10)
        self.softmax = nn.Softmax()
        self.dropout = nn.Dropout(0.5)

    def forward(self, x):
        x = self.vgg3(self.vgg2(self.vgg1(x)))
        x = nn.Flatten()(x)
        x = self.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.relu(self.fc2(x))
        x = self.dropout(x)
        x = self.softmax(self.fc3(x))

        return x

Then I train it and visualize loss and accuracy:
import matplotlib.pyplot as plt
from IPython.display import clear_output

def plot_history(train_history, val_history, title='loss'):
    plt.figure()
    plt.title('{}'.format(title))
    plt.plot(train_history, label='train', zorder=1)

    points = np.array(val_history)
    steps = list(range(0, len(train_history) + 1, int(len(train_history) / len(val_history))))[1:]

    plt.scatter(steps, val_history, marker='*', s=180, c='red', label='val', zorder=2)
    plt.xlabel('train steps')

    plt.legend(loc='best')
    plt.grid()

    plt.show()

def train_model(model, optimizer, train_dataloader, test_dataloader):
      criterion = nn.CrossEntropyLoss()

      train_loss_log = []
      train_acc_log = []
      val_loss_log = []
      val_acc_log = []

      for epoch in range(NUM_EPOCH):

        model.train()

        train_loss = 0.
        train_size = 0
        train_acc = 0.

        for inputs, labels in train_dataloader:

          inputs, labels = inputs.to(device), labels.to(device)

          optimizer.zero_grad()
          y_pred = model(inputs)

          loss = criterion(y_pred, labels)
          loss.backward()
          optimizer.step()

          train_loss += loss.item()
          train_size += y_pred.size(0)
          train_loss_log.append(loss.data / y_pred.size(0))

          _, pred_classes = torch.max(y_pred, 1)
          train_acc += (pred_classes == labels).sum().item()
          train_acc_log.append(np.mean((pred_classes == labels).cpu().numpy()))

        # блок validation
        val_loss = 0.
        val_size = 0
        val_acc = 0.

        model.eval()
        with torch.no_grad():
            for inputs, labels in test_dataloader:
                inputs, labels = inputs.to(device), labels.to(device)
                y_pred = model(inputs)
                loss = criterion(y_pred, labels)
                val_loss += loss.item()
                val_size += y_pred.size(0)

                _, pred_classes = torch.max(y_pred, 1)
                val_acc += (pred_classes == labels).sum().item()

        val_loss_log.append(val_loss/val_size)
        val_acc_log.append(val_acc/val_size)

        clear_output()
        plot_history(train_loss_log, val_loss_log, 'loss')
        plot_history(train_acc_log, val_acc_log, 'accuracy')

        print('Train loss:', train_loss / train_size)
        print('Train acc:', train_acc / train_size)
        print('Val loss:', val_loss / val_size)
        print('Val acc:', val_acc / val_size)

Then I train the model:
first_model = FirstModel()
first_model.to(device)

optimizer = optim.RMSprop(first_model.parameters(), lr=0.001, momentum=0.9)

train_model(first_model_rms, optimizer, train_dataloader, test_dataloader)

The loss and accuracy do not change (accuracy at level of 0.1). However, if the optimizer is SGD with momentum everything works fine (loss and accuracy change). I've already tried to change momentum and lr, but it does not help.
What should be fixed? Would be grateful for any possible advice!


title: openCv and PyTorch inverser Transform not working

body:
I have a transforms class which only does:
        if transform is None:
            transform = transforms.Compose([
                transforms.Resize((256, 256)),
                transforms.ToTensor()
            ])
        root = os.path.join(PROJECT_ROOT_DIR, "data")
        super(AttributesDataset, self).__init__()
        self.data = torchvision.datasets.CelebA(
            root=root,
            split=split,
            target_type='attr',
            download=True,
            transform=transform
        )

From the documentation, I understand that this implies just a scale-down of values in the range 0,1 ie all pixel values shall lie between [0,1] (I have verified this as well).
I want to visualize some of the outputs coming from the model. As such, I created a simple method which does:-
    for img, label in dataloader:
        img.squeeze_(0)
        # permute the channels. cv2 expects image in format (h, w, c)
        unscaled_img = img.permute(1, 2, 0)
        # move images to cpu and convert to numpy as required by cv2 library
        unscaled_img = torch.round(unscaled_img * 255)
        unscaled_img = unscaled_img.to(torch.uint8)
        # unscaled_img = np.rint(unscaled_img * 255).astype(np.uint8)
        unscaled_img = cv2.cvtColor(unscaled_img, cv2.COLOR_RGB2BGR)
        cv2.imshow(unscaled_img.numpy())

However, all the images that are created have an unusually blue shade. For instance,

Can someone please tell me what exactly am I doing wrong here? Your help would be highly appreciated


title: What is the difference between these backward training methods in Pytorch?

body:
I am a 3-month DL freshman who is doing small NLP projects with Pytorch.
Recently I am trying to reappear a GAN network introduced by a paper, using my own text data, to generate some specific kinds of question sentences.

Here is some background... If you have no time or interest about it, just kindly read the following question is OK.
As that paper says, the generator is firstly trained normally with normal question data to make that the output at least looks like a real question. Then by using an auxiliary classifier's result (of classifying the outputs), the generator is trained again to just generate the specific (several unique categories) questions.
However, as the paper do not reveal its code, I have to do the code all myself. I have these three training thoughts, but I do not know their differences, could you kindly tell me about it?
If they have almost the same effect, could you tell me which is more recommended in Pytorch's grammar? Thank you very much!

Suppose the discriminator loss to generator is loss_G_D, the classifier loss to generator is loss_G_C, and loss_G_D and loss_G_C has the same shape, i.e. [batch_size, loss value], then what is the difference?
1.
optimizer.zero_grad()
loss_G_D = loss_func1(discriminator(generated_data))
loss_G_C = loss_func2(classifier(generated_data))
loss = loss_G+loss_C
loss.backward()
optimizer.step()

optimizer.zero_grad()
loss_G_D = loss_func1(discriminator(generated_data))
loss_G_D.backward()
loss_G_C = loss_func2(classifier(generated_data))
loss_G_C.backward()
optimizer.step()

optimizer.zero_grad()
loss_G_D = loss_func1(discriminator(generated_data))
loss_G_D.backward()
optimizer.step()

optimizer.zero_grad()
loss_G_C = loss_func2(classifier(generated_data))
loss_G_C.backward()
optimizer.step()

Additional info: I observed that the classifier's classification loss is always very big compared with generator's loss, like -300 vs 3. So maybe the third one is better?


title: How to use a learnable parameter in pytorch, constrained between 0 and 1?

body:
I want to use a learnable parameter that only takes values between 0 and 1. How can I do this in pytorch?
Currently I am using:
self.beta = Parameter(torch.Tensor(1))
#initialize
zeros(self.beta)

But I am getting zeros and NaN for this parameter, as I train.


I am trying to train a CNN in PyTorch on MNIST data. However, I am getting ValueError: Expected input batch_size (500) to match target batch_size (1000). This occurs when I run the test() command in the code below. I have looked up solutions to this problem but none of them help fix this issue.
My code is as follows:
n_epochs = 20
batch_size_train = 64
batch_size_test = 1000
learning_rate = 1e-4
log_interval = 50

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 64, kernel_size=5)
        self.conv2 = nn.Conv2d(64, 128, kernel_size=1)
        self.fc1 = nn.Linear(9216, 100)
        self.fc2 = nn.Linear(100, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2(x), 2))
        x = x.view(-1, 9216)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return F.log_softmax(x, dim=1)

    def loss_function(self, out, target):
        return F.cross_entropy(out, target)

def init_weights(m):
    if type(m) == nn.Linear or type(m) == nn.Conv2d:
        torch.nn.init.xavier_uniform_(m.weight)
        m.bias.data.fill_(0.01)

network = Net()
network.apply(init_weights)
network.cuda()

optimizer = optim.Adam(network.parameters(), lr=1e-4)

def train(epoch):
  network.train()
  for batch_idx, (data, target) in enumerate(train_loader):
    data = data.cuda()
    target = target.cuda()
    optimizer.zero_grad()
    output = network(data)
    loss = network.loss_function(output, target)
    loss.backward()
    optimizer.step()
    if batch_idx % log_interval == 0:
      print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
        epoch, batch_idx * len(data), len(train_loader.dataset),
        100. * batch_idx / len(train_loader), loss.item()))

def test():
  network.eval()
  test_loss = 0
  correct = 0
  with torch.no_grad():
    for data, target in test_loader:
      data = data.cuda()
      target = target.cuda()
      target = target.view(batch_size_test)
      output = network(data)
      test_loss += network.loss_function(output, target).item()
      pred = output.data.max(1, keepdim=True)[1]
      correct += pred.eq(target.data.view_as(pred)).sum()
  test_loss /= len(test_loader.dataset)
  print('\nTest set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
    test_loss, correct, len(test_loader.dataset),
    100. * correct / len(test_loader.dataset)))

test()
for epoch in range(1, n_epochs + 1):
  train(epoch)
  test()

Full error log:
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-12-ef6e122ea50c> in <module>()
----> 1 test()
      2 for epoch in range(1, n_epochs + 1):
      3   train(epoch)
      4   test()

3 frames
<ipython-input-9-23a4b65d1ae9> in test()
      9       target = target.view(batch_size_test)
     10       output = network(data)
---> 11       test_loss += network.loss_function(output, target).item()
     12       pred = output.data.max(1, keepdim=True)[1]
     13       correct += pred.eq(target.data.view_as(pred)).sum()

<ipython-input-5-d97bf44ef6f0> in loss_function(self, out, target)
     91
     92     def loss_function(self, out, target):
---> 93         return F.cross_entropy(out, target)

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce, reduction)
   2466     if size_average is not None or reduce is not None:
   2467         reduction = _Reduction.legacy_get_string(size_average, reduce)
-> 2468     return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
   2469
   2470

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2260     if input.size(0) != target.size(0):
   2261         raise ValueError('Expected input batch_size ({}) to match target batch_size ({}).'
-> 2262                          .format(input.size(0), target.size(0)))
   2263     if dim == 2:
   2264         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

ValueError: Expected input batch_size (500) to match target batch_size (1000).

Please let me know how to fix this.
Thanks,
Vinny


title: How to sample along with another dataloader in PyTorch

body:
Assume I have train/valid/test dataset with batch_size and shuffleed as normal.
When I do train/valid/test, I want to sample a certain number (called memory_size) of new samples from the entire dataset for each sample.
For example, I set batch_size as 256, let dataset shuffled, and memory_size as 80.
In every forward step, not only use each sample from dataset, but sample data from entire original dataset which size is memory_size and I want to use it inside forward. Let new samples as Memory (Yeah, I want to adopt idea from Memory Networks). Memory can be overlapped between each sample in train set.
I'm using PyTorch and PyTorch-Lightning. Can I create new memory dataloader per each train_dataloader, val_dataloader, and test_dataloader then load it with original dataloader? or is there a better way to achieve what I want?


title: Pytorch C++ API : CMake Issue

body:
I want to include the pytorch C++ API to the large C++ software I am working on.
For legacy reasons, I must use find_package  and the associated find_path and find_library functions, instead of the proposed target_link_libraries.
Here's my FindTORCH.cmake :
include( FindPackageHandleStandardArgs )

find_path( TORCH_INCLUDE_DIR torch/torch.h
           PATHS
           /path/to/libtorch/include/torch/csrc/api/include/
           NO_DEFAULT_PATH )

find_library( TORCH_LIBRARIES libtorch.so
              PATHS
              /path/to/libtorch/lib/
              NO_DEFAULT_PATH )

FIND_PACKAGE_HANDLE_STANDARD_ARGS( TORCH REQUIRED_VARS TORCH_INCLUDE_DIR TORCH_LIBRARIES )

if ( TORCH_FOUND )
message( STATUS "Torch found" )
endif( TORCH_FOUND )

mark_as_advanced( TORCH_LIBRARIES TORCH_INCLUDE_DIR )

At the compilation, the torch files are found and I can include <torch/torch.h> in a random .cxx in the project.
However, if I add to the .cxx :
torch::Tensor tensor = torch::rand({2, 3});
cout << tensor << std::endl;

Then I cannot compile anymore and I get the following error :
/path/to/libtorch/include/torch/csrc/api/include/torch/utils.h:4:10: fatal error: ATen/record_function.h: No such file or directory
 #include <ATen/record_function.h>
          ^~~~~~~~~~~~~~~~~~~~~~~~
compilation terminated.

I'm working Ubuntu 18, C++ 14, and the cmake version is 3.10.2 .
Thanks in advance


title: Fastai predict on collaboative learning model

body:
I have a Fast ai collaborative filtering model. I would like to predict on this model for a new tuple.
I am having trouble with the predict function
From their documentation,
Signature: learn.predict(item, rm_type_tfms=None, with_input=False)
Docstring: Prediction on `item`, fully decoded, loss function decoded and probabilities
File:      ~/playground/virtualenv/lib/python3.8/site-packages/fastai/learner.py
Type:      method

How do I define the Item that I need to pass. Lets say for a movielens dataset, for a user already with in the dataset, we would like to recommend a set of movies, how do we pass the userID?
I have tried to follow somewhat of an answer here - https://forums.fast.ai/t/making-predictions-with-collaborative-filtering/3900
learn.predict( [np.array([3])] )
I seem to get an error: TypeError: list indices must be integers or slices, not list


title: Getting Runtime error while making API using YOLOV4-Pytorch

body:
Trying to create a API using yolov4-pytorch but facing error
RuntimeError: shape '[256, 384, 3, 3]' is invalid for input of size 767781.
Using yolov4-tiny with pretrained weights
yolo.py file
import cv2
import matplotlib.pyplot as plt
from utils import *
from Darknet import Darknet

def process(uploads_dir, output_dir, filename):

    # Set the location and name of the cfg file
    cfg_file = './cfg/yolov4-tiny.cfg'

    # Set the location and name of the pre-trained weights file
    weight_file = './weights/yolov4-tiny.weights'

    # Set the location and name of the COCO object classes file
    namesfile = 'data/coco.names'

    # Load the network architecture
    m = Darknet(cfg_file)   #

    # Load the pre-trained weights
    m.load_weights(weight_file)

    # Load the COCO object classes
    class_names = load_class_names(namesfile)

    # Set the default figure size
    plt.rcParams['figure.figsize'] = [24.0, 14.0]

    # Set the NMS threshold
    nms_thresh = 0.6

    # Set the IOU threshold
    iou_thresh = 0.4

    # Set the default figure size
    plt.rcParams['figure.figsize'] = [24.0, 14.0]

    # Load the image
    img = cv2.imread(uploads_dir + '/' + filename + '.jpg')

    # Convert the image to RGB
    original_image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    # We resize the image to the input width and height of the first layer of the network.
    resized_image = cv2.resize(original_image, (m.width, m.height))

    # Set the IOU threshold. Default value is 0.4
    iou_thresh = 0.4

    # Set the NMS threshold. Default value is 0.6
    nms_thresh = 0.6

    # Detect objects in the image
    boxes = detect_objects(m, resized_image, iou_thresh, nms_thresh)

    # Print and save the objects found and their confidence levels
    objects_count, objects_confidence = print_objects(boxes, class_names)

    # Plot the image with bounding boxes and corresponding object class labels
    plot_boxes(original_image, boxes, class_names, output_dir, filename)

    return objects_count, objects_confidence

app.py
from flask import Flask, request, jsonify
import os
from werkzeug.utils import secure_filename
from flask_cors import CORS
from yolo import process
from datetime import datetime
from random import randint
from Darknet import *

app = Flask(__name__)
CORS(app)
uploads_dir = os.path.join(app.instance_path, 'uploads')
output_dir = os.path.join(app.instance_path, 'output')

@app.route('/upload', methods=['POST'])
def upload_image():
    try:
        os.mkdir(uploads_dir)
        os.mkdir(output_dir)
    except:
        pass

    file = request.files['file']
    if not file:
        return {'error': 'Missing file'}, 400

    now = datetime.now()
    filename = now.strftime("%Y%m%d_%H%M%S") + "_" + str(randint(000, 999))
    file.save(os.path.join(uploads_dir, secure_filename(filename + '.jpg')))
    objects_count, objects_confidence = process(uploads_dir, output_dir, filename)

    response = {
        'objects_count': objects_count,
        'objects_confidence': objects_confidence,
        'filename': filename + '.jpg'
    }

    return jsonify({"data": response}), 200

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=5000, debug=True)

Iam using this repo https://github.com/Tianxiaomo/pytorch-YOLOv4, I have renamed tool>darknet2pytorch as Darknet


title: Computing the loss of a function of predictions with pytorch

body:
I have a convolutional neural network that predicts 3 quantities: Ux, Uy, and P. These are the x velocity, y-velocity, and pressure field. They are all 2D arrays of size [100,60], and my batch size is 10.
I want to compute the loss and update the network by calculating the CURL of the predicted velocity with the CURL of the target velocity. I have a function that does this: v = curl(Ux_pred, Uy_pred). Given the predicted Ux and Uy, I want to compute the loss by comparing it to ground truth targets that I have: true_curl = curl(Ux_true, Uy_true) - I've already computed the true curl and added it to my Y data, as the fourth channel.
However, I want my network to only predict Ux, Uy, and P. I want my NN parameters to update based on the LOSS of the curls to improve the accuracy of Ux and Uy. The loss of the curl has to be in terms of Ux and Uy. I have been trying to do this using Pytorch autograd, and have already read many similar questions, but I just can't get it to work. This is my code so far:
        print("pred_Curl shape:", np.shape(pred_curl))
        print("pred_Ux shape:", np.shape(pred[:,0,:,:]))
        print("pred_Uy shape:", np.shape(pred[:,1,:,:]))
        true_curl = torch.from_numpy(y[:,3,:,:]) # not sure where to use the true curl?

        pred_curl = Variable(pred_curl, requires_grad=True)

        pred_ux = pred[:,0,:,:]
        pred_uy = pred[:,1,:,:]

        pred_ux = Variable(pred_ux, requires_grad=True)
        pred_uy = Variable(pred_uy, requires_grad=True)

        grad_tensor = torch.autograd.grad(outputs=pred_curl, inputs=(pred_ux, pred_uy),
                       grad_outputs=torch.ones_like(pred_curl),
                       retain_graph=True,create_graph=True)

        loss = torch.sum(grad_tensor)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

This has the following output:
pred_Curl shape: torch.Size([10, 100, 60])
pred_Ux shape: torch.Size([10, 100, 60])
pred_Uy shape: torch.Size([10, 100, 60])

RuntimeError: One of the differentiated Tensors appears to not have been used in the graph.
Set allow_unused=True if this is the desired behavior.

Any help would be appreciated!
Edit: Here is my curl function:
    def discrete_curl(self,x,y,new_arr):
            for m in range(100):
                for n in range(60):
                    if n <= 58:
                        if m <= 98:
                            if x[m,n] != 0 and y[m,n] != 0:
                                new_arr[m,n] = ((y[m+1,n] - y[m-1,n]) / 2*1) - ((x[m,n+1] - x[m,n-1]) / 2*1)
            return new_arr

Where x and y are Ux ad Uy, and new_arr is the curl output.


title: How to speed up add/sub using GPU cuda.FloatTensor?

body:
I am trying to speed up the PSO algorithm. I have two question, want can I do to speed up the following:

1) The creation of the particles;

2) The movement equation.

  1) x = (xmax - xmin)*torch.rand(pop).dtype + xmin

  2) vnext = w*torch.rand(pop).dtype + c1*torch.rand(pop).dtype*(b - x) + c2*torch.rand(pop).dtype(bg - x)

When I use dtype = torch.cuda.FloatTensor is slower then using dtype = torch.FloatTensor.
How can I speed up the math using the GPU?


title: Policy network for the game 2048

body:
I'm trying to implement a policy network agent for the game 2048 according to Karpathy's RL tutorial. I know the algorithm will need to play some batch of games, remember the inputs and actions taken, normalize and mean center the ending scores. However, I got stuck at the design of the loss function. How to correctly encourage actions that lead to the better final scores and discourage those that lead to worse scores?

When using softmax at the output layer, I devised something along this:

loss = sum((action - net_output) * reward)

where action is in one hot format. However, this loss doesn't seem to do much, the network doesn't learn. My full code (without the game environment) in PyTorch is here.


title: Getting different output in the Pytorch NLP example Part-of-Speech Tagging

body:
I am following the NLP tutorials on Pytorch’s tutorials website. I am getting different output than what it should show, so I just copy pasted the whole code as it is and still the output is different.

My code is shared in this gist:

Example: An LSTM for Part-of-Speech Tagging

For the 1st sentence

[‘The’, ‘dog’, ‘ate’, ‘the’, ‘apple’]
[‘DET’, ‘NN’, ‘V’, ‘DET’, ‘NN’]

the output is coming as below:

tensor([[-0.7662, -0.6405, -4.8002],
[-2.7163, -0.0698, -6.6515],
[-3.1324, -5.7668, -0.0479],
[-0.0528, -3.3832, -4.0481],
[-2.4527, -0.0931, -5.8702]])

I am getting the sequence: 1 1 2 0 1 rather than 0 1 2 0 1

Can anyone please check this and point out why I am getting different output?


title: Is this the best way to compute kernel in latest version of PyTorch (TensorFlow to PyTorch conversion)?

body:
I’m trying to convert a MMD-VAE implementation from TensorFlow to PyTorch. I’ve got most of the model built just fine but I just want to make sure that I am converting the following functions correctly (Everything is working but I’m not getting the results I expect so I thought maybe I am computing the kernel incorrectly as I am not so strong in TensorFlow).

In TensorFlow:

def compute_kernel(x, y):
    x_size = tf.shape(x)[0]
    y_size = tf.shape(y)[0]
    dim = tf.shape(x)[1]
    tiled_x = tf.tile(tf.reshape(x, tf.stack([x_size, 1, dim])), tf.stack([1, y_size, 1]))
    tiled_y = tf.tile(tf.reshape(y, tf.stack([1, y_size, dim])), tf.stack([x_size, 1, 1]))
    return tf.exp(-tf.reduce_mean(tf.square(tiled_x - tiled_y), axis=2) / tf.cast(dim, tf.float32))

And what I have for PyTorch:

def compute_kernel(x, y):
    x_size = x.size(0)
    y_size = y.size(0)
    dim = x.size(1)
    x = x.unsqueeze(1)
    y = y.unsqueeze(0)
    tiled_x = x.expand(x_size, y_size, dim)
    tiled_y = y.expand(x_size, y_size, dim)
    kernel_input = (tiled_x - tiled_y).pow(2).mean(2)/float(dim)
    return torch.exp(-kernel_input)

Thanks for your help!!


title: Register_hook with respect to a subtensor of a tensor

body: Assuming that we would like to modify gradients only of a part of the variable values, is it possible to register_hook in pytorch only to a subtensor (of a tensor that is a pytorch variable)?


title: Error installing feeder/feedr library in python

body: I am trying to run a code which uses feeder (feedr on git repository) library. I am getting error while installing it. I have tried pip, cloning git, and previous versions but still getting same error on all PC. Please tell how to solve it or any alternative for feeder. Original code link is https://github.com/nashory/DeLF-pytorch/tree/master/notebook enter image description here


title: Densenet with Hinge loss on CIFAR dataset

body:
I am trying to use Hinge loss with densenet on the CIFAR 100 dataset. The learning converges to some point and after that there is no learning. The accuracy is much less than Densenet with CrossEntropy loss function. I tried with different learning rates and weight decays.

Any ideas on why I am unable to train properly Densenet with Hinge loss? I am able to use Hinge loss with Resnet without any problem.


title: output:\ntorch-1.1.0-cp27-cp27mu-linux_x86_64.whl is not a supported wheel on this platform- Pytorch /cloud functions

body:
I'm trying to deploy a function on Cloud functions and I'm having trouble getting pytorch to work. I need either version 1.1, 1.2 or 1.3 (any version that has torch.hub functionality)

Here is what I have been trying in the requirments.txt for my function:

numpy==1.17.2
https://download.pytorch.org/whl/cpu/torch-1.1.0-cp27-cp27mu-linux_x86_64.whl

Which results in the error:

Build failed: {"error": {"canonicalCode": "INVALID_ARGUMENT", "errorMessage": "`pip_download_wheels` had stderr output:\ntorch-1.1.0-cp27-cp27mu-linux_x86_64.whl is not a supported wheel on this platform.\n\nerror: `pip_download_wheels` returned code: 1", "errorType": "InternalError", "errorId": XXX}}

And of course trying the various other url for 1.2 and 1.3 and the same thing happens.

What can I do to fix this/ what am I doing wrong??

Appreciate the help.

-if this affects anything, I'm using Python 3.7 for my function and have been using the linux packages.

__________________________________________________________________________

Edit: I've tried this in my requirments.txt:

numpy==1.17.0
torch==1.3.0
torchvision===0.4.1

And I now get the error:

Build failed: {"cacheStats": [{"status": "MISS", "hash": "1f6ebb5b3667b3d677184dbf04b82666XXX", "type": "docker_layer_cache", "level": "global"}, {"status": "MISS", "hash": "1f6ebb5b3667b3d677184dbf04b826660b67c784608d4e4XXXXX", "type": "docker_layer_cache", "level": "project"}]}

I've never had this error with any other library on cloud functions. If anyone has other suggestions it would be greatly appreciated.


title: GRU included CNN for text Generator

body:
I try to integrate CNN to GRU. My model gets the image through CNN. The features from CNN will pass to the GRU frame by frame. The structure is shown in the picture.

This is my example code that implemented follows the above structure.
encoder :
### input Image size [batch,seq,colorch,hight,weight]
### expext output

class CNNencoder(nn.Module):
def __init__(self, input_size, hidden_size,batch_size =5):
    super(CNNencoder, self).__init__()
    self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    self.hidden_size = hidden_size
    self.modelVGG = models.vgg11(pretrained = False)
    self.modelVGG = self.modelVGG.to(self.device)

    self.adaptor = nn.Linear(8192, self.hidden_size)
    self.adaptor = self.adaptor.to(self.device)

    self.gru = nn.GRU(hidden_size, hidden_size, batch_first=True, bidirectional=False)  ## (inputSize, hidden_size)
    self.gru = self.gru.to(self.device)
    self.batch_size = batch_size

def forward(self, input, hidden):
    seqs = input.size()[1]
    for indexseq in range(0, seqs):
        inputImageBatch = input[:, indexseq,:,:,:].view(-1,3,128,128)
        features = self.modelVGG.features(inputImageBatch)

        flat_features = features.view(features.size(0), 1,-1)  # flatten
        if indexseq == 0:
            output = flat_features
        else:
            output = torch.cat((output, flat_features), dim=1)

    # output = flat_features    ## expected   [batch,seq,features]
    outputAdaptor = self.adaptor(output)
    outputGru, hidden = self.gru(outputAdaptor, hidden)
    return outputGru, hidden

def initHidden(self):
    return torch.zeros(1, self.batch_size, self.hidden_size, device=self.device)

I would like to know. How CNN parameter will get gradient through time?
If I create a model from this class and execute loss.backword().
loss values get from decoder section.


title: How to change weight in loss function without affecting the back propagation?

body:
I have a ground truth tensor with zeros and integers, I want to change the weight by multiplying the nonzero values by 5 in the loss function, but it will affect the back propagation and causes error, Is there any other way I can achieve this in the Loss function?

def MSE(output,truth,batch_size):
   for i in range(batch_size):
    for pos,j in enumerate(truth[i]):
        if j != 0:
            output[i][pos] = output[i][pos]*5

   mse = ((truth.sub(output))**2).mean()
   return mse

Traning sections below

for epoch in range(EPOCH):
        losses = []
        for step,(im,pointlist) in enumerate(train_loader):

            im = im.to(device)
            output = cnn(im)
            y = torch.stack(pointlist,dim=1)
            y = y.type(torch.FloatTensor)
            y = y.to(device)
            loss = MSE(output,y,batch_size)
            optimizer.zero_grad()
            loss.backward()                 # backpropagation, compute gradients
            optimizer.step()

            losses.append(loss.item())
        scheduler.step()
        print("Epoch={0},Loss={1:.6F}".format(epoch, np.average(np.array(losses))))

The error occurred at loss.backward()

    Traceback (most recent call last):
      File "CNN_Icarus.py", line 213, in <module>
        main()
    File "CNN_Icarus.py", line 203, in main
    loss.backward()# backpropagation, compute gradients
    File "C:\Users\et302\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\tensor.py", line 166, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
    File "C:\Users\et302\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\autograd\__init__.py", line 99, in backward
    allow_unreachable=True)  # allow_unreachable flag

RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [4, 20]], which is output 0 of SigmoidBackward, is at version 54; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).


title: How do I know whether an instance is stored on GPU with Pytorch?

body:
I'm learning Pytorch recently, and this question comes up.
For example, if I have a net inheriting the "torch.nn.Module".

class Net(torch.nn.Module):
    def __init__(self, something):
        super(net, self).__init__()
        self.p1=something

    def forward():
        pass

net1=Net(123)
net1.cuda()  ##Here I can't see what is changed.

Then how can I know whether net1 (and that something) is stored on GPU.

I've read how the *.cuda() works, seems like let all the "children" run the *.cuda(). I tried to see what the "children" are. It seems the net1 above has no children.


title: Can I combine Monte Carlo policy gradient algorithm with other policy gradient algorithms

body:
I know that Monte Carlo REINFORCE policy gradient algorithm is different in how it calculates the reward values by calculating discounted cumulative future reward at each step.

here is the peace of code to calculate the discounted cumulative future reward at each time step.

G = np.zeros_like(self.reward_memory, dtype=np.float64)
        for t in range(len(self.reward_memory)):
            G_sum = 0
            discount = 1
            for k in range(t, len(self.reward_memory)):
                G_sum += self.reward_memory[k] * discount
                discount *= self.gamma
            G[t] = G_sum

another example for increasing accuracy is to calculate the reward after the action called "reward to go".
another example is to add the entropy bonus.

Is it possible to add the entropy bonus and the rewards to go or either one to the Monte Carlo method.

Also another step is taken in the Monte Carlo after the reward calculation is to normalize the values.

  “In practice it can can also be important to normalize these. For example, suppose we compute [discounted cumulative reward] for all of the 20,000 actions in the batch of 100 Pong game rollouts above. One good idea is to “standardize” these returns (e.g. subtract mean, divide by standard deviation) before we plug them into backprop. This way we’re always encouraging and discouraging roughly half of the performed actions. Mathematically you can also interpret these tricks as a way of controlling the variance of the policy gradient estimator”.

Does that effect the accuracy if both or either of the entropy bonus or reward to go modification is added?

That is from the research PDF https://arxiv.org/pdf/1506.02438.pdf

I am Studying Policy Gradient Algorithms and I want to know how to Improve these algorithms. I would greatly appreciated if you could help me out.

Edit:

I would also like to add on whether the advantage function could also be added

The A(s,a) is the advantages function; is it possible to add this to the Monte Carlo approach assuming we also add both reward to go and the entropy bonus?


title: Should we train the original data point when we do data augmentation?

body:
I am confused about the definition of data augmentation. Should we train the original data points and the transformed ones or just the transformed? If we train both, then we will increase the size of the dataset while the second approach won't.

I got this question when using the function RandomResizedCrop.

'train': transforms.Compose([
        transforms.RandomResizedCrop(224),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),

If we resize and crop some of the dataset randomly, we don't actually increase the size of the dataset for data augmentation. Is that correct? Or data augmentation just requires the change/modification of original dataset rather than increase the size of it?

Thanks.


title: About using RNN in pytorch

body:
I am trying to use RNN to do a binary classification. But when my model is training, it gets stuck at loss.backward().
Here is my model:

class RNN2(nn.Module):
    def __init__(self, input_size, hidden_size, output_size=2, num_layers=1):
        super(RNN2, self).__init__()
        self.rnn = nn.RNN(input_size, hidden_size, num_layers)
        self.reg = nn.Linear(hidden_size, output_size)
        #self.softmax = nn.LogSoftmax(dim=1)

    def forward(self,x):
        x, hidden = self.rnn(x)
        return self.reg(x[:,2])

rnn = RNN2(13,10)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(rnn.parameters(), lr=learning_rate)
for e in range(10):
    out = rnn(train_X)
    optimizer.zero_grad()
    print(out[0])
    print(out.shape)
    print(train_Y.shape)
    loss = criterion(out, train_Y)
    print(loss)
    loss.backward()
    print("1")
    optimizer.step()
    print("2")

The shape of train_X is 420000*3*13 and the shape of train_Y is 420000
So it can print loss. Can anyone tell me why it gets stuck at loss.backward(). It can't print 1.


title: Enumerate array placed in torch.autograd.Variable

body:
I have variable xy that loads data from file and is used to create x_data and y_data variables of type torch.autograd.Variable . I suppose  x_data and y_data content are arrays. But how to enumerate that array in order to print content?

d = r"C:\gdrive\python\diabetes.csv"
import torch
from torch.autograd import Variable
import numpy as np
import torch.nn.functional as F
xy = np.loadtxt(d,delimiter=',',dtype=np.float32 )
print("***")
print(xy)
x_data = Variable(torch.from_numpy(xy[:,0:-1]))
y_data=  Variable(torch.from_numpy(xy[:,[-1]]))


title: Label value of -1 which is outside the valid range

body:
I’m working through the Pytorch text tokenization example here using the tensorflow version run_tf_ner.py: https://github.com/huggingface/transformers/tree/master/examples/token-classification

The PyTorch version using run_ner.py works. I believe the difference is in https://github.com/huggingface/transformers/blob/master/examples/token-classification/utils_ner.py starting at line 139 where it has different logic for tensorflow. Narrowing down to utils_ner.py line 149:

pad_token_label_id: int = -1

The Pytorch version uses -100. This seems to be the only labeling difference that I can tell. I tried to change this line to -100 as well. But tensorflow code doesn’t seem to accept -1 or -100.

Anyone know how to get this example to work using the tensorflow version? Thanks\

I created a Colab for the example here:
https://colab.research.google.com/drive/10GrYSYx5sUVMXplgUS79fIV73bFtoiHX?usp=sharing

This is the error I keep getting:
\
    05/31/2020 00:07:45 - INFO - utils_ner -   tokens: [CLS] dar ##aus en ##t ##wick ##elt ##e sic ##h im ro ##ko ##ko die sit ##te des gem ##ein ##sam ##en wei ##nen ##s im theater , das die stand ##es ##gren ##zen inner ##hal ##b des pub ##lik ##ums uber ##bruck ##en sol ##lt ##e . [SEP]
    05/31/2020 00:07:45 - INFO - utils_ner -   input_ids: 101 18243 20559 4372 2102 7184 20042 2063 14387 2232 10047 20996 3683 3683 3280 4133 2618 4078 17070 12377 21559 2368 11417 10224 2015 10047 4258 1010 8695 3280 3233 2229 13565 10431 5110 8865 2497 4078 9047 18393 18163 19169 28985 2368 14017 7096 2063 1012 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    05/31/2020 00:07:45 - INFO - utils_ner -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    05/31/2020 00:07:45 - INFO - utils_ner -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
    05/31/2020 00:07:45 - INFO - utils_ner -   label_ids: -1 24 -1 24 -1 -1 -1 -1 24 -1 24 6 -1 -1 24 24 -1 24 24 -1 -1 -1 24 -1 -1 24 24 24 24 24 24 -1 -1 -1 24 -1 -1 24 24 -1 -1 24 -1 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
    05/31/2020 00:07:54 - WARNING - transformers.training_args -   Using deprecated --per_gpu_train_batch_size argument which will be removed in a future version. Using --per_device_train_batch_size is preferred.
    05/31/2020 00:07:54 - WARNING - transformers.training_args -   Using deprecated --per_gpu_train_batch_size argument which will be removed in a future version. Using --per_device_train_batch_size is preferred.
    05/31/2020 00:07:54 - INFO - transformers.trainer_tf -   Created an/a adam optimizer
    05/31/2020 00:07:54 - INFO - transformers.trainer_tf -   ***** Running training *****
    05/31/2020 00:07:54 - INFO - transformers.trainer_tf -     Num examples = 24000
    05/31/2020 00:07:54 - INFO - transformers.trainer_tf -     Num Epochs = 3
    05/31/2020 00:07:54 - INFO - transformers.trainer_tf -     Total optimization steps = 750
    WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/transformers/trainer_tf.py:360: StrategyBase.experimental_run_v2 (from tensorflow.python.distribute.distribute_lib) is deprecated and will be removed in a future version.
    Instructions for updating:
    renamed to run
    05/31/2020 00:07:54 - WARNING - tensorflow -   From /usr/local/lib/python3.6/dist-packages/transformers/trainer_tf.py:360: StrategyBase.experimental_run_v2 (from tensorflow.python.distribute.distribute_lib) is deprecated and will be removed in a future version.
    Instructions for updating:
    renamed to run
    WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.init (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
    Instructions for updating:
    If using Keras pass *_constraint arguments to layers.
    05/31/2020 00:08:02 - WARNING - tensorflow -   From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.init (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
    Instructions for updating:
    If using Keras pass *_constraint arguments to layers.
    /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.
      "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "
    2020-05-31 00:08:52.364556: W tensorflow/core/framework/op_kernel.cc:1753] OP_REQUIRES failed at sparse_xent_op.cc:90 : Invalid argument: Received a label value of -1 which is outside the valid range of [0, 25).  Label values: -1 24 -1 -1 -1 -1 24 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 24 24 2 -1 -1 -1 24 0 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 -1 -1 -1 24 24 -1 -1 24 -1 -1 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 -1 -1 -1 -1 24 -1 24 -1 24 -1 24 -1 -1 24 -1 24 24 6 24 -1 24 -1 24 -1 24 -1 -1 -1 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 0 24 3 -1 24 24 24 24 -1 -1 1 24 -1 -1 -1 -1 9 21 -1 -1 24 0 -1 -1 -1 24 24 24 24 -1 -1 24 24 -1 -1 24 24 -1 -1 -1 3 15 15 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 24 24 -1 -1 24 -1 24 24 -1 -1 24 -1 24 -1 -1 -1 -1 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 6 24 24 24 6 18 18 24 24 6 24 -1 -1 24 24 24 -1 -1 -1 24 24 9 21 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 -1 24 -1 24 -1 24 -1 24 -1 24 -1 24 24 24 -1 -1 24 24 24 -1 24 -1 24 -1 24 -1 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 24 -1 -1 -1 24 -1 -1 -1 24 -1 9 24 9 21 24 24 -1 24 24 0 -1 -1 24 -1 -1 24 24 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 24 -1 24 -1 24 24 -1 -1 -1 24 -1 24 24 -1 24 -1 24 -1 24 -1 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 24 -1 -1 24 -1 0 -1 24 2 -1 -1 -1 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 -1 -1 -1 -1 -1 24 24 3 -1 -1 -1 -1 -1 -1 24 -1 24 -1 -1 24 24 -1 -1 24 24 24 -1 -1 -1 24 24 -1 -1 -1 -1 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 24 24 24 -1 24 -1 24 -1 24 -1 24 24 -1 24 -1 -1 24 -1 24 24 24 24 -1 -1 -1 -1 24 24 -1 -1 -1 -1 24 -1 -1 24 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 -1 24 24 -1 24 24 24 -1 -1 24 -1 -1 -1 24 -1 24 -1 -1 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 24 -1 -1 -1 24 -1 -1 24 -1 24 -1 -1 24 24 -1 -1 24 -1 -1 -1 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 24 24 -1 24 -1 24 24 24 24 -1 9 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 -1 24 -1 24 24 -1 24 -1 24 -1 24 -1 24 24 24 24 -1 -1 24 -1 24 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 24 -1 -1 24 -1 24 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 24 -1 24 24 -1 24 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 6 18 24 -1 -1 24 -1 -1 24 -1 -1 24 24 -1 -1 -1 -1 -1 24 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 8 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 24 -1 -1 -1 24 24 -1 -1 -1 8 -1 -1 -1 -1 -1 -1 -1 24 24 -1 24 -1 24 -1 24 24 -1 -1 -1 24 -1 -1 24 24 24 24 8 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 -1 -1 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 6 -1 -1 18 18 -1 -1 -1 18 18 18 -1 18 18 -1 -1 -1 24 24 -1 -1 -1 -1 24 24 0 -1 24 -1 -1 -1 24 -1 -1 24 -1 -1 -1 24 24 -1 -1 24 24 -1 24 24 0 -1 24 -1 -1 24 24 -1 24 5 -1 -1 -1 -1 -1 -1 24 0 12 12 24 -1 -1 -1 24 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 24 24 24 24 24 -1 -1 -1 -1 -1 24 -1 -1 -1 24 -1 -1 24 24 -1 24 24 24 24 -1 -1 24 24 -1 -1 -1 -1 0 -1 -1 -1 24 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 24 24 24 -1 24 -1 24 -1 6 -1 -1 24 24 1 -1 24 -1 -1 -1 -1 -1 -1 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 -1 -1 24 3 -1 -1 24 24 24 24 24 24 -1 24 -1 24 24 24 -1 24 -1 -1 -1 -1 24 24 -1 -1 -1 -1 24 24 -1 -1 -1 -1 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 -1 24 24 -1 24 24 24 -1 -1 -1 24 24 -1 24 -1 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 24 -1 -1 -1 -1 -1 24 24 24 -1 -1 -1 -1 24 -1 24 -1 -1 24 24 3 15 15 -1 -1 -1 -1 -1 24 0 12 12 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 0 -1 -1 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 24 24 24 -1 0 -1 -1 24 -1 24 -1 24 -1 24 -1 -1 24 24 -1 -1 -1 24 24 -1 -1 24 -1 -1 -1 -1 24 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 -1 9 21 -1 24 9 -1 21 24 9 21 24 24 -1 -1 -1 -1 24 -1 -1 -1 -1 24 -1 -1 24 24 -1 24 -1 24 24 -1 24 24 -1 -1 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 24 24 -1 24 -1 24 -1 24 24 1 -1 -1 24 24 24 -1 24 -1 -1 24 24 -1 -1 -1 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 9 -1 24 -1 -1 -1 -1 24 -1 24 24 -1 24 24 9 24 24 24 24 -1 -1 -1 -1 24 24 -1 -1 -1 24 0 24 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 9 -1 24 24 -1 24 24 -1 24 -1 -1 -1 -1 24 -1 -1 -1 24 -1 -1 -1 -1 -1 24 24 24 -1 24 -1 -1 24 24 -1 24 -1 -1 -1 24 24 6 -1 -1 24 24 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 24 -1 -1 24 -1 -1 -1 -1 24 -1 24 24 -1 -1 -1 -1 -1 24 24 -1 -1 -1 -1 24 -1 -1 24 24 -1 24 24 -1 24 24 -1 -1 -1 24 -1 -1 -1 -1 24 -1 24 -1 24 24 -1 -1 24 24 -1 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 24 -1 -1 24 24 -1 -1 -1 -1 24 -1 -1 24 -1 -1 -1 -1 -1 -1 24 -1 24 -1 -1 24 24 24 -1 -1 -1 -1 -1 24 24 24 -1 -1 -1 24 -1 -1 24 -1 -1 24 -1 24 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1 -1
    Traceback (most recent call last):
      File "run_tf_ner.py", line 281, in
        main()
      File "run_tf_ner.py", line 213, in main
        trainer.train()
      File "/usr/local/lib/python3.6/dist-packages/transformers/trainer_tf.py", line 277, in train
        for training_loss in self._training_steps():
      File "/usr/local/lib/python3.6/dist-packages/transformers/trainer_tf.py", line 321, in _training_steps
        for i, loss in enumerate(self._accumulate_next_gradients()):
      File "/usr/local/lib/python3.6/dist-packages/transformers/trainer_tf.py", line 354, in _accumulate_next_gradients
        yield _accumulate_next()
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py", line 580, in call
        result = self._call(*args, **kwds)
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py", line 708, in _call
        return function_lib.defun(fn_with_cond)(*canon_args, **canon_kwds)
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 2420, in call
        return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 1665, in _filtered_call
        self.captured_inputs)
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 1746, in _call_flat
        ctx, args, cancellation_manager=cancellation_manager))
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py", line 598, in call
        ctx=ctx)
      File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py", line 60, in quick_execute
        inputs, attrs, num_outputs)
    tensorflow.python.framework.errors_impl.InvalidArgumentError:  Received a label value of -1 which is outside the valid range of [0, 25).  [[{{node cond/else/_1/StatefulPartitionedCall/sparse_categorical_crossentropy/SparseSoftmaxCrossEntropyWithLogits/SparseSoftmaxCrossEntropyWithLogits}}]] [Op:__inference_fn_with_cond_30948]

Function call stack:
fn_with_cond


title: Point Cloud classification bz using Pytorch geometric

body:
I try to run the benchmarks for point cloud classification.I don't know how to use the benchmarks.
For example, i want to reproduce point cloud for epoch 100 on ModelNet10 dataset.
https://github.com/rusty1s/pytorch_geometric/tree/master/benchmark/points
How do I operate?


title: Can't load models from torchvision on GPU clusters in my company

body:
codes:
from torchvision.models import vgg16
vgg_model = vgg16(pretrained=True).features[:16]

the log output:
Traceback (most recent call last):
  File "train.py", line 95, in <module>
    vgg_model = vgg16(pretrained=True).features[:16]
  File "/opt/conda/lib/python3.7/site-packages/torchvision/models/vgg.py", line 150, in vgg16
    return _vgg('vgg16', 'D', False, pretrained, progress, **kwargs)
  File "/opt/conda/lib/python3.7/site-packages/torchvision/models/vgg.py", line 93, in _vgg
    progress=progress)
  File "/opt/conda/lib/python3.7/site-packages/torch/hub.py", line 480, in load_state_dict_from_url
    os.makedirs(model_dir)
  File "/opt/conda/lib/python3.7/os.py", line 213, in makedirs
    makedirs(head, exist_ok=exist_ok)
  File "/opt/conda/lib/python3.7/os.py", line 213, in makedirs
    makedirs(head, exist_ok=exist_ok)
  File "/opt/conda/lib/python3.7/os.py", line 223, in makedirs
    mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/.cache'

I have to run my codes on GPU clusters, and I have no rights to modify these docker images on GPU clusters. What can I do?


title: PyTorch - Basic math operations on different shaped numpy arrays

body:
1:
I have two numpy arrays with different shapes, I intent to get the mean squared error from the resulting operation. The numpy arrays are of different shapes, as part of the training, I have a validation set separate from the prediction numpy array, but the prediction set was built from the numpy array.
2:
I have tried to use torch.Tensor.repeat() as a way to enlarge the the prediction set, but I lack the correct math to do the repeat efficiently.
3:
the training method:
def train(net, x_train, x_opt, BATCH_SIZE, EPOCHS, input_dim):
    outputs = 0
    mse = 0
    optimizer = optim.SGD(net.parameters(), lr=0.001)
    loss_function = nn.MSELoss()
    loss = 0
    for epoch in range(EPOCHS):
        for i in tqdm(range(0, len(x_train), BATCH_SIZE)):

            batch_y = x_opt[i:i + BATCH_SIZE]

            net.zero_grad()

            outputs = net(batch_y)

            loss = loss_function(outputs, batch_y)
            loss.backward()
            optimizer.step()  # Does the update

        print(f"Epoch: {epoch}. Loss: {loss}")

    outputs = torch.Tensor.repeat(outputs, *****....***?, 1)
    return np.mean(np.power(x_opt - outputs.data.numpy(), 2), axis=1)

4:
the error:
ValueError: operands could not be broadcast together with shapes (92334,10) (46,10)


title: Bert Sequence Classifier returning same response for al sequence

body:
I was doing fine tuning by BertForSequenceClassification from transformers using pytorch. But the problem is during evaluation, it returns the same output for every sequence. Can not figure out why?
Tokenization:
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

# Model parameter
MAX_SEQ_LEN = 128
PAD_INDEX = tokenizer.convert_tokens_to_ids(tokenizer.pad_token)
UNK_INDEX = tokenizer.convert_tokens_to_ids(tokenizer.unk_token)

# Fields

label_field = Field(sequential=False, use_vocab=False, batch_first=True, dtype=torch.float)
text_field = Field(use_vocab=False, tokenize=tokenizer.encode, lower=False, include_lengths=False, batch_first=True,
                   fix_length=MAX_SEQ_LEN, pad_token=PAD_INDEX, unk_token=UNK_INDEX)

#fields = [('Slogan', label_field), ('Sentence', text_field)]
fields = [('Sentence', text_field), ('Slogan', label_field)]
#fields = [('Sentence', data.Field(sequential=True)), ('Slogan', data.Field())]

train, valid, test = TabularDataset.splits(path= '/content/drive/My Drive/research/slogan/data', train='train.csv', validation= 'val.csv', test='test.csv', format='CSV', fields=fields, skip_header=True)
#train = TabularDataset(path= '/content/drive/My Drive/Data/train.csv',  format='CSV', fields=[('Sentence', data.Field(sequential=True)), ('Slogan', data.Field())] )

Generator:
train_iter = BucketIterator(train, batch_size=16, sort_key=lambda x: len(x.Sentence),
                            device=device, train=True, sort=True, sort_within_batch=True)
valid_iter = BucketIterator(valid, batch_size=16, sort_key=lambda x: len(x.Sentence),
                            device=device, train=True, sort=True, sort_within_batch=True)
test_iter = Iterator(test, batch_size=16, device=device, train=False, shuffle=False, sort=False)

Bert Initialization:
#build the model
class BERT(nn.Module):

    def __init__(self):
        super(BERT, self).__init__()

        options_name = "bert-base-uncased"
        self.encoder = BertForSequenceClassification.from_pretrained(options_name)

    def forward(self, Sentence, Slogan):
        loss, Sentence_fea = self.encoder(Sentence, labels=Slogan)[:2]
        return loss, Sentence_fea

Save and Load Functions
def save_checkpoint(save_path, model, valid_loss):

    if save_path == None:
        return

    state_dict = {'model_state_dict': model.state_dict(),
                  'valid_loss': valid_loss}

    torch.save(state_dict, save_path)
    print(f'Model saved to ==> {save_path}')

def load_checkpoint(load_path, model):

    if load_path==None:
        return

    state_dict = torch.load(load_path, map_location=device)
    print(f'Model loaded from <== {load_path}')

    model.load_state_dict(state_dict['model_state_dict'])
    return state_dict['valid_loss']

def save_metrics(save_path, train_loss_list, valid_loss_list, global_steps_list):

    if save_path == None:
        return

    state_dict = {'train_loss_list': train_loss_list,
                  'valid_loss_list': valid_loss_list,
                  'global_steps_list': global_steps_list}

    torch.save(state_dict, save_path)
    print(f'Model saved to ==> {save_path}')

def load_metrics(load_path):

    if load_path==None:
        return

    state_dict = torch.load(load_path, map_location=device)
    print(f'Model loaded from <== {load_path}')

    return state_dict['train_loss_list'], state_dict['valid_loss_list'], state_dict['global_steps_list']

Train:
def train(model,
          optimizer,
          criterion = nn.BCELoss(),
          train_loader = train_iter,
          valid_loader = valid_iter,
          num_epochs = 5,
          eval_every = len(train_iter) // 2,
          file_path = destination_folder,
          best_valid_loss = float("Inf")):

    # initialize running values
    running_loss = 0.0
    valid_running_loss = 0.0
    global_step = 0
    train_loss_list = []
    valid_loss_list = []
    global_steps_list = []

    # training loop
    model.train()
    for epoch in range(num_epochs):
        for ( Sentence, Slogan), _ in train_loader:
            Slogan = Slogan.type(torch.LongTensor)
            Slogan = Slogan.to(device)
            Sentence = Sentence.type(torch.LongTensor)
            Sentence = Sentence.to(device)
            output = model(Sentence, Slogan)
            loss, _ = output

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            # update running values
            running_loss += loss.item()
            global_step += 1

            # evaluation step
            if global_step % eval_every == 0:
                model.eval()
                with torch.no_grad():

                    # validation loop
                    for (Sentence, Slogan), _ in valid_loader:
                        Slogan = Slogan.type(torch.LongTensor)
                        Slogan = Slogan.to(device)
                        Sentence = Sentence.type(torch.LongTensor)
                        Sentence = Sentence.to(device)
                        output = model(Sentence, Slogan)
                        loss, _ = output

                        valid_running_loss += loss.item()

                # evaluation
                average_train_loss = running_loss / eval_every
                average_valid_loss = valid_running_loss / len(valid_loader)
                train_loss_list.append(average_train_loss)
                valid_loss_list.append(average_valid_loss)
                global_steps_list.append(global_step)

                # resetting running values
                running_loss = 0.0
                valid_running_loss = 0.0
                model.train()

                # print progress
                print('Epoch [{}/{}], Step [{}/{}], Train Loss: {:.4f}, Valid Loss: {:.4f}'
                      .format(epoch+1, num_epochs, global_step, num_epochs*len(train_loader),
                              average_train_loss, average_valid_loss))

                # checkpoint
                if best_valid_loss > average_valid_loss:
                    best_valid_loss = average_valid_loss
                    save_checkpoint(file_path + '/' + 'model.pt', model, best_valid_loss)
                    save_metrics(file_path + '/' + 'metrics.pt', train_loss_list, valid_loss_list, global_steps_list)

    save_metrics(file_path + '/' + 'metrics.pt', train_loss_list, valid_loss_list, global_steps_list)
    print('Finished Training!')

model = BERT().to(device)
optimizer = optim.Adam(model.parameters(), lr=2e-5)

train(model=model, optimizer=optimizer)

During train, the log:
Epoch [1/5], Step [572/5725], Train Loss: 0.3253, Valid Loss: 2.2138
Model saved to ==> /content/drive/My Drive/research/slogan/data/model.pt
Model saved to ==> /content/drive/My Drive/research/slogan/data/metrics.pt
Epoch [1/5], Step [1144/5725], Train Loss: 0.4064, Valid Loss: 1.6462
Model saved to ==> /content/drive/My Drive/research/slogan/data/model.pt
Model saved to ==> /content/drive/My Drive/research/slogan/data/metrics.pt
Epoch [2/5], Step [1716/5725], Train Loss: 0.4750, Valid Loss: 1.7286
Epoch [2/5], Step [2288/5725], Train Loss: 0.5080, Valid Loss: 1.7306
Epoch [3/5], Step [2860/5725], Train Loss: 0.7156, Valid Loss: 1.1750
Model saved to ==> /content/drive/My Drive/research/slogan/data/model.pt
Model saved to ==> /content/drive/My Drive/research/slogan/data/metrics.pt
Epoch [3/5], Step [3432/5725], Train Loss: 0.5992, Valid Loss: 1.4372
Epoch [4/5], Step [4004/5725], Train Loss: 0.7525, Valid Loss: 0.8883
Model saved to ==> /content/drive/My Drive/research/slogan/data/model.pt
Model saved to ==> /content/drive/My Drive/research/slogan/data/metrics.pt
Epoch [4/5], Step [4576/5725], Train Loss: 0.6275, Valid Loss: 1.2206
Epoch [5/5], Step [5148/5725], Train Loss: 0.7644, Valid Loss: 0.8501
Model saved to ==> /content/drive/My Drive/research/slogan/data/model.pt
Model saved to ==> /content/drive/My Drive/research/slogan/data/metrics.pt
Epoch [5/5], Step [5720/5725], Train Loss: 0.6230, Valid Loss: 1.2039
Model saved to ==> /content/drive/My Drive/research/slogan/data/metrics.pt

Evaluation:
def evaluate(model, test_loader):
    y_pred = []
    y_true = []

    model.eval()
    with torch.no_grad():
        for (Sentence, Slogan), _ in test_loader:

                Sentence = Sentence.type(torch.LongTensor)
                Sentence = Sentence.to(device)
                Slogan = Slogan.type(torch.LongTensor)
                Slogan = Slogan.to(device)
                print(Sentence, Slogan)
                output = model(Sentence, Slogan)
                print(output)

                _, output = output
                y_pred.extend(torch.argmax(output, 1).tolist())
                y_true.extend(Slogan.tolist())

    print('Classification Report:')
    print(classification_report(y_true, y_pred, labels=[1,0], digits=4))
    cm = confusion_matrix(y_true, y_pred, labels=[1,0])
    ax= plt.subplot()
    sns.heatmap(cm, annot=True, ax = ax, cmap='Blues', fmt="d")

    ax.set_title('Confusion Matrix')

    ax.set_xlabel('Predicted Labels')
    ax.set_ylabel('True Labels')

    ax.xaxis.set_ticklabels(['Slogan', 'Not Slogan'])
    ax.yaxis.set_ticklabels(['Slogan', 'Not Slogan'])

First print function above print the following, which means tokenization is okay.
tensor([[  101,  2021,  9483,  ...,     0,     0,     0],
        [  101,  2024,  2017,  ...,     0,     0,     0],
        [  101,  6343,  2064,  ...,     0,     0,     0],
        ...,
        [  101,  2026,  2028,  ...,     0,     0,     0],
        [  101, 19067,  2102,  ...,     0,     0,     0],
        [  101,  1045,  5959,  ...,     0,     0,     0]], device='cuda:0') tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], device='cuda:0')

But the output is ambiguous:
(tensor(1.3949, device='cuda:0'), tensor([[-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470],
        [-0.3630,  0.7470]], device='cuda:0'))


title: Value error while converting tensor to numpy array

body:
I'm using the following code to extract the features from image.
def ext():
    imgPathList = glob.glob("images/"+"*.JPG")
    features = []
    for i, path in enumerate(tqdm(imgPathList)):
        feature = get_vector(path)
        feature = feature[0] / np.linalg.norm(feature[0])
        features.append(feature)
        paths.append(path)
    features = np.array(features, dtype=np.float32)
    return features, paths

However, the above code throws the following error,
 features = np.array(features, dtype=np.float32)
ValueError: only one element tensors can be converted to Python scalars

How can I be able to fix it?