title: how to reduce the second dimension in pytorch python tensor operation

body:
I have the below tensor with the following shape
    seq = dataset['features'][...]
    print(f'shape of seq before unsequeeze {seq.shape}')
    shape of seq before unsequeeze (461, 1024)

I am trying to convert the shape in (461, 512) How should I achieve this in  pytorch tensor operation.
examples feature x as below,
import torch
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
x = torch.tensor([[0.1,0.8,0.8,0.4,0.9,0.4,0.4,0.5,0.4,0.2,0.3,0.8,0.2,0.7,0.6,0.1,0.2,0.1,0.1,0.4,0.2,0.1,0.0,0.5,0.2,0.4,0.3,0.3,0.7,0.1,0.4,0.6,0.5,1.0,0.1,0.8,0.9,0.0,0.2,0.9,0.8,0.0,0.9,0.7,0.2,0.2,0.9,0.6,0.1,0.2,0.6,0.0,0.1,0.1,0.3,0.5,0.8,0.8,0.4,0.4,0.7,0.7,0.4,0.2,0.1,1.0,0.3,0.8,0.1,0.7,0.7,0.9,0.6,0.3,0.8,0.2,0.9,0.6,0.7,0.8,0.2,0.1,1.0,0.6,0.5,0.5,0.5,0.8,0.8,0.3,0.1,0.2,0.5,0.9,0.6,0.8,0.0,0.6,0.2,0.1,0.8,0.4,0.8,0.5,0.8,0.4,0.7,0.6,0.8,0.1,0.4,0.8,1.0,0.9,0.4,0.4,0.4,0.1,0.7,0.3,0.8,0.6,0.4,0.5,0.9,0.1,0.9,0.7,0.4,0.7,0.1,0.8,0.2,0.2,0.7,0.2,0.9,0.6,0.2,0.9,0.1,0.9,0.2,1.0,0.9,0.6,0.3,0.6,0.9,0.6,0.0,0.3,0.4,0.6,0.7,0.9,0.2,0.6,0.2,0.5,0.3,0.3,0.4,0.4,0.1,0.2,0.6,0.0,0.7,0.5,0.5,0.2,0.5,0.6,0.5,0.5,0.7,0.8,0.4,0.5,0.8,0.8,0.1,0.5,0.7,0.8,0.1,0.1,0.8,0.6,0.6,0.4,1.0,0.4,0.6,0.9,0.1,0.6,0.3,1.0,0.7,0.2,0.5,0.5,1.0,0.5,0.4,0.3,0.7,0.1,1.0,0.9,0.4,0.6,0.6,0.6,0.2,0.0,0.9,0.9,0.2,0.1,0.5,0.5,0.8,0.7,0.8,0.0,0.0,0.1,0.5,0.5,0.5,0.8,0.1,0.5,1.0,0.3,0.2,0.8,0.9,0.4,0.4,0.9,0.2,0.4,0.9,0.9,0.3,0.7,0.4,0.9,0.5,0.7,0.8,0.5,0.5,0.5,0.8,0.7,0.9,0.2,0.8,1.0,0.1,0.9,0.6,0.5,0.0,0.2,0.8,0.2,0.8,0.5,0.9,0.9,0.5,0.6,0.1,0.8,1.0,0.3,0.1,0.5,0.9,0.1,0.0,0.5,0.3,0.1,0.5,0.8,0.3,0.4,0.4,0.3,0.2,0.8,0.7,0.6,0.3,0.5,0.1,0.7,0.4,0.2,0.1,0.1,0.4,0.2,0.8,0.8,0.4,0.1,0.0,0.3,0.2,0.0,1.0,0.2,0.6,0.5,0.7,0.7,0.7,0.1,0.2,0.1,0.1,0.9,0.6,0.5,1.0,0.4,0.4,0.8,0.7,0.5,0.6,0.9,0.0,0.8,0.3,0.1,0.5,0.9,0.9,0.9,0.7,0.7,1.0,0.6,0.6,1.0,0.8,1.0,0.4,0.3,0.2,1.0,0.9,0.2,0.7,0.1,0.3,0.1,0.1,0.7,0.6,0.8,0.8,0.7,0.7,0.4,0.8,0.4,0.1,0.0,1.0,0.2,0.6,0.8,0.3,0.9,0.3,0.6,0.6,0.4,0.7,0.0,0.2,0.9,0.2,0.1,0.4,0.9,0.5,0.2,0.4,1.0,0.1,0.3,0.8,0.8,0.2,0.2,0.6,0.8,0.1,0.0,0.5,1.0,0.5,0.7,0.3,0.5,0.0,0.2,0.6,0.7,0.6,0.4,0.2,0.0,0.4,0.4,0.0,0.3,0.3,0.8,0.5,0.7,0.4,0.1,0.8,0.4,0.1,0.3,1.0,0.3,0.6,0.5,0.6,0.2,0.9,0.4,0.4,0.8,0.0,0.3,0.8,0.3,0.1,0.0,0.5,0.5,0.8,0.6,1.0,0.7,0.8,0.7,0.7,0.6,0.0,0.6,0.6,0.3,0.7,0.2,1.0,0.6,0.4,0.8,0.4,0.7,0.3,0.8,0.8,0.1,0.1,0.2,0.2,0.7,0.1,0.8,0.4,1.0,0.6,1.0,0.3,0.9,0.9,0.9,0.9,1.0,0.2,0.3,0.9,0.5,0.5,0.4,0.1,0.4,0.0,0.7,0.2,0.6,0.8,0.2,0.8,0.2,0.6,0.9,0.1,0.3,0.4,0.2,0.9,0.3,0.9,0.1,0.1,0.7,1.0,0.4,0.2,0.9,0.2,0.5,0.1,0.3,0.6,0.5,0.6,0.5,0.3,0.4,0.3,0.9,0.7,0.1,0.2,0.8,1.0,0.5,0.0,0.8,0.2,0.2,0.0,1.0,0.2,1.0,0.5,1.0,0.9,0.5,0.2,0.5,0.8,0.4,0.9,0.9,0.2,0.5,0.5,0.2,0.6,0.3,0.3,0.8,0.3,0.5,0.4,0.2,0.7,0.8,0.9,0.2,0.9,0.6,0.0,0.3,0.8,0.5,0.3,0.9,0.9,0.7,0.4,0.9,0.3,0.7,0.4,0.3,0.5,0.8,0.9,0.7,0.6,0.5,0.1,0.9,0.6,0.5,0.2,0.7,0.3,0.3,0.1,0.0,0.2,0.5,0.9,0.7,0.3,0.3,1.0,0.3,0.6,0.9,0.1,0.9,0.3,0.7,0.1,0.7,0.6,0.6,0.5,0.1,0.1,0.3,0.5,0.7,0.1,0.7,0.4,0.8,0.4,0.6,0.8,0.7,0.6,0.0,0.1,0.3,0.8,0.2,0.5,0.7,0.0,0.4,1.0,0.2,0.2,0.4,0.3,0.9,0.2,0.4,0.3,0.4,0.2,0.5,0.6,0.6,0.8,0.7,0.3,0.1,0.7,0.5,0.1,0.4,1.0,0.2,0.8,0.5,0.7,0.3,0.7,0.6,0.7,0.5,1.0,0.2,0.8,0.0,0.1,0.2,0.6,0.0,0.2,0.1,0.2,0.4,0.6,0.2,1.0,0.3,0.1,0.1,0.7,0.0,0.7,0.0,0.7,0.9,0.1,0.2,0.8,0.7,0.5,0.3,0.8,0.3,0.0,0.1,0.1,0.8,0.9,0.2,0.5,0.5,0.4,0.4,0.8,0.9,0.4,1.0,0.8,0.4,0.2,0.1,0.3,0.1,0.7,0.9,0.2,0.9,0.8,0.7,0.2,0.7,0.4,0.0,1.0,0.7,0.3,0.6,0.9,0.1,0.5,0.2,0.5,0.7,0.3,0.9,0.7,0.2,1.0,0.6,0.4,0.3,0.1,0.1,0.0,0.3,0.9,0.7,0.5,0.9,0.8,0.6,0.8,0.1,0.4,0.5,0.8,0.7,0.4,0.8,0.4,0.1,0.6,0.8,0.0,0.9,0.7,0.7,0.7,0.7,0.3,0.4,0.4,0.2,0.6,0.3,0.4,1.0,0.2,0.3,0.0,0.5,1.0,0.8,0.7,0.3,0.2,0.7,0.1,0.5,0.2,0.3,0.4,0.8,0.4,0.2,0.3,0.9,0.5,0.1,0.7,0.0,0.3,0.3,0.1,0.1,0.8,0.2,0.6,0.2,0.0,0.3,0.6,0.4,0.7,0.6,0.2,0.8,0.4,0.3,0.7,0.3,0.7,0.9,0.4,0.8,0.9,0.4,0.5,0.4,0.6,0.7,0.5,0.6,0.6,0.4,0.4,0.8,0.3,0.9,0.8,0.9,0.6,0.1,0.9,1.0,1.0,0.8,0.8,0.2,0.1,0.1,0.4,0.9,0.9,0.9,0.6,0.4,0.8,0.6,0.6,0.4,0.6,0.6,0.8,1.0,0.2,0.3,0.4,0.9,0.3,0.7,0.9,0.6,1.0,0.5,0.3,0.5,0.9,0.1,0.9,0.6,0.4,0.9,0.9,0.7,0.9,0.0,0.3,0.7,0.2,0.1,0.2,0.6,0.1,0.6,0.3,0.5,0.1,0.5,0.7,0.1,0.9,0.4,0.1,0.4,1.0,0.1,0.7,0.5,0.6,0.1,0.4,1.0,0.3,0.8,0.3,0.9,0.8,0.9,0.4,0.2,0.2,0.7,0.0,0.8,0.7,0.3,0.2,0.2,0.3,0.9,0.8,0.2,0.3,0.4,0.2,0.9,0.4,0.6,0.2,0.5,0.6,0.0,0.3,0.2,0.9,0.7,0.5,0.7,0.8,0.8,0.2,0.7,0.7,0.5,0.1,0.0,0.3,0.6,0.4,1.0,1.0,0.1,0.2,0.4,0.5,0.0,0.2,0.6,0.8,0.7,0.5,0.2,0.3,0.7,0.4,0.7,0.8,0.2,0.7,0.8,0.9,0.7,0.2,0.5,0.7,0.9,0.7,0.5,0.1,1.0,0.5,0.6,0.9,0.5,0.7,0.3,0.9,0.8],
[0.5,0.6,0.0,0.9,0.9,0.4,0.4,0.9,0.1,0.7,0.8,0.7,1.0,0.5,0.6,0.5,0.9,0.7,0.2,0.4,0.6,0.7,0.4,0.2,0.3,0.3,0.9,1.0,0.0,0.5,0.5,0.6,0.1,0.6,0.1,1.0,0.8,0.4,0.2,0.6,0.9,0.2,0.1,0.5,0.0,0.5,0.3,0.9,0.5,0.0,0.9,0.4,0.4,0.5,0.7,0.9,0.1,0.9,0.0,0.2,0.6,0.8,0.7,0.1,0.6,0.2,0.2,0.8,0.7,0.2,0.1,0.2,0.6,0.8,0.6,0.4,0.8,0.8,0.9,0.7,0.8,0.4,0.5,0.1,0.7,0.9,0.2,0.3,0.0,0.7,0.0,0.1,0.7,0.8,0.9,0.7,0.6,0.3,0.7,0.7,0.2,0.1,0.3,0.7,0.3,0.8,0.2,0.1,0.8,0.9,0.2,0.4,0.5,0.5,0.9,0.9,0.3,0.7,0.1,0.6,0.7,0.2,0.6,0.9,0.8,0.7,0.0,0.4,0.1,0.6,0.5,0.1,0.8,0.7,0.9,0.7,0.5,0.7,0.8,0.8,0.2,0.5,0.3,0.4,0.8,0.4,0.1,0.3,0.4,0.3,0.4,0.7,0.4,0.7,0.9,0.2,0.8,0.3,0.8,0.3,0.8,0.7,0.3,0.4,0.4,0.6,0.1,0.3,0.6,0.5,0.9,0.7,0.3,0.6,0.5,0.3,0.4,0.2,0.8,0.3,0.1,0.9,0.9,0.6,0.1,0.4,0.2,0.4,0.8,0.9,0.1,0.4,0.8,0.5,0.4,0.8,0.9,1.0,0.1,0.8,0.8,0.8,0.8,0.8,0.3,0.1,1.0,0.2,0.9,0.2,0.9,0.7,0.9,1.0,0.4,0.2,0.5,0.4,0.3,0.2,0.1,0.1,0.8,0.7,0.0,0.3,1.0,1.0,0.0,0.5,0.0,0.5,0.6,0.8,0.2,0.4,0.0,0.8,0.5,0.8,0.6,0.3,0.4,0.7,0.9,0.0,0.8,0.7,0.9,0.9,0.2,0.3,0.3,0.9,0.3,0.3,0.3,0.6,0.8,0.5,0.5,0.0,0.5,0.8,1.0,0.4,1.0,0.3,0.5,0.5,0.6,0.6,0.7,0.1,0.3,0.6,0.4,0.2,0.8,1.0,0.6,0.9,0.7,0.5,0.1,0.7,0.6,1.0,0.4,0.9,0.3,0.6,0.1,1.0,0.8,0.7,0.7,0.5,0.0,0.6,0.5,1.0,0.6,0.9,0.8,0.9,0.7,1.0,0.9,1.0,0.3,0.2,0.5,0.3,0.8,0.1,0.9,0.6,0.9,0.9,0.3,0.4,0.1,0.6,0.0,0.0,0.2,0.2,0.9,0.9,0.6,1.0,0.2,0.7,1.0,0.8,1.0,0.2,0.3,0.3,0.9,0.5,0.1,0.2,0.5,0.9,0.1,0.5,0.2,1.0,0.7,0.4,0.2,0.1,0.4,0.4,0.7,0.8,0.3,0.6,0.0,1.0,0.8,1.0,0.1,0.2,0.9,0.4,0.8,0.0,0.0,1.0,0.1,0.3,0.0,0.7,0.6,0.9,0.4,0.4,0.9,0.4,0.8,0.7,0.7,0.5,0.3,0.6,0.5,0.5,0.5,0.9,0.8,0.4,0.8,0.6,0.4,0.2,0.9,1.0,0.8,0.2,0.2,0.8,0.9,0.7,0.1,0.8,0.7,0.3,0.1,0.2,0.3,0.6,0.6,0.6,0.7,0.4,0.1,0.9,0.5,0.5,0.5,0.4,0.6,0.2,0.7,0.6,0.3,0.3,0.2,0.4,0.2,0.9,0.9,0.9,0.7,0.8,0.3,0.0,0.4,0.1,0.9,0.6,0.3,0.0,0.7,0.1,0.8,0.6,0.3,0.6,0.8,0.2,0.1,0.4,0.8,0.9,1.0,0.7,0.8,0.1,0.4,0.1,0.4,0.9,0.4,0.6,0.7,0.2,0.5,0.6,0.8,0.6,0.6,0.9,0.7,0.4,0.3,0.5,0.1,0.8,0.9,0.4,0.0,0.4,0.0,0.3,0.6,0.8,0.1,0.4,0.1,0.6,0.7,0.1,0.0,0.0,0.0,0.8,0.7,0.6,0.8,0.6,0.9,0.1,0.4,0.0,0.4,0.0,0.4,0.7,0.5,0.1,0.9,0.3,0.3,0.1,0.3,0.6,0.6,0.8,0.8,0.9,0.2,0.0,0.6,0.3,1.0,0.6,0.7,1.0,1.0,0.9,0.4,0.1,0.6,0.9,0.1,0.1,0.1,0.2,0.5,0.0,0.8,0.5,0.0,0.8,0.4,0.1,0.2,0.2,0.8,0.9,0.6,0.3,0.2,0.5,0.0,0.1,0.1,0.8,0.9,1.0,0.8,0.2,0.8,0.3,0.8,0.2,0.0,0.1,1.0,0.7,0.1,0.8,0.2,0.5,0.3,0.6,0.1,0.7,0.7,0.5,0.2,0.3,0.5,0.5,1.0,0.2,0.3,0.4,0.1,0.1,0.7,1.0,0.7,0.6,0.9,1.0,0.4,0.8,0.1,0.4,0.1,0.9,0.7,0.4,0.0,0.0,0.3,0.3,0.5,0.6,0.3,0.8,0.5,0.3,0.1,0.9,0.5,0.1,0.3,0.9,0.4,0.3,0.4,0.2,0.9,0.5,0.4,0.9,0.8,0.9,0.9,0.9,0.6,0.6,0.3,0.4,0.3,0.3,0.4,0.4,0.2,0.3,0.7,0.1,0.4,0.1,0.7,0.2,0.7,0.7,0.1,0.3,1.0,0.4,0.4,0.0,0.1,0.4,0.6,0.9,0.5,0.1,0.6,0.9,0.1,0.2,0.4,0.5,0.5,0.1,0.7,0.0,0.1,1.0,0.6,0.1,0.5,0.7,0.2,0.7,0.1,0.1,0.5,0.5,0.2,0.7,0.0,0.9,0.3,0.2,0.9,0.2,0.2,0.5,0.5,0.6,0.3,0.4,0.9,0.4,0.5,0.8,0.1,0.4,0.5,0.9,0.5,0.4,0.3,1.0,0.7,0.5,0.1,0.0,0.3,0.0,0.5,0.5,0.9,0.6,0.3,0.7,0.1,0.9,0.1,0.9,0.1,0.8,0.0,0.9,0.0,0.0,0.7,0.6,1.0,0.5,0.9,0.7,0.4,0.5,0.6,0.3,0.6,0.9,0.4,0.3,0.3,1.0,0.2,1.0,0.3,0.7,0.9,0.8,0.8,0.7,0.6,0.6,0.8,0.5,0.3,0.4,0.5,0.1,0.3,0.4,0.0,0.2,0.8,0.3,1.0,0.5,0.0,0.7,0.9,0.3,0.3,0.9,0.9,0.5,0.0,0.0,0.6,0.7,0.6,0.5,0.1,0.8,0.3,0.3,0.1,0.7,0.0,0.6,0.0,0.1,0.9,0.1,0.4,0.1,0.5,1.0,0.3,0.2,0.8,0.6,0.3,0.5,0.3,0.1,0.9,0.1,0.9,0.9,0.1,0.8,0.7,0.8,0.3,0.5,1.0,0.1,0.7,0.4,0.7,0.7,0.9,0.9,1.0,0.3,0.8,0.3,0.3,0.5,0.2,0.6,0.4,0.5,0.7,0.8,0.9,0.8,0.9,0.2,0.0,0.5,0.2,1.0,0.7,0.4,0.1,0.6,0.6,0.0,0.4,0.6,0.6,0.4,0.1,0.7,1.0,0.1,0.4,0.3,0.9,0.1,0.0,0.1,0.6,0.1,1.0,0.1,0.3,0.3,0.4,0.3,0.8,0.2,0.5,0.1,0.3,0.8,0.7,0.0,0.4,0.5,0.2,0.0,0.5,0.8,0.2,0.6,0.9,0.8,0.9,0.5,0.7,0.5,0.9,0.9,0.3,0.5,0.3,1.0,0.8,0.7,0.9,0.6,0.6,0.5,0.8,0.2,0.7,0.6,0.3,0.1,0.9,0.2,0.4,0.9,0.3,0.2,0.5,0.5,0.9,0.2,1.0,0.9,0.8,0.2,0.2,1.0,0.4,0.4,0.6,0.8,0.3,0.2,0.6,0.0,0.5,0.9,0.6,0.3,0.4,0.8,0.5,0.6,0.7,0.6,0.0,0.1,0.3,0.7,0.4,0.1,0.2,0.7,0.2,0.3,0.8,0.2,0.4,0.2,1.0,1.0,0.7,0.8,0.2,0.5,0.3,0.5,0.4,0.6,0.5,0.3,0.6,0.5,1.0,0.7,0.8,0.9,0.0,0.6,0.3,0.9,0.3,0.9,0.5,0.7,0.5,0.1,0.1,0.3,0.7,0.8,0.1,0.0,0.7,0.5,1.0,0.3,0.8,0.7,0.7,0.2,0.9,0.5,0.6,0.1,0.5,0.5,0.0,0.2,0.7,0.9,0.1,0.9,0.3,0.2]]).to(
    device, dtype=torch.int64
)

x.shape
torch.Size([2, 1024])

I need to reduce  the feature size to 512 keeping the first dim batch size in tact,
x.shape
torch.Size([2, 512])

Thanks


title: Running Pytorch with torch.distributed.launch requires more memory?

body:
I can run my model with few GPUs perfectly (by few I mean anything less than 5). When I increased the number of GPUs, let's say 10, I got Cuda out of memory error:
RuntimeError: CUDA out of memory. Tried to allocate 12.50 MiB (GPU 0; 10.92 GiB total capacity; 8.57 MiB already allocated; 9.28 GiB free; 4.68 MiB cached)

I used torch.distributed.launch and DistributedDataParallel to execute my script. I would like to know, what is the reason for this behavior and how can I avoid it? Does the master node require more memory? if so, how much it needs? How can I calculate it? How can I run my code with 10 GPUs?
Few things to consider:

This issue happens on V100 GPUs as well
reducing batch-size doesn't help since my batch size is already 1


title: How to "cut" a tensor into half in Pytorch?

body: I have a tensor with shape [1, 2, 96, 96] and would like two tensors with the shape [1, 1, 96, 96], is there a quick way of doing this? Thanks in advance


title: Pytorch: Weights not changing during training

body:
Basically the same question as this one here, which was never answered: Why the first convolutional layer weights don't change during training?
I just want to watch the weights of my convolutional layers as they change during training. How can I do this? No matter what I do, the weights seem to stay the same even though loss is decreasing.
I'm trying to follow this tutorial here although the model is slightly different: https://pytorch.org/tutorials/beginner/blitz/cifar10_tutorial.html#sphx-glr-beginner-blitz-cifar10-tutorial-py
Model
class CNN(nn.Module):
    def __init__(self):
        super(Digit_Classifier, self).__init__()
        self.conv1 = nn.Conv2d(1,6,3)
        self.pool1 = nn.MaxPool2d(2)
        self.conv2 = nn.Conv2d(6,16,3)
        self.pool2 = nn.MaxPool2d(2)
        self.out = nn.Linear(400, 10)

    def forward(self, inputs):
        x = self.pool1(F.relu(self.conv1(inputs)))
        x = self.pool2(F.relu(self.conv2(x)))
        x = torch.flatten(x, start_dim=1)
        x = self.out(x)
        return x

Training
def train(epochs=100):
    criterion = nn.CrossEntropyLoss()
    net = CNN()
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
    losses = []

    for epoch in range(epochs):  # loop over the dataset multiple times

        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            # get the inputs; data is a list of [inputs, labels]
            inputs, labels = data

            # zero the parameter gradients
            optimizer.zero_grad()

            outputs = net(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            running_loss += loss.item()
            w = model.conv1._parameters['weight']
            print(w)

            losses.append(running_loss / z)
            if i % 2000 == 1999:    # print every 2000 mini-batches
                print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0
    return net


title: What does does if self.transforms mean?

body:
What does
if self.transforms:
            data = self.transforms(data)

do? I don't understand the logic behind this line - what is the condition the line is using?
I'm reading an article on creating a custom dataset with pytorch based on the below implementation:
#custom dataset
class MNISTDataset(Dataset):
    def __init__(self, images, labels=None, transforms=None):
        self.X = images
        self.y = labels
        self.transforms = transforms

    def __len__(self):
        return (len(self.X))

    def __getitem__(self, i):
        data = self.X.iloc[i, :]
        data = np.asarray(data).astype(np.uint8).reshape(28, 28, 1)

        if self.transforms:
            data = self.transforms(data)

        if self.y is not None:
            return (data, self.y[i])
        else:
            return data
train_data = MNISTDataset(train_images, train_labels, transform)
test_data = MNISTDataset(test_images, test_labels, transform)
# dataloaders
trainloader = DataLoader(train_data, batch_size=128, shuffle=True)
testloader = DataLoader(test_data, batch_size=128, shuffle=True)

thank you! i'm basically trying to understand why it works & how it applies transforms to the data.


title: How to replace all subsets of values in a PyTorch tensor?

body:
Let's say I have two PyTorch tensors:
t_1d = torch.Tensor([9, 3, 0, 3, 5, 7, 3, 3, 5, 3, 1, 11]])
t = torch.Tensor([3, 5])
t_replace = torch.Tensor([10, 8])

I want to replace all occurrences of the second tensor t in t_1d with the new tensor t_replace.
The output should be something like:
torch.Tensor([9, 3, 0, 10, 8, 7, 3, 10, 8, 3, 1, 11]])

Preferably on GPU for large Tensors, so no loops or Numpy casts.


title: Weights of pre-trained BERT model not initialized

body:
I am using the Language Interpretability Toolkit (LIT) to load and analyze a BERT model that I pre-trained on an NER task.
However, when I'm starting the LIT script with the path to my pre-trained model passed to it, it fails to initialize the weights and tells me:
    modeling_utils.py:648] loading weights file bert_remote/examples/token-classification/Data/Models/results_21_03_04_cleaned_annotations/04.03._8_16_5e-5_cleaned_annotations/04-03-2021 (15.22.23)/pytorch_model.bin
    modeling_utils.py:739] Weights of BertForTokenClassification not initialized from pretrained model: ['bert.pooler.dense.weight', 'bert.pooler.dense.bias']
    modeling_utils.py:745] Weights from pretrained model not used in BertForTokenClassification: ['bert.embeddings.position_ids']

It then simply uses the bert-base-german-cased version of BERT, which of course doesn't have my custom labels and thus fails to predict anything. I think it might have to do with PyTorch, but I can't find the error.
If relevant, here is how I load my dataset into CoNLL 2003 format (modification of the dataloader scripts found here):
    def __init__(self):

        # Read ConLL Test Files

        self._examples = []

        data_path = "lit_remote/lit_nlp/examples/datasets/NER_Data"
        with open(os.path.join(data_path, "test.txt"), "r", encoding="utf-8") as f:
            lines = f.readlines()

        for line in lines[:2000]:
            if line != "\n":
                token, label = line.split(" ")
                self._examples.append({
                    'token': token,
                    'label': label,
                })
            else:
                self._examples.append({
                    'token': "\n",
                    'label': "O"
                })

    def spec(self):
        return {
            'token': lit_types.Tokens(),
            'label': lit_types.SequenceTags(align="token"),
        }

And this is how I initialize the model and start the LIT server (modification of the simple_pytorch_demo.py script found here):
    def __init__(self, model_name_or_path):
        self.tokenizer = transformers.AutoTokenizer.from_pretrained(
            model_name_or_path)
        model_config = transformers.AutoConfig.from_pretrained(
            model_name_or_path,
            num_labels=15,  # FIXME CHANGE
            output_hidden_states=True,
            output_attentions=True,
        )
        # This is a just a regular PyTorch model.
        self.model = _from_pretrained(
            transformers.AutoModelForTokenClassification,
            model_name_or_path,
            config=model_config)
        self.model.eval()

## Some omitted snippets here

    def input_spec(self) -> lit_types.Spec:
        return {
            "token": lit_types.Tokens(),
            "label": lit_types.SequenceTags(align="token")
        }

    def output_spec(self) -> lit_types.Spec:
        return {
            "tokens": lit_types.Tokens(),
            "probas": lit_types.MulticlassPreds(parent="label", vocab=self.LABELS),
            "cls_emb": lit_types.Embeddings()


title: Pytorch Siamese NN with BERT for sentence matching

body:
I'm trying to build a Siamese neural network using pytorch in which I feed BERT word embeddings and trying to find whether two sentences are similar or not (imagine duplicate posts matching, product matching etc). Here's the model:
class SiameseNetwork(torch.nn.Module):
    def __init__(self):
        super(SiameseNetwork, self).__init__()
        self.brothers = torch.nn.Sequential(
            torch.nn.Linear(512 * 768, 512),
            torch.nn.BatchNorm1d(512),
            torch.nn.ReLU(inplace=True),
            torch.nn.Linear(512, 256),
            torch.nn.BatchNorm1d(256),
            torch.nn.ReLU(inplace=True),
            torch.nn.Linear(256, 32),
        )

        self.final = torch.nn.Sequential(
            torch.nn.Linear(32, 16),
            torch.nn.ReLU(inplace=True),
            torch.nn.Linear(16, 2),
        )

    def forward(self, left, right):
        outputLeft = self.brothers(left)
        outputRight = self.brothers(right)
        output = self.final((outputLeft - outputRight) ** 2)
        return output

bros = SiameseNetwork()
bros = bros.to(device)

Criterion and optimizer:
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(params=bros.parameters(), lr=0.001)

The training loop:
for batch in tqdm(tLoader, desc=f"Train epoch: {epoch+1}"):
        a = batch[0].to(device)
        b = batch[1].to(device)
        y = torch.unsqueeze(batch[2].type(torch.FloatTensor), 1).to(device)

        optimizer.zero_grad()

        output = bros(a,b)
        loss = criterion(output, y)
        loss.backward()

        trainingLoss += loss.item()

        optimizer.step()

Now, this seems to be working, as it produces results that are reasonable, but the validation error stops dropping at 0.13 after just a of epochs. Can't find a lot of things on this kind of NNs using Pytorch. Are there ways to optimize it? Am I doing something wrong?


title: How can I emulate/run PyTorch model that uses ATen STFT implementation on arm based cpu?

body:
I am trying to run my PyTorch model for ASR on an arm based device without gpu. As far as I know, arm does not support MKL which ATen uses. Naturally, I am getting the following error when I try to make inference:
RuntimeError: fft: ATen not compiled with MKL support

How can I solve this problem? Are there any alternatives that I can use?


title: How to load weights from Pytorch to Keras layer by layer?

body:
I have tried tons of methods like onnx2keras, pytorch2keras and so on. But there would always be something wrong...
Since my model is not really complicated: just a ResNet18-encoder + Decoder with some skip-connections. I'm considering to simply transfer them one layer by another, from pytorch to Keras.
Before I try I'd like to ask if you have similar experience? I know there's set_weights method, but that's for keras-to-keras so nothing special. However, Keras is object-based model, so how can I assign name-based weights, e.g. 'encoder.bn1.bias', 'encoder.bn1.running_mean', 'encoder.bn1.running_var' to a BN? I don't want TF1.x solutions because all of my work is on TF2.x.
So In my opinion, it would be something like:
# 1. Save weights and names from pytorch model
weights_dict = torch_mode.static_dict()

# 2. Construct Keras model
keras_model = tf.keras.models.Model(...)

# 3. Now load weights for each layer in Keras model
for var_name, weight in weights_dict.items():
    # Assign conv with weight with'encoder.conv1.weight'
    # Assign BN with 'encoder.bn1.weight', 'encoder.bn1.bias', 'encoder.bn1.running_mean', 'encoder.bn1.running_var', 'encoder.bn1.num_batches_tracked'

But I don't know how... Look forward to your opinions!


title: How to differentiate a gradient in Pytorch

body:
I'm trying to differentiate a gradient in PyTorch. I found this link but can't get it to work.
My code looks as follows:
import torch
from torch.autograd import grad
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module):
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(2, 20)
            self.fc2=nn.Linear(20, 20)
            self.out=nn.Linear(20, 4)

        def forward(self, x):
            x=self.fc1(x)
            x=self.fc2(x)
            x=self.out(x)
            return x

nx = net_x()
r = torch.tensor([1.0,2.0])
nx(r)
>>>tensor([-0.2356, -0.7315, -0.2100, -0.6741], grad_fn=<AddBackward0>)

But when I try to differentiate the function with respect to the first parameter
grad(nx, r[0])

I get the error
TypeError: 'net_x' object is not iterable

Update
Trying to extend this to tensors:
For some reason the gradient is the same for all inputs.
a = torch.rand((8,2), requires_grad=True)
s = []
s_t = []
for input_tensor in a:
    output_tensor = nx(input_tensor)
    s.append(output_tensor[0])
    s_t_value = grad(output_tensor[0], input_tensor)[0][0]
    s_t.append(s_t_value)
print(s_t)

But the output is:
[tensor(-0.1326), tensor(-0.1326), tensor(-0.1326), tensor(-0.1326), tensor(-0.1326), tensor(-0.1326), tensor(-0.1326), tensor(-0.1326)]


title: Converting ResNet50 Decoder code to ResNet18 Decoder

body:
I'm trying to convert ResNet50 encoder to ResNet18 encoder for U-Net model from this repository https://github.com/kevinlu1211/pytorch-unet-resnet-50-encoder/blob/master/u_net_resnet_50_encoder.py. Its all new to me, I don't understand why changing the network like this does not work.
self.bridge = Bridge(512, 512)

up_blocks.append(UpBlockForUNetWithResNet50(512, 256))
up_blocks.append(UpBlockForUNetWithResNet50(256, 128))
up_blocks.append(UpBlockForUNetWithResNet50(128,64))

The error received is:
Given groups=1, weight of size [128, 192, 3, 3], expected input[2, 256, 64, 64] to have 192 channels, but got 256 channels instead.


title: RuntimeError: mat1 and mat2 shapes cannot be multiplied (20x27 and 3x30)

body:
I am new to AI and have followed a tutorial to build a machine learner which predicts championship game scores. I have completed the tutorial but when I go to run the code, I get a run time error which just says mat1 and mat2 shapes cannot be multiplied. I have looked through all the code and cant seem to find an issue with it. Does anyone know why this error is coming up? This is the code for the program:
import pandas
import torch
import matplotlib.pyplot as plt
import numpy as np

filepath = "2010_training.csv"
df = pandas.read_csv(filepath)

extract = [5, 6, 7, 8, 9, 10, 13, 14, 16, 17, 18, 20, 21, 23, 24, 25, 26, 27, 29, 30, 31, 32, 33, 35,
 36, 37, 38, 47, 48, 49]
 df = df.iloc[:, extract]

training = df.iloc[:-20]
test = df.iloc[-20:]

for e in range(len(training.columns) - 3):
num = max(training.iloc[:, e].max(), test.iloc[:, e].max())
if num < 10:
  training.iloc[:, e] /=10
  test.iloc[:, e] /=10
elif num < 100:
  training.iloc[:, e] /=100
  test.iloc[:, e] /=100
elif num < 1000:
  training.iloc[:, e] /= 1000
  test.iloc[:, e] /=1000
else:
  print("Error in normalisation! Please check!")

training = training.sample(frac=1)
test = test.sample(frac=1)

training_input = training.iloc[:, -3:]
training_output = training.iloc[:, -3:]
test_input = test.iloc[:, :-3]
test_output = test.iloc[:, -3:]

def convert_output_win(source):
  target = source.copy()
  target['new'] = 2
  for i, rows in target.iterrows():
    if rows['win'] == 1:
      rows['new'] = 1
    if rows['draw'] == 1:
      rows['new'] = 0
    if rows['defeat'] == 1:
      rows['new'] = 0
   return target.iloc[:, -1]

training_output = convert_output_win(training_output)
test_output = convert_output_win(test_output)

class Net(torch.nn.Module):
  def __init__(self, input_size, hidden_size):
    super(Net, self).__init__()
    self.input_size = input_size
    self.hidden_size = hidden_size
    self.fc1 = torch.nn.Linear(self.input_size, self.hidden_size)
    self.relu = torch.nn.ReLU()
    self.fc2 = torch.nn.Linear(self.hidden_size, 1)
    self.sigmoid = torch.nn.Sigmoid()

  def forward(self, x):
    hidden = self.fc1(x)
    relu = self.relu(hidden)
    output = self.fc2(relu)
    output = self.sigmoid(output)
    return output

training_input = torch.FloatTensor(training_input.values)
training_output = torch.FloatTensor(training_output.values)
test_input = torch.FloatTensor(test_input.values)
test_output = torch.FloatTensor(test_output.values)

input_size = training_input.size()[1]
hidden_size = 30

model = Net(input_size, hidden_size)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(model.parameters(), lr = 0.9, momentum=0.2)

model.eval()
y_pred = model(test_input)
before_train = criterion(y_pred.squeeze(), test_output)
print('Test loss before training', before_train.item())

model.train()
epochs = 5000
errors = []
for epoch in range(epochs):
optimizer.zero_grad()
y_pred = model(training_input)
loss = criterion(y_pred.squeeze(), training_output)
errors.append(loss.item())

print('Epoch {}: train loss: {}'.format(epoch, loss.item()))
loss.backward
optimizer.step()

model.eval()
y_pred = model(test_input)
after_train = criterion(y_pred.squeeze(), test_output)
print('Test loss after Training', after_train.item())

def plotcharts(errors):
errors = np.array(errors)

plt.figure(figsize=(12, 5))

graf02 = plt.subplot(1, 2, 1)
graf02.set_title('Errors')
plt.plot(errors, '-')
plt.xlabel('Epochs')

graf03 = plt.subplot(1, 2, 2)
graf03.set_title('Tests')
a = plt.plot(test_output.numpy(), 'yo', label='Real')
plt.setp(a, markersize=10)
a = plt.plot(y_pred.detach().numpy(), 'b+', label='Predicted')
plt.setp(a, markersize=10)
plt.legend(loc=7)
plt.show()

plotcharts(errors)


title: torch.tensor object has no attribute 'dim' with a basic network

body:
I m having some issues with pytorch after I train my model, it seems that the type i pass into my network changes, i.e when i try to do.a single prediction my forward function has an error as it has no ".dim" function even though I pass a torch.tensor object as input. this is my basic network architecture
class Net(nn.Module):
  def __init__(self,input_shape):
    super(Net,self).__init__()
    self.fc1 = nn.Linear(input_shape,2)
    self.fc2 = nn.Dropout(p=0.2)
    self.fc3 = nn.Linear(2,2)
  def forward(self,x):
    x = torch.relu(self.fc1(x))
    x = torch.softmax(self.fc3(x), 1)
    return x

it trains and works as expected on the training batch but when i pass a single tensor
torch.Size([1, 100])

it gives me this error
x = torch.relu(self.fc1(x))  AttributeError: 'builtin_function_or_method' object has no attribute 'dim'

The network was trained with batches of
torch.Size([64, 100])

And it worked during training on the batches of 64 and output the results I expected when when I perform a single prediction using
prediction = model(X[0])

I encounter this error. I have tried a few things such as printing the type to verify that it was a valid object being passed in but it seems that somehow during the forward pass the types I have passed become invalid and I cannot understand why if my input is a singular element of the original training data . I also tried wrapping the X[0] inside a Variable() as Im unsure exactly what pytorch seems to expect as input given this error. I have also read other answers but they seem to address changes mostly in other layers I am not using as returning different values and I am unsure how to verify if the output of one of my layers is changing the type passed into the next layer.
Any help would be appreciated.
EDITED------------------------------------------------------------------------

AttributeError                            Traceback (most recent call last)
<ipython-input-602-1f45fd5f7727> in <module>
---> 12 y_pred = model(X[0])

/opt/conda/envs/pytorch-py35/lib/python3.5/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-595-74840ffbc280> in forward(self, x)
     13     self.fc3 = nn.Linear(2,2)
     14   def forward(self,x):
---> 15     x = torch.relu(self.fc1(x))
     16     x = torch.softmax(self.fc3(x), 1)
     17     return x

/opt/conda/envs/pytorch-py35/lib/python3.5/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

/opt/conda/envs/pytorch-py35/lib/python3.5/site-packages/torch/nn/modules/linear.py in forward(self, input)
     65     @weak_script_method
     66     def forward(self, input):
---> 67         return F.linear(input, self.weight, self.bias)
     68
     69     def extra_repr(self):

/opt/conda/envs/pytorch-py35/lib/python3.5/site-packages/torch/nn/functional.py in linear(input, weight, bias)
   1348         - Output: :math:`(N, *, out\_features)`
   1349     """
-> 1350     if input.dim() == 2 and bias is not None:
   1351         # fused op is marginally faster
   1352         ret = torch.addmm(torch.jit._unwrap_optional(bias), input, weight.t())

AttributeError: 'builtin_function_or_method' object has no attribute 'dim'

Here is my training batching also incase this is useful.
from torch.utils.data import Dataset, DataLoader #Define a dataset and trainer for pytorch
class dataset(Dataset):
  def __init__(self,x,y):
    self.x = x.clone().detach().requires_grad_(True)
    self.y = y.clone().detach().requires_grad_(False)
    self.length = self.x.shape[0]

  def __getitem__(self,idx):
    return self.x[idx],self.y[idx]
  def __len__(self):
    return self.length

trainset = dataset(X,y)
trainloader = DataLoader(trainset,batch_size=64,shuffle=False)

EDITED------------------------------------------------------------------------

To add more details:
If i print(X)
tensor([[ 1.0430, -0.8477,  0.9932,  ..., -0.8662,  0.6748, -0.4958],
        [ 2.0728, -1.3987,  2.0863,  ..., -1.2584,  0.9423, -1.1652],
        [ 1.1594, -0.2600,  1.1863,  ..., -0.2268, -0.0596, -1.2434],
        ...,
        [ 1.2905, -2.1808,  1.2301,  ..., -2.4004,  2.1426,  0.8747],
        [-0.4722, -1.2714, -0.4748,  ..., -1.3785,  1.6691,  1.8886],
        [ 0.6362, -1.1497,  0.6117,  ..., -1.1174,  1.0634,  0.2901]],
       dtype=torch.float64)

and to actually train the network im using:
output = model(x_train)
loss = loss_fn(output,y_train)
predicted = model(X.clone().detach().requiresgrad(True).float())
optimizer.zero_grad()
loss.backward()
optimizer.step()

EDITED------------------------------------------------------------------------

If i add the type passed into forward i get this during training
def forward(self,x):
    print(type(X))
    x = torch.relu(self.fc1(x))
    x = torch.softmax(self.fc3(x), 1)
    return x

<class 'torch.Tensor'>
<class 'torch.Tensor'>


there is a cuda runtime error(59)  when i train my model with pytorch. I get the error as: "assertion ‘THCNumerics::ge(val,zero)’ failed"

I get a cuda runtime error(59) when i train model, just like that picture. The problem  appeared when i use 128000 pieces of data for training, but didn't appear when i used 12800 pieces of data for training.Also, I used the DataLoader to load 128 piece of data per batch.the code that prompts for the problem is :

  for i in range(batch_size):
            idxs_data = idxs[i].data
            for j in range(seq_len):
                if mask[i][j] == 1:
                    pass

"if mask[i][j] == 1:" is the error code


title: What is the PyTorch equivalent to torch's module.backward()?

body: Torch has a module.backward function that takes an input and gradient output. Is there an equivalent call in PyTorch that I can use instead of calling .backward() on some output variable?


title: Stuck on building a customer support chatbot from scratch using reddit dataset

body:
I've trained a model who can mimic day to day conversations occurring on reddit. But my problem is that I want it to reply to a specific use case based on the vocabulary it has learned.

Summary: I am building a chatbot and there are many use cases. I've trained a model with a reddit dataset and now I have a model who can mimic reddit conversation. I want it to map it's vocabulary to one of my use cases. How should I tackle this scenario?

Any idea? Please help, I've tried searching the internet for days, but no answer.


title: Pytorch simulation fails to converge on convex loss function when not initialized with 0

body:
My code works when the weights initialized with 0. When I initialize them according to some seed, they fail to converge. This should be a bug since the loss function is convex.

I filtered two labels from MNIST (0 and 1), and then I trained a logistic regression model using pytorch. Since I use only 200 training samples (and 784 parameters), the model should quickly converge to 100% accuracy on the training set. This is not the case when the weights initialize by some seed.

I had some problem to share my code on stackoverflow, so here is a link to the code: https://drive.google.com/file/d/1ELe8TIWrXMiXgsB63B0Ss43GPr719rGc/view?usp=sharing


title: VS Code debugger shows infinitely deep PyTorch Tensor

body: I'm working through some PyTorch code (not mine) in the VS Code debugger and it has this weird property that some (but not all) tensors have this infinite recursion property, pictured. Has anyone encountered this before? Is it a VS Code bug or a strange property of the tensor itself? The code works as expected.


title: Adding extra data to standard convolution neural network in pytorch

body:
I am trying to extra data during training Resnet18.
In my case it is gender information expressed as gender=Tensor([g]), where g =[-1|0|1] for [female|unknown|male], and some shape descriptor obtained in previous steps, which is of the form desc=Tensor([...]),  desc.shape = ([64,]).
I followed the hints which i had found at this stack overflow thread and successfully added the gender information, however after i add the desc tensor, the Loss becomes NaN. I tried to normalize the resulting layer but to no avail. Here is the modified Resnet18 forward function that I use:

def forward(self, xy):
    x = xy[0] #torch.Size([4, 3, 224, 224])
    desc = xy[1] #torch.Size([4, 64])
    genders = xy[2] #torch.Size([1, 4])

    x = self.conv1(x) #torch.Size([4, 64, 112, 112])
    x = self.bn1(x) #torch.Size([4, 64, 112, 112])
    x = self.relu(x) # torch.Size([4, 64, 112, 112])
    x = self.maxpool(x) #torch.Size([4, 64, 56, 56])

    x = self.layer1(x) #torch.Size([4, 64, 56, 56])
    x = self.layer2(x) #torch.Size([4, 128, 28, 28])
    x = self.layer3(x) #torch.Size([4, 256, 14, 14])
    x = self.layer4(x) #torch.Size([4, 512, 7, 7])

    x = self.avgpool(x) #torch.Size([4, 512, 1, 1])
    x = torch.flatten(x, 1) #torch.Size([4, 512])
    x = self.fc(x) #torch.Size([4, 35]); torch.norm(x,dim=1) == tensor([4.4604, 1.9574, 2.0123, 1.9583])

    y = genders.transpose(1,0) #torch.Size([4, 1]);  torch.norm(y,dim=1) == tensor([1., 1., 1., 1.])
    desc = desc.type(torch.float32) #torch.Size([4, 64]); torch.norm(desc,dim=1) == tensor([1., 1., 1., 1.]

    z1 = torch.cat([x,y,desc],dim=1) #torch.Size([4, 100]); torch.norm(z1,dim=1) == tensor([3.6243, 2.5945, 3.1520, 3.0312])
    #z2 = self.bn2(z1)
    z1 = self.fc2(z1) #torch.Size([4, 35]); torch.norm(z1,dim=1) == tensor([1.2954, 0.9399, 1.0137, 1.1230]
    return z1

Would you know how to properly add the extra data to the layer?

Best

Chris


title: Finding parameters with backpropagation and gradient descent in PyTorch

body:
I am experimenting with PyTorch and autodifferentiation and gradient descent

To that end I would like to estimate the parameters that would produce a certain value to an arbitrary linear in the parameters function.

My code is here:

import torch

X = X.astype(float)

X = np.array([[3.], [4.], [5.]])

X = torch.from_numpy(X)

X.requires_grad = True

W = np.random.randn(3,3)

W = np.triu(W, k=0)

W = torch.from_numpy(W)

W.requires_grad = True

out = 10 - (X@torch.transpose(X, 1,0) * W).sum()

out is :

My objective is to make out close to 0 (within an interval of [-.00001 , 0.0001]) by adjusting W using the gradient of W.

How should I proceed from here to achieve this end with pytorch?

Update

@Umang: this is what I get when I run the code you propose:

In fact the algorithm diverges.


title: broadcasting across tensors in `pytorch`

body:
I am using pytorch as an array processing language (not for the traditional deep learning purposes), and I am wondering what the canonical way is to do "batching" parallelism.

For example, suppose I want to compute svds of two dimensional layers of a 3-d tensor (using torch.svd(), say), and I want to return a tuple of stacked us, stacked s, stacked v.

Presumably, through the magic of SIMD parallelism, this should be doable in roughly the same time as a single layer svd (on gpu), but how to program it?


title: Pytorch - Project each row of a tensor to the column space of another tensor

body:
Currently, I have a tensor A, and a tensor U where U is an orthogonal matrix and is of full rank(so that its columns is a set of basis of U's column space, and all columns, say, u_i, have a norm of 1).

I am trying to compute the projection of each row of A onto the column space of U, using the formula from this post.

Which is, to compute Proj(A).
Is there any convenient functions or better operations to achieve this?
Thanks.


title: GAN does not learn when using symmetric outputs from generator to disciminator

body:
I'm currently trying to implement the paper Generative modeling for protein structures and I have succesfully been able to train a model following Pytorch's DCGAN Tutorial which has a similar model structure to the paper. The two implementations differ when it comes to output of the generator.
In the tutorial's model, the generator simply passes a normal output matrix to the discriminator. This works fine when I implement the paper's model (ommiting the symmetry and clamping) but the paper specifies:

During training, we enforce that G(z) be
positive by clamping output values above zero and symmetric

when I put this into my training loop I receive a loss graph that indicates that the generator isn't learning.
Here is my training loop:
# Training Loop
# Lists to keep track of progress
img_list = []
G_losses = []
D_losses = []
iters = 0

print("Starting Training Loop...")
# For each epoch
for epoch in range(num_epochs):
    # For each batch in the dataloader
    for i, data in enumerate(dataloader, 0):
        ############################
        # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
        ###########################
        ## Train with all-real batch
        netD.zero_grad()
        # Format batch
        # Unsqueezed dim one to convert [128, 64, 64] to [128, 1, 64, 64] to conform to D architecture
        real_cpu = (data.unsqueeze(dim=1).type(torch.FloatTensor)).to(device)
        b_size = real_cpu.size(0)
        label = torch.full((b_size,), real_label, device=device)
        # Forward pass real batch through D
        output = netD(real_cpu).view(-1)
        # Calculate loss on all-real batch
        errD_real = criterion(output, label)
        # Calculate gradients for D in backward pass
        errD_real.backward()
        D_x = output.mean().item()

        ## Train with all-fake batch
        # Generate batch of latent vectors
        noise = torch.randn(b_size, nz, 1, 1, device=device)
        # Generate fake image batch with G
        fake = netG(noise)
        label.fill_(fake_label)
        # Make Symmetric
        sym_fake = (fake.detach().clamp(min=0) + fake.detach().clamp(min=0).permute(0, 1, 3, 2)) / 2
        # Classify all fake batch with D
        output = netD(sym_fake).view(-1)
        # Calculate D's loss on the all-fake batch
        errD_fake = criterion(output, label)
        # Calculate the gradients for this batch
        errD_fake.backward()
        D_G_z1 = output.mean().item()
        # Add the gradients from the all-real and all-fake batches
        errD = errD_real + errD_fake
        # Update D
        optimizerD.step()
        #adjust_optim(optimizerD, iters)
        ############################
        # (2) Update G network: maximize log(D(G(z)))
        ###########################
        netG.zero_grad()
        label.fill_(real_label)  # fake labels are real for generator cost
        # Since we just updated D, perform another forward pass of all-fake batch through D
        output = netD(fake.detach()).view(-1)
        # Calculate G's loss based on this output
        errG = criterion(output, label)
        # Calculate gradients for G
        errG.backward()
        D_G_z2 = output.mean().item()
        # Update G
        optimizerG.step()
        adjust_optim(optimizerG, iters)

        # Output training stats
        if i % 50 == 0:
            print('[%d/%d][%d/%d]\tLoss_D: %.4f\tLoss_G: %.4f\tD(x): %.4f\tD(G(z)): %.4f / %.4f'
                  % (epoch, num_epochs, i, len(dataloader),
                     errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))

        # Save Losses for plotting later
        G_losses.append(errG.item())
        D_losses.append(errD.item())

        # Check how the generator is doing by saving G's output on fixed_noise
        if (iters % 500 == 0) or ((epoch == num_epochs-1) and (i == len(dataloader)-1)):
            with torch.no_grad():
                fake = netG(fixed_noise).detach().cpu()
            img_list.append(vutils.make_grid(fake, padding=2, normalize=True))

        iters += 1

Here is the training loss.

Here is my expected loss.

I make the output symmetric with the following line
sym_fake = (fake.detach().clamp(min=0) + fake.detach().clamp(min=0).permute(0, 1, 3, 2)) / 2

and I pass it to the discriminator on the lines that call sym_fake
Question
Is my implementation in pytorch wrong or is there something I'm missing? I don't understand why the paper makes the matrix symmetric and clamps if the network is capable of generating images without the need for symmetry and clamping.


title: how can i retrain huggingface fine tuned model

body:
thanks in advance.
I want to train first with more data, it means more label.
(its like pre training in this case)
int this example, label number is 19 and its done well.
after that I choose some label only, and want to train again.
(only 10 label)
But error happened during below code
tokenizer = RobertaTokenizer.from_pretrained(pretrained_path, do_lower_case=False)
model = RobertaForSequenceClassification.from_pretrained(pretrained_path, num_labels=10)
error message is like below.
Error(s) in loading state_dict for RobertaForSequenceClassification:
size mismatch for classifier.out_proj.weight: copying a param with shape torch.Size([19, 768]) from checkpoint, the shape in current model is torch.Size([10, 768]).
size mismatch for classifier.out_proj.bias: copying a param with shape torch.Size([19]) from checkpoint, the shape in current model is torch.Size([10]).
how can I do it??


title: Perform a different neighbourhood operation for specified pixels

body:
I have an HxW "feature map", F. Let us assume that it is a HxWx1 map. Through some other operation, I have a set of pixels that are of interest to me, (say N pixels). Each of these pixels is associated with a different value, thus my set is of the form Nx3 where each pixel is of the form x, y and val. Note that this val is different from the feature map value at the location.
Here is my question. Is it possible to vectorize a neighbourhood operation for each of these points? For each pixel n from N, I wish to multiply the corresponding val to its 3x3 neighbourhood in the feature map F. For the 3x3 neighbourhood, this gives a new 3x3 set of elements new val. I want to replace the x y with the pixel with the maximum of new val (multiplied feature map) in the 3x3 window.
This sounds similar to a convolution (slight abuse of terminology here) followed by a max pool operation, but not exactly since each pixel location has a different val to be multiplied.
Sample input and output, and walkthrough for required solution
Let us assume H=10 and W=10
Here is a sample F
0.635955  0.922379  0.993406  0.007837  0.818661  0.983730  0.199866  0.757519  0.073152  0.015831
0.397718  0.097353  0.231351  0.177886  0.343099  0.419940  0.017342  0.087294  0.402266  0.366337
0.978686  0.476594  0.067836  0.148977  0.058994  0.810586  0.542894  0.797419  0.386559  0.225982
0.479860  0.033354  0.353366  0.431562  0.336208  0.674272  0.398151  0.713732  0.598623  0.829230
0.940838  0.869564  0.287100  0.669844  0.631836  0.748982  0.762292  0.597999  0.540236  0.758802
0.925995  0.141296  0.466772  0.672663  0.929746  0.544029  0.991860  0.197474  0.762866  0.798973
0.543519  0.128332  0.624323  0.876569  0.050709  0.223705  0.708381  0.380842  0.818092  0.163447
0.283125  0.329618  0.283481  0.672950  0.136922  0.897785  0.385479  0.764824  0.132671  0.091148
0.661984  0.369459  0.501181  0.352681  0.554113  0.133283  0.593048  0.108534  0.397813  0.836065
0.654929  0.928576  0.539204  0.931213  0.344114  0.591214  0.126809  0.456681  0.036531  0.725228

My structure of pixels, let us say N=3
The three values in the order of row,col,val: (for simplicity I assume x is rows, and y is cols, though it isn't necessarily the case). This is completely independent of the feature map in the previous step.
3,2,0.38
4,4,0.602
7,5,0.9647

The neighborhood around (3,2) is:
[[0.4765941 , 0.06783561, 0.14897662],
[0.03335438, 0.35336647, 0.4315618 ],
[0.86956374, 0.28709952, 0.66984412]]

Thus val * neighborhood yields. (here val is 0.38)
[[0.18110576, 0.02577753, 0.05661112],
[0.01267466, 0.13427926, 0.16399349],
[0.33043422, 0.10909782, 0.25454077]]

The location of max value here is (2,0) i.e. (1,-1) with respect to center pixel. Thus my updated (x,y) should be (3,2) + (1,-1) = (4,1).
Similarly for the other two, the updated pixels are : (5,4) and (7,5)
How can I parallelize this entire thing?
(Hopefully to be loaded onto a GPU using Pytorch, but not necessarily, I have not come to that stage yet.)
Note: I had asked this question a few days ago, but it was poorly framed without proper info. Hopefully this solves the issue.
Edit: For this specific instance, F can be produced as a random array:
F = np.random.rand(10,10)


title: pytorch exports sequential model to onnx as functional model

body:
I am exporting a pytorch model to keras by using torch.onnx.export and then the onnx2keras tool. The pytorch model is:
    model = torch.nn.Sequential(
            torch.nn.Conv2d(1, 32, kernel_size=5, padding=0, stride=1),
            torch.nn.ReLU(),
            torch.nn.MaxPool2d(kernel_size=2),
            torch.nn.Conv2d(32, 32, kernel_size=5, padding=0, stride=1),
            torch.nn.ReLU(),
            torch.nn.MaxPool2d(kernel_size=2),
            )

Then I train the model and export it.
    torch.onnx.export(model, dummy_input, "alice.onnx", verbose=True)
    o_model = onnx.load("onnxmodels/alice.onnx")
    k_model = onnx_to_keras(o_model, ['input.1'])
    json_config = k_model.to_json()
    with open("k_alice.json", 'w') as f:
         f.write(json_config)
    k_model.save_weights("k_alice.h5")

The resulting keras model is a functional model, rather than a sequential model.
    {"class_name": "Functional", "config": {"name": "functional_1", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 1, 28, 28],.....

I need the keras model to be either a sequential model, or simply a part of the model class (I am using a tool that only supports those classes). The strangest part is, I did this exact same workflow with the exact same model a few days ago, and the tool returned a model class model!
    {"class_name": "Model", "config": {"name": "model", "layers": [{"class_name": "InputLayer",........

How do I get torch.onnx.export and then onnx2keras to output a model of the same class that the original pytorch model was? Thanks!


title: Deprecated error when creating PyTorch Deep Learning VM from the Google Cloud Marketplace

body:
I followed the instructions here to create a GCP Deep Learning VM with pytorch, but get the following error:
tensorflow has resource level warnings
ferracane-pytorch-chiefcomplaint-vm: The resource 'projects/click-to-deploy-images/global/images/pytorch-1-4-cu101-20200615' is deprecated. A suggested replacement is 'projects/click-to-deploy-images/global/images/pytorch-1-4-cu101-v20200701'.


title: After enough(200) epochs, LSTM LM return only , , 'the' token with Nan loss

body:
I constructed Bi-LSTM Language Model by pytorch, and found that after 200 epochs, the model suddenly returned only meaningless tokens with Nan loss, while it returned reasonable tokens before.
Please refer to the code of model below:
# optimizer = torch.optim.Adam(model.parameters(), lr=0.009,  amsgrad=False)

class BiLSTM(nn.Module):

    def __init__(self, voc_size, hidn_size, emb_size=300):
        super().__init__()
        self.voc_size = voc_size
        self.emb_size = emb_size
        self.hidn_size = hidn_size
        self.emb = nn.Embedding(num_embeddings=voc_size, embedding_dim=emb_size)
        self.lstm = nn.LSTM(input_size=emb_size, hidden_size=hidn_size, bidirectional=True)
        self.lm_out = nn.Linear(hidn_size*2, voc_size)
        self.dropout = nn.Dropout(p=0.3)

    def forward(self, x, prev_state):
        state_h, state_c = prev_state
        bs = len(x)

        emb = self.emb(x)
        emb = emb.permute(1,0,-1)
        out, (state_h, state_c) = self.lstm(emb, (state_h[:,:bs,:].contiguous(), state_c[:,:bs,:].contiguous()))

        forward_out = out[:, :, :self.hidn_size]
        backward_out = out[:, :, self.hidn_size:]
        concat_h = torch.cat([forward_out[:-2], backward_out[2:]], dim=2)

        final_out = self.lm_out(self.dropout(concat_h.permute(1,0,2)))

        return final_out.view(final_out.size()[0]*final_out.size()[1], final_out.size()[-1]), (state_h, state_c)


title: Training a LSTM/RNN using Maximum Likelihood Estimation (MLE)

body:
I am trying to do a time-series forecasting task with LSTM and RNN. My goal is to train the model to input a sequence of 128-time-step [x_1, x_2, ..., x_128] to predict an outcome x_129 (regression task)
I have tried MSE and Mean Absolute Error loss and now I am to estimate using MLE. I wonder if this can be done with a simple loss function implementation or there should be any step in between? All MLE-based models I have read is used to maximize the probability of each class given the dataset, which seems to be more suitable for classification tasks
Any help is very appreciated :D


title: How to plot Fastai loss function after each learning cycle

body:
https://github.com/BorutFlis/predictor/blob/master/soccer_predictor.ipynb
The problem is available in the notebook above.
learn.recorder.plot_loss()

I use this command and the image(check notebook) plots the validation loss by every cycle of learning, while it plots a lot more data from the train-set loss. Can I change this somehow so that the train loss will only be plotted after each cycle?


title: PyTorch Geometric: What is the function utility to split train/validation/test for node classification

body: I am new to PyTorch Geometric. I know there is a function that gives you the train, test, and validation node mask of a custom ratio in the node classification task. Does anyone know it?


title: How to get a column from a tensor?

body:
Let's say I have a tensor consisting of 1 and 0's as shown below. How can I get the index of a specific column to replace with new values ? If I want to replace the values of column 1 with the [3.,4.,5.,6.], how do I accomplish this ?
a = torch.tensor([[[1., 0., 0., 0.]],
        [[0., 1., 0., 0.]],
        [[1., 0., 0., 0.]],
        [[0., 0., 0., 1.]],
        [[1., 0., 0., 0.]],
        [[0., 0., 0., 1.]],
        [[1., 0., 0., 0.]]])


title: GAN convergence failure, generator loss keeps increasing

body:
I'm trying to build an SGAN which is propose by this paper. In particular, this SGAN can take only one image as training input, and randomly crop it to form a minibatch for each training iteration.
Here is my training sample:

I trained my model for 320 iterations, and keep generator output from a fixed noise every 10 iteraion, the result I get is horrible:

Most importantly, my generator losses keeps increasing with some sudden drop:

Does anyone have any idea on how to fix this problem? Thanks in advance.
P.S., here is my model:
Discriminator(
  (main): Sequential(
    (0): Conv2d(1, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), 640 params
    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), 128
params
    (2): LeakyReLU(negative_slope=0.2, inplace=True), 0 params
    (3): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), 73,856 params
    (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), 256 params
    (5): LeakyReLU(negative_slope=0.2, inplace=True), 0 params
    (6): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), 295,168 params
    (7): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), 512 params
    (8): LeakyReLU(negative_slope=0.2, inplace=True), 0 params
    (9): Conv2d(256, 1, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), 2,305 params
    (10): Sigmoid(), 0 params
  ), 372,865 params
), 372,865 params

Generator(
  (main): Sequential(
    (0): ConvTranspose2d(20, 256, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False), 250,880 params
    (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), 512 params
    (2): ReLU(inplace=True), 0 params
    (3): ConvTranspose2d(256, 128, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False), 1,605,632 params
    (4): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), 256 params
    (5): ReLU(inplace=True), 0 params
    (6): ConvTranspose2d(128, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False), 401,408 params
    (7): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True), 128
params
    (8): ReLU(inplace=True), 0 params
    (9): ConvTranspose2d(64, 1, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False), 3,136 params
    (10): Tanh(), 0 params
  ), 2,261,952 params
), 2,261,952 params

And my training code:
for iteration in range(ITERATIONS):
############################
# (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
###########################
miniBatch = getRandomMiniBatchFromSingleImage(trainSample, BATCH_SIZE)
miniBatch = miniBatch.to(DEVICE)
# train with real
netD.zero_grad()

output = netD(miniBatch)
errD_real = criterion(output, output.detach() * 0.0 + REAL_LABEL)
errD_real.backward()
D_x = output.mean().item()

# train with fake.
noise = torch.randn(BATCH_SIZE, Z_NO_OF_CHANNELS, Z_DIMENSION, Z_DIMENSION, device=DEVICE)
fake = netG(noise)
output = netD(fake.detach())
errD_fake = criterion(output, output.detach() * 0.0 + FAKE_LABEL)
errD_fake.backward()

D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()

############################
# (2) Update G network: maximize log(D(G(z)))
###########################
netG.zero_grad()
output = netD(fake)
errG = criterion(output, output.detach() * 0.0 + REAL_LABEL)
errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()


title: Can't create conda environment with tensorflow and pytorch

body:
I am trying to create a conda environment that contains both tensorflow and pytorch. As a minimal example, I am running
conda env create -n test -f test-env.yaml
using the following test-env.yaml file:
channels:
- conda-forge
- defaults
- pyviz

dependencies:
- python>=3.7,<3.8
- tensorflow>=2.0.0,<2.4.0
- pytorch>=1.0.0,<2.0.0
name: my_environment

I get the following long error complaining about conflicts:
$ conda env create -n test -f test-env.yaml
Collecting package metadata (repodata.json): done
Solving environment: -
Found conflicts! Looking for incompatible packages.
This can take several minutes.  Press CTRL-C to abort.
failed
Solving environment: \
Found conflicts! Looking for incompatible packages.
This can take several minutes.  Press CTRL-C to abort.
failed

UnsatisfiableError: The following specifications were found to be incompatible with each other:

Output in format: Requested package -> Available versions

Package _libgcc_mutex conflicts for:
pytorch[version='>=1.0.0,<2.0.0'] -> _openmp_mutex[version='>=4.5'] -> _libgcc_mutex[version='*|0.1',build='conda_forge|main|main']
python[version='>=3.7,<3.8'] -> libgcc-ng[version='>=9.3.0'] -> _libgcc_mutex[version='*|0.1',build='conda_forge|main']

Package python conflicts for:
tensorflow[version='>=2.0.0,<2.4.0'] -> python[version='2.7.*|3.6.*|3.7.*|3.8.*']
python[version='>=3.7,<3.8']
tensorflow[version='>=2.0.0,<2.4.0'] -> tensorboard[version='>=2.3.0'] -> python[version='>=2.7,<2.8.0a0|>=3.6,<3.7.0a0|>=3.8,<3.9.0a0|>=3|>=3.2|>=3.5|>=3.7,<3.8.0a0']
pytorch[version='>=1.0.0,<2.0.0'] -> python[version='>=2.7,<2.8.0a0|>=3.6,<3.7.0a0|>=3.7,<3.8.0a0|>=3.8,<3.9.0a0|>=3.9,<3.10.0a0']
pytorch[version='>=1.0.0,<2.0.0'] -> cffi -> python[version='2.7.*|3.5.*|3.6.*|3.6.12|3.6.12|3.7.9|3.6.9|3.6.9|3.6.9|3.6.9|>=3.5,<3.6.0a0|3.4.*|3.9.*|>=3.5|3.8.*|3.7.*|>=3.7|>=3.6,<3.7|>=3',build='1_73_pypy|3_73_pypy|4_73_pypy|5_73_pypy|2_73_pypy|0_73_pypy']

Package setuptools conflicts for:
tensorflow[version='>=2.0.0,<2.4.0'] -> tensorboard[version='>=2.3.0'] -> setuptools[version='>=41.0.0|>=41.4']
python[version='>=3.7,<3.8'] -> pip -> setuptools

Package numpy conflicts for:
tensorflow[version='>=2.0.0,<2.4.0'] -> tensorboard[version='>=2.3.0'] -> numpy[version='>=1.12.0|>=1.16|>=1.16.6,<2.0a0|>=1.16.1']
pytorch[version='>=1.0.0,<2.0.0'] -> numpy[version='>=1.11.3,<2.0a0|>=1.16.6,<2.0a0|>=1.19.5,<2.0a0|>=1.19.4,<2.0a0|>=1.14.6,<2.0a0']

Package cudatoolkit conflicts for:
pytorch[version='>=1.0.0,<2.0.0'] -> cudnn[version='>=7.6,<8.0a0'] -> cudatoolkit[version='11.0|11.0.*|>=10.0,<10.1|>=10.1,<10.2|>=9.0,<9.1|>=9.2,<9.3|>=10.2,<10.3|11.2|11.2.*|11.1|11.1.*|9.0.*|8.0.*']
pytorch[version='>=1.0.0,<2.0.0'] -> cudatoolkit[version='10.0.*|10.0|10.0.*|10.1|10.1.*|10.2|10.2.*|9.2|9.2.*|>=10.1.243,<10.2.0a0|>=9.2,<9.3.0a0|>=10.0.130,<10.1.0a0|9.2.*|>=9.0,<9.1.0a0|>=8.0,<8.1.0a0']

Package _openmp_mutex conflicts for:
pytorch[version='>=1.0.0,<2.0.0'] -> _openmp_mutex[version='>=4.5']
pytorch[version='>=1.0.0,<2.0.0'] -> mkl[version='>=2020.4,<2021.0a0'] -> _openmp_mutex=[build=*_llvm]The following specifications were found to be incompatible with your system:

  - feature:/linux-64::__glibc==2.17=0
  - pytorch[version='>=1.0.0,<2.0.0'] -> __glibc[version='>=2.17,<3.0.a0']
  - pytorch[version='>=1.0.0,<2.0.0'] -> magma[version='>=2.5.4,<2.5.5.0a0'] -> __glibc[version='>=2.17']

Your installed version is: 2.17

Note that strict channel priority may have removed packages required for satisfiability.

If I set channel_priority to flexible instead of strict, it does work. However, I would really like this to work with strict channel priority if possible. Is there any way to make it work?
Here is the output of conda info:
$ conda info

     active environment : base
    active env location : /home/kgullikson/miniconda3
            shell level : 1
       user config file : /home/kgullikson/.condarc
 populated config files : /home/kgullikson/.condarc
          conda version : 4.9.2
    conda-build version : 3.20.3
         python version : 3.6.11.final.0
       virtual packages : __cuda=11.2=0
                          __glibc=2.17=0
                          __unix=0=0
                          __archspec=1=x86_64
       base environment : /home/kgullikson/miniconda3  (writable)
           channel URLs : https://conda.anaconda.org/conda-forge/linux-64
                          https://conda.anaconda.org/conda-forge/noarch
                          https://conda.anaconda.org/pyviz/linux-64
                          https://conda.anaconda.org/pyviz/noarch
                          https://repo.anaconda.com/pkgs/main/linux-64
                          https://repo.anaconda.com/pkgs/main/noarch
                          https://repo.anaconda.com/pkgs/r/linux-64
                          https://repo.anaconda.com/pkgs/r/noarch
                          https://conda.anaconda.org/kgullikson/linux-64
                          https://conda.anaconda.org/kgullikson/noarch
          package cache : /home/kgullikson/miniconda3/pkgs
                          /home/kgullikson/.conda/pkgs
       envs directories : /home/kgullikson/miniconda3/envs
                          /home/kgullikson/.conda/envs
               platform : linux-64
             user-agent : conda/4.9.2 requests/2.25.1 CPython/3.6.11 Linux/3.10.0-1160.11.1.el7.x86_64 centos/7.6.1810 glibc/2.17
                UID:GID : 207402735:207400513
             netrc file : None
           offline mode : False

And my .condarc:
channels:
  - conda-forge
  - pyviz
  - defaults
  - kgullikson
conda-build:
  root-dir: ~/conda-builds
  skip_existing: true
pip_interop_enabled: false
channel_priority: strict

UPDATE:
Reading more closely through the documentation on conda, it seems like this is actually the expected behavior of strict channel priority.
https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-channels.html
Namely:

With strict channel priority, packages in lower
priority channels are not considered if a package with the same name
appears in a higher priority channel

So the problem here is that there are in fact tensorflow packages in conda-forge, just none that satisfy my requirement of using tensorflow 2. Given that they are pushing people to use strict and that they plan to make it the default in conda 5, that seems problematic...


title: How to use batch correctly with a simple Neural Network - Basic Pytorch

body:
Ok, first of all, sorry about my English and thanks for your help.
I think I have a problem with the term "batch". I know what it means and I think understand how it works, but I don't know how to translate it into python-pytorch neural network.
I have a variable "input_train" what contains (obviously) the input data to the neural network. This is a vector of 216 values. Then, I have "label_train" what contains the expected output, all defined here:
f = h5py.File('./Train.h5', 'r')
input_train = f['data'][...]
label_train = f['label'][...]
f.close()

f = h5py.File('./Test.h5', 'r')
input_test = f['data'][...]
label_test = f['label'][...]
f.close()

I tried to print the values like "print(input_train[0])" and it works perfectly, and the values are correct.
So, now I call a function what uses this vectors and some parameters more to do the "train loop".
def train_loop(train, label, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    for batch_size, (X, y) in enumerate(train, label):
        # Compute prediction and loss
        pred = model(X)
        loss = loss_fn(pred, y)

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

There is the problem. My code was written with Dataloaders, and for that reason is there that enumerate function. But I need to change that to don't use enumerate, so I have no idea how to do it. Because I really dont understand how can I say to a for bucle "ey, these are 27 inputs of my dataset, do this by one by". Because, I'm sure I'm wrong with it, but... where is the difference between that and putting batch_size = 1? I really don't understand the logic of this and how to use it correctly.
Again, thanks you all.
EDIT:
I have been working on delete the Dataloader and try to use data directly on model, and I have some archives but not the necessary ones.
Now I give to the train_loop function the data and labels separately(this last one is the correct answer that the model must predict).
epochs = 10
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    train_loop(data_train, label_train, model, loss_fn, optimizer)
    test_loop(data_test, label_test, model, loss_fn)
print("Done!")

And now the fucntion is defined here:
def train_loop(X, y, model, loss_fn, optimizer):

    examples = len(X)
    data_foreach_example = len(X[0])

    print("Num examples: " + str(examples))
    print("Num data_foreach_example: " + str(data_foreach_example))

    size = examples
    for i in range(size):

        XX = np.array(X)
        yy = np.array(y)

        print(XX[i])
        print(yy[i])

        # Compute prediction and loss
        pred = model(XX[i])
        loss = loss_fn(pred, yy[i])

        # Backpropagation
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if batch % 100 == 0:
            loss, current = loss.item(), batch * len(X)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

The prints says that I have 65000 examples to train and 216 data for each example, and that's correct.
But, my function crashes on pred = model(XX[i]), and here is the error:

AttributeError: 'numpy.ndarray' object has no attribute 'dim'

Can someone help me? -.-


title: PyTorch CPU memory leak but only when running on a specific machine

body:
I'm running a model and I've noticed that the RAM usage slowly increases during the training of the model. It's around 200mb-400mb per epoch, but over time it fills up all the RAM on my machine which eventually leads the OS to kill the job. However, the strange thing about this is it's only when running on a specific machine. I've run the same code on a different machine and there's no memory leak whatsoever.
The difference between the two machines is one is running PyTorch 1.7.1 with cuda 10.2 (the machine without the memory leak) and the other machine (the one with the memory leak) is running PyTorch 1.8.1· Both are run with conda and only on the CPU. I have tried using older versions of PyTorch on the machine with the memory leak, but the memory leak still exists so I doubt it's due to a PyTorch version.
I've been using psutil to monitor the RAM usage on the CPU and I've been using the tracemalloc package to print out snapshots during the training loop to see how the memory usage changes from one epoch to the next. Yet when printing out these differences there isn't any within my code that matches anywhere near the 200mb-400mb RAM increase...
An example of this would something like...
Epoch     25/  1000 Loss 1428.8508 RAM:  9.910GB
~/Model.py:46: size=1136 B (-96 B), count=2 (+0), average=568 B
~/utils.py:14: size=1040 B (+24 B), count=8 (+1), average=130 B
~/calc_loss.py:79: size=2128 B (+0 B), count=24 (+0), average=89 B
~/calc_loss.py:78: size=2056 B (+0 B), count=23 (+0), average=89 B
~/utils.py:6: size=1920 B (+0 B), count=21 (+0), average=91 B
Epoch     26/  1000 Loss 1426.0033 RAM: 10.254GB
~/Model.py:46: size=1232 B (+96 B), count=2 (+0), average=616 B
~/utils.py:14: size=1016 B (-24 B), count=7 (-1), average=145 B
~/calc_loss.py:79: size=2128 B (+0 B), count=24 (+0), average=89 B
~/calc_loss.py:78: size=2056 B (+0 B), count=23 (+0), average=89 B
~/utils.py:6: size=1920 B (+0 B), count=21 (+0), average=91 B
~/Layers.py:71: size=992 B (+0 B), count=11 (+0), average=90 B
Epoch     27/  1000 Loss 1436.8241 RAM: 10.606GB
~/utils.py:14: size=1040 B (+24 B), count=8 (+1), average=130 B
~/calc_loss.py:79: size=2128 B (+0 B), count=24 (+0), average=89 B
~/calc_loss.py:78: size=2056 B (+0 B), count=23 (+0), average=89 B
~/utils.py:6: size=1920 B (+0 B), count=21 (+0), average=91 B
~/Model.py:46: size=1232 B (+0 B), count=2 (+0), average=616 B
~/Layers.py:71: size=992 B (+0 B), count=11 (+0), average=90 B
Epoch     28/  1000 Loss 1428.6560 RAM: 10.968GB
~/calc_loss.py:79: size=2128 B (+0 B), count=24 (+0), average=89 B
~/calc_loss.py:78: size=2056 B (+0 B), count=23 (+0), average=89 B
~/utils.py:6: size=1920 B (+0 B), count=21 (+0), average=91 B
~/Model.py:46: size=1232 B (+0 B), count=2 (+0), average=616 B
~/utils.py:14: size=1040 B (+0 B), count=8 (+0), average=130 B
Epoch     29/  1000 Loss 1435.2988 RAM: 11.321GB
~/calc_loss.py:79: size=2128 B (+0 B), count=24 (+0), average=89 B
~/calc_loss.py:78: size=2056 B (+0 B), count=23 (+0), average=89 B
~/utils.py:6: size=1920 B (+0 B), count=21 (+0), average=91 B
~/Model.py:46: size=1232 B (+0 B), count=2 (+0), average=616 B
~/utils.py:14: size=1040 B (+0 B), count=8 (+0), average=130 B
~/Layers.py:71: size=992 B (+0 B), count=11 (+0), average=90 B

The information printed out in between the lines showing the current Epoch are created via the use of this function
def compare_snaps(snap1, snap2, limit=50):
  top_stats=snap1.compare_to(snap2, "lineno")

  for stat in top_stats[:limit]:
    line=str(stat)
    if("~/" in line): #filter only lines from my own code
      print(line)

this function takes two snapshots from  tracemalloc.take_snapshot() from the current epoch and the previous epoch and compares how the memory usage changes. It takes the top 50 memory intensive operations, and filters only the ones that I've written (i.e. it excludes any changes from within anaconda3) and prints these changes to the screen. As can be seen, the changes in memory are negligible. In fact, when comparing the snap shotoutput from both machines, they're near identical.
It seems really weird that PyTorch code would have a memory leak on one machine and not on another... Could this perhaps be a conda environemnt issue? I have tried running this from a pip install and the leak still persists. I understand that this a bit vague (as there's no source code) but are there any ways to check where the leaks coming from? Things like Valgrind comes to mind for compile code but not much for interpretative code. This memory leak issue seems to be outside of my skill-set, so any help would be appreciated.
Thank you! :)


title: Issue installing Facebook NeuralProphet

body:
I'm trying to install NeuralProphet via Anaconda Prompt (Acnadconda3), however I'm getting the following errors:
ERROR: Could not find a version that satisfies the requirement torch<1.7.0,>=1.4.0 (from neuralprophet) (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2, 1.7.0, 1.7.1, 1.8.0, 1.8.1)
ERROR: No matching distribution found for torch<1.7.0,>=1.4.0 (from neuralprophet)
I've checked my version of python (as below) from what I can see it's running 64bit - so I don't think this is the issue? Any help would be greately apprecaited!
Python 3.7.6 (default, Jan  8 2020, 20:23:39) [MSC v.1916 64 bit (AMD64)] :: Anaconda, Inc. on win32```


title: Pytorch Memory efficient implementation of indexed matrix multiplication

body:
i am searching for a memory efficient implementation of indexed matrix multiplication in pytorch. Lets say we have given a tensor A with shape b1xnxd, a tensor B with shape b2xmxd, and two index vectors ind and ind2 with shape b3.
I want to calculate the following memory efficiently:
A2 = A[ind] # shape b3xnxd
B2 = B[ind2] # shape b3xmxd
result = torch.matmul(A2,B2.transpose(2,1)) # shape b3xnxm

The problem is when b3>>b or b3>>b2, then the first two lines consume a lot of memory but a lot of the entries of that tensors are the same. Is there a method in pytorch that prevents that? Is there maybe a version of the first two lines where the resulting tensor only stores pointers to the entries of A and B?


title: How to use PyTorch's autograd efficiently with tensors?

body:
In my previous question I found how to use PyTorch's autograd to differentiate. And it worked:
#autograd
import torch
from torch.autograd import grad
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module):
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(1, 20)
            self.fc2=nn.Linear(20, 20)
            self.out=nn.Linear(20, 4)

        def forward(self, x):
            x=torch.tanh(self.fc1(x))
            x=torch.tanh(self.fc2(x))
            x=self.out(x)
            return x

nx = net_x()
r = torch.tensor([1.0], requires_grad=True)
print('r', r)
y = nx(r)
print('y', y)
print('')
for i in range(y.shape[0]):
    # prints the vector (dy_i/dr_0, dy_i/dr_1, ... dy_i/dr_n)
    print(grad(y[i], r, retain_graph=True))

>>>
r tensor([1.], requires_grad=True)
y tensor([ 0.1698, -0.1871, -0.1313, -0.2747], grad_fn=<AddBackward0>)

(tensor([-0.0124]),)
(tensor([-0.0952]),)
(tensor([-0.0433]),)
(tensor([-0.0099]),)

The problem that I currently have is that I have to differentiate a very large tensor and iterating through it like I'm currently doing (for i in range(y.shape[0])) is taking forever.
The reason I'm iterating is that from understanding, grad only knows how to propagate gradients from a scalar tensor, which y is not. So I need to compute the gradients with respect to each coordinate of y.
I know that TensorFlow is capable of differentiating tensors, from here:
tf.gradients(
    ys, xs, grad_ys=None, name='gradients', gate_gradients=False,
    aggregation_method=None, stop_gradients=None,
    unconnected_gradients=tf.UnconnectedGradients.NONE
)
"ys and xs are each a Tensor or a list of tensors. grad_ys is a list of Tensor, holding the gradients received by the ys. The list must be the same length as ys.

gradients() adds ops to the graph to output the derivatives of ys with respect to xs. It returns a list of Tensor of length len(xs) where each tensor is the sum(dy/dx) for y in ys and for x in xs."

And was hoping that there's a more efficient way to differentiate tensors in PyTorch.
For example:
a = range(100)
b = range(100)
c = range(100)
d = range(100)
my_tensor = torch.tensor([a,b,c,d])

t = range(100)

#derivative = grad(my_tensor, t) --> not working

#Instead what I'm currently doing:
for i in range(len(t)):
    a_grad = grad(a[i],t[i], retain_graph=True)
    b_grad = grad(b[i],t[i], retain_graph=True)
    #etc.

I was told that it might work if I could run autograd on the forward  pass rather than the backwards pass, but from here it seems like it's not currently a feature PyTorch has.
Update 1:
@jodag mentioned that what I'm looking for might be just the diagonal of the Jacobian. I'm following the link he attached and trying out the faster method. Though, this doesn't seem to work and gives me an error:
RuntimeError: grad can be implicitly created only for scalar outputs.
Code:
nx = net_x()
x = torch.rand(10, requires_grad=True)
x = torch.reshape(x, (10,1))
x = x.unsqueeze(1).repeat(1, 4, 1)
y = nx(x)
dx = torch.diagonal(torch.autograd.grad(torch.diagonal(y, 0, -2, -1), x), 0, -2, -1)


title: pytorch: memorize output from several layers of sequencial

body:
I have a model in pytorch and from one forward pass want to extract the outputs of several layers. Is that possible?

e.g. the outputs of the first five conv layers of vgg

import torchvision
vgg = torchvision.models.vgg19_bn()


title: what happens when we "install" pytorch

body: i thought tools like opencv or pytorch are just some codes for developer to import and use the functions of them,in this degree, we just need to copy the source , of course ,things definately are not that straight-forward. So what extra operation do we need to use the tools(install them),and why?


title: PyTorch derivative has no degree

body:
I am studying the tutorial from PyTorch official docs. I am trying to understand the content. Start from You can do many crazy things with autograd!

x = torch.randn(3, requires_grad=True)
y = x * 2

i = 0
while y.data.norm() < 100:
    y = y * 2
    i+= 1

print(x)
print(y)
print(i)

Output:

tensor([-0.6933,  0.1126,  0.3913], requires_grad=True)
tensor([-88.7455,  14.4082,  50.0871], grad_fn=<MulBackward>)
6

Find the derivative w.r.t to x at point [0.1, 1.0, 0.0001]

gradients = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)
y.backward(gradients)
print(x.grad)

Output:

tensor([ 12.8000, 128.0000,   0.0128])

In my understanding i equals to 6. Then y = (2x)^7 and derivative is different from PyTorch. It has 7 as a factor when substitute the value
to my derivative.

The answer from the PyTorch is simply substitute x with given point to the dy/dx = 2^7 * x

Question:

How to derive the derivative?

References:

How to use PyTorch to calculate partial derivatives?

PyTorch Autograd automatic differentiation feature


title: Pytorch RNNLearner predict causes mem leak in docker

body:
I am trying to run a simple predict with pre-trained model, but it keeps eating memory.

[ENV]

docker: docker Ubuntu18.04, python3.6 container
pytorch:
https://download.pytorch.org/whl/cpu/torch-1.1.0-cp36-cp36m-linux_x86_64.whl
fastai: 1.0.46

this is the python code:

from fastai.basic_train import load_learner
predictor = load_learner(path, "file.pkl")
pred_class, pred_idx, outputs = predictor.predict("string")

I can not reproduce the memory leak on my mac or linux machine, so I think it is docker related problem, that maybe some missing library in docker or sth else. Any thought?


title: The architecture used in the deep image prior

body:
From Deep Image Prior, there is a figure of the architecture used in the experiment, shown as follows:
decoder-encoder architecture,

The figure mentions clearly what layers are to be used for each downsampling, upsampling, and skip networks.

My question is, are these layers in each network really represents the layers in the programming code?

For an easy understanding, let's take a look at the skip network. In the figure, it says the skip network uses convolution, BN, and LeakyReLU layers. But does the code really uses only these three layers to perform the skip network?


title: Can you pass a variable to a data member in Python as done in Pytorch?

body:
Here is the sample code from the nn.Module of pytorch documentation:

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 20, 5)

    def forward(self, x):
       x = F.relu(self.conv1(x))
       return F.relu(self.conv2(x))

Here, we are passing x to self.conv1() in the forward function.
However, self.conv1 is a variable as can been from this line self.conv1 = nn.Conv2d(1, 20, 5) in the __init__ function.

How is this possible?


title: Why does one Python script raise a qt.qpa.plugin error, while another identical script does not?

body:
I have two virtually identical scripts running in the same PyCharm IDE. They both call into a third script that uses matplotlib to output a Numpy array to a PNG. One of the scripts works fine and outputs the PNG. The other script raises the following error:

  qt.qpa.plugin: Could not load the Qt platform plugin "windows" in "" even though it was found.

The differences between the scripts are minimal - they only vary in that they each import a different pytorch model (both models created by me). This leads me to think there might be some sort of environmental difference, but they are both launched from the same IDE and Conda environment.

Does anyone have any suggestions?

EDIT:

Here is a stripped down version of the two files. The bits I have removed are identical to each other down to the character and whitespace.

File 1:

import argparse
from average_meter import AverageMeter
import criteria  # -> NOTE: This is not used, but was not removed in this file
import dataset
from densenet_mapmaker_model import DensenetMapMakerModel  # -> Different model imported
import json
import math
import os
import time
import torch
from torch.autograd import Variable
import torch.nn as nn
from torchvision import transforms
import visualizer

parser = argparse.ArgumentParser(description='Densenet Map Maker Trainer for Steel')

# ...Define args...

##########################################################################################
# These args were defined here, but not in the other file. They are not actually used here, just defined and ignored.
parser.add_argument('--shrinkage_coef', dest='shrinkage_coef', type=int,
                    help='Use a shrinking gaussian loss and multiply its result by this amount.')

parser.add_argument('--multiply', dest='multiply', type=int,
                    help='If 1, in the shrinking gaussian loss, ' +
                         'multiply by 1000 before comparing and then divide the loss by 1000.')

parser.add_argument('--diff_coef', dest='diff_coef', type=int,
                    help='Use the count difference loss and multiply its result by this amount.')

parser.add_argument('--clustering_coef', dest='clustering_coef', type=int,
                    help='Use the clustering loss and multiply its result by this amount.')
##########################################################################################

use_cuda = torch.cuda.is_available()

def main():
    best_mse = 1e6

    version = 1

    args = parser.parse_args()
    # ...set up default arg values...
    args.loss_calc_method = 'DenseNet'  # Defined here, but not in the other file. Not used here and could be removed.

    # ...load dataset lists...

    model = DensenetMapMakerModel()  # -> Note: Different model used
    if use_cuda:
        model = model.cuda()

    # ...Create criteria and optimizer...

    # ...Load weights from last training run...

    for epoch in range(args.start_epoch, args.epochs):
        train(train_list, model, criterion, optimizer, epoch, args)
        mse = validate(val_list, model, criterion, epoch, args)

        # ...Check result and save weights...

def train(...):

    # ...Set up counters and dataset...

    model.train()
    end = time.time()

    # img: torch.Tensor      # Image data
    # location : torch.Tensor. Target annotation locations for the image.
    # image_id : str. Unique ID of image.
    for i, (img, location, image_id) in enumerate(train_loader):

        # ...Run input through model, check result and backpropagate...

        if i % args.print_freq == 0:
            # Note: Print statement has different whitespace formatting from other file...
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                .format(
                epoch, i, len(train_loader), batch_time=batch_time,
                data_time=data_time, loss=losses))

def validate(...) -> float:

    # ...Set up dataset...

    # img: torch.Tensor      # Image data
    # location : torch.Tensor. Target annotation locations for the image.
    # image_id : str. Unique ID of image.
    for i, (img, location, image_id) in enumerate(test_loader):

        # ...Run input through model and check result...

        # Output target and source the first time...
        output_dir = args.output_dir
        if args.output_images == 1 and epoch % 1 == 0 and i == 2:  # -> NOTE: Other file does NOT have additional check i == 2
            visualizer.display_tensor(output, output_dir + "/epoch_" + str(epoch) + "_" + str(i) + "_output")

if __name__ == '__main__':
    main()

File 2:

import argparse
from average_meter import AverageMeter
import dataset
from steel_scanner_model import SteelScanner  # -> Different model imported
import json
import math
import os
import time
import torch
from torch.autograd import Variable
import torch.nn as nn
from torchvision import transforms
import visualizer

parser = argparse.ArgumentParser(description='Densenet Map Maker Trainer for Steel')

# ...Define args...

use_cuda = torch.cuda.is_available()

def main():
    best_mse = 1e6

    version = 1

    args = parser.parse_args()
    # ...set up default arg values...

    # ...load dataset lists...

    model = SteelScanner()  # -> Note: Different model used
    if use_cuda:
        model = model.cuda()

    # ...Create criteria and optimizer...

    # ...Load weights from last training run...

    for epoch in range(args.start_epoch, args.epochs):
        train(train_list, model, criterion, optimizer, epoch, args)
        mse = validate(val_list, model, criterion, epoch, args)

        # ...Check result and save weights...

def train(...):

    # ...Set up counters and dataset...

    model.train()
    end = time.time()

    # img: torch.Tensor      # Image data
    # location : torch.Tensor. Target annotation locations for the image.
    # image_id : str. Unique ID of image.
    for i, (img, location, image_id) in enumerate(train_loader):

        # ...Run input through model, check result and backpropagate...

        if i % args.print_freq == 0:
            # Note: Print statement has different whitespace formatting from other file...
            print('Epoch: [{0}][{1}/{2}]\t'
                  'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
                  'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
                  'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
                  .format(epoch, i, len(train_loader), batch_time=batch_time, data_time=data_time, loss=losses))

def validate(...) -> float:

    # ...Set up dataset...

    # img: torch.Tensor      # Image data
    # location : torch.Tensor. Target annotation locations for the image.
    # image_id : str. Unique ID of image.
    for i, (img, location, image_id) in enumerate(test_loader):

        # ...Run input through model and check result...

        # Output target and source the first time...
        output_dir = args.output_dir
        if args.output_images == 1 and epoch % 1 == 0:  # -> NOTE: Other file has additional check i == 2
            # NOTE: The error occurs inside 'display_tensor'
            visualizer.display_tensor(output, output_dir + "/epoch_" + str(epoch) + "_" + str(i) + "_output")

if __name__ == '__main__':
    main()

The error occurs inside visualizer.display_tensor when called from the second file.


title: How to convert the following CNN written in tensor-flow to the model in Pytorch?

body:
I have a existing CNN network model written in tensorflow. I am changing my project to pytorch. I am a newbie in Pytorch. I believe that I have done mostly right in the conversion process of the model. However I am getting an error saying: invalid argument 0: Sizes of tensors must match except in dimension 3. Got 144 and 72 in dimension 1

#My tensorflow network that works perfectly
def FluidNet( nClasses, nClasses1 ,  input_height=128, input_width=128):
    ## input_height and width must be devisible by 32 because maxpooling with filter size = (2,2) is operated 5 times,
    ## which makes the input_height and width 2^5 = 32 times smaller
    assert input_height%32 == 0
    assert input_width%32 == 0
    IMAGE_ORDERING =  "channels_last"

    img_input = Input(shape=(input_height,input_width, 6), name='combined_input') ## Assume 128,128,6

    ## Block 1 128x128
    x = Conv2D(18, (2, 2), activation='relu', padding='same', name='block1_conv1', data_format=IMAGE_ORDERING )(img_input)
    x = Conv2D(18, (2, 2), activation='relu', padding='same', name='block1_conv2', data_format=IMAGE_ORDERING )(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool', data_format=IMAGE_ORDERING )(x)
    f1 = x

    # Block 2 64x64
    x = Conv2D(36, (2, 2), activation='relu', padding='same', name='block2_conv1', data_format=IMAGE_ORDERING )(x)
    x = Conv2D(36, (2, 2), activation='relu', padding='same', name='block2_conv2', data_format=IMAGE_ORDERING )(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool', data_format=IMAGE_ORDERING )(x)
    f2 = x

    # Block 3 32x32
    x = Conv2D(72, (2, 2), activation='relu', padding='same', name='block3_conv1', data_format=IMAGE_ORDERING )(x)
    x = Conv2D(72, (2, 3), activation='relu', padding='same', name='block3_conv2', data_format=IMAGE_ORDERING )(x)
    x = Conv2D(72, (2, 2), activation='relu', padding='same', name='block3_conv3', data_format=IMAGE_ORDERING )(x)
    x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool', data_format=IMAGE_ORDERING )(x)
    pool3 = x

    # Block 4 16x16
    x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block4_conv1', data_format=IMAGE_ORDERING )(x)
    x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block4_conv2', data_format=IMAGE_ORDERING )(x)
    x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block4_conv3', data_format=IMAGE_ORDERING )(x)
    pool4 = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool', data_format=IMAGE_ORDERING )(x)

    # Block 5 8x8
    x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block5_conv1', data_format=IMAGE_ORDERING )(pool4)
    x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block5_conv2', data_format=IMAGE_ORDERING )(x)
    x = Conv2D(144, (2, 2), activation='relu', padding='same', name='block5_conv3', data_format=IMAGE_ORDERING )(x)
    pool5 = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool', data_format=IMAGE_ORDERING )(x)

    # Block Transpose <DECODER> : Depth
    #1st deconv layer 4x4
    x = (Conv2DTranspose( 72, kernel_size=(4,4) ,  strides=(2,2) , padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool5" ) (pool5))

    #concatinate x and pool4 for 2nd Deconv layer 8X8
    x = concatenate ([x, pool4],axis = 3)
    x = (Conv2DTranspose( 36 , kernel_size=(6,6) ,  strides=(2,2) ,padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool4")(x))

    #concatinate x and pool3 for 3rd Deconv layer 28x28
    x = concatenate ([x, pool3],axis = 3)
    x= (Conv2DTranspose( 18 , kernel_size=(4,4) ,  strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool3" )(x))

    #concatinate x and f2 for 4th Deconv layer
    x = concatenate ([x, f2],axis = 3)
    x = (Conv2DTranspose( 9 , kernel_size=(4,4) ,  strides=(2,2) ,  padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool2" )(x))

    #concatinate x and f1 for 5th Deconv layer

    x = concatenate ([x, f1],axis = 3)
    x = (Conv2DTranspose( nClasses + nClasses1 , kernel_size=(3,3) ,  strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool1" )(x))

    o = x
    o = (Activation('sigmoid', name="depth_out"))(o)

    x2 = (Conv2DTranspose( 72, kernel_size=(4,4) ,  strides=(2,2) , padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool5_2" ) (pool5))

    x2 = concatenate ([x2, pool4],axis = 3)
    x2 = (Conv2DTranspose( 36 , kernel_size=(6,6) ,  strides=(2,2) ,padding='same', dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool4_2")(x2))

    x2 = concatenate ([x2, pool3],axis = 3)
    x2= (Conv2DTranspose( 18 , kernel_size=(4,4) ,  strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool3_2" )(x2))

    x2 = concatenate ([x2, f2],axis = 3)
    x2 = (Conv2DTranspose( 9 , kernel_size=(4,4) ,  strides=(2,2) ,  padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool2_2" )(x2))

    x2 = concatenate ([x2, f1],axis = 3)
    x2 = (Conv2DTranspose( 7 , kernel_size=(3,3) ,  strides=(2,2) , padding='same',dilation_rate = (1,1), use_bias=False, data_format=IMAGE_ORDERING, name="Transpose_pool1_2" )(x2))

    o2 = x2
    o2 = (Activation('sigmoid', name="scale_out"))(o2)

    singleOut = concatenate([o,o2],axis = 3, name="single_out")

    #model creation
    model = Model(img_input, singleOut)

    return model

# My pytorch network that gives an error
class FluidNet (nn.Module):
def __init__(self,nClasses=4,nClasses1=7):
    super(FluidNet,self).__init__()

    # conv layers: (in_channel size, out_channels size, kernel_size, stride, padding)
    self.conv1_1 = nn.Conv2d(6, 18, kernel_size=2, padding=1)
    self.conv1_2 = nn.Conv2d(18, 18, kernel_size=2, padding=1)

    self.conv2_1 = nn.Conv2d(18, 36, kernel_size=2, padding=1)
    self.conv2_2 = nn.Conv2d(36, 36, kernel_size=2, padding=1)

    self.conv3_1 = nn.Conv2d(36, 72, kernel_size=2, padding=1)
    self.conv3_2 = nn.Conv2d(72, 72, kernel_size=2, padding=1)
    self.conv3_3 = nn.Conv2d(72, 72, kernel_size=2, padding=1)

    self.conv4_1 = nn.Conv2d(72, 144, kernel_size=2, padding=1)
    self.conv4_2 = nn.Conv2d(144, 144, kernel_size=2, padding=1)
    self.conv4_3 = nn.Conv2d(144, 144, kernel_size=2, padding=1)

    self.conv5_1 = nn.Conv2d(144, 144, kernel_size=2, padding=1)
    self.conv5_2 = nn.Conv2d(144, 144, kernel_size=2, padding=1)
    self.conv5_3 = nn.Conv2d(144, 144, kernel_size=2, padding=1)

    # max pooling (kernel_size, stride)
    self.pool = nn.MaxPool2d(kernel_size=2, stride=2,padding=0)

    # Block Transpose <DECODER> : Depth and Normal : nClasses = 4
    self.up5_a = nn.ConvTranspose2d(144,72,4, stride=2,  padding=2, bias=False)
    self.up4_a = nn.ConvTranspose2d(72,36,6, stride=2,  padding=2, bias=False)
    self.up3_a = nn.ConvTranspose2d(36,18,4, stride=2,  padding=2, bias=False)
    self.up2_a = nn.ConvTranspose2d(18,9,4, stride=2, padding=2, bias=False)
    self.up1_a = nn.ConvTranspose2d(9,nClasses,3,stride=2, padding=2, bias=False)

    # Block Transpose <DECODER> : Image+Ref+Scale: nClasses1 = 7
    self.up5_b = nn.ConvTranspose2d(144,72,4, stride=2, padding=2, bias=False)
    self.up4_b = nn.ConvTranspose2d(72,36,6, stride=2,  padding=2, bias=False)
    self.up3_b = nn.ConvTranspose2d(36,18,4, stride=2,  padding=2, bias=False)
    self.up2_b = nn.ConvTranspose2d(18,9,4, stride=2,  padding=2, bias=False)
    self.up1_b = nn.ConvTranspose2d(9,nClasses1,3, stride=2,  padding=2, bias=False)

def forward(self, x, training=True):
    x = F.relu(self.conv1_1(x))
    x = F.relu(self.conv1_2(x))
    x = self.pool(x)
    pool1 = x

    x = F.relu(self.conv2_1(x))
    x = F.relu(self.conv2_2(x))
    x = self.pool(x)
    pool2 = x

    x = F.relu(self.conv3_1(x))
    x = F.relu(self.conv3_2(x))
    x = F.relu(self.conv3_3(x))
    x = self.pool(x)
    pool3 = x

    x = F.relu(self.conv4_1(x))
    x = F.relu(self.conv4_2(x))
    x = F.relu(self.conv4_3(x))
    x = self.pool(x)
    pool4 = x

    x = F.relu(self.conv5_1(x))
    x = F.relu(self.conv5_2(x))
    x = F.relu(self.conv5_3(x))
    x = self.pool(x)
    pool5 = x

    x = self.up5_a(x)
    print(x.size())
    print(pool4.size())
    x = torch.cat((x,pool4),3)
    x = self.up4_a(x)
    x = torch.cat((x,pool3),3)
    x = self.up3_a(x)
    x = torch.cat((x,pool2),3)
    x = self.up2_a(x)
    x = torch.cat((x,pool1),3)
    x = self.up1_a(x)
    o = x
    o = torch.sigmoid(o)

    x1 = self.up5_a(pool5)
    x1 = torch.cat((x1,pool4),3)
    x1 = self.up4_a(x1)
    x1 = torch.cat((x1,pool3),3)
    x1 = self.up3_a(x1)
    x1 = torch.cat((x1,pool2),3)
    x1 = self.up2_a(x1)
    x1 = torch.cat((x1,pool1),3)
    x1 = self.up1_a(x1)
    o1 = x1
    o1 = torch.sigmoid(o1)

    return o,o1

These print statements in my pytorch code (print(x.size()), print(pool4.size())) prints out to be: torch.Size([1, 72, 10, 10])
torch.Size([1, 144, 10, 10]). Please help me make this transfer. Thank you.


title: Categorical Classification probabilites as a graph

body:
What is the best way to take the categorical classification probabilities typically output from a neural network and plot them as a graph using something like networkx.

Given 8 categories, say ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] and the predictions across 5 samples with shape (8, 5) I'd like to plot a graph where the node label is the category and the probability for all indices where the probability is greater than 0.0.

array([[0.        , 0.        , 0.        , 0.        , 0.        ],
       [0.        , 0.        , 0.        , 0.        , 0.        ],
       [0.17817138, 0.11233618, 0.12554741, 0.16154018, 0.16248149],
       [0.        , 0.        , 0.        , 0.        , 0.        ],
       [0.70687366, 0.86215913, 0.85997397, 0.8285762 , 0.828603  ],
       [0.        , 0.        , 0.        , 0.        , 0.        ],
       [0.        , 0.        , 0.        , 0.        , 0.        ],
       [0.        , 0.        , 0.        , 0.        , 0.        ]],
      dtype=float32)

Valid edges are between all the non zero values of consecutive columns in the array.


title: Adjust input shape for my pytorch problem

body:
I am trying to create a neural net to feed in an input of shape 249561, 80, 1 and y labels are (249561, 2).

def __init__(self):
    super(Net1, self).__init__()
    self.conv1 = nn.Conv1d(80, 16, kernel_size=1)

    self.conv2_drop = nn.Dropout()
    self.fc1 = nn.Linear(1,256)
    self.fc2 = nn.Linear(256, 64)
    self.fc3 = nn.Linear(64,32)
    self.fc4 = nn.Linear(32,2)

def forward(self, x):
    print(type(x))
    x = F.relu(F.max_pool1d(self.conv1(x), 1))
    print(x.shape)
    x.reshape(-1)
    e1 = F.relu(self.fc1(x))
    x = F.dropout(e1, training=self.training)
    x = F.relu(self.fc2(x))
    x = F.dropout(x, training=self.training)
    x = F.relu(self.fc3(x))
    x = F.dropout(x, training=self.training)
    x = self.fc4(x)

    return x

My training loop looks something like this

losses = [];
batch_size = 16
for epoch in range(10):
      permutation = torch.randperm(x2.size()[0])
      for i in range(0,len(x2), batch_size):
        indices = permutation[i:i+batch_size]
        batch_x, batch_y = x2[indices], onehot_encoded[indices]
        #images = Variable(images.float())
        #labels = Variable(labels)

        # Forward + Backward + Optimize
        optimizer.zero_grad()
        outputs = model(batch_x)
        loss = criterion(outputs, batch_y)
        loss.backward()
        optimizer.step()

I have a batch of 16 and I feed in a tensor with a shape [16, 80, 1].I get the following error. RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target' in call to _thnn_nll_loss2d_forward. I suspect that it is the problem with an output layer but it returns a tensor of shape 2 which is the same as my labels. The output x size torch.Size([16, 16, 2])