title: Numpy outperforms Tensorflow and Pytorch with same hyperparameters

body:
I have made 3 Neural Networks for Joel's FizzBuzz implementation, each in Numpy, TensorFlow and Pytorch. With the same hyperparameters and 1k epochs, my numpy script converges to 0.002 loss yet my pytorch and tensorflow is still jumping around 0.6. Could someone please help me figure out whats happening. I dont think Google and [Facebook + Nvidia] made something dump than Numpy just for GPU boost. My code below

Numpy

import numpy as np

input_size = 10
epochs = 1000
batches = 64
lr = 0.01

def sig(val):
    return 1 / (1 + np.exp(-val))

def sig_d(val):
    sig_val = sig(val)
    return sig_val * (1 - sig_val)

def binary_enc(num):
    ret = [int(i) for i in '{0:b}'.format(num)]
    return [0] * (input_size - len(ret)) + ret

def binary_dec(array):
    ret = 0
    for i in array:
        ret = ret * 2 + int(i)
    return ret

def training_test_gen(x, y):
    assert len(x) == len(y)
    indices = np.random.permutation(range(len(x)))
    split_size = int(0.9 * len(indices))
    trX = x[indices[:split_size]]
    trY = y[indices[:split_size]]
    teX = x[indices[split_size:]]
    teY = y[indices[split_size:]]
    return trX, trY, teX, teY

def x_y_gen():
    x = []
    y = []
    for i in range(1000):
        x.append(binary_enc(i))
        if i % 15 == 0:
            y.append([1, 0, 0, 0])
        elif i % 5 == 0:
            y.append([0, 1, 0, 0])
        elif i % 3 == 0:
            y.append([0, 0, 1, 0])
        else:
            y.append([0, 0, 0, 1])
    return training_test_gen(np.array(x), np.array(y))

def check_fizbuz(i):
    if i % 15 == 0:
        return 'fizbuz'
    elif i % 5 == 0:
        return 'buz'
    elif i % 3 == 0:
        return 'fiz'
    else:
        return 'number'

trX, trY, teX, teY = x_y_gen()

w1 = np.random.randn(10, 100)
w2 = np.random.randn(100, 4)

b1 = np.zeros((1, 100))
b2 = np.zeros((1, 4))

no_of_batches = int(len(trX) / batches)
for epoch in range(epochs):
    for batch in range(no_of_batches):
        # forward
        start = batch * batches
        end = start + batches
        x = trX[start:end]
        y = trY[start:end]
        a2 = x.dot(w1) + b1
        h2 = sig(a2)
        a3 = h2.dot(w2) + b2
        hyp = sig(a3)
        error = hyp - y
        loss = (error ** 2).mean()

        # backward
        outerror = error
        outgrad = outerror * sig_d(a3)
        outdelta = h2.T.dot(outgrad)
        outbiasdelta = np.ones([1, batches]).dot(outgrad)

        hiddenerror = outerror.dot(w2.T)
        hiddengrad = hiddenerror * sig_d(a2)
        hiddendelta = x.T.dot(hiddengrad)
        hiddenbiasdelta = np.ones([1, batches]).dot(hiddengrad)

        w1 -= hiddendelta * lr
        b1 -= hiddenbiasdelta * lr
        w2 -= outdelta * lr
        b2 -= outbiasdelta * lr
    print(epoch, loss)

# test
a2 = teX.dot(w1) + b1
h2 = sig(a2)
a3 = h2.dot(w2) + b2
hyp = sig(a3)
outli = ['fizbuz', 'buz', 'fiz', 'number']
for i in range(len(teX)):
    num = binary_dec(teX[i])
    print(
        'Number: {} -- Actual: {} -- Prediction: {}'.format(
            num, check_fizbuz(num), outli[hyp[i].argmax()]))
print('Test loss: ', np.mean(teY - hyp))

Pytorch

import numpy as np
import torch as th
from torch.autograd import Variable

input_size = 10
epochs = 1000
batches = 64
lr = 0.01

def binary_enc(num):
    ret = [int(i) for i in '{0:b}'.format(num)]
    return [0] * (input_size - len(ret)) + ret

def binary_dec(array):
    ret = 0
    for i in array:
        ret = ret * 2 + int(i)
    return ret

def training_test_gen(x, y):
    assert len(x) == len(y)
    indices = np.random.permutation(range(len(x)))
    split_size = int(0.9 * len(indices))
    trX = x[indices[:split_size]]
    trY = y[indices[:split_size]]
    teX = x[indices[split_size:]]
    teY = y[indices[split_size:]]
    return trX, trY, teX, teY

def x_y_gen():
    x = []
    y = []
    for i in range(1000):
        x.append(binary_enc(i))
        if i % 15 == 0:
            y.append([1, 0, 0, 0])
        elif i % 5 == 0:
            y.append([0, 1, 0, 0])
        elif i % 3 == 0:
            y.append([0, 0, 1, 0])
        else:
            y.append([0, 0, 0, 1])
    return training_test_gen(np.array(x), np.array(y))

def check_fizbuz(i):
    if i % 15 == 0:
        return 'fizbuz'
    elif i % 5 == 0:
        return 'buz'
    elif i % 3 == 0:
        return 'fiz'
    else:
        return 'number'

trX, trY, teX, teY = x_y_gen()
if th.cuda.is_available():
    dtype = th.cuda.FloatTensor
else:
    dtype = th.FloatTensor
x = Variable(th.from_numpy(trX).type(dtype), requires_grad=False)
y = Variable(th.from_numpy(trY).type(dtype), requires_grad=False)

w1 = Variable(th.randn(10, 100).type(dtype), requires_grad=True)
w2 = Variable(th.randn(100, 4).type(dtype), requires_grad=True)

b1 = Variable(th.zeros(1, 100).type(dtype), requires_grad=True)
b2 = Variable(th.zeros(1, 4).type(dtype), requires_grad=True)

no_of_batches = int(len(trX) / batches)
for epoch in range(epochs):
    for batch in range(no_of_batches):
        start = batch * batches
        end = start + batches
        x_ = x[start:end]
        y_ = y[start:end]

        a2 = x_.mm(w1)
        a2 = a2.add(b1.expand_as(a2))
        h2 = a2.sigmoid()

        a3 = h2.mm(w2)
        a3 = a3.add(b2.expand_as(a3))
        hyp = a3.sigmoid()

        error = hyp - y_
        loss = error.pow(2).sum()
        loss.backward()

        w1.data -= lr * w1.grad.data
        w2.data -= lr * w2.grad.data
        b1.data -= lr * b1.grad.data
        b2.data -= lr * b2.grad.data
        w1.grad.data.zero_()
        w2.grad.data.zero_()
    print(epoch, error.mean().data[0])

TensorFlow

import tensorflow as tf
import numpy as np

input_size = 10
epochs = 1000
batches = 64
learning_rate = 0.01

def binary_enc(num):
    ret = [int(i) for i in '{0:b}'.format(num)]
    return [0] * (input_size - len(ret)) + ret

def binary_dec(array):
    ret = 0
    for i in array:
        ret = ret * 2 + int(i)
    return ret

def training_test_gen(x, y):
    assert len(x) == len(y)
    indices = np.random.permutation(range(len(x)))
    split_size = int(0.9 * len(indices))
    trX = x[indices[:split_size]]
    trY = y[indices[:split_size]]
    teX = x[indices[split_size:]]
    teY = y[indices[split_size:]]
    return trX, trY, teX, teY

def x_y_gen():
    x = []
    y = []
    for i in range(1000):
        x.append(binary_enc(i))
        if i % 15 == 0:
            y.append([1, 0, 0, 0])
        elif i % 5 == 0:
            y.append([0, 1, 0, 0])
        elif i % 3 == 0:
            y.append([0, 0, 1, 0])
        else:
            y.append([0, 0, 0, 1])
    return training_test_gen(np.array(x), np.array(y))

def check_fizbuz(i):
    if i % 15 == 0:
        return 'fizbuz'
    elif i % 5 == 0:
        return 'buz'
    elif i % 3 == 0:
        return 'fiz'
    else:
        return 'number'

trX, trY, teX, teY = x_y_gen()

x = tf.placeholder(tf.float32, [None, 10], name='x')
y = tf.placeholder(tf.float32, [None, 4], name='y')

lr = tf.placeholder(tf.float32, [], name='lr')

w1 = tf.Variable(tf.truncated_normal([10, 100]))
w2 = tf.Variable(tf.truncated_normal([100, 4]))

b1 = tf.Variable(tf.zeros(100))
b2 = tf.Variable(tf.zeros(4))

a2 = tf.sigmoid(tf.add(tf.matmul(x, w1), b1))
hyp = tf.sigmoid(tf.add(tf.matmul(a2, w2), b2))

cost = tf.reduce_mean(tf.square(hyp - y))
optmizer = tf.train.GradientDescentOptimizer(lr).minimize(cost)

prediction = tf.argmax(hyp, 1)

no_of_batches = int(len(trX) / batches)
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    for epoch in range(epochs):
        p = np.random.permutation(range(len(trX)))
        trX = trX[p]
        trY = trY[p]
        for batch in range(no_of_batches):
            start = batch * batches
            end = start + batches
            input_batch = trX[start: end]
            target_batch = trY[start: end]
            sess.run(
                optmizer, feed_dict={x: input_batch, y: target_batch, lr: learning_rate})
            if epoch % 100 == 0:
                a = np.argmax(teY, axis=1)
                b = sess.run(prediction, feed_dict={x: teX})
                acc = np.mean(a == b)
                out_cost = sess.run(
                    cost, feed_dict={x: input_batch, y: target_batch, lr: learning_rate})
                print('cost - {} --- accuracy - {}'.format(out_cost.mean(), acc))


title: BiLSTM Pytorch model.cuda() gives error in GPU

body:
import torch
import torchwordemb
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
torch.manual_seed(1)

class Net(nn.Module):
    def __init__(self):
    super(Net, self).__init__()
    self.embedding = nn.Embedding(1000, 100)
    self.lstm = nn.LSTM(100, 30, num_layers=1, bidirectional=True)
    self.out_layer = nn.Linear(60, 2)

def forward(self, x):
    x = x.t()
    batch_size = x.size(1)
    emb = self.embedding(x)
    print emb.size()
    hidden = self.init_hidden(batch_size)
    lstm_out, hidden = self.lstm(emb.view(len(emb), batch_size, -1), hidden)
    print 'LSTM Out: ', lstm_out[-1].size()
    out_layer = self.out_layer(lstm_out[-1])
    return out_layer
def init_hidden(self, batch_size):
    return (create_variable(torch.zeros(2, batch_size, 30)),
            create_variable(torch.zeros(2, batch_size, 30)))

x = [[2, 30, 40, 1, 0], [20, 3, 5, 10, 3], [5, 2, 4, 80, 1]]
def create_variable(tensor):
# Do cuda() before wrapping with variable
    if torch.cuda.is_available():
        return Variable(tensor.cuda())
    else:
        return Variable(tensor)
x = create_variable(torch.LongTensor(x))
y = create_variable(torch.LongTensor([0,1,1]))

model = Net()
loss_function = nn.NLLLoss()
#print 'Model parameter ', model.parameters()
optimizer = optim.Adam(model.parameters(), lr=1e-3)
if torch.cuda.device_count() > 1:
    print("Let's use", torch.cuda.device_count(), "GPUs!")
    model = nn.DataParallel(model)

if torch.cuda.is_available():
    model.cuda()

for epoch in range(10):

    output = model(x)
    print 'output view: ', output.size()
    print 'y: ', y.size()
    loss = loss_function(output, y.view(-1))

   # total_loss += loss.data[0]
    print 'Loss function: ', loss.data[0]
    model.zero_grad()
    loss.backward()
    optimizer.step()

  Traceback (most recent call last):   File "cuda_test.py", line 52, in

      model.cuda()   File "/home1/dhanachandra/anaconda3/envs/my_env27/lib/python2.7/site-packages/torch/nn/modules/module.py",
  line 216, in cuda
      return self._apply(lambda t: t.cuda(device))   File "/home1/dhanachandra/anaconda3/envs/my_env27/lib/python2.7/site-packages/torch/nn/modules/module.py",
  line 146, in _apply
      module._apply(fn)   File "/home1/dhanachandra/anaconda3/envs/my_env27/lib/python2.7/site-packages/torch/nn/modules/module.py",
  line 146, in _apply
      module._apply(fn)   File "/home1/dhanachandra/anaconda3/envs/my_env27/lib/python2.7/site-packages/torch/nn/modules/rnn.py",
  line 123, in _apply
      self.flatten_parameters()   File "/home1/dhanachandra/anaconda3/envs/my_env27/lib/python2.7/site-packages/torch/nn/modules/rnn.py",
  line 111, in flatten_parameters
      params = rnn.get_parameters(fn, handle, fn.weight_buf)   File "/home1/dhanachandra/anaconda3/envs/my_env27/lib/python2.7/site-packages/torch/backends/cudnn/rnn.py",
  line 165, in get_parameters
      assert filter_dim_a.prod() == filter_dim_a[0] AssertionError

Code runs in cpu without any error but gives error in GPU


title: getting the classification labels for torchvision's pretrained networks

body:
Pytorch's torchvision package provides pre-trained neural networks for image classification. I've been using the following code to classify an image using Alexnet (note: some of this code is from this webpage):

from PIL import Image
import torch
from torchvision import transforms
from torchvision import models

# function to transform image
transform = transforms.Compose([
    transforms.Resize(224),
    transforms.CenterCrop(224),
    transforms.ToTensor(),
    transforms.Normalize(
        mean=[0.485, 0.456, 0.406],
        std=[0.229, 0.224, 0.225])])

# image
img = Image.open('/path/to/image.jpg')
img = transform(img)
img = torch.unsqueeze(img, 0)

# alexnet
alexnet = models.alexnet(pretrained=True)
alexnet.eval()
out = alexnet(img)
percents = torch.nn.functional.softmax(out, dim=1)[0] * 100
top5_vals, top5_inds = percents.topk(5)

There are 1,000 total classes, and the top5_inds variable gives me the indices of the top 5 classes. But how do I get the associated labels (e.g. snail, basketball, banana)? I can't seem to find any sort of list as part of Pytorch's documentation or the alexnet variable.


title: Is it possible to add own function in transform.compose in pytorch

body:
I am using a pre-trained Alex model. I am running this model on some random image dataset. I want to convert RGB images to YCbCr images before training.
I am wondering is it possible to add a function on my own to transform.compose, For example:
transform = transforms.Compose([
  ycbcr(), #something like this
  transforms.Resize((224, 224)),
  transforms.ToTensor(),
  transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])

where,
def ycbcr(img):
   img = cv2.imread(img)
   img = cv2.cvtColor(img, cv2.COLOR_BGR2ycbcr)
   t = torch.from_numpy(img)
 return t

training_dataset = datasets.ImageFolder(link_train ,transform = transform_train)

training_loader = torch.utils.data.DataLoader(training_dataset, batch_size=96, shuffle=True)

Is this process correct? Please help me on how to proceed?


title: Train OpenNMT with pytorch and `TypeError: NoneType object is not callable`

body:
I was trying to train OpenNMT example on mac with cpu with the following steps:

Env: python3.5, Pytorch 0.1.10.1

Step 1

preprocess data and shrink src and tgt to have only the first 100 sentences by inserting the following lines after line133 in preprocess.py

    shrink = True
    if shrink:
        src = src[0:100]
        tgt = tgt[0:100]

then, I ran

python preprocess.py -train_src data/src-train.txt -train_tgt data/tgt-train.txt -valid_src data/src-val.txt -valid_tgt data/tgt-val.txt -save_data data/demo

Step 2

then I train using python train.py -data data/demo.train.pt -save_model demo_model

Then it rans ok for a while before an error appeared:

(dlnd-tf-lab)  ->python train.py -data data/demo.train.pt -save_model demo_model
Namespace(batch_size=64, brnn=False, brnn_merge='concat', curriculum=False, data='data/demo.train.pt', dropout=0.3, epochs=13, extra_shuffle=False, gpus=[], input_feed=1, layers=2, learning_rate=1.0, learning_rate_decay=0.5, log_interval=50, max_generator_batches=32, max_grad_norm=5, optim='sgd', param_init=0.1, pre_word_vecs_dec=None, pre_word_vecs_enc=None, rnn_size=500, save_model='demo_model', start_decay_at=8, start_epoch=1, train_from='', train_from_state_dict='', word_vec_size=500)
Loading data from 'data/demo.train.pt'
 * vocabulary size. source = 24999; target = 35820
 * number of training sentences. 100
 * maximum batch size. 64
Building model...
* number of parameters: 58121320
NMTModel (
  (encoder): Encoder (
    (word_lut): Embedding(24999, 500, padding_idx=0)
    (rnn): LSTM(500, 500, num_layers=2, dropout=0.3)
  )
  (decoder): Decoder (
    (word_lut): Embedding(35820, 500, padding_idx=0)
    (rnn): StackedLSTM (
      (dropout): Dropout (p = 0.3)
      (layers): ModuleList (
        (0): LSTMCell(1000, 500)
        (1): LSTMCell(500, 500)
      )
    )
    (attn): GlobalAttention (
      (linear_in): Linear (500 -> 500)
      (sm): Softmax ()
      (linear_out): Linear (1000 -> 500)
      (tanh): Tanh ()
    )
    (dropout): Dropout (p = 0.3)
  )
  (generator): Sequential (
    (0): Linear (500 -> 35820)
    (1): LogSoftmax ()
  )
)

Train perplexity: 29508.9
Train accuracy: 0.0216306
Validation perplexity: 4.50917e+08
Validation accuracy: 3.57853

Train perplexity: 1.07012e+07
Train accuracy: 0.06198
Validation perplexity: 103639
Validation accuracy: 0.944334

Train perplexity: 458795
Train accuracy: 0.031198
Validation perplexity: 43578.2
Validation accuracy: 3.42942

Train perplexity: 144931
Train accuracy: 0.0432612
Validation perplexity: 78366.8
Validation accuracy: 2.33598
Decaying learning rate to 0.5

Train perplexity: 58696.8
Train accuracy: 0.0278702
Validation perplexity: 14045.8
Validation accuracy: 3.67793
Decaying learning rate to 0.25

Train perplexity: 10045.1
Train accuracy: 0.0457571
Validation perplexity: 26435.6
Validation accuracy: 4.87078
Decaying learning rate to 0.125

Train perplexity: 10301.5
Train accuracy: 0.0490849
Validation perplexity: 24243.5
Validation accuracy: 3.62823
Decaying learning rate to 0.0625

Train perplexity: 7927.77
Train accuracy: 0.062812
Validation perplexity: 7180.49
Validation accuracy: 5.31809
Decaying learning rate to 0.03125

Train perplexity: 4573.5
Train accuracy: 0.047421
Validation perplexity: 6545.51
Validation accuracy: 5.6163
Decaying learning rate to 0.015625

Train perplexity: 3995.7
Train accuracy: 0.0549085
Validation perplexity: 6316.25
Validation accuracy: 5.4175
Decaying learning rate to 0.0078125

Train perplexity: 3715.81
Train accuracy: 0.0540765
Validation perplexity: 6197.91
Validation accuracy: 5.86481
Decaying learning rate to 0.00390625

Train perplexity: 3672.46
Train accuracy: 0.0540765
Validation perplexity: 6144.18
Validation accuracy: 6.01392
Decaying learning rate to 0.00195312

Train perplexity: 3689.7
Train accuracy: 0.0528286
Validation perplexity: 6113.55
Validation accuracy: 6.31213
Decaying learning rate to 0.000976562
Exception ignored in: <function WeakValueDictionary.__init__.<locals>.remove at 0x118b19b70>
Traceback (most recent call last):
  File "/Users/Natsume/miniconda2/envs/dlnd-tf-lab/lib/python3.5/weakref.py", line 117, in remove
TypeError: 'NoneType' object is not callable

Could you tell me how to fix it? Thanks!


title: pytorch - why to use Tensor.requires_grad_(True) instead of Tensor.requires_grad=True

body: I'm reading introduction into PyTorch and a bit confused by the Tensor.requires_grad_(...) function. Why would one would prefer it instead of just using the setter: Tensor.requires_grad = ... ? Setter syntax seems to be more clean and readable. Why to introduce this strange function with this funny underscore suffix?


title: Running PyTorch multiprocessing in a Docker container with Gunicorn worker manager

body:
I am trying to deploy a service on GCP. It's a Docker container that uses Gunicorn for worker management.
The code runs a torch.multiprocessing.process to run a POST response as a background process.

This works if I run the script using a python3 command. But hangs when using Gunicorn.

My understanding is that CUDA needs threadsafe multiprocessing and that is why torch has its own implementation. When we set up Gunicorn to manage workers, this may be causing some conflict or thread safety issues.

Has anyone come across this before. Would there be a different worker manager that I could be using?

In Dockerfile: CMD gunicorn -w 1 -t 6000 -b 0.0.0.0:8080 --timeout 6000 --preload app_script:app - this is how i am running the app in docker. So yes I am using preload. And the issue happens even if I run the docker container locally so its not just a gcp situation

p=torch.multiprocessing.Process(target=my_function args=(args, )) . p.start() - this is how a post call is getting handled.


title: Transformer Pipeline for NER returns partial words with ##s

body:
How should I interpret the partial words with '##'s in them returned by the Transformer NER pipelines? Other tools like Flair and SpaCy return the word and their tag. I have worked with the CONLL dataset before and never noticed anything like this. Moreover, why are words being divided like this?

Example from the HuggingFace:

from transformers import pipeline

nlp = pipeline("ner")

sequence = "Hugging Face Inc. is a company based in New York City. Its headquarters are in DUMBO, therefore very" \
           "close to the Manhattan Bridge which is visible from the window."

print(nlp(sequence))

Output:

[
    {'word': 'Hu', 'score': 0.9995632767677307, 'entity': 'I-ORG'},
    {'word': '##gging', 'score': 0.9915938973426819, 'entity': 'I-ORG'},
    {'word': 'Face', 'score': 0.9982671737670898, 'entity': 'I-ORG'},
    {'word': 'Inc', 'score': 0.9994403719902039, 'entity': 'I-ORG'},
    {'word': 'New', 'score': 0.9994346499443054, 'entity': 'I-LOC'},
    {'word': 'York', 'score': 0.9993270635604858, 'entity': 'I-LOC'},
    {'word': 'City', 'score': 0.9993864893913269, 'entity': 'I-LOC'},
    {'word': 'D', 'score': 0.9825621843338013, 'entity': 'I-LOC'},
    {'word': '##UM', 'score': 0.936983048915863, 'entity': 'I-LOC'},
    {'word': '##BO', 'score': 0.8987102508544922, 'entity': 'I-LOC'},
    {'word': 'Manhattan', 'score': 0.9758241176605225, 'entity': 'I-LOC'},
    {'word': 'Bridge', 'score': 0.990249514579773, 'entity': 'I-LOC'}
]


title: Integrating ROS with pycharm

body:
I wanted to run ROS in PyCharm, but could not find that .desktop file as mentioned here in which changes should be made. Moreover, I want to use the same environment that is created for PyTorch, do not want to change the interpreter. Can someone help me out with this?

Regards.


title: How do I remove elements from a 3D tensor with PyTorch?

body: I have a tensor with shape torch.Size([4, 161, 325]). How do I remove the first element along dim=2 so that the resulting tensor has a shape of torch.Size([4, 161, 324])?


title: Display misclassified images in pytorch

body:
I'm new to pytorch and numpy so this may be a dumb question. I'd like to see some images misclassified by my net, with the correct label and the predicted label. Here is my code
valid_and_test_set = torchvision.datasets.MNIST("./mnist", train=False, download=True)
dataset_valid, dataset_test = torch.utils.data.random_split(valid_and_test_set,[5000, 5000])
dataset_test.dataset.transform = transform #transform is composed by unsqueeze, normalize, view and gaussian noise with randn
dataset_test.dataset.target_transform = OneHot() #OneHot return the label
dataloader_test = torch.utils.data.DataLoader(dataset_test.dataset, batch_size=5000, num_workers=num_workers, pin_memory=True)

def test(dataset, dataloader):
    net.eval()
    with torch.no_grad():
        for batch in dataloader:
            inputs = batch[0]
            inputs = inputs.to(device, non_blocking=True)
            outputs = net(inputs)
            predictions = torch.argmax(outputs, dim=1)
            return predictions

Thank you in advance


title: TypeError: forward() missing 1 required positional argument with Tensorboard PyTorch

body:
I am trying to write my model to tensorboard with the following code:

model = SimpleLSTM(4, HIDDEN_DIM, HIDDEN_LAYERS, 1, BATCH_SIZE, device)
writer = tb.SummaryWriter(log_dir=tb_path)
sample_data = iter(trainloader).next()[0]
writer.add_graph(model, sample_data.to(device))

I get the error: TypeError: forward() missing 1 required positional argument: 'batch_size'

My model looks like this:

class SimpleLSTM(nn.Module):

    def __init__(self, input_dims, hidden_units, hidden_layers, out, batch_size, device):
        super(SimpleLSTM, self).__init__()
        self.input_dims = input_dims
        self.hidden_units = hidden_units
        self.hidden_layers = hidden_layers
        self.batch_size = batch_size
        self.device = device
        self.lstm = nn.LSTM(self.input_dims, self.hidden_units, self.hidden_layers,
                            batch_first=True, bidirectional=False)
        self.output_layer = nn.Linear(self.hidden_units, out)

    def init_hidden(self, batch_size):

        hidden = torch.rand(self.hidden_layers, batch_size, self.hidden_units, device=self.device, dtype=torch.float32)
        cell = torch.rand(self.hidden_layers, batch_size, self.hidden_units, device=self.device, dtype=torch.float32)
        hidden = nn.init.xavier_normal_(hidden)
        cell = nn.init.xavier_normal_(cell)
        return (hidden, cell)

    def forward(self, input, batch_size):
        hidden = self.init_hidden(batch_size)  incomplete batch
        lstm_out, (h_n, c_n) = self.lstm(input, hidden)
        raw_out = self.output_layer(h_n[-1])
        return raw_out

How can I write this model to TensorBoard?


title: Discrepancy between tensorflow's conv1d and pytorch's conv1d

body:
I am trying to import some pytorch code to tensorflow, I came to know that torch.nn.functional.conv1d() is  tf.nn.conv1d() but I am afraid there are still some discrepancies in tf's versions. Specifically, I cannot find the group parameter in tf.conv1d. For example: the following codes output two different results:

Pytorch:

inputs = torch.Tensor([[[1, 1, 1, 1],[2, 2, 2, 2],[3, 3, 3, 3]]]) #batch_sizex seq_length x embed_dim,
inputs = inputs.transpose(2,1) #batch_size x embed_dim x seq_length
batch_size, embed_dim, seq_length = inputs.size()

kernel_size = 3
in_channels = 2
out_channels = in_channels
weight = torch.ones(out_channels, 1, kernel_size)

inputs = inputs.contiguous().view(-1, in_channels, seq_length) #batch_size*embed_dim/in_channels x in_channels x seq_length
inputs = F.pad(inputs, (kernel_size-1,0), 'constant', 0)
output = F.conv1d(inputs, weight, padding=0, groups=in_channels)
output = output.contiguous().view(batch_size, embed_dim, seq_length).transpose(2,1)

Output:

tensor([[[1., 1., 1., 1.],
         [3., 3., 3., 3.],
         [6., 6., 6., 6.]]])

Tensorflow:

inputs = tf.constant([[[1, 1, 1, 1],[2, 2, 2, 2],[3, 3, 3, 3]]], dtype=tf.float32) #batch_sizex seq_length x embed_dim
inputs = tf.transpose(inputs, perm=[0,2,1])
batch_size, embed_dim, seq_length = inputs.get_shape()

print(batch_size, seq_length, embed_dim)
kernel_size = 3
in_channels = 2
out_channels = in_channels
weight = tf.ones([kernel_size, in_channels, out_channels])

inputs = tf.reshape(inputs, [(batch_size*embed_dim)//in_channels, in_channels, seq_length], name='inputs')
inputs = tf.transpose(inputs, perm=[0, 2, 1])
padding = [[0, 0], [(kernel_size - 1), 0], [0, 0]]
padded = tf.pad(inputs, padding)

res = tf.nn.conv1d(padded, weight, 1, 'VALID')
res = tf.transpose(res, perm=[0, 2, 1])
res = tf.reshape(res, [batch_size, embed_dim, seq_length])
res = tf.transpose(res, perm=[0, 2, 1])
print(res)

Output:

[[[ 2.  2.  2.  2.]
  [ 6.  6.  6.  6.]
  [12. 12. 12. 12.]]], shape=(1, 3, 4), dtype=float32)


title: How to fetch image dataset from Google Drive to Colab?

body:
I have this very weird problem. I have searched across internet, read documentation but am not able to figure out how to do it. So what I want to do is train a classifier using Colab. And for that I have a image dataset of dogs on my local machine.

So what I did was I packed that dataset folder of images into a zip file and uploaded it onto Drive. Then from Colab I mounted the drive and from there I tried to unzip the files. Everything good. But I've realised that after sometime some of the extracted files get deleted. And thing is that those files aren't on Colab storage, but instead on Drive and I dunno why they are getting deleted after sometime. Like about an hour.

So far I've used the following commands to do the extraction -

from google.colab import drive
drive.mount('/content/drive')

from zipfile import ZipFile
filename = 'Stanford Dogs Dataset.zip'

with ZipFile(filename, 'r') as zip:
  zip.extractall()
  print('Done')

and also tried this -

!unzip filename -d destination

Not sure where I am going wrong. And also, dunno why the extracted files though being extracted to a subfolder within drive, also starts showing up on the main root directory. And no I am not talking about the recent section, because when I want to check their location then they points to the root of the drive. It's all so confusing.


title: Truncated backpropagation in PyTorch (code check)

body:
I am trying to implement truncated backpropagation through time in PyTorch, for the simple case where K1=K2. I have an implementation below that produces reasonable output, but I just want to make sure it is correct. When I look online for PyTorch examples of TBTT, they do inconsistent things around detaching the hidden state and zeroing out the gradient, and the ordering of these operations. Please let me know if I have made a mistake.
In the code below, H maintains the current hidden state, and model(weights, H, x) outputs the prediction and the new hidden state.
while i < NUM_STEPS:
    # Grab x, y for ith datapoint
    x = data[i]
    target = true_output[i]

    # Run model
    output, new_hidden = model(weights, H, x)
    H = new_hidden

    # Update running error
    error += (output - target)**2

    if (i+1) % K == 0:
        # Backpropagate
        error.backward()
        opt.step()
        opt.zero_grad()
        error = 0
        H = H.detach()

    i += 1


title: how to manage torch.Tensor with variable length in some dimenson?

body:
I am programming a custom optimizer now, and the legth of bias in first dimension is not certain because last batch have no enough data to bulid a batch. So the initialization of weights with fixed batch_size do not satisfy  for torch.add between the last batch and the fixed length weights.

bias = torch.randn(batch_size,units)
batch_data = generator(path)
# for example
weights.shape # is (128,256)
# but the last batch has only 50 samples.
out = sigmoid(x*weights+bias) # where the length of first dimension is not mathed.

So, I wonder whether I can create a tensor where the lenght of some dimension could be variable, like variable length list.


title: How necessary are activation functions after dense layer in neural networks?

body:
I'm currently training multiple recurrent convolutional neural networks with deep q-learning for the first time.

Input is a 11x11x1 matrix, each network consists of 4 convolutional layer with dimensions 3x3x16, 3x3x32, 3x3x64, 3x3x64. I use stride=1 and padding=1. Each convLayer is followed by ReLU activation. The output is fed into a feedforward fully-connected dense layer with 128 units and after that into an LSTM layer, also containing 128 units. Two following dense layer produce separate advantage and value steams.

So training is running for a couple of days now and now I've realized (after I've read some related paper), I didn't add an activation function after the first dense layer (as in most of the papers). I wonder if adding one would significantly improve my network? Since I'm training the networks for university, I don't have unlimited time for training, because of a deadline for my work. However, I don't have enough experience in training neural networks, to decide on what to do...
What do you suggest? I'm thankful for every answer!


title: pytorch with mkl-dnn backend performance on small conv with multi thread

body:
I'm trying pytorch model with mkl-dnn backend. But i got a problem that the multi thread performance is slower than expected, runing on small conv. Please see this table.
mkl-dnn performace table

Runs big conv, the performance is obvious faster on 8 threads compared with single thread.  But runs small conv, the speed has no big differences with 8 threads and single thread.

So my question is:
1. why 8 threads is not obviously faster than 1 thread with small conv?
2. How to improve the 8 threads performance on small conv?

My code here

import time
import torch
import torch.nn as nn
from torch.nn.utils import weight_norm

class MyConv(nn.Module):
    def __init__(self, *args, **kwargs):
        super().__init__()
        self.cell = nn.Conv1d(*args, **kwargs)
        self.cell.weight.data.normal_(0.0, 0.02)

    def forward(self, x):
        return self.cell(x)

def main():
    #print(*torch.__config__.show().split("\n"), sep="\n")
    torch.set_num_threads(1)
    dim = 32
    kernels = 3
    seq = 100000
    MyCell = MyConv(dim, dim, kernel_size=kernels, stride=1)
    MyCell.eval()
    inputs = []
    iter = 1000
    for i in range(iter):
        inputs.append(torch.rand(1, dim, seq))

    start = time.time() * 1000
    for i in range(iter):
        print(i)
        y = MyCell(inputs[i])
        #print(y)
    end = time.time() * 1000
    print('cost %d ms per iter\n' % ((end - start) / iter))

if __name__ == "__main__":
    main()

Mkldnn verbose info (running with "export MKLDNN_VERBOSE=1")

mkldnn_verbose,exec,reorder,jit:uni,undef,in:f32_nchw out:f32_nChw8c,num:1,1x32x1x107718,2.18091
mkldnn_verbose,exec,reorder,jit:uni,undef,in:f32_oihw out:f32_OIhw8i8o,num:1,32x32x1x1,0.00195312
mkldnn_verbose,exec,convolution,jit_1x1:avx2,forward_training,fsrc:nChw8c fwei:OIhw8i8o fbia:x fdst:nChw8c,alg:convolution_direct,mb1_ic32oc32_ih1oh1kh1sh1dh0ph0_iw107718ow107718kw1sw1dw0pw0,3.87793
mkldnn_verbose,exec,reorder,jit:uni,undef,in:f32_nChw8c out:f32_nchw,num:1,1x32x1x107718,4.69116

Thanks a lot!


title: pytorch parallelize for loop of Cross Validation

body:
I have a cuda9-docker with tensorflow and pytorch installed, I am doing cross validation on an image dataset. Currently I am using a for loop to do the cross validation. Something like

for data_train, data_test in sklearn.kfold(5, all_data):
  train(data_train)
  test(data_test)

But the for loop takes too long, will the following code work to parallelize the for loop? Maybe there is already a solution. But this is not Data Parallelization.

from multiprocessing import Pool

def f(trainset, testset):
    train_result = train(trainset)
    test_result = test(testset)
    save_train_result()
    save_test_result()

if __name__ == '__main__':
    with Pool(5) as p:
        print(p.map(f, sklearn.cvfold(5, all_data)))

I am not sure if the multiprocessing will only paralize the cpu or both cpu and gpu? This might be easiler than doing parallel in side  a model i guess like https://discuss.pytorch.org/t/parallelize-simple-for-loop-for-single-gpu/33701
since in my case, there is no need to communicate across each process?


title: int8 data type in Pytorch

body:
What is the best way to run a qauntized model using int8 data types in Pytorch? I know in pytorch I can define tensors as int8, however, when I actually want to use int8, I get:

RuntimeError: _thnn_conv2d_forward is not implemented for type torch.CharTensor

So I am confused, how to run quantized model in pytorch that uses for instance int8 when the datatype is not supported for computational blocks such as convolutions? I am using pytorch version 1.0.1.post2.


title: Max pooling layers have weights. interesting work

body:
In the pytorch tutorial step of " Deep Learning with PyTorch: A 60 Minute Blitz > Neural Networks "

I have a question that what dose mean params[1] in the networks?

The reason why i have this think is because of max polling dose not have any weight values.

for example.

If you write some codes like that
'
def init(self) :
    self.conv1 = nn.Conv2d(1 , 6 , 5)
'

this means input has 1 channel, 6 output channel, conv(5,5)

So i understood that params[0] has  6 channel, 5 by 5 matrix random mapping values when init.

for the same reason

params[2] has like same form, but 16 channel. i understood this too.

but params[1], what dose mean?

Maybe it is just presentation method of existence for max polling.

but at the end of this tutorial, in step of the " update the weights "
It's likely updated by this code below.

learning_rate = 0.01
for f in net.parameters():
    f.data.sub_(f.grad.data * learning_rate)

this is code for construct a network

import torch
import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        # 1 input image channel, 6 output channels, 5x5 square convolution
        # kernel
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        # an affine operation: y = Wx + b
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        # Max pooling over a (2, 2) window
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        # If the size is a square you can only specify a single number
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(-1, self.num_flat_features(x))
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def num_flat_features(self, x):
        size = x.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features

params = list(net.parameters())
print(params[1])

Parameter containing:
tensor([-0.0614, -0.0778,  0.0968, -0.0420,  0.1779, -0.0843],
       requires_grad=True)

please visit this pytorch tutorial site.
https://pytorch.org/tutorials/beginner/blitz/neural_networks_tutorial.html#sphx-glr-beginner-blitz-neural-networks-tutorial-py

Summary

I have a one question.

that is why max pooling layers has four weights which can be updated ?

I think they shouldn't have any weights right?

Am I wrong?

Please help me. I'm a korean.


title: Converting Pytorch .ckpt(GAN) model into onnx

body:
I have one pre-trained GAN model into format of .ckpt extension. I want to convert that into onnx model. But I am not finding any way to do that.

I trained 10 categories in RaFD mode with https://github.com/yunjey/StarGAN and get pre-trained model , so I convert this model to onnx.

Then, I got the error below.

I can not solve this error without knowing it. please tell me.

there is my code.

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision
import numpy as np
from torch.autograd import Variable
from collections import OrderedDict

class ResidualBlock(nn.Module):
    def __init__(self, dim_in, dim_out):
        super(ResidualBlock, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(dim_in, dim_out, kernel_size=3,
                      stride=1, padding=1, bias=False),
            nn.InstanceNorm2d(dim_out, affine=True),
            nn.ReLU(inplace=True),
            nn.Conv2d(dim_out, dim_out, kernel_size=3,
                      stride=1, padding=1, bias=False),
            nn.InstanceNorm2d(dim_out, affine=True))

    def forward(self, x):
        return x + self.main(x)

class Generator(nn.Module):

    def __init__(self, conv_dim=64, c_dim=10, repeat_num=6):
        super(Generator, self).__init__()

        layers = []
        layers.append(nn.Conv2d(3+c_dim, conv_dim, kernel_size=7,
                                stride=1, padding=3, bias=False))
        layers.append(nn.InstanceNorm2d(conv_dim, affine=True))
        layers.append(nn.ReLU(inplace=True))

        curr_dim = conv_dim
        for i in range(2):
            layers.append(nn.Conv2d(curr_dim, curr_dim*2,
                                    kernel_size=4, stride=2, padding=1, bias=False))
            layers.append(nn.InstanceNorm2d(curr_dim*2, affine=True))
            layers.append(nn.ReLU(inplace=True))
            curr_dim = curr_dim * 2

        for i in range(repeat_num):
            layers.append(ResidualBlock(dim_in=curr_dim, dim_out=curr_dim))

        for i in range(2):
            layers.append(nn.ConvTranspose2d(curr_dim, curr_dim //
                                             2, kernel_size=4, stride=2, padding=1, bias=False))
            layers.append(nn.InstanceNorm2d(curr_dim//2, affine=True))
            layers.append(nn.ReLU(inplace=True))
            curr_dim = curr_dim // 2

        layers.append(nn.Conv2d(curr_dim, 3, kernel_size=7,
                                stride=1, padding=3, bias=False))
        layers.append(nn.Tanh())
        self.main = nn.Sequential(*layers)

    def forward(self, x, c):
        c = c.view(c.size(0), c.size(1), 1, 1)
        c = c.repeat(1, 1, x.size(2), x.size(3))
        x = torch.cat([x, c], dim=1)
        return self.main(x)

model = Generator().cuda()
state_dict = torch.load('../models/300000-G.ckpt')
model.load_state_dict(state_dict, strict=False)
dummy_input = Variable(torch.randn(32, 3, 256, 256)).cuda()

dummy_input = Variable(torch.randn(32, 3, 256, 256)).cuda()
torch.onnx.export(model, , 'model.onnx', verbose=False)

there is my error:

Traceback (most recent call last):
  File "test.py", line 85, in <module>
    torch.onnx.export(model, x, 'model.onnx', verbose=False)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/onnx/__init__.py", line 25, in export
    return utils.export(*args, **kwargs)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/onnx/utils.py", line 84, in export
    _export(model, args, f, export_params, verbose, training, input_names, output_names)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/onnx/utils.py", line 134, in _export
    trace, torch_out = torch.jit.get_trace_graph(model, args)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/jit/__init__.py", line 255, in get_trace_graph
    return LegacyTracedModule(f, nderivs=nderivs)(*args, **kwargs)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/nn/modules/module.py", line 491, in __call__
    result = self.forward(*input, **kwargs)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/jit/__init__.py", line 288, in forward
    out = self.inner(*trace_inputs)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/nn/modules/module.py", line 489, in __call__
    result = self._slow_forward(*input, **kwargs)
  File "~/.pyenv/versions/anaconda3-5.1.0/lib/python3.6/site-packages/torch/nn/modules/module.py", line 479, in _slow_forward
    result = self.forward(*input, **kwargs)
TypeError: forward() missing 1 required positional argument: 'c'


title: Creating and train only specified weights in TensorFlow or PyTorch

body:
I am wondering if there is a way in TensorFlow, PyTorch or some other library to selectively connect neurons. I want to make a network with a very large number of neurons in each layer, but that has very few connections between layers.

Note that I do not think this is a duplicate of this answer: Selectively zero weights in TensorFlow?. I implemented a custom keras layer using essentially the same method that appears in that question - essentially by creating a dense layer where all but the specified weights are ignored in training and evaluation. This fulfills part of what I want to do by not training specified weights, and not using them for prediction. But, the problems is that I still waste memory saving the untrained weights, and I waste time calculating the gradients of the zeroed weights. What I would like is for the computation of the gradient matrices to involve only sparse matrices, so that I do not waste time and memory.

Is there a way to selectively create and train weights without wasting memory? If my question is unclear or there is more information that it would be helpful for me to provide, please let me know. I would like to be helpful as a question-asker.


title: Is it a good idea to Multiply loss().item by batch_size to get the loss of a batch when batch size is not a factor of train_size?

body:
Suppose we have problem where we have 100 images and a batch size of 15. We have 15 images in all of out batches except our last batch which contains 10 images.

Suppose we have network training as:

network = Network()
optimizer = optim.Adam(network.parameters(),lr=0.001)

for epoch in range(5):

    total_loss = 0

    train_loader = torch.utils.data.DataLoader(train_set,batch_size=15)

    for batch in train_loader:
        images,labels = batch

        pred = network(images)
        loss = F.cross_entropy(pred,labels)

        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        total_loss+= loss.item()*15

Is'nt the last batch always supposed to give us a increased value of loss because we will be multiplying by 15 where we were supposed to multiply by 10 in the last batch? Should't it be
total_loss+= loss.item()*len(images) in place of 15 or batch_size??

Can we use

for every epoch:
    for every batch:
        loss = F.cross_entropy(pred,labels,reduction='sum')
        total_loss+=loss.item()

    avg_loss_per_epoch = (total_loss/len(train_set))

can someone please explain that multiplying by batch_size a good idea and how am I wrong?


title: How to change channel dimension of am image?

body: Suppose I have torch tensor with shape = [x,y,z,21] where x = batch_size, y = image_width, z= image_height. The above tensor represents batch of images with 21 channels. How should I convert it to size = [ x,y,z,3 ] ?


title: what is the default weight initialization used in Pytorch embedding layer?

body: When we create an embedding layer using the class torch.nn.Embedding, how are the weights initialized ? Is uniform, normal or initialization techniques like He or Xavier used by default?


I am running the model below and after splitting the image and trying to assign the image back I get the error message stating.
    sample[:, :, 0] = 0
TypeError: 'Image' object does not support item assignment

I have tried different approaches to assigning the image back to the sample but that didn't help. Any help would be greatly appreciated. Many Thanks
Here's the code I am using to do the data loading:
import os

import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset
from torchvision import transforms, utils
from torchvision.transforms import functional
#from torchvision.transforms import Grayscalei
import pandas as pd
import pdb
import cv2
from cv2 import imread
from cv2 import resize

class CellsDataset(Dataset):
    # a very simple dataset

    def __init__(self, root_dir, transform=None, return_filenames=False):
        self.root = root_dir
        self.transform = transform
        self.return_filenames = return_filenames
        self.files = [os.path.join(self.root,filename) for filename in os.listdir(self.root)]
        self.files = [path for path in self.files
                      if os.path.isfile(path) and os.path.splitext(path)[1]=='.png']

    def __len__(self):
        return len(self.files)

        path = self.files[idx]
        sample = Image.open(path)
        sample = np.asarray(sample, dtype=np.uint8)
        sample = cv2.resize(src=sample, dsize=(1024, 1024))
        sample = functional.to_grayscale(sample, num_output_channels=3)

        sample[:, :, 0] = 0
        sample[:, :, 1] = 0

        if self.transform:
            sample = self.transform(sample)

        if self.return_filenames:
            return sample, path
        else:
            return sample

    def duplicate_and_transform(self,transforms):
        currfilelen = self.__len__()

        for pind in range(0,currfilelen-1):
            path = self.files[pind]
            sample = Image.open(path)
            samplet = transforms(sample)
            # save the transformed images and save the associated counts:
            newpath = path[:-4]+'trans.png' #path[:-4] is the path without the .png extension.
            samplet.save(newpath,"PNG")
            metadata = pd.read_csv(os.path.join(self.root,'gt_red.csv'))
            metadata.set_index('filename',inplace=True)
            count = metadata['count'][os.path.split(path)[-1]]
            samplet.save(newpath,"PNG")
            f = open(self.root+'gt.csv','a')
            basepath = os.path.basename(path) # strip away the directory list and just get the filename
            f.write(basepath[:-4]+'trans.png,'+str(count)+'\n')
            # return the new file names and add to self.files.
            self.files.append(newpath)
            #pdb.set_trace()


title: When I run deep learning training code on Google Colab, do the resulting weights and biases get saved somewhere?

body:
I am training some deep learning code from this repository on a Google Colab notebook. The training is ongoing and seems like it is going to take a day or two.

I am new to deep learning, but my question:

Once the Google Colab notebook has finished running the training script, does this mean that the resulting weights and biases will be hard written to a model somewhere (in the repository folder that I have on my Google Drive), and therefore I can then run the code on any test data I like at any point in the future? Or, once I close the Google Colab notebook, do I lose the weight and bias information and would have to run the training script again if I wanted to use the neural network?

I realise that this might depend on the details of the script (again, the repository is here), but I thought that there might be a general way that these things work also.

Any help in understanding would be greatly appreciated.


title: Get indices of elements in tensor a that are present in tensor b

body:
For example, I want to get the indices of elements valued 0 and 2 in tensor a. These values, (0 and 2) are stored in tensor b. I have devised a pythonic way to do so (shown below), but I don't think list comprehensions are optimized to run on GPU, or maybe there is a more PyTorchy way to do it that I am unaware of.

import torch
a = torch.tensor([0, 1, 0, 1, 1, 0, 2])
b = torch.tensor([0, 2])
torch.tensor([x in b for x in a]).nonzero()

>>>> tensor([[0],
             [2],
             [5],
             [6]])

Any other suggestions or is this an acceptable way?


How should I fix this error?
[jalal@goku GoodNews]$ python train.py --cnn_weight data/resnet152-b121ed2d.pth
DataLoader loading json file:  data/data_news.json
vocab size is  37200
DataLoader loading h5 file:  data/data_news_label.h5 /scratch2/goodnewsdata/data_news_image.h5
read 489229 images of size 3x256x256
max sequence length in data is 31
assigned 445433 images to split train
assigned 19376 images to split val
assigned 24420 images to split test
WARNING:tensorflow:From train.py:49: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.

THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch-nightly_1553749772122/work/aten/src/THC/THCGeneral.cpp line=51 error=3 : initialization error
Traceback (most recent call last):
  File "train.py", line 280, in <module>
    train(opt)
  File "train.py", line 81, in train
    cnn_model.cuda()
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 263, in cuda
    return self._apply(lambda t: t.cuda(device))
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 190, in _apply
    module._apply(fn)
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 196, in _apply
    param.data = fn(param.data)
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py", line 263, in <lambda>
    return self._apply(lambda t: t.cuda(device))
  File "/scratch/sjn-p3/anaconda/anaconda3/lib/python3.6/site-packages/torch/cuda/__init__.py", line 163, in _lazy_init
    torch._C._cuda_init()
RuntimeError: cuda runtime error (3) : initialization error at /opt/conda/conda-bld/pytorch-nightly_1553749772122/work/aten/src/THC/THCGeneral.cpp:51
Closing remaining open files:data/data_news_label.h5...done/scratch2/goodnewsdata/data_news_image.h5...done

I am using the code and instruction from this GitHub repository: https://github.com/furkanbiten/GoodNews


title: Update specific vector elements in PyTorch

body:
I have a large vector that I would like to update. I'll update it by adding an offset to specific elements in the vector. I specify a vector of indices that I want to update (call the index vector ix), and for each index, I specify a value that I want to add to that element (call the value vector vals). If all entries of the index vector are unique, then the following code suffices:

vec = torch.zeros(4, dtype=torch.float)
ix = torch.tensor([0,2], dtype=torch.long)
vals = torch.tensor([0.2, 0.5], dtype=torch.float)
vec[ix] += vals

However, this does not work if there are repeated indices in ix. A naive approach for the case of repeated indices is as follows:

for i in range(len(ix)):
    vec[ix[i]] += vals[i]

But this doesn't scale well - it is very slow when ix is large. Are there any faster ways to do this? If there were a fast way to sum all entries of vals that have the same index in ix, then the solution should be easy.

Update:
I found one solution that works pretty well, described below. I'd still love feedback for better solutions.

# get unique indices
ix_unique = torch.unique(ix)

# for each unique index, get sum of all vals with that index
vals_unique = torch.stack([
    torch.sum(torch.where(ix==i, vals, torch.zeros_like(vals)))
    for i in ix_unique
])

# update vec
vec[ix_unique] += vals_unique


title: Variable length of text with cnn pytorch

body:
I am a newbie with pytorch and I am wondering what is the best practice for variable length sentences sequences in CNNs.

I want to use CNN for feature selection on top of word emmbeding generated by fasttext and then feed the output into LSTM .

Now I know pytorch has a dynamic graph and I was wondering if there is a way to this except padding


title: MSELoss when mask is used

body:
I'm trying to calculate MSELoss when mask is used. Suppose that I have tensor with batch_size of 2: [2, 33, 1] as my target, and another input tensor with the same shape. Since sequence length might differ for each instance, I have also a binary mask indicating the existence of each element in the input sequence. So here is what I'm doing:

mse_loss = nn.MSELoss(reduction='none')

loss = mse_loss(input, target)
loss = (loss * mask.float()).sum() # gives \sigma_euclidean over unmasked elements

mse_loss_val = loss / loss.numel()

# now doing backpropagation
mse_loss_val.backward()

Is loss / loss.numel() a good practice? I'm skeptical, as I have to use reduction='none' and when calculating final loss value, I think I should calculate the loss only considering those loss elements that are nonzero (i.e., unmasked), however, I'm taking the average over all tensor elements with torch.numel(). I'm actually trying to take 1/n factor of MSELoss into account. Any thoughts?


title: How to install PyTorch in Coral Dev Board

body:
I was trying to install PyTorch in Coral Dev Board. I could not find any online reference for the installation.

Can I install PyTorch Mobile (formerly Lite) in Coral Dev Board?
Does the architecture support the installation?


title: Difference between versions 9.2,10.1,10.2,11.0 of cuda for PyTorch 1.7

body:
Recently,I need to install pytorch ,when I check out the website :
install pytorch website
It shows four different version 9.2,10.1,10.2,11.0 to choose ,And I have cuda version 10.0 and driver version 450 installed on my computer,I thought it would fail to enable gpu when using pytorch
,After I choose 10.1 and  try torch.cuda.is_available() and it returns True
I have two questions:

Why does everything turn out to be working even my cuda version is not the same as any of one I mentioned ?

What's the difference between choosing cuda verison 9.2,10.1,10.2,11.0 ?


title: How can I simplify a nested loop into torch tensor operations?

body:
I'm trying to convert some code I have written in numpy which contains a nested-loop into tensor operations found in PyTorch. However, after trying to implement my own version I'm not getting the same value on the output. I have managed to do the same with a single loop, so I'm not entirely sure what I'm doing wrong.

    #(Numpy Version)
    #calculate Kinetic Energy
    summation = 0.0
    for i in range(0,len(k_values)-1):
        summation += (k_values[i]**2.0)*wavefp[i]*(((self.hbar*kp_values[i])**2.0)/(2.0*self.mu))*wavef[i]
    Ek = step*(4.0*np.pi)*summation

    #(Numpy Version)
    #calculate Potential Energy
    summation = 0.0
    for i in range(0,len(k_values)-1):
        for j in range(0,len(kp_values)-1):
            summation+= (k_values[i]**2.0)*wavefp[i]*(kp_values[j]**2.0)*wavef[j]*self.MTV[i,j]
    Ep = (step**2.0)*(4.0*np.pi)*(2.0/np.pi)*summation

            #####################################################

    #(PyTorch Version)
    #calcualte Kinetic Energy
    Ek = step*(4.0*np.pi)*torch.sum( k_values.pow(2)*wavefp.mul(wavef)*((kp_values.mul(self.hbar)).pow(2)/(2.0*self.mu)) )

    #(PyTorch Version)
    #calculate Potential Energy
    summation = 0.0
    for i in range(0,len(k_values)-1):
        summation += ((k_values[i].pow(2)).mul(wavefp[i]))*torch.sum( (kp_values.pow(2)).mul(wavef).mul(self.MTV[i,:]) )
    Ep = (step**2.0)*(4.0*np.pi)*(2.0/np.pi)*summation

The arrays/tensors k_values, kp_values, wavef, and wavefp have dimensions of (1000,1). The values self.hbar, and self.mu, and step are scalars. The variable self.MTV is a matrix of size (1000,1000).

I would expect that both methods would give the same output but they don't. The code for calculating the Kinetic Energy (in both Numpy and PyTorch) give the same value. However, the potential energy calculation differ, and I'm not entirely sure why.

Many Thanks in advance!


title: PyTorch C++ Extensions: Accessing data for Half Tensors

body:
I'm trying to write a C++/CUDA extension for PyTorch using the C++ Tensor API, and I would like my code to work with both float32 and float16 (half precision). I'm not sure how to access the data pointer for half tensors coming from Python.

Here's how I do it for float tensors:

// Access data pointer for float Tensor A
torch::Tensor A;
float* ptr = A.data<float>();

Here's what I've tried for half tensors:

// CUDA float 16 type
// undefined symbol: _ZNK2at6Tensor4dataI6__halfEEPT_v
A.data<__half>();

// PyTorch float16 type
// error: no instance of function template "at::Tensor::data"
A.data<torch::ScalarType::Half>();

// Casting to __half*
// This compiles but throws and error if the requested pointer type doesn't match the Tensor type:
// RuntimeError: expected scalar type Float but found Half
(__half*)(A.data<float>());

I tried looking into the C++ api source code, but couldn't find anything else that looks like a float16 type.

System information:
Python 3.6.2
PyTorch 1.0.1


title: How to use Datasets with the Pytorch C++ API with simple float I/O Data?

body:
I'm trying to use the Pytorch C++ API to train a simple neural net that has float input and output values. The Values are saved in "input.csv" and a output.csv" files. There are just 10x10 inputs and outputs that represent the negation. So if the input is 1 the output is 0.

I was able to build a simple neural net by creating input and output tensors manually. The code for this is on my repo in the master branch that you can find here: master branch repo.

But now I wanted to use torch::data::make_data_loader and torch::data::datasets to make my own custom dataset and being able to shuffle my data and also to create batch sizes for the training loop.

Unfortunately all of the examples and tutorials for the pytorch cpp API only show how to do it for complex image recognition. These examples are way to complicated for a programming beginner like me. (Just started programming a few months ago)

I've tried it myself by creating a custom dataset class as shown in most examples and then using this class to make a dataloader. You can find it here: dataset test branch.

I've been following a Chinese tutorial since this was the only tutorial that was somewhat beginner friendly: Chinese Tutorial.

I think what I want to accomplish is something like in this python tutorial:
pytorch dataset loader with python

Here is my Class defined in src/owndata.cpp:

#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <torch/torch.h>
#include <vector>
//#include "include/owndata.h"

//----------------------Funktionsprototyp Onelinevector----------------------

std::vector<float>
onelinevector(const std::vector<std::vector<float>> &invector);

//------------------------Funktionsprototyp csv2Dvector----------------------

std::vector<std::vector<float>> csv2Dvector(std::string inputFileName);

//-------------------Custom Dataset Class------------------------------------

class CustomDataset : public torch::data::datasets::Dataset<CustomDataset> {

private:
std::vector<std::vector<float>> xdata, ydata;
torch::Tensor outputtensor, inputtensor;
public:
CustomDataset(std::string xpath, std::string ypath) {
    xdata = csv2Dvector(xpath);
    ydata = csv2Dvector(ypath);

    //create input tensor:
    unsigned int ivsize = xdata.size() * xdata.front().size();
    std::vector<float> olinevec(ivsize);
    olinevec = onelinevector(xdata);

    inputtensor = torch::from_blob(
      olinevec.data(), {static_cast<unsigned int>(xdata.size()),
                        static_cast<unsigned int>(xdata.front().size())});
    //create output tensor:
    unsigned int ovsize = ydata.size() * ydata.front().size();
    std::vector<float> iovec(ovsize);
    iovec = onelinevector(ydata);

    outputtensor = torch::from_blob(
      iovec.data(), {static_cast<unsigned int>(ydata.size()),
                        static_cast<unsigned int>(ydata.front().size())});
};

torch::data::Example<> get(size_t index) override {

    torch::Tensor sample_input = inputtensor[index];
    torch::Tensor sample_output = outputtensor[index];

    return {sample_input.clone(), sample_output.clone()};
};

// Return the length of data
torch::optional<size_t> size() const override {
    return ydata.size();
  };

};

size_t batch_size = 5;

//-----------------------------NETZDEFINITION-----------------------------
struct MeinNetz : torch::nn::Module {
  MeinNetz() {
    fc1 = register_module("fc1", torch::nn::Linear(10, 10));
    fc2 = register_module("fc2", torch::nn::Linear(10, 10));
  }

  torch::Tensor forward(torch::Tensor x) {
    x = torch::relu(fc1->forward(x));
    x = fc2->forward(x);
    return x;
  }

  torch::nn::Linear fc1{nullptr}, fc2{nullptr};
};

//-------------------------------------main-Funktion-------------------------

int main() {

  auto custom_dataset = CustomDataset("input.csv","output.csv");
  auto data_loader = torch::data::make_data_loader<torch::data::samplers::SequentialSampler>(std::move(custom_dataset),batch_size);

  auto dataset_size = custom_dataset.size().value();
  int n_epochs = 50;

  auto net = std::make_shared<MeinNetz>();

  torch::optim::SGD optimizer(net->parameters(), 0.2);

  for(int epoch=1; epoch<=n_epochs; epoch++) {
    for(auto& batch: *data_loader) {
      auto data = batch.data;
      auto target = batch.target;

      data = data.to(torch::kF32);
      target = target.to(torch::kF32);

      optimizer.zero_grad();

      auto prediction = net->forward(data);
      auto loss = torch::mse_loss(prediction, target);

      loss.backward();

      optimizer.step();

      std::cout << "Epoch: " << epoch << " Loss: "
      << loss.item<float>() << std::endl;
    }
  }

  return 0;
}

//--------------------------Funktionen--------------------------------------------

std::vector<float>
onelinevector(const std::vector<std::vector<float>> &invector) {

  std::vector<float> v1d;
  if (invector.size() == 0)
    return v1d;
  v1d.reserve(invector.size() * invector.front().size());

  for (auto &innervector : invector) {
    v1d.insert(v1d.end(), innervector.begin(), innervector.end());
  }

  return v1d;
}

//-------------------------csv2vector Funktionsdefinition--------------------------

std::vector<std::vector<float>> csv2Dvector(std::string inputFileName) {
  using namespace std;

  vector<vector<float>> data;
  ifstream inputFile(inputFileName);
  int l = 0;

  while (inputFile) {
    l++;
    string s;
    if (!getline(inputFile, s))
      break;
    if (s[0] != '#') {
      istringstream ss(s);
      vector<float> record;

      while (ss) {
        string line;
        if (!getline(ss, line, ','))
          break;
        try {
          record.push_back(stof(line));
        } catch (const std::invalid_argument e) {
          cout << "NaN found in file " << inputFileName << " line " << l
               << endl;
          e.what();
        }
      }

      data.push_back(record);
    }
  }

  if (!inputFile.eof()) {
    cerr << "Could not read file " << inputFileName << "\n";
    throw invalid_argument("File not found.");
  }

  return data;
}

This is not working at all.

PS: there is also an example with Python from the pytorch repo but it's way more complicated than the Chinese example: python example for image import 2.


title: Conv2D and Conv3D: increase the accuracy

body: I have 10000 RGB images with 18 * 18 pixels (yes, images are small sized) for each of 5 classes. I wanted to make deep learning model for classification of images into 5 labels. I tried both Conv2D and Conv3D, with kernels (3,3) and (3,3,3) respectively. I tried adding multiple layers (up to 8) with different filters (16, 32, 64, 128, 256). I tried different activators (relu, sigmoid) and optimizers (adam, sgd). I also tried # epochs upto 500. However, I could not get more than 0.5 overall accuracy by any means. Please help me guys how can I increase the accuracy.


title: saving and loading pytorch neural nets

body:
So I created a neural net and I would like to save it and load it whenever I want. Specifically, I want to take pictures and do real time processing. I am using the neural net created here

I read that the standard way is to create the net then use torch.save(net,'mynet') to save it and then load it with torch.load('mynet').

However if I open a new python3 terminal and use:

>>import torch
>>torch.load('mynet')

It gives me the error:

  File "<stdin>", line 1, in <module>
  File "/home/tim/anaconda3/lib/python3.7/site-packages/torch/serialization.py", line 367, in load
    return _load(f, map_location, pickle_module)
  File "/home/tim/anaconda3/lib/python3.7/site-packages/torch/serialization.py", line 538, in _load
    result = unpickler.load()
AttributeError: Can't get attribute 'Net' on <module '__main__' (built-in)>

I think this is from not having the Net class defined. Adding

import torch.nn as nn
import torch.nn.functional as F
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 15, 3)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(15, 15, 5)
        self.conv3 = nn.Conv2d(15, 10, 3)
        self.fc1 = nn.Linear(10*4*4, 100)
        self.fc2 = nn.Linear(100, 24)
        self.fc3 = nn.Linear(24, 4)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = self.pool(F.relu(self.conv3(x)))
        x = x.view(-1, 10*4*4)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

is what you need to do , but why do we need to define the neural net class? What If I load a neural net with a different architecture to the one i specify in the class will the architecture defined in the class get overwritten? surely the object im loading have all the architecture and class information incapsulated in it?

Update:
Actually it doesn't even work when i define the Net class.


title: LSTM getting caught up in loop

body:
I recently implemented a name generating RNN "from scratch" which was doing ok but far from perfect. So I thought about trying my luck with pytorch's LSTM class to see if it makes a difference. Indeed it does and the outpus looks way better for the first 7 ~ 8 characters. But then the networks gets caught in a loop and outputs things like "laulaulaulau" or "rourourourou" (it is supposed the generate french names).

Is it a often occuring problem ? If so do you know a way to fix it ? I'm concern about the fact the network doesn't produce EOS tokens...
This is an issue which has already been asked here Why does my keras LSTM model get stuck in an infinite loop?
but not really answered hence my post.

here is the model :

class pytorchLSTM(nn.Module):
    def __init__(self,input_size,hidden_size):
        super(pytorchLSTM,self).__init__()
        self.input_size = input_size
        self.hidden_size = hidden_size
        self.lstm = nn.LSTM(input_size, hidden_size)
        self.output_layer = nn.Linear(hidden_size,input_size)
        self.tanh = nn.Tanh()
        self.softmax = nn.LogSoftmax(dim = 2)

    def forward(self, input, hidden)
            out, hidden = self.lstm(input,hidden)
            out = self.tanh(out)
            out = self.output_layer(out)
            out = self.softmax(out)
        return out, hidden

The input and target are two sequences of one-hot encoded vectors respectively with a start of sequence and end of sequence vector at the start and the end. They represent the characters inside of a name taken from the name list (database).
I use a  and  token on each name from the database. here are the function I use

def inputTensor(line):
#tensor starts with <start of sequence> token.
    tensor = torch.zeros(len(line)+1, 1, n_letters)
    tensor[0][0][n_letters - 2] = 1
    for li in range(len(line)):
        letter = line[li]
        tensor[li+1][0][all_letters.find(letter)] = 1
    return tensor

# LongTensor of second letter to end (EOS) for target
def targetTensor(line):
    letter_indexes = [all_letters.find(line[li]) for li in range(len(line))]
    letter_indexes.append(n_letters - 1) # EOS
    return torch.LongTensor(letter_indexes)

training loop :

def train_lstm(model):
    start = time.time()
    criterion = nn.NLLLoss()
    optimizer = torch.optim.Adam(model.parameters())
    n_iters = 20000
    print_every = 1000
    plot_every = 500
    all_losses = []
    total_loss = 0
    for iter in range(1,n_iters+1):
        line = randomChoice(category_line)
        input_line_tensor = inputTensor(line)
        target_line_tensor = targetTensor(line).unsqueeze(-1)
        optimizer.zero_grad()
        loss = 0
        output, hidden = model(input_line_tensor)
        for i in range(input_line_tensor.size(0)):
            l = criterion(output[i], target_line_tensor[i])
            loss += l
        loss.backward()
        optimizer.step()

the sampling function :

def sample():
    max_length = 20
    input = torch.zeros(1,1,n_letters)
    input[0][0][n_letters - 2] = 1
    output_name = ""
    hidden = (torch.zeros(2,1,lstm.hidden_size),torch.zeros(2,1,lstm.hidden_size))

    for i in range(max_length):
        output, hidden = lstm(input)
        output = output[-1][:][:]
        l = torch.multinomial(torch.exp(output[0]),num_samples = 1).item()
        if l == n_letters - 1:
            break
        else:
            letter = all_letters[l]
            output_name += letter
        input = inputTensor(letter)
    return output_name

The typical sampled output looks something like that :

Laurayeerauerararauo
Leayealouododauodouo
Courouauurourourodau

Do you know how I can improve that ?


title: EnsembleVoteClassifier with neural network

body:
I have a trained neural networks in which I am trying to average their prediction using EnsembleVoteClassifier from mlxtend.classifier. The problem is my neural network don't share the same input, (I performed feature reduction and feature select algorithms randomly and stored the results on new different variables, so I have something like X_test_algo1, X_test_algo2 and X_test_algo3 and Y_test).

I am trying to average the weights, but as I said, I don't have the same X, and I didn't any example on the documentation. How can I average the predictions for my three models model1, model2 and model3

eclf = EnsembleVoteClassifier(clfs=[model1, model2, model3], weights=[1,1,1], refit=False)
names = ['NN1', 'NN2', 'NN2', 'Ensemble']

eclf.fit(X_train_algo1, Ytrain) #????

If it's not possible, that is okay. I am only interested on how to calculate the formulas of Hard Voting, Hard Voting and Weighted Voting, or if there is anther library that is more flexible or the explicit expressions of the formulas could be helpful too.


title: Forward Propagate RNN using Pytorch

body:
I am trying to create an RNN forward pass method that can take a variable input, hidden, and output size and create the rnn cells needed. To me, it seems like I am passing the correct variables to self.rnn_cell -- the input values of x and the previous hidden layer. However, the error I receive is included below.

I have also tried using x[i] and x[:,i,i] (as suggested by my professor) to no avail. I am confused and just looking for guidance as to whether or not I am doing the right thing here. My prof suggested that since I keep receiving errors, I should restart the kernel in jupyter notebook and rerun code. I have, and I receive the same errors...

Please let me know if you need additional context.

class RNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(RNN, self).__init__()
        self.hidden_size = hidden_size

        self.rnn_cell = nn.RNNCell(input_size, hidden_size)
        self.fc = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, x):
        """
        x: size [seq_length, 1, input_size]
        """
        h = torch.zeros(x.size(1), self.hidden_size)

        for i in range(x.size(0)):
            ### START YOUR CODE ###
            h = self.rnn_cell(x[:,:,i], h)
            ### END YOUR CODE ###

        ### START YOUR CODE ###
        # Hint: first call fc, then call softmax
        out = self.softmax(self.fc(self.hidden_size, h.size(0)))
        ### END YOUR CODE ###

        return out
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)


title: Process stuck when training on multiple nodes using PyTorch DistributedDataParallel

body:
I am trying to run the script mnist-distributed.py from Distributed data parallel training in Pytorch. I have also pasted the same code here. (I have replaced my actual MASTER_ADDR with a.b.c.d for posting here).
import os
import argparse
import torch.multiprocessing as mp
import torchvision
import torchvision.transforms as transforms
import torch
import torch.nn as nn
import torch.distributed as dist

class ConvNet(nn.Module):
    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))
        self.layer2 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))
        self.fc = nn.Linear(7*7*32, num_classes)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.reshape(out.size(0), -1)
        out = self.fc(out)
        return out

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--nodes', default=1, type=int, metavar='N')
    parser.add_argument('-g', '--gpus', default=1, type=int,
                        help='number of gpus per node')
    parser.add_argument('-nr', '--nr', default=0, type=int,
                        help='ranking within the nodes')
    parser.add_argument('--epochs', default=2, type=int, metavar='N',
                        help='number of total epochs to run')
    args = parser.parse_args()
    args.world_size = args.gpus * args.nodes
    os.environ['MASTER_ADDR'] = 'a.b.c.d'
    os.environ['MASTER_PORT'] = '8890'
    mp.spawn(train, nprocs=args.gpus, args=(args,))

def train(gpu, args):
    rank = args.nr * args.gpus + gpu
    dist.init_process_group(
        backend='nccl',
        init_method='env://',
        world_size=args.world_size,
        rank=rank
    )

    torch.manual_seed(0)
    model = ConvNet()
    torch.cuda.set_device(gpu)
    model.cuda(gpu)
    batch_size = 100
    # define loss function (criterion) and optimizer
    criterion = nn.CrossEntropyLoss().cuda(gpu)
    optimizer = torch.optim.SGD(model.parameters(), 1e-4)

    # Wrap the model
    model = nn.parallel.DistributedDataParallel(model,
                                                device_ids=[gpu])

    # Data loading code
    train_dataset = torchvision.datasets.MNIST(
        root='./data',
        train=True,
        transform=transforms.ToTensor(),
        download=True
    )
    train_sampler = torch.utils.data.distributed.DistributedSampler(
        train_dataset,
        num_replicas=args.world_size,
        rank=rank
    )

    train_loader = torch.utils.data.DataLoader(
        dataset=train_dataset,
       batch_size=batch_size,
       shuffle=False,
       num_workers=0,
       pin_memory=True,
      sampler=train_sampler)

    total_step = len(train_loader)
    for epoch in range(args.epochs):
        for i, (images, labels) in enumerate(train_loader):
            images = images.cuda(non_blocking=True)
            labels = labels.cuda(non_blocking=True)
            # Forward pass
            outputs = model(images)
            loss = criterion(outputs, labels)

            # Backward and optimize
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            if (i + 1) % 100 == 0 and gpu == 0:
                print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(
                    epoch + 1,
                    args.epochs,
                    i + 1,
                    total_step,
                    loss.item())
                   )

if __name__ == '__main__':
    main()

There are 2 nodes with 2 GPUs each. I run this command from the terminal of the master node-

python mnist-distributed.py -n 2 -g 2 -nr 0

, and then this from the terminal of the other node-

python mnist-distributed.py -n 2 -g 2 -nr 1

But then my process gets stuck with no output on either terminal.
Running the same code on a single node using the following command works perfectly fine-

python mnist-distributed.py -n 1 -g 2 -nr 0


title: Pytorch in V.S. Code: torch.tensor is not callable

body:
I am running my pytorch program in V.S. Code, but, I am getting the following errors:-
torch.tensor is not callable

I have followed this answer https://stackoverflow.com/a/53572939/13218692 and added the following lines in my Settings.json file in V.S. Code, but, I am still getting an error. Any help would be highly appreciated! Thanks!


title: Beta Distribution in PyTorch for a, b>1?

body:
PyTorch supports Beta distributions however, when alpha or beta is greater than 1, it doesn't work:

m = Beta(torch.tensor([2]), torch.tensor([2]))
m.sample()


title: Unity ML-Agents Running Very Slowly

body: Using the Python API, whether I run using a build or in the Editor the simulation is much slower than when using the provided mlagents-learn methods. I'm running something similar to this, using a PyTorch implementation of DDPG and CUDA 9.0. Is this expected behaviour?


title: Different `grad_fn` for similar looking operations in Pytorch (1.0)

body:
I am working on an attention model, and before running the final model, I was going through the tensor shapes which flow through the code. I have an operation where I need to reshape the tensor. The tensor is of the shape torch.Size([[30, 8, 9, 64]])  where 30 is the batch_size, 8 is the number of attention head (this is not relevant to my question) 9 is the number of words in the sentence and 64 is some intermediate embedding representation of the word. I have to reshape the tensor to a size of torch.size([30, 9, 512]) before processing it further. So I was looking into some reference online and they have done the following x.transpose(1, 2).contiguous().view(30, -1, 512)
whereas I was thinking that this should work x.transpose(1, 2).reshape(30, -1, 512).

In the first case the grad_fn is <ViewBackward>, whereas in my case it is <UnsafeViewBackward>. Aren't these two the same operations? Will this result in a training error?


title: Pytorch's packed_sequence/pad_sequence pads tensors vertically for list of tensors

body:
I am trying to pad sequence of tensors for LSTM mini-batching, where each timestep in the sequence contains a sub-list of tensors (representing multiple features in a single timestep).

For example, sequence 1 would have 3 timesteps and within each timestep there are 2 features. An example below would be:

Sequence 1 = [[1,2],[2,2],[3,3],[3,2],[3,2]]

Sequence 2 = [[4,2],[5,1],[4,4]]

Sequence 3 = [[6,9]]

I run pytorch's pad_sequence function (this goes for pack_sequence too) like below:

import torch
import torch.nn.utils.rnn as rnn_utils

a = torch.tensor([[1,2],[2,2],[3,3],[3,2],[3,2]])
b = torch.tensor([[4,2],[5,1],[4,4]])
c = torch.tensor([[6,9]])
result = rnn_utils.pad_sequence([a, b, c])

My expected output is as follows:

Sequence 1 = [[1,2],[2,2],[3,3],[3,2],[3,2]]

Sequence 2 = [[4,2],[5,1],[4,4],[0,0],[0,0]]

Sequence 3 = [[6,9],[0,0],[0,0],[0,0],[0,0]]

However, the output I got is as follows:

tensor([[[1, 2],
         [4, 2],
         [6, 9]],

        [[2, 2],
         [5, 1],
         [0, 0]],

        [[3, 3],
         [4, 4],
         [0, 0]],

        [[3, 2],
         [0, 0],
         [0, 0]],

        [[3, 2],
         [0, 0],
         [0, 0]]])

The padding seems to go vertically rather than what I expect. How do I go about getting the correct padding that I need?


title: PyTorch nn.Transformer learns to copy target

body:
I’m trying to train a Transformer Seq2Seq model using nn.Transformer class. I believe I am implementing it wrong, since when I train it, it seems to fit too fast, and during inference it repeats itself often. This seems like a masking issue in the decoder, and when I remove the target mask, the training performance is the same. This leads me to believe I am doing the target masking wrong. Here is my model code:
class TransformerModel(nn.Module):
    def __init__(self,
        vocab_size, input_dim, heads, feedforward_dim, encoder_layers, decoder_layers,
        sos_token, eos_token, pad_token, max_len=200, dropout=0.5,
        device=(torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu"))):

        super(TransformerModel, self).__init__()
        self.target_mask = None
        self.embedding = nn.Embedding(vocab_size, input_dim, padding_idx=pad_token)
        self.pos_embedding = nn.Embedding(max_len, input_dim, padding_idx=pad_token)
        self.transformer = nn.Transformer(
            d_model=input_dim, nhead=heads, num_encoder_layers=encoder_layers,
            num_decoder_layers=decoder_layers, dim_feedforward=feedforward_dim,
            dropout=dropout)
        self.out = nn.Sequential(
            nn.Linear(input_dim, feedforward_dim),
            nn.ReLU(),
            nn.Linear(feedforward_dim, vocab_size))

        self.device = device
        self.max_len = max_len
        self.sos_token = sos_token
        self.eos_token = eos_token

    # Initialize all weights to be uniformly distributed between -initrange and initrange
    def init_weights(self):
        initrange = 0.1
        self.encoder.weight.data.uniform_(-initrange, initrange)
        self.decoder.bias.data.zero_()
        self.decoder.weight.data.uniform_(-initrange, initrange)

    # Generate mask covering the top right triangle of a matrix
    def generate_square_subsequent_mask(self, size):
        mask = (torch.triu(torch.ones(size, size)) == 1).transpose(0, 1)
        mask = mask.float().masked_fill(mask == 0, float('-inf')).masked_fill(mask == 1, float(0.0))
        return mask

    def forward(self, src, tgt):
        # src: (Max source seq len, batch size, 1)
        # tgt: (Max target seq len, batch size, 1)

        # Embed source and target with normal and positional embeddings
        embedded_src = (self.embedding(src) +
           self.pos_embedding(
           torch.arange(0, src.shape[1]).to(self.device).unsqueeze(0).repeat(src.shape[0], 1)))
        # Generate target mask
        target_mask = self.generate_square_subsequent_mask(size=tgt.shape[0]).to(self.device)
        embedded_tgt = (self.embedding(tgt) +
            self.pos_embedding(
            torch.arange(0, tgt.shape[1]).to(self.device).unsqueeze(0).repeat(tgt.shape[0], 1)))
        # Feed through model
        outputs = self.transformer(src=embedded_src, tgt=embedded_tgt, tgt_mask=target_mask)
        outputs = F.log_softmax(self.out(outputs), dim=-1)
        return outputs


title: How can I visualize network architectures effectively?

body:
Is there some sort of software that can do so? Specifically, I would like to visualize Resnet18. Is there no other way other than to just draw it myself? Here is an example of what I want to see:
Sample Architecture Visualization


title: pytorch：RuntimeError: dimension out of range (expected to be in range of [-1, 0], but got 1)

body:
I want to train a 100-classes model by using pretrain inceptionV3，but in my  training code, I got a strange error, show the error below:

my training code is:

step = -1
print_inter=50
val_inter=400
train_size = ceil(len(data_set['train']) / dataloader['train'].batch_size)
for epoch in range(50):
    # train phase
    exp_lr_scheduler.step(epoch)
    inception_v3.train(True)
    for batch_cnt, data in enumerate(dataloader['train']):
        step += 1
        inception_v3.train(True)

        inputs, labels = data

        inputs = torch.autograd.Variable(inputs.cuda())
        labels = torch.autograd.Variable(torch.from_numpy(np.array(labels)).long().cuda())
        outputs = inception_v3(inputs)
        # zero the parameter gradients
        optimizer.zero_grad()

        outputs = inception_v3(inputs)
        print(inputs.shape)
        print(outputs[0].shape)
        print(outputs[1].shape)
        print(labels.shape)

        loss = criterion(outputs[0], labels)
        loss += criterion(outputs[1], labels)
        outputs = (outputs[0] + outputs[1]) / 2

        _, preds = torch.max(outputs, 1)
        loss.backward()
        optimizer.step()
        # batch loss

        inception_v3.train(False)  # Set model to evaluate mode

        for batch_cnt_val, data_val in enumerate(dataloader['val']):
            # print data
            inputs, labels = data_val

            inputs = Variable(inputs.cuda())
            labels = Variable(torch.from_numpy(np.array(labels)).long().cuda())

            # forward
            outputs = inception_v3(inputs)
            print(inputs.shape)
            print(outputs[0].shape)
            print(outputs[1].shape)
            print(labels.shape)
            loss = criterion(outputs[0], labels)
            loss += criterion(outputs[1], labels)
            outputs = (outputs[0] + outputs[1]) / 2

After running this code, Got result below:

torch.Size([8, 3, 299, 299])
torch.Size([8, 100])
torch.Size([8, 100])
torch.Size([8])
torch.Size([8, 3, 299, 299])
torch.Size([100])
torch.Size([100])
torch.Size([8])
        ---------------------------------------------------------------------------
    RuntimeError                              Traceback (most recent call last)
    <ipython-input-21-0cfbe57aae73> in <module>()
         49             print(outputs[1].shape)
         50             print(labels.shape)
    ---> 51             loss = criterion(outputs[0], labels)
         52             loss += criterion(outputs[1], labels)
         53             outputs = (outputs[0] + outputs[1]) / 2

    ~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
        489             result = self._slow_forward(*input, **kwargs)
        490         else:
    --> 491             result = self.forward(*input, **kwargs)
        492         for hook in self._forward_hooks.values():
        493             hook_result = hook(self, input, result)

    ~/anaconda3/lib/python3.6/site-packages/torch/nn/modules/loss.py in forward(self, input, target)
        757         _assert_no_grad(target)
        758         return F.cross_entropy(input, target, self.weight, self.size_average,
    --> 759                                self.ignore_index, self.reduce)
        760
        761

    ~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py in cross_entropy(input, target, weight, size_average, ignore_index, reduce)
       1440         >>> loss.backward()
       1441     """
    -> 1442     return nll_loss(log_softmax(input, 1), target, weight, size_average, ignore_index, reduce)
       1443
       1444

    ~/anaconda3/lib/python3.6/site-packages/torch/nn/functional.py in log_softmax(input, dim, _stacklevel)
        942     if dim is None:
        943         dim = _get_softmax_dim('log_softmax', input.dim(), _stacklevel)
    --> 944     return torch._C._nn.log_softmax(input, dim)
        945
        946

    RuntimeError: dimension out of range (expected to be in range of [-1, 0], but got 1)

you can see the same model get the same input size, but my second outputs miss the batch dimension. forgive my broken Englis


title: Sample code for model-parallel and pipelined training in TensorFlow

body:
Naive model-partitioning across several GPUs results in the workload moving from GPU to GPU during the forward and backward pass. At any instant, one GPU is busy. Here's the naive version.

with tf.device('/gpu:0'):
    model.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

with tf.device('/gpu:1'):
    model.add(Conv2D(128, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Conv2D(64, (3, 3), activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))

with tf.device('/gpu:2'):
    model.add(Dropout(0.25))
    model.add(Flatten())
    model.add(Dense(1500, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes, activation='softmax'))

We need sample code (template) that pipelines the work and keeps all GPUs busy by sending waves of batches and coordinates the work on each GPU (forward, gradient calc, parameter updates).

A hint is provided here via the use of a data_flow_ops.StagingArea but a concrete example would be helpful.

I understand that data-partitioning (or data-parallel) is the way to go, but there exist use-cases where the model needs to be partitioned across CPU+GPUs.

Grateful for any pointer or sample (pseudo)code.