title: ResNet model of pytorch and tensorflow give different results when stride=2

body:
class BasicBlock(nn.Module):
    def __init__(self, in_planes, out_planes, stride, dropRate=0.0):
        super(BasicBlock, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.relu1 = nn.ReLU(inplace=True)
        self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                               padding=1, bias=False) # 1
        self.bn2 = nn.BatchNorm2d(out_planes)
        self.relu2 = nn.ReLU(inplace=True)
        self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1,
                               padding=1, bias=False)
        self.droprate = dropRate
        self.equalInOut = (in_planes == out_planes)
        self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
                                                                padding=0, bias=False) or None
    def forward(self, x):
        if not self.equalInOut:
            x = self.relu1(self.bn1(x))
        else:
            out = self.relu1(self.bn1(x))
        out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x)))
        if self.droprate > 0:
            out = F.dropout(out, p=self.droprate, training=self.training)
        out = self.conv2(out)
        if self.convShortcut is not None:
        return torch.add(x if self.equalInOut else self.convShortcut(x), out)

class NetworkBlock(nn.Module):
    def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0):
        super(NetworkBlock, self).__init__()
        self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate)

    def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate):
        layers = []
        for i in range(int(nb_layers)):
            layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate))
        return nn.Sequential(*layers)

    def forward(self, x):
        return self.layer(x)

class WideResNet(nn.Module):
    def __init__(self, depth=34, num_classes=10, widen_factor=10, dropRate=0.0):
        super(WideResNet, self).__init__()
        nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor]
        assert ((depth - 4) % 6 == 0)
        n = (depth - 4) / 6
        block = BasicBlock
        # 1st conv before any network block
        self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1,
                               padding=1, bias=False)
        # 1st block
        self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
        # 1st sub-block
        self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
        # 2nd block
        self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) # 2
        # 3rd block
        self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) # 2
        # global average pooling and classifier
        self.bn1 = nn.BatchNorm2d(nChannels[3])
        self.relu = nn.ReLU(inplace=True)
        self.fc = nn.Linear(nChannels[3], num_classes)
        self.nChannels = nChannels[3]

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                m.bias.data.zero_()

    def forward(self, x):
        out = self.conv1(x)
        out = self.block1(out)
        out = self.block2(out)
        out = self.block3(out)
        out = self.relu(self.bn1(out))
        out = F.avg_pool2d(out, 8)
        out = out.view(-1, self.nChannels)
        return self.fc(out)

  def _conv(self, name, x, filter_size, in_filters, out_filters, strides, padding='SAME'):
    """Convolution."""
    with tf.variable_scope(name):
      n = filter_size * filter_size * out_filters
      kernel = tf.get_variable(
          'DW', [filter_size, filter_size, in_filters, out_filters],
          tf.float32, initializer=tf.random_normal_initializer(
              stddev=np.sqrt(2.0/n)))
      return tf.nn.conv2d(x, kernel, strides, padding=padding)

  def _residual(self, x, in_filter, out_filter, stride,
                activate_before_residual=False, is_log=False):
    """Residual unit with 2 sub layers."""
    if activate_before_residual:
        x = self._batch_norm('bn1', x)
        x = self._relu(x)
        orig_x = x
    else:
        orig_x = x
        x = self._batch_norm('bn1', x)
        x = self._relu(x)

    x = self._conv('conv1', x, 3, in_filter, out_filter, stride)
    x = self._batch_norm('bn2', x)
    x = self._relu(x)
    x = self._conv('conv2', x, 3, out_filter, out_filter, [1, 1, 1, 1])
    if in_filter != out_filter:
        orig_x = self._conv('shortcut_conv', orig_x, filter_size=1, in_filters=in_filter, out_filters=out_filter,
                            strides=stride, padding="VALID")
    x += orig_x
    return x

  def _build_model(self):
    assert self.mode == 'train' or self.mode == 'eval'
    with tf.variable_scope('input'):

      self.x_input = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
      self.y_input = tf.placeholder(tf.float32, shape=[None, 10])
      self.is_training = tf.placeholder(tf.bool, shape=None)
    x = self._conv('conv1.weight', self.x_input, 3, 3, 16, self._stride_arr(1))

    strides = [1, 2, 2]
    activate_before_residual = [True, True, True]
    res_func = self._residual
    # wide residual network (https://arxiv.org/abs/1605.07146v1)
    filters = [16, 160, 320, 640]

    with tf.variable_scope('block1.layer.0'):
      x = res_func(x, filters[0], filters[1], self._stride_arr(strides[0]),
                   activate_before_residual[0])
    for i in range(1, 5):
      with tf.variable_scope('block1.layer.%d' % i):
        x = res_func(x, filters[1], filters[1], self._stride_arr(1), False)

    with tf.variable_scope('block2.layer.0'):
      x = res_func(x, filters[1], filters[2], self._stride_arr(strides[1]),
                   activate_before_residual[1], is_log=True)
    for i in range(1, 5):
      with tf.variable_scope('block2.layer.%d' % i):
        x = res_func(x, filters[2], filters[2], self._stride_arr(1), False)

    with tf.variable_scope('block3.layer.0'):
      x = res_func(x, filters[2], filters[3], self._stride_arr(strides[2]),
                   activate_before_residual[2])
    for i in range(1, 5):
      with tf.variable_scope('block3.layer.%d' % i):
        x = res_func(x, filters[3], filters[3], self._stride_arr(1), False)

    x = self._batch_norm('bn1', x)
    x = self._relu(x)
    x = self._global_avg_pool(x)

    with tf.variable_scope('fc'):
      self.pre_softmax = self._fully_connected(x, 10)

I'm doing experiment on "adversarial defense", and I checked that the performances of pytorch and tensorflow is different with same weights (I exported it as numpy and loaded to pytorch and tensorflow) I printed out each result of WideResNet34 and calculate the difference of each output, then, the above output of below image comes out

The results start to be different from block2. Then, I only change the stride of each block to all 1 (stride of block 2 and 3),  the below output of above image comes out

The differences are negligible at all layers, so I think the difference appear only when stride=2. I don't know why there is no difference when stride=1 but different when stride=2... Who knows about this thing?


title: How to fix the dimension error in the loss function/softmax?

body:
I am implementing a logistic regression in PyTorch for XOR (I don't expect it to work well it's just a demonstration). For some reason I am getting an error 'IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)'. It is not clear to me where this originates. The error points to log_softmax during training.

import torch.nn as nn
import torch.nn.functional as F

class LogisticRegression(nn.Module):
  # input_size: Dimensionality of input feature vector.
  # num_classes: The number of classes in the classification problem.
  def __init__(self, input_size, num_classes):
    # Always call the superclass (nn.Module) constructor first!
    super(LogisticRegression, self).__init__()
    # Set up the linear transform
    self.linear = nn.Linear(input_size, num_classes)

  # Forward's sole argument is the input.
  # input is of shape (batch_size, input_size)
  def forward(self, x):
    # Apply the linear transform.
    # out is of shape (batch_size, num_classes)
    out = self.linear(x)
    # Softmax the out tensor to get a log-probability distribution
    # over classes for each example.
    out_distribution = F.softmax(out, dim=-1)
    return out_distribution

# Binary classifiation
num_outputs = 1
num_input_features = 2

# Create the logistic regression model
logreg_clf = LogisticRegression(num_input_features, num_outputs)

print(logreg_clf)

lr_rate = 0.001

X = torch.Tensor([[0,0],[0,1], [1,0], [1,1]])
Y = torch.Tensor([0,1,1,0]).view(-1,1) #view is similar to numpy.reshape()

# Run the forward pass of the logistic regression model
sample_output = logreg_clf(X) #completely random at the moment
print(X)

loss_function = nn.CrossEntropyLoss() # computes softmax and then the cross entropy
optimizer = torch.optim.SGD(logreg_clf.parameters(), lr=lr_rate)

from torch.autograd import Variable
#training loop:

epochs = 201 #how many times we go through the training set
steps = X.size(0) #steps = 4; we have 4 training examples

for i in range(epochs):
    for j in range(steps):
        #sample from the training set:
        data_point = np.random.randint(X.size(0))
        x_var = Variable(X[data_point], requires_grad=False)
        y_var = Variable(Y[data_point], requires_grad=False)

        optimizer.zero_grad() # zero the gradient buffers
        y_hat = logreg_clf(x_var) #get the output from the model
        loss = loss_function.forward(y_hat, y_var) #calculate the loss
        loss.backward() #backprop
        optimizer.step() #does the update

    if i % 500 == 0:
        print ("Epoch: {0}, Loss: {1}, ".format(i, loss.data.numpy()))


title: Cannot load trace file using libtorch 1.5 in C++

body:
After training a model, I created a trace file in Python side using Pytorch 1.5. I tried to load that file in C++ side using libtorch 1.5 writing that code:

int main(int argc, const char* argv[]) {

    //Deserialize the ScriptModule from a file using torch::jit::load().
    torch::jit::script::Module module = torch::jit::load("D:/Downloads/deneme18.pt");
    module.to(torch::kCUDA);

}

It gave me this error message:

  "Unhandled exception at 0x00007FF8C2A0A799 in DL_Predicton.exe:
  Microsoft C++ exception: c10::Error at memory location
  0x0000009B6118EFE0."


title: Backpropagation issue with custom loss function

body:
Context
I am building a super-resolution imaging model (a U-Net). My data consists of microscopic blood smears. I am using fast ai.
I want to try a new loss function that would "focus" on the cells' components.
What I have done
I have created a pseudo_mask function that takes two arguments, an image and an integer β ranging from 0 to 255. This function first computes a cell segmenting mask, converts the 0 values of the mask to β and applies the result to the original image.
Pseudo_mask output of the same image, with β values of respectively 0, 122 and 255 (e.g. the original image)
I want to create a custom loss function that would compute the MSE on the output of the pseudo_mask function and backpropagate it to the original image.
My code is fairly trivial as I simply "completed" the Pytorch MSE loss function.
class MSE_Mask_Loss(_Loss):

   def __init__(self, size_average=None, reduce=None, reduction: str = 'mean') -> None:
       super(MSE_Mask_Loss, self).__init__(size_average, reduce, reduction)

   def forward(self, input: Tensor, target: Tensor) -> Tensor:

        input = tensor_pseudo_mask(input,255)
        target = tensor_pseudo_mask(target,255)

       return F.mse_loss(input, target, reduction=self.reduction)

The code runs fine but I assume there are some issues with the backpropagation since results obtained with this loss and β=255 are terrible, while they should be equivalent to the (good) ones obtained with original images.
What I am doing wrong?

Edit
The tensor_pseudo_mask is defined as follows:
def tensor_pseudo_mask(tns,val):

  for i in range(len(tns)):

    tns[i,:,:,:] = pseudo_mask(tns[i,:,:,:],val).data

  return tns.cuda()

where
def convert(x, val):

  if x == 0:
    return val
  else:
    return x

vecfunc = np.vectorize(convert)

def apply_mask(img, mask):

  tensor_mask = torch.unsqueeze(torch.Tensor(mask) / 255, 2) #For broadcasting
  img_masked = tensor_mask * torch.Tensor(convert_Image_RGB(img))

  return Image(convert_RGB_Image(img_masked))

def pseudo_mask(img, val):

  mask_value = torch.Tensor(np.array(mask(img)))
  pseudo_mask = vecfunc(mask_value, val)
  applied_pseudo_mask = apply_mask(img,pseudo_mask)

  return applied_pseudo_mask


title: type 'NoneType' is not iterable error when training pytorch model with ray tunes Trainable API

body:
I wrote a simple pytorch script to train MNIST and it worked fine. I reimplemented my script to be with Trainable class:
import numpy as np
import torch
import torch.optim as optim
import torch.nn as nn
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
import torch.nn.functional as F
import ray

from ray import tune

# Change these values if you want the training to run quicker or slower.
EPOCH_SIZE = 512
TEST_SIZE = 256

class ConvNet(nn.Module):

    def __init__(self):
        super(ConvNet, self).__init__()
        # In this example, we don't change the model architecture
        # due to simplicity.
        self.conv1 = nn.Conv2d(1, 3, kernel_size=3)
        self.fc = nn.Linear(192, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 3))
        x = x.view(-1, 192)
        x = self.fc(x)
        return F.log_softmax(x, dim=1)

class AlexTrainer(tune.Trainable):

    def setup(self, config):

        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

        # Data Setup
        mnist_transforms = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])

        self.train_loader = DataLoader(
            datasets.MNIST("~/data", train=True, download=True, transform=mnist_transforms),
            batch_size=64,
            shuffle=True)
        self.test_loader = DataLoader(
            datasets.MNIST("~/data", train=False, transform=mnist_transforms),
            batch_size=64,
            shuffle=True)

        self.model = ConvNet()
        self.optimizer = optim.SGD(self.model.parameters(), lr=config["lr"], momentum=config["momentum"])

        print('finished setup')

    def step(self):

        self.train()
        print("after train")
        acc = self.test()

        return {'acc': acc}

    def train(self):

        print("in train")

        self.model.train()
        for batch_idx, (data, target) in enumerate(self.train_loader):

            # We set this just for the example to run quickly.
            if batch_idx * len(data) > EPOCH_SIZE:
                return

            data, target = data.to(self.device), target.to(self.device)
            self.optimizer.zero_grad()

            print(type(data))

            output = self.model(data)
            loss = F.nll_loss(output, target)
            loss.backward()

            self.optimizer.step()

    def test(self):
        self.model.eval()

        correct = 0
        total = 0
        with torch.no_grad():
            for batch_idx, (data, target) in enumerate(self.test_loader):
                # We set this just for the example to run quickly.
                if batch_idx * len(data) > TEST_SIZE:
                    break
                data, target = data.to(self.device), target.to(self.device)
                outputs = self.model(data)
                _, predicted = torch.max(outputs.data, 1)
                total += target.size(0)
                correct += (predicted == target).sum().item()

        return correct / total

if __name__ == '__main__':
    ray.init()
    analysis = tune.run(
        AlexTrainer,
        stop={"training_iteration": 2},
        # verbose=1,
        config={
            "lr": tune.sample_from(lambda spec: 10 ** (-10 * np.random.rand())),
            "momentum": tune.uniform(0.1, 0.9)
        }
    )

How ever when i try to run, this time it fails:
Traceback (most recent call last):
  File "/hdd/raytune/venv/lib/python3.6/site-packages/ray/tune/trial_runner.py", line 473, in _process_trial
    is_duplicate = RESULT_DUPLICATE in result
TypeError: argument of type 'NoneType' is not iterable
Traceback (most recent call last):
  File "/hdd/raytune/test_3.py", line 116, in <module>
    "momentum": tune.uniform(0.1, 0.9)
  File "/hdd/raytune/venv/lib/python3.6/site-packages/ray/tune/tune.py", line 356, in run
    raise TuneError("Trials did not complete", incomplete_trials)
ray.tune.error.TuneError: ('Trials did not complete', [AlexTrainer_9b3cd_00000])

What could be the reason for this?


title: PyTorch: understanding RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation

body:
Here is a problem setting:
Function that converts from rgb to hsv:
def rbg2hsv(img_rgb):
    batch_size, channel, height, width = img_rgb.size()

    r, g, b = img_rgb[:, 0, :, :], img_rgb[:, 1, :, :], img_rgb[:, 2, :, :]

    t = torch.min(img_rgb, dim=1, keepdim=False)[0]
    v = torch.max(img_rgb, dim=1, keepdim=False)[0]

    s = (v - t) / (v + 1e-6)
    s[v == 0] = 0

    # v==r
    hr = 60 * (g - b) / (v - t + 1e-6)
    # v==g
    hg = 120 + 60 * (b - r) / (v - t + 1e-6)
    # v==b
    hb = 240 + 60 * (r - g) / (v - t + 1e-6)

    h = torch.zeros(batch_size, height, width, requires_grad=False)
    if torch.cuda.is_available():
        h = h.cuda()
    h = h.flatten()

    hr = hr.flatten()
    hg = hg.flatten()
    hb = hb.flatten()

    h[(v == b).flatten()] = hb[(v == b).flatten()]
    h[(v == g).flatten()] = hg[(v == g).flatten()]
    h[(v == r).flatten()] = hr[(v == r).flatten()]

    h[h < 0] += 360

    h = torch.reshape(h, (batch_size, height, width))

    img_hsv = torch.stack([h, s, v])
    img_hsv = img_hsv.permute(1, 0, 2, 3)

    return img_hsv

And here is minimum code that reproduce a problem:
img_rgb = torch.rand(2, 3, 32, 32)
img_rgb.requires_grad = True
instance_norm = nn.InstanceNorm2d(1, affine=False)

gamma = 2
beta = 1

img_hsv = rbg2hsv(img_rgb)

# 1: OK
#img_hsv[:,2:3,:,:] = img_hsv[:,2:3,:,:] * gamma + beta

# 2: FAIL
img_hsv[:,2:3,:,:] = instance_norm(img_hsv[:,2:3,:,:])

# 3: OK
#img_hsv[:,2:3,:,:] = instance_norm(img_hsv[:,2:3,:,:].clone())

img_hsv.mean().backward() # Fails on this line

print('img_rgb.grad.size()', img_rgb.grad.size())

Variant 2: FAIL fails with error: RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.FloatTensor [1, 2, 32, 32]], which is output 0 of ViewBackward, is at version 1; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True).
If I change rbg2hsv to some dummy add_fn all three variants work:
def add_fn(x):
    x = x + 1
    return x

img_rgb = torch.rand(2, 3, 32, 32)
img_rgb.requires_grad = True
instance_norm = nn.InstanceNorm2d(1, affine=False)

gamma = 2
beta = 1

img_hsv = add_fn(img_rgb)

# 1: OK
#img_hsv[:,2:3,:,:] = img_hsv[:,2:3,:,:] * gamma + beta

# 2: OK
img_hsv[:,2:3,:,:] = instance_norm(img_hsv[:,2:3,:,:])

# 3: OK
#img_hsv[:,2:3,:,:] = instance_norm(img_hsv[:,2:3,:,:].clone())

img_hsv.mean().backward()

print('img_rgb.grad.size()', img_rgb.grad.size())

So my question are:

Why variant 1 is not failing (seems it's also inplace modification of tensor)?
Why changing rbg2hsv to add_fn makes all variants work? (i.e. rbg2hsv have some problems?)

Here is some similar question https://discuss.pytorch.org/t/encounter-the-runtimeerror-one-of-the-variables-needed-for-gradient-computation-has-been-modified-by-an-inplace-operation/836 but I still don't understand the rules when we need to use .clone() and when not, i.e. when some operation considered inplace or not?


title: How to concatenate like List comprehension

body:
I want to do without define input_list and output_list befor for like "List comprehension"
Please tell me how to do in this case?
Thank you.
e.g.
inputs = torch.rand(1,5)
weights = torch.rand(5,5)

def training(self,inputs,weights,steps=50):
    input_list=torch.empty(1,5)
    output_list=torch.empty(5,5)

    for i in range(steps):
        outputs = torch.matmul(inputs,weights)

        input_list = torch.cat([input_list,inputs], axis=0)
        output_list = torch.cat([output_list,outputs], axis=0)

        inputs = outputs

    return inputs,input_list,output_list

them
input_list is
tensor([[1.8367e-36, 0.0000e+00, 3.3631e-44, 0.0000e+00,        nan],
[3.9109e-01, 8.9431e-01, 6.8890e-01, 8.3894e-01, 1.7802e-01]])
I dont need the first Row. How to concatenate in this case?


title: Insufficient cluster resources to launch trial - has only 0 GPUs

body:
I am following this tutorial (which is basically this) in order to use ray tune for hyperparemeter optimization. My model is training fine on the GPU without the optimization but now I want to optimize.
I applied the tutorial to my code but when I try to kick off the thing:
result = tune.run(
    train,
    resources_per_trial={"gpu": 1},
    config=config,
    num_samples=10,
    scheduler=scheduler,
    progress_reporter=reporter,
    checkpoint_at_end=False,
)

I'm stuck with:

TuneError: Insufficient cluster resources to launch trial: trial requested 1 CPUs, 1 GPUs, but the cluster has only 6 CPUs, 0 GPUs, 12.74 GiB heap, 4.39 GiB objects (1.0 node:XXX).

But then again, when I take a look at the ray dashboard:

there clearly are both GPUs listed.
Why isn't ray tune seeing my GPUs? How do I make this work?
Specs:
GPU 0: TITAN Xp
GPU 1: GeForce GTX 1080 Ti
CUDA 10.1
Python 3.7
PyTorch 1.7
Debian 9.12
ray tune 1.0.1.post1

//edit:
ray.init(num_gpus=1)
ray.get_gpu_ids()

[]


title: Loading pretrained BERT model issue

body:
I am using Huggingface to further train a BERT model. I saved the model using two methods: step (1) Saving the entire model using this code: model.save_pretrained(save_location), and step (2) save the state_dict of the model using this code: torch.save(model.state_dict(),'model.pth')
However, when I try to load this pretrained BERT model using the following code bert_mask_lm = BertForMaskedLM.from_pretrained('save_location') for step (1) and torch.load('model.pth') for step (2), I am getting this following error in both steps:
AttributeError                            Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/torch/serialization.py in _check_seekable(f)
    307     try:
--> 308         f.seek(f.tell())
    309         return True

AttributeError: 'torch._C.PyTorchFileReader' object has no attribute 'seek'

During handling of the above exception, another exception occurred:

Detailed stacktrace of step (1) is as follows:
AttributeError                            Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/torch/serialization.py in _check_seekable(f)
    307     try:
--> 308         f.seek(f.tell())
    309         return True

AttributeError: 'torch._C.PyTorchFileReader' object has no attribute 'seek'

During handling of the above exception, another exception occurred:

AttributeError                            Traceback (most recent call last)
~/anaconda3/lib/python3.6/site-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
   1037             try:
-> 1038                 state_dict = torch.load(resolved_archive_file, map_location="cpu")
   1039             except Exception:

~/anaconda3/lib/python3.6/site-packages/torch/serialization.py in load(f, map_location, pickle_module, **pickle_load_args)
    593                     return torch.jit.load(opened_file)
--> 594                 return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
    595         return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)

~/anaconda3/lib/python3.6/site-packages/moxing/framework/file/file_io_patch.py in _load(f, map_location, pickle_module, **pickle_load_args)
    199
--> 200     _check_seekable(f)
    201     f_should_read_directly = _should_read_directly(f)

~/anaconda3/lib/python3.6/site-packages/torch/serialization.py in _check_seekable(f)
    310     except (io.UnsupportedOperation, AttributeError) as e:
--> 311         raise_err_msg(["seek", "tell"], e)
    312     return False

~/anaconda3/lib/python3.6/site-packages/torch/serialization.py in raise_err_msg(patterns, e)
    303                                 + " try to load from it instead.")
--> 304                 raise type(e)(msg)
    305         raise e

AttributeError: 'torch._C.PyTorchFileReader' object has no attribute 'seek'. You can only torch.load from a file that is seekable. Please pre-load the data into a buffer like io.BytesIO and try to load from it instead.

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
~/work/algo-FineTuningBert3/FineTuningBert3.py in <module>()
      1 #Model load checking
----> 2 loadded_model = BertForMaskedLM.from_pretrained('/cache/raw_model/')

~/anaconda3/lib/python3.6/site-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
   1039             except Exception:
   1040                 raise OSError(
-> 1041                     f"Unable to load weights from pytorch checkpoint file for '{pretrained_model_name_or_path}' "
   1042                     f"at '{resolved_archive_file}'"
   1043                     "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. "

OSError: Unable to load weights from pytorch checkpoint file for '/cache/raw_model/' at '/cache/raw_model/pytorch_model.bin'If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True.

I am using the latest torch (1.7.1) and transformers (4.3.3) packages. I do not clearly understand what causes this error and how to solve this issue.


title: pytorch: RuntimeError: CUDA out of memory. with enough GPU memory

body:
Torch Error:
RuntimeError: CUDA out of memory. Tried to allocate 392.00 MiB (GPU 0; 10.73 GiB total capacity; 9.47 GiB already allocated; 347.56 MiB free; 9.51 GiB reserved in total by PyTorch)

I checked GPU resource by nvidia-smi, showing no other running process and memory-usage: 10/10989MiB.
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 418.56       Driver Version: 418.56       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  GeForce RTX 208...  On   | 00000000:04:00.0 Off |                  N/A |
| 22%   30C    P8    10W / 230W |     10MiB / 10989MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+

I have tried following 3 methods:

with torch.no_grad();
torch.cuda.empty_cache();
reduce batch_size;

All of them did not work.


title: Implementing Loss Function for FCN on Pytorch

body:
I am trying to implement a loss function for an FCN. My output is a tensor of shape (n, c, h, w). My target is of shape (h, w). I would like to calculate a loss between the output and the tensor but the problem is that I have a mask. There is only a certain portion of the image that I am interested in calculating the loss for and training on (I would like to ignore the rest). I am trying to achieve my goal by unwrapping the image into arrays and then applying the mask on it. Then I would calculate the loss. When I do this, I receive an error:

  RuntimeError: Assertion `cur_target >= 0 && cur_target < n_classes’ failed. at /pytorch/aten/src/THNN/generic/ClassNLLCriterion.c:93

Please see my code (There may be an easier way to do this as I am new to this):

def Loss(inp, target, mask):
    mask=torch.from_numpy(np.array(mask, dtype=np.uint8))
    target=target.contiguous().view(-1,1) #Flattening the Target Image
    mask = mask.contiguous().view(-1, 1) #Flattening Mask
    target = target[~mask] #Masking Target
    n, c, h, w = inp.size()
    inp1=np.zeros((target.shape[0],c)) #Creating new empty array with dimensions of (masked_region, c)
    inp1=torch.from_numpy(inp1)
    for i in range( c):
        inp1[:,i]=inp[0,i,:,:].view(-1,1)[~mask] #Masking the input and filling in the array created
    log_p = F.log_softmax(inp1, dim=1)
    criterion=nn.NLLLoss()
    loss = criterion(log_p, target)
    return loss


title: Difference in Model Performance when using Validation set/ Testing set

body:
I have implemented a PyTorch NN code for classification and regression.

Classification:
a) Use stratifiedKfolds for cross-validation (K=10- means 10 fold-cross validation)

I divided the data: as follows:
Suppose I have 100 data: 10 for testing, 18 for validation, 72 for training.

b) Loss function = CrossEntropy

c) Optimization = SGD

d) Early Stopping where waittime = 100 epochs.

Problem is:
Baseline Accuracy = 51%
Accuracy on Training set = 100%
Accuracy on validation set = 90%
Accuracy on testing set = 72%

I don’t understand what are the reasons behind the huge performance difference in Testing data/ Validation data?

How can I solve this problem?

Regression:

a) use the same network structure

b) loss function = MSELoss

c) Optimization = SGD

d) Early Stopping where wait-time = 100 epochs.

e) Use K-fold for cross-validation.

I divided the data: as follows:

Suppose I have 100 data: 10 for testing, 18 for validation, 72 for training.

Problem is:
Baseline MSE= 14.0
Accuracy on Training set = 0.0012
Accuracy on validation set = 6.45
Accuracy on testing set = 17.12

I don’t understand what are the reasons behind the huge performance difference in Testing data/ Validation data?

How can I solve these problems? or Is this an obvious thing for NN/ depend on particular dataset?


title: Interpret GAN loss

body: I am currently training the standard DCGAN network on my dataset. After 40 epochs, the loss of both generator and discriminator is 45-50. Can someone please explain the reason and possible solution for this?


title: Num_workers > 0 creates a memory error in SLURM?

body:
I'm trying to run my code on a SLURM cluster with the following configuration - 1 node, 2 Nvidia 1080Ti GPUs, 8 CPUs and 8GBs of RAM per CPU.

I'm implementing ResNeXt, with a dataset that contains about 1million 32x32 images. When I try running this code with torchvision.datasets.ImageFolder and num_workers = 4-8, it throws an "exceeded virtual memory" error by requesting for 341GB of data! That seems a little absurd. This error is thrown at the first trainLoader loop as it is preparing the first batch for the code.

Initially, I assumed this was an error with my program, but my program works just fine with num_workers = 8 on Google Colab. My program only works when I set num_workers = 0. At num_workers = 2, it works for two epochs before throwing the same error.

#!/bin/sh

#SBATCH --partition=gpu
#SBATCH --nodelist=compute-2-3
#SBATCH --cpus-per-task=16
#SBATCH --time=24:00:00
#SBATCH --mem=100000M
#SBATCH --job-name="test"
#SBATCH --output=srun_output.out
#SBATCH --mail-user=vsanil@andrew.cmu.edu
#SBATCH --mail-type=FAIL
#SBATCH --gres=gpu:2

srun python -u next.py > srun_output.out


title: Why test loss fluctuates so much using Resnet?

body:
Here is a typical plot of train/test losses behaviour as epoch increases.

I'm not an expert but I have read several topics on similar problems. Well, let me explain what I'm doing.

First, I have used implementation given by https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py for resnet18 & resnet50,  and by https://github.com/akamaster/pytorch_resnet_cifar10 for resnet32, resnet56. For all these nets I got the same kind of test-loss hieratic behaviour.

Second, my inputs are images 5x64x64, so I have adapted the first Convolutional Layer, and the output of the last Full-connected consist of 180 neurons. I have used either 64, 128, 256 batch sizes for the training, and 128 for the test: the same behaviour persists. I have also both used 300k or 100k images in input training (100k for the test): same  behaviour persists too. The images are not of "standard" RGB photos: first, as you probably have already, ,  remarked there are 5 channels, second the pixel values can be negative (eg. spanning the range  (-0.01, 500))

Third, I am aware of the model.train() statement for the training phase, as well as the model.eval() statement (coupled with the  with torch.no_grad():) for the testing phase.  It is clear that if I do not use model.eval() during the test phase, the test loss is gently decrasing as the traing loss. But, this is not allowed, isn't it?

I have tried several things after reading post concerning Batch Norm behaviour wo any success

I have used SGD, Adam (& SWATS)
I have tryied lr = 0.1 to lr= 1e-5
I have modified the BN momentum (default = 0.1) : 0.5 and 0.01; as well as the eps parameter.

Now, I have managed to get nice results (ie; good training & testing losses) with a classical CNN (ie. wo any Batch normalization, & short-cuts) but I would like to study  Resnet behaviour against adversarial attack. So, I would like to get Resnet fit my images :slight_smile:

Any idea ?
Thanks

After making some tests, I have found something: I have used the standard resnet20 (h=1). Then, I have used as test set the same samples (100,000 images) as for the train set. BUT, for the test set 1) I do not use the shuffling, and 2) I do not make any Horizontal/vertical flip or Rot90deg, Rot180deg or Rot270deg. I observe the same kind of fluctuations for the test loss.

Moreover, when I switch OFF complety the transformations of the train set, and uses the same set for train & test, I got the same behviour:

And finaly, if I switch off the suffling and random transforms (flips & Rotations) of train set, and I use the same set for test, then I get:

Seems that the test loss is converging towards a value, but different from the train loss. Why ???


title: Is there a difference between Pytorch 0.4.1 and 1.1.0

body: When I finished my training task of semantic segmentation (pytorch 0.4.1 GPU CUDA9.0), and successed in inference of the model(pytorch 0.4.1), however when I switched my pytorch version to 1.1.0, I got slightly different result. What's the problem???


title: In pytorch, what situations the loss function need to inherit nn.module?

body: I am confused about the loss function in PyTorch. Some people define the loss function as a normal python function while others define the loss function by defining a class that inherits nn.Module. So I want to know what situations we need to define the loss function by inheriting nn.Module? Many thanks.


title: Pytorch: all-but-one summation?

body:
I am in the process of moving some message passing code from Numpy to Pytorch. I am unsure of how to do this single step of a much larger algorithm. Below is the simplest explanation of the step.
Given the following:
index = [[2,0,1], [2,2,0]]
value = [[0.1, 1.2, 2.3], [3.4, 4.5, 5.6]]

I would like to compute the "all-but-one" sum of messages to each index.  Here is a graphical representation:

The answer I am looking for is:
ans = [[7.9, 5.6, 0], [4.6, 3.5, 1.2]]

The explanation is that, for example, index[0][0] points at node 2. The sum of all messages at node 2 is 0.1+3.4+4.5=8. However we want to exclude the message we are considering (value[0][0]=0.1), so we obtain ans[0][0]=7.9. If only a single index points at a node then the answer is 0 (eg. node 1 with ans[0][2]).
I would be happy with computing the sums for each node, and then subtracting out the individual messages.  I am aware that this can lead to loss of significance, but I believe that my use case is very well-behaved (eg. no floating point infinities).
I can also provide the minimal numpy code, but even the minimal example is a bit long. I have looked at pytorch's scatter and gather commands, but I don't think that they're appropriate here.


title: Pytorch: Google cloud storage, persistent disk for training models on DLVM?

body:
I'm wondering what's the best way to go about
a)reading in the data
b)writing data while training a model?
My data is in GCS bucket, about 1TB, from a dataflow job.
For writing data (all I want is model checkpoints, and logs) do people just write to a zonal persistent disk? Don't write to google cloud storage. it is a large model so the checkpoints take up a fair bit of space.
I can't seem to write data to cloud storage, without writing say a context manager and byte string code all the places I want to write.
Now for reading in data:
pytorch doesn't have a good way to read in the data from GCS bucket like tensorflow?
So what should I do, I've tride gcsfuse which I think could work, when I 'mount' the bucket, I can only see inside the repo I selected, not sub directories. Is this normal?
would gcsfuse be the right way to load in data from GCS?
Thanks.


title: Pytorch: Trained using DataParallel and Testing on cpu

body:
I have a trained model using one of my other pre-trained model in pytorch. For testing, I would like to test on cpu rather than gpu. The problem is that since I have used Dataparallel function during training, it is not straight forward for me to convert it to cpu.
My code for trained model is as follow:
class DGCNN_tl(nn.Module):
    def __init__(self):
        super(DGCNN_tl, self).__init__()
        self.model = DGCNN()
        if torch.cuda.device_count() > 1:
            print(str(torch.cuda.device_count())+" GPUs detected!")
            self.model = nn.DataParallel(self.model)
            patch_replication_callback(self.model) #sync_batch_norm
            self.model.to(device)
        self.model.load_state_dict(torch.load("teeth_seg2_pretrained.pt"))

        for param in self.model.parameters(): #making all layers fine tuneable
            param.requires_grad = True

    def forward(self, inputs, l):
        outputs,t,feature = self.model(inputs, l)
        return outputs, t, feature

Below is how I tried to convert it to cpu for testing:
dgcnn_lower = DGCNN_tl()
# original saved file with DataParallel
model_old = torch.load('teeth_seg2_lower_0.953iou.pt', map_location=torch.device('cpu'))

# create new OrderedDict that does not contain `module.`
from collections import OrderedDict

new_state_dict = OrderedDict()
for k, v in model_old.items():
    name = k[7:] # remove `module.`
    new_state_dict[name] = v

# load params
dgcnn_lower.load_state_dict(new_state_dict)

This gives an error saying
    Missing key(s) in state_dict: "model.module.transform_net.bn1.weight", "model.module.transform_net.bn1.bias"

"teeth_seg2_pretrained.pt": This is my pre-trained model which was used to train my current model
"teeth_seg2_lower_0.953iou.pt": This is my current model which I would like to test it on cpu
Can somebody give me an advice or help to solve this problem?
Thank you!!


title: Add column and row to multidimensional torch.tensor (kind of wrap-up or padding)

body:
The tensor should be updated with additional row-zeros (bottom) and column-zeros (on the right side).
My solution will be provided below. Is there any better (actually simpler) one?
input: («ones» are just for clarification - figures might be different, because in my case there is a tensor exactly the same size but with real values in it)
tensor([[[[1., 1., 1.],
          [1., 1., 1.],
          [1., 1., 1.]],

         [[1., 1., 1.],
          [1., 1., 1.],
          [1., 1., 1.]],

         [[1., 1., 1.],
          [1., 1., 1.],
          [1., 1., 1.]]]])

output:
tensor([[[[1., 1., 1., 0.],
          [1., 1., 1., 0.],
          [1., 1., 1., 0.],
          [0., 0., 0., 0.]],

         [[1., 1., 1., 0.],
          [1., 1., 1., 0.],
          [1., 1., 1., 0.],
          [0., 0., 0., 0.]],

         [[1., 1., 1., 0.],
          [1., 1., 1., 0.],
          [1., 1., 1., 0.],
          [0., 0., 0., 0.]]]])

possible solution:
x1 = torch.ones(1, 3, 3, 3)

z2 = torch.cat((torch.cat((x1[0, 0, :], torch.zeros(1, 3)), 0), torch.zeros(4, 1)), 1)
z3 = torch.cat((torch.cat((x1[0, 1, :], torch.zeros(1, 3)), 0), torch.zeros(4, 1)), 1)
z4 = torch.cat((torch.cat((x1[0, 2, :], torch.zeros(1, 3)), 0), torch.zeros(4, 1)), 1)

output_t = torch.zeros(1, 3, 4, 4)

output_t[0, 0, :] = z2
output_t[0, 1, :] = z3
output_t[0, 2, :] = z4

output_t


title: RuntimeError: Input, output and indices must be on the current device. (fill_mask("Random text .")

body:
I am getting "RuntimeError: Input, output and indices must be on the current device."
when I run this line.
fill_mask("Auto Car .")
I am running it on Colab.
My Code:
from transformers import BertTokenizer, BertForMaskedLM
from pathlib import Path
from tokenizers import ByteLevelBPETokenizer
from transformers import BertTokenizer, BertForMaskedLM

paths = [str(x) for x in Path(".").glob("**/*.txt")]
print(paths)

bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')

from transformers import BertModel, BertConfig

configuration = BertConfig()
model = BertModel(configuration)
configuration = model.config
print(configuration)

model = BertForMaskedLM.from_pretrained("bert-base-uncased")

from transformers import LineByLineTextDataset
dataset = LineByLineTextDataset(
    tokenizer=bert_tokenizer,
    file_path="./kant.txt",
    block_size=128,
)

from transformers import DataCollatorForLanguageModeling
data_collator = DataCollatorForLanguageModeling(
    tokenizer=bert_tokenizer, mlm=True, mlm_probability=0.15
)

from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir="./KantaiBERT",
    overwrite_output_dir=True,
    num_train_epochs=1,
    per_device_train_batch_size=64,
    save_steps=10_000,
    save_total_limit=2,
    )

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=dataset,
)

trainer.train()

from transformers import pipeline

fill_mask = pipeline(
    "fill-mask",
    model=model,
    tokenizer=bert_tokenizer
)

fill_mask("Auto Car <mask>.")

The last line is giving me the error mentioned above. Please let me know what I am doing wrong or what I have to do in order to remove this error.


title: Compile PyTorch with additional linker options

body:
I have a modified version of the gloo library. I am able to compile and run programs that use this library (similar to what you can find in gloo/gloo/examples).

Now, I want to build pytorch with my library.
I replaced the third_party/gloo folder in PyTorch with my version of gloo and I am trying to compile it.

However, my version of gloo requires some additional libraries and special linker options. Where should these linker options be added in the pytorch build system?

Without these linker options, my compilation stops with the linker error:

/pytorch/build/lib/libcaffe2_gpu.so: undefined reference to <my code>
/pytorch/build/lib/libcaffe2.so: undefined reference to <my code>


title: Randomness in VGG16 prediction

body:
I am using the VGG-16 network available in pytorch out of the box to predict some image index. I found out that for same input file, if i predict multiple time, I get different outcome. This seems counter-intuitive to me. Once the weights are predicted ( since I am using the pretrained model) there should not be any randomness at any step, and hence multiple run with same input file shall return same prediction.

Here is my code:

import torch
import torchvision.models as models
VGG16 = models.vgg16(pretrained=True)
def VGG16_predict(img_path):
  transformer = transforms.Compose([transforms.CenterCrop(224),transforms.ToTensor()])
  data = transformer(Image.open(img_path))
  output = softmax(VGG16(data.unsqueeze(0)), dim=1).argmax().item()
  return output # predicted class index
VGG16_predict(image)

Here is the image


title: Android application with pytorch

body: I made some andorid application. Also I have python files that take some jpg files as input and return some values. These python files use pytorch library. How can I run my python files in android application?


title: Tensorflow get_variable into Pytorch

body:
I am trying to convert this TensorFlow code into PyTorch. For example, I converted the below TF code

tf.get_variable("char_embeddings", [len(data.char_dict), data.char_embedding_size]),  char_index)  # [num_sentences, max_sentence_length, max_word_length, emb]

into

class CharEmbeddings(nn.Module):
    def __init__(self, config, data):
          ....
          self.embeddings = nn.init.xavier_uniform_(torch.empty(len(data.char_dict), data.char_embedding_size))

    def forward(self, char_index):
        # [num_sentences, max_sentence_length, max_word_length, emb]
        char_emb = self.embeddings[char_index]

I don't understand 100% what TF is doing there. Is it supposed to first initialize char_embeddings, gather (which I understand) and then backprogate gradients to update the char_embeddings value so that in the next iteration, the char_embeddings will be updated?

If so, I tried to convert that into PyTorch and from what I read, if no initializer is passed to the get_variable here, the glorot_uniform_initializer will be used which is I think it equivalent to pytorch xavier_uniform_

Two questions here:

Is my interpretation of TF code correct?
Is that conversion valid?
Should I expect the original embeddings self.embeddings to backpropagate and update its values? Is that the expected behavior from the tensorflow version as well? and how to achieve that in Pytorch? I added requires_grad to the embeddings tensor but that doesn't update the values.

Those might be newbie's question but I am new to this. Thanks!


title: Mask_RCNN setup RuntimeError,

body:
I'm building env for Mask_RCNN with Pytorch
And when I in the maskrcnn-benchmark setup step.
I use the commend
python setup.py build develop

But error show:
RuntimeError: Error compiling objects for extension

sometimes is
ImportError: libgthread-2.0.so.0: cannot open shared object file: No such file or directory

I use
cuda 10.1

pytorch 1.5.1

I searched the Internet for a while and also looked at maskrcnn for related issues,
but I didn't see the answer.
Hope everyone can help me.thanks~


title: train yolov3 by mmdetection with VOCstyle-dataset

body:
When I try to train yolo3 with command
python mmdetection/tools/train.py config/yolo-darknet53.py
and everything goes ok until first epoch begin and error happens that
it says "ValueError: need at least one array to concatenate" I am searching for a long time on net. But no use. Please help or try to give some ideas how to achieve this.

here is the yolo3 configuration file
#!user/bin/env python3
# -*- coding: utf-8 -*-
_base_ = '../mmdetection/configs/_base_/default_runtime.py'
classes = ('hand',)
# model settings
model = dict(
    type='YOLOV3',
    pretrained='D:/workplace/srtp/handdetection/checkpoints/darknet53-a628ea1b.pth',
    backbone=dict(type='Darknet', depth=53, out_indices=(3, 4, 5)),
    neck=dict(
        type='YOLOV3Neck',
        num_scales=3,
        in_channels=[1024, 512, 256],
        out_channels=[512, 256, 128]),
    bbox_head=dict(
        type='YOLOV3Head',
        num_classes=1,
        in_channels=[512, 256, 128],
        out_channels=[1024, 512, 256],
        anchor_generator=dict(
            type='YOLOAnchorGenerator',
            base_sizes=[[(116, 90), (156, 198), (373, 326)],
                        [(30, 61), (62, 45), (59, 119)],
                        [(10, 13), (16, 30), (33, 23)]],
            strides=[32, 16, 8]),
        bbox_coder=dict(type='YOLOBBoxCoder'),
        featmap_strides=[32, 16, 8],
        loss_cls=dict(
            type='CrossEntropyLoss',
            use_sigmoid=True,
            loss_weight=1.0,
            reduction='sum'),
        loss_conf=dict(
            type='CrossEntropyLoss',
            use_sigmoid=True,
            loss_weight=1.0,
            reduction='sum'),
        loss_xy=dict(
            type='CrossEntropyLoss',
            use_sigmoid=True,
            loss_weight=2.0,
            reduction='sum'),
        loss_wh=dict(type='MSELoss', loss_weight=2.0, reduction='sum')))
# training and testing settings
train_cfg = dict(
    assigner=dict(
        type='GridAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0))
test_cfg = dict(
    nms_pre=1000,
    min_bbox_size=0,
    score_thr=0.05,
    conf_thr=0.005,
    nms=dict(type='nms', iou_threshold=0.45),
    max_per_img=100)
# dataset settings
dataset_type = 'VOCDataset'
dataset_root = 'D:/workplace/srtp/handdetection/VOC_HandDataSetVOC2007'
img_norm_cfg = dict(mean=[0, 0, 0], std=[255., 255., 255.], to_rgb=True)
train_pipeline = [
    dict(type='LoadImageFromFile', to_float32=True),
    dict(type='LoadAnnotations', with_bbox=True),
    dict(type='PhotoMetricDistortion'),
    dict(
        type='Expand',
        mean=img_norm_cfg['mean'],
        to_rgb=img_norm_cfg['to_rgb'],
        ratio_range=(1, 2)),
    dict(
        type='MinIoURandomCrop',
        min_ious=(0.4, 0.5, 0.6, 0.7, 0.8, 0.9),
        min_crop_size=0.3),
    dict(type='Resize', img_scale=[(320, 320), (608, 608)], keep_ratio=True),
    dict(type='RandomFlip', flip_ratio=0.5),
    dict(type='Normalize', **img_norm_cfg),
    dict(type='Pad', size_divisor=32),
    dict(type='DefaultFormatBundle'),
    dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels'])
]
test_pipeline = [
    dict(type='LoadImageFromFile'),
    dict(
        type='MultiScaleFlipAug',
        img_scale=(608, 608),
        flip=False,
        transforms=[
            dict(type='Resize', keep_ratio=True),
            dict(type='RandomFlip'),
            dict(type='Normalize', **img_norm_cfg),
            dict(type='Pad', size_divisor=32),
            dict(type='ImageToTensor', keys=['img']),
            dict(type='Collect', keys=['img'])
        ])
]
data = dict(
    samples_per_gpu=1,
    workers_per_gpu=1,
    train=dict(
        type=dataset_type,
        ann_file=f'{dataset_root}/ImageSets/Main/hand_train.txt',
        img_prefix=f'{dataset_root}',
        pipeline=train_pipeline),
    val=dict(
        type=dataset_type,
        ann_file=f'{dataset_root}/ImageSets/Main/hand_val.txt',
        img_prefix=f'{dataset_root}',
        pipeline=test_pipeline),
    test=dict(
        type=dataset_type,
        ann_file=f'{dataset_root}/ImageSets/Main/hand_test.txt',
        img_prefix=f'{dataset_root}',
        pipeline=test_pipeline))
# optimizer
optimizer = dict(type='SGD', lr=0.001, momentum=0.9, weight_decay=0.0005)
optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2))
# learning policy
lr_config = dict(
    policy='step',
    warmup='linear',
    warmup_iters=2000,  # same as burn-in in darknet
    warmup_ratio=0.1,
    step=[218, 246])
# runtime settings
total_epochs = 273
evaluation = dict(interval=1, metric=['bbox'])


title: Pytorch CNN NotImplementedError

body:
Here is my network class:
class network(nn.Module):
    def __init__(self):
        super(network, self).__init__()
        self.conv1 = nn.Conv2d(3, 32, 3, stride=1, padding=1, padding_mode='zeros')
        self.conv2 = nn.Conv2d(32, 32, 3, stride=1, padding=1, padding_mode='zeros')
        self.conv3 = nn.Conv2d(32, 64, 3, stride=1, padding=1, padding_mode='zeros')
        self.conv4 = nn.Conv2d(64, 64, 3, stride=1, padding=1, padding_mode='zeros')
        self.maxpool1 = nn.MaxPool2d(2)     # 14 * 14
        self.conv5 = nn.Conv2d(64, 128, 3, stride=1, padding=1, padding_mode='zeros')
        self.conv6 = nn.Conv2d(128, 128, 3, stride=1, padding=1, padding_mode='zeros')
        self.conv7 = nn.Conv2d(128, 256, 3, stride=1, padding=1, padding_mode='zeros')
        self.conv8 = nn.Conv2d(256, 256, 3, stride=1, padding=1, padding_mode='zeros')
        self.maxpool2 = nn.MaxPool2d(2)     # 7 * 7
        self.gap = nn.AvgPool2d(7)
        self.fc1 = nn.Linear(256 * 1 * 1, 256)
        self.fc2 = nn.Linear(256, len(classes))

    def foward(self, x):
        x = torch.nn.ReLU(self.conv1(x))
        x = torch.nn.ReLU(self.conv2(x))
        x = torch.nn.ReLU(self.conv3(x))
        x = torch.nn.ReLU(self.conv4(x))

        x = self.maxpool1(x)

        x = torch.nn.ReLU(self.conv5(x))
        x = torch.nn.ReLU(self.conv6(x))
        x = torch.nn.ReLU(self.conv7(x))
        x = torch.nn.ReLU(self.conv8(x))

        x = self.maxpool2(x)
        x = self.gap(x)
        x = torch.nn.ReLU(self.fc1(x))
        x = torch.nn.Softmax(self.fc2(x))
        print(f"output shape: {x.shape}")

        return x

Main code
network = network()
print(network)
loss = nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)

for epoch in range(nb_epochs + 1):
    iter = -1
    for batch_index, sample in enumerate(dataloader):
        x_train, y_train = sample

        # Upload Dataset in GPU device
        X = x_train.to(device, dtype=torch.float)
        Y = y_train.to(device, dtype=torch.long)

        optimizer.zero_grad()
        output = network(X)
        cost = loss(output, torch.max(Y, 1)[1])
        cost.backward()
        optimizer.step()
        iter = iter + 1

        if iter % 100 == 99 or iter == dataloader.__len__():
            print(f"Epoch {epoch + 1}/{nb_epochs}   "
                  f"Iteration {iter + 1}/{dataloader.__len__()}     "
                  f"Loss {str(float(cost))[0:7]}")

print('Finished Training')

PATH = './cifar10_vgg.pth'
torch.save(model.state_dict(), PATH)

And Error displayed:
Traceback (most recent call last):
    File "D:/Files/works/0+Development/Python/0+DNN/0+torch_dc/tutorial_train_save.py", line 198, in <module>
        output = network(X)
    File "C:\Users\bolero\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 727, in _call_impl
        result = self.forward(*input, **kwargs)
    File "C:\Users\bolero\anaconda3\lib\site-packages\torch\nn\modules\module.py", line 175, in _forward_unimplemented
        raise NotImplementedError
NotImplementedError

I just implemented a very simple Network with Pytorch, but it doesn't work.
I wrote the "init" function and the forward function separately.
I watched the pytorch tutorial, and I wrote code similarly in that tutorial.
What I am aware of is that when writing a network class, I should inherit nn.Module and create init and forward functions.
I know that the init function defines the layer type, and the forward function defines the actual operation part.
If anyone knows about this issue, please comment.


title: Pytorch: No training effect after deepcopy

body:
I tried to make a copy of a neural network in pytorch and subsequently train the copied network, but training does not seem to change the weights in the network after copying. This post suggests that deepcopy is a convenient way to make a copy of a neural network, so I tried using that in my code.
The code below works just fine and shows that the weights and accuracy of the network are different after training from before training. However, when I toggle so that network_cp=deepcopy(network) and optimizer_cp=deepcopy(optimizer), the accuracy and weights before and after training are exactly the same.
# torch settings
torch.backends.cudnn.enabled = True
device = torch.device("cpu")

# training settings
learning_rate = 0.01
momentum = 0.5
batch_size_train = 64
batch_size_test = 1000

# get MNIST data set
train_loader, test_loader = load_mnist(batch_size_train=batch_size_train,
    batch_size_test=batch_size_test)

# make a network
network = Net()
optimizer = optim.SGD(network.parameters(), lr=learning_rate,
    momentum=momentum)
network.to(device)

# train network
train(network, optimizer, train_loader, device)

# copy network
network_cp = network
#network_cp = deepcopy(network)
optimizer_cp = optimizer
#optimizer_cp = deepcopy(optimizer)

# get edge weights and accuracy of the copied network
acc1 = float(test(network_cp, optimizer_cp, test_loader, device))
weights1 = np.array(get_edge_weights(network_cp))

# train copied network
train(network_cp, optimizer_cp, train_loader, device)

# get edge weights and accuracy of the copied network after training
acc2 = float(test(network_cp, optimizer_cp, test_loader, device))
weights2 = np.array(get_edge_weights(network_cp))

# compare edge weights and accuracy of copied network before and after training
print('accuracy', acc1, acc2)
print('abs diff of weights for net1 and net2', np.sum(np.abs(weights1-weights2)))

To run the code above, include these imports and function definitions:
import torch
import torchvision
import torchvision.transforms as transforms
import torch.optim as optim
import torch.nn as tnn
import torch.nn.functional as tnf
from copy import deepcopy
import numpy as np

def load_mnist(batch_size_train = 64, batch_size_test = 1000):
    train_loader = torch.utils.data.DataLoader(
        torchvision.datasets.MNIST('temp/', #'/data/users/alice/pytorch_training_files/',
                                   train=True, download=True,
                                   transform=torchvision.transforms.Compose([
                                       torchvision.transforms.ToTensor(),
                                       torchvision.transforms.Normalize(
                                           (0.1307,), (0.3081,))
                                       ])),
        batch_size=batch_size_train, shuffle=True)

    test_loader = torch.utils.data.DataLoader(
        torchvision.datasets.MNIST('temp/', #'/data/users/alice/pytorch_training_files/',
                                   train=False, download=True,
                                   transform=torchvision.transforms.Compose([
                                       torchvision.transforms.ToTensor(),
                                       torchvision.transforms.Normalize(
                                           (0.1307,), (0.3081,))
                                     ])),
        batch_size=batch_size_test, shuffle=True)

    return(train_loader, test_loader)

def train(network, optimizer, train_loader, device, n_epochs=5):
    network.train()
    for epoch in range(1, n_epochs + 1):
        for batch_idx, (data, target) in enumerate(train_loader):
            data, target = data.to(device), target.to(device)
            optimizer.zero_grad()
            output = network(data)
            loss = tnf.nll_loss(output, target)
            loss.backward()
            optimizer.step()

def test(network, optimizer, test_loader, device):
    network.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = network(data)
            test_loss += tnf.nll_loss(output, target, size_average=False).item()
            pred = output.data.max(1, keepdim=True)[1]
            correct += pred.eq(target.data.view_as(pred)).sum()
    test_loss /= len(test_loader.dataset)
    print('\nTest set: Avg. loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

    return(float(correct)/float(len(test_loader.dataset)))

def get_edge_weights(network):
    layers = [module for module in network.modules()][1:]
    output = np.zeros(1)
    for j, layer in enumerate(layers):
        weights = list(layer.parameters())[0]
        weights_arr = weights.detach().numpy()
        weights_arr = weights_arr.flatten()
        output = np.concatenate((output,weights_arr))
    return output[1:]

class Net(tnn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 =tnn.Linear(784,264)
        self.fc2 = tnn.Linear(264,10)

    def forward(self, x):
        x = tnf.relu(self.fc1(x.view(-1,784)))
        x = tnf.relu(self.fc2(x))
        return tnf.log_softmax(x)


title: How can I install Pytorch version 1.1 on Windows x64?

body:
I need to install an older version of Pytorch, version 1.1, on my Windows 10 x64 machine.
The instructions here tell me to download the wheel and install it. However, pip refuses to install this wheel, claiming it is not supported on my platform. I suppose that is because, judging by the the name of the wheel (torch-1.1.0-cp37-cp37m-win_amd64.whl) it is meant for AMD, and I'm on an Intel i7. I found no better matching wheel on the Pytorch site.
What is the easiest way to install Pytorch 1.1 on my machine?


title: How to optimize parameters in a specified range in PyTorch?

body:
A trainable parameter p initialized to 0.5:
class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.p = nn.Parameter(torch.tensor(0.5))

    def forward(self, x):
        y = x * self.p
        return y

model = Model()
optimizer = torch.optim.Adam(model.parameters())
loss_func = torch.nn.MSELoss()
y = torch.tensor([0.1,0.2,0.3])
x = torch.tensor([1.,2.,3.])
model.train()
for i in range(3):
    pred = model(x)
    optimizer.zero_grad()
    loss = loss_func(pred, y)
    loss.backward()
    optimizer.step()

I want to optimize p in a certain range. For example, the value of p must between 0.2 to 0.6 in each epoch.


title: Pytorch class torch._C.TensorBase information?

body: I am wondering from where the tensor._c.TensorBase comes from?Can Someone please tell me the c source code for this.


title: What is the point of the comma operator when a tuple only has one entry in python?

body:
This section of code is taken from one of the Pytorch tutorials, I have just removed the non-essential parts so it doesn't error out and added some print statements. The question I have is why the two print statements I provided have slightly different results? Is this a tuple with nothing in the second half of it? I am confused by the comma without anything after it before the assignment operator.

import torch

class MyReLU(torch.autograd.Function):
    @staticmethod
    def forward(ctx, input):
        ctx.save_for_backward(input)
        return input.clamp(min=0)

    @staticmethod
    def backward(ctx, grad_output):
        input, = ctx.saved_tensors
        print("ctx ", ctx.saved_tensors)
        print("inputs ", input)
        grad_input = grad_output.clone()
        grad_input[input < 0] = 0
        return grad_input

relu = MyReLU.apply
relu = MyReLU.apply
y_pred = relu(x.mm(w1)).mm(w2)
loss = (y_pred - y).pow(2).sum()
loss.backward()

Output

ctx  (tensor([[-34.2381,  18.6334,   8.8368,  ...,  13.7337, -31.5657, -11.8838],
        [-25.5597,  -6.2847,   9.9412,  ..., -75.0621,   5.0451, -32.9348],
        [-56.6591, -40.0830,   2.4311,  ...,  -2.8988, -18.9742, -74.0132],
        ...,
        [ -6.4023, -30.3526, -73.9649,  ...,   1.8587, -23.9617, -11.6951],
        [ -3.6425,  34.5828,  27.7200,  ..., -34.3878, -19.7250,  11.1960],
        [ 16.0137, -24.0628,  14.4008,  ...,  -5.4443,   9.9499, -18.1259]],
       grad_fn=<MmBackward>),)
inputs  tensor([[-34.2381,  18.6334,   8.8368,  ...,  13.7337, -31.5657, -11.8838],
        [-25.5597,  -6.2847,   9.9412,  ..., -75.0621,   5.0451, -32.9348],
        [-56.6591, -40.0830,   2.4311,  ...,  -2.8988, -18.9742, -74.0132],
        ...,
        [ -6.4023, -30.3526, -73.9649,  ...,   1.8587, -23.9617, -11.6951],
        [ -3.6425,  34.5828,  27.7200,  ..., -34.3878, -19.7250,  11.1960],
        [ 16.0137, -24.0628,  14.4008,  ...,  -5.4443,   9.9499, -18.1259]],
       grad_fn=<MmBackward>)


title: Loading mobilenet_v2 from hub conf gives error: cannot import name 'resnext50_32x4d'

body:
model = torch.hub.load('pytorch/vision:v0.5.0', 'mobilenet_v2', pretrained=True)

Importing the mobile net gives an error because its dependency 'resnext50_32x4d' cannot be imported. Is this the only way to use the mobile network in torch or how can I make it work anyhow?

Thanks in advance


title: CNN training loss regular spikes at the end of the epoch

body:
I am training a CNN in PyTorch with Adam and the initial learning rate is 1e-5. I have 5039 samples in my epoch and the batch size is 1. I have observed that I have a regular spike pattern of training loss at the end of an epoch. Here is a plot of the training loss:

From the plot one can see the clear patter of spikes which are happening exactly at the end of the epoch. My epoch contains 5039 samples. Interestingly enough the spikes do not only shoot down but sometimes up as well.

What I don't think it is:

Those spikes could have been explained if one did not shuffle the
data-set. However, I shuffle my data-set every epoch.
This behavior is known to happen when final batch of the epoch is
smaller than other batches which results in different magnitude of
loss
(Why does my training loss have regular spikes?).
However, it's not my case since my batch size is 1.

One potential hack could be to apply gradient clipping before the update step. However, it doesn't seem like a good way to treat this issue for me.

What are your thoughts on reasons for this spike pattern?
How bad
is it to have such a pattern?


title: With PyTorch, how can I pass the output of a bidirectional LSTM to a Linear layer?

body:
I have:

    def forward(self, x, hidden=None):
        lstm_out, hidden = self.lstm(x, hidden)
        print('lstm_out.size', lstm_out.size())
        lstm_out = lstm_out.view(-1, lstm_out.shape[2])
        out = self.linear(lstm_out)
        print('out', out.size())

And this does not work. My self.linear is self.linear = nn.Linear(64 * seq_length, 5). I can change the 5 to whatever later.

So the size of my lstm_out is torch.Size([64, 20, 322]). But then I get an error when doing the self.linear: RuntimeError: size mismatch, m1: [1280 x 322], m2: [1280 x 5] at. What am I doing wrong?


title: How To Use Pytorch In Xcode

body:
I want to use Pytorch with C++, so I followed the instructions listed here:
https://pytorch.org/cppdocs/installing.html
However, I want to use Pytorch in Xcode. So basically I downloaded LibTorch as described in the link above using the following commands in Terminal

wget https://download.pytorch.org/libtorch/nightly/cpu/libtorch-shared-with-deps-latest.zip

unzip libtorch-shared-with-deps-latest.zip

All I had to do, I thought, was go into Xcode and create a project with something like:

#include <torch/torch.h>
#include <iostream>

int main()
{
  torch::Tensor tensor = torch::rand({2, 3});
  std::cout << tensor << std::endl;
}

And change the "Header Search Paths" in the Build Settings of Xcode to where LibTorch was unzipped. However, I'm still getting the error "torch/torch.h file not found". What else do I need to do to get this to work?

Also, this is similar to another question listed here:
I want to run Libtorch (Pytorch C++ Frontend) in OpenFrameworks


title: How can I install Auto-PyTorch on Windows 10 from requirements.txt?

body:
I have been trying to install Auto-PyTorch, an automatic Neural Network tuning system (more info about installation here: https://github.com/automl/Auto-PyTorch), in a Windows 10 system. The installation steps are as follow:
$ cd install/path
$ git clone https://github.com/automl/Auto-PyTorch.git
$ cd Auto-PyTorch
$ cat requirements.txt | xargs -n 1 -L 1 pip install
$ python setup.py install

However, I can't get around the following line of code:
cat requirements.txt | xargs -n 1 -L 1 pip install

Once I get there, the following Windows PowerShell error is raised:
xargs : The term 'xargs' is not recognized as the name of a cmdlet, function, script file, or

operable program.
Check the spelling of the name, or if a path was included, verify that the path is correct and try again.
At line:1 char:86
+ ... read.CurrentUICulture = 'en-US'; cat requirements.txt | xargs -n 1 -L ...
+                                                             ~~~~~
    + CategoryInfo          : ObjectNotFound: (xargs:String) [], CommandNotFoundException
    + FullyQualifiedErrorId : CommandNotFoundException

Note that I passed the following line of code so that the error is thrown in English (I am Spanish):
[Threading.Thread]::CurrentThread.CurrentUICulture = 'en-US'; cat requirements.txt | xargs -n 1 -L 1 pip install

I'm not an expert on PowerShell, so I would greatly appreciate your help.
Best regards!


title: 'datasets.ImageFolder' doesnot working in python

body:
when I run this piece of code I get this error:
NameError: name 'datasets' is not defined
how can I solve this?
Many Thanks
import numpy as np
import torch
from torchvision import transforms
from torchvision.datasets.folder import default_loader
np.random.seed(random_seed)
torch.manual_seed(random_seed)

train_dataset = datasets.ImageFolder("/content/gdrive/My Drive/val2017",
transform)


title: Can't install pytorch with pip3 on Ubuntu 20.04

body:
I want to install torch version 1.2.0, as I want to use biobert-embeddings, which seems to require pytorch 1.2.0, as when I am trying to install biobert-embeddings using
pip3 install biobert-embedding, I am getting an error saying

ERROR: Could not find a version that satisfies the requirement torch==1.2.0 (from versions: 0.1.2, 0.1.2.post1, 0.1.2.post2, 1.4.0, 1.4.0+cpu, 1.4.0+cu100, 1.4.0+cu92, 1.5.0, 1.5.0+cpu, 1.5.0+cu101, 1.5.0+cu92, 1.5.1, 1.5.1+cpu, 1.5.1+cu101, 1.5.1+cu92, 1.6.0, 1.6.0+cpu, 1.6.0+cu101, 1.6.0+cu92)
ERROR: No matching distribution found for torch==1.2.0

When I am trying to install torch==1.2.0, am getting the same error.


I want to profile my entire training and eval pytorch code. I am using custom dataloaders (e.g. torchmeta library) and novel pytorch libraries (e.g. higher library) and I see very significant performance slow down from what other libraries reported (despite me using better GPUs e.g. I use v100 vs titan xp). They take 2.5 hours while mine is taking 16h or more.
Instead of sharing the code I want to profile the two ENTIRE scripts and pin point what is slowing things down when I compare the profilers output.
Unfortuantely, I see a lot of profilers and it hard to chose and what is worse is that most examples seem focused on profiling a specific model and not include the dataloader. For me the dataloader and the entire code is crucial.
These are the resources I've found:
https://pytorch.org/tutorials/recipes/recipes/profiler.html
python -m cProfile -s cumtime meta_learning_experiments_submission.py > profile.txt
https://github.com/adityaiitb/pyprof2
https://pytorch-lightning.readthedocs.io/en/latest/profiler.html
https://discuss.pytorch.org/t/understanding-pytorch-profiler/70851
which one is recommended for profiling the entire code so that it works even with the presence of GPU? is:
python -m cProfile -s cumtime meta_learning_experiments_submission.py > profile.txt

the best way to do this
(btw profiling seems better than changing my code randomly until it speeds up)

cross-posted:

https://discuss.pytorch.org/t/how-to-profiling-entire-pytorch-code-when-gpus-are-present/102866
How to profiling ENTIRE pytorch code when GPUs are present?


title: How to handle odd resolutions in Unet architecture PyTorch

body:
I'm implementing a U-Net based architecture in PyTorch. At train time, I've patches of size 256x256 which doesn't cause any problem. However at test time, I've full HD images (1920x1080). This is causing a problem during skip connections.
Downsampling 1920x1080 3 times gives 240x135. If I downsample one more time, the resolution becomes 120x68 which when upsampled gives 240x136. Now, I cannot concatenate these two feature maps. How can I solve this?
PS: I thought this is a fairly common problem, but I didn't get any solution or even mentioning of this problem anywhere on the web. Am I missing something?


title: RuntimeError: 1D target tensor expected, multi-target not supported Python: NumPy

body:
I am dealing with a CNN and I get the following error on the line loss = criterion(outputs, data_y):
Here is the relevant code snippet:
def run(model, X_train, Y_train, X_test, Y_test, learning_rate=0.01,
          num_epochs=100, minibatch_size=8, print_cost=True):

    seed = 0                                         # to keep results consistent (numpy seed)
    (m, n_H0, n_W0, n_C0) = X_train.shape
    costs = []                                       # To keep track of the cost

    criterion = nn.NLLLoss()
    optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

    # Training loop
    model.train() # Turn on the training mode
    for epoch in range(num_epochs):

        minibatch_cost = 0.
        num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set
        seed = seed + 1
        minibatches = utils.generate_minibatch(X_train, Y_train, minibatch_size, seed)

        for minibatch in minibatches:
            (batch_x, batch_y) = minibatch
            data_x = torch.Tensor(batch_x)
            data_y = torch.LongTensor(batch_x)

            ### START YOUR CODE ###
            # Zero the gradients
            optimizer.zero_grad() # Hint: call zero_grad()

            # Forward pass and compute loss
            outputs = model(data_x) # Hint: use model as a callable
            loss = criterion(outputs, data_y)  # Hint: use criterion as a callable

            # Backward and optimize
            loss.backward() # Hint: call backward()
            optimizer.step()  # Hint: call step()
            ### END YOUR CODE ###

            minibatch_cost += loss.item()

        # Print the cost every epoch
        minibatch_cost /= num_minibatches
        if print_cost and epoch % 5 == 0:
            print ("Cost after epoch %i: %f" % (epoch, minibatch_cost))
        costs.append(minibatch_cost)

    # Calculate accuracy on the train and test datasets
    data_x = torch.Tensor(X_test)
    data_y = torch.LongTensor(Y_test)
    model.eval() # Turn on the evaluation mode
    with torch.no_grad():
        test_pred = model(data_x)
        num_correct = (torch.argmax(test_pred, dim=1).view(data_y.size()).data == data_y.data).float().sum()
        test_acc = num_correct / test_pred.size()[0]
    print("Test Accuracy:", test_acc.item())

model = CNN_Model()

torch.manual_seed(0)
run(model, X_train, Y_train, X_test, Y_test)

And here is the error I'm getting:
RuntimeError                              Traceback (most recent call last)
<ipython-input-6-9839fc42e5c2> in <module>
      3
      4 torch.manual_seed(0)
----> 5 run(model, X_train, Y_train, X_test, Y_test)
      6
      7 # NOTE: It could be slow to run 100 epochs. Make sure that your costs for after each epoch

<ipython-input-5-05ddcdc9ddf5> in run(model, X_train, Y_train, X_test, Y_test, learning_rate, num_epochs, minibatch_size, print_cost)
     40             # Forward pass and compute loss
     41             outputs = model(data_x) # Hint: use model as a callable
---> 42             loss = criterion(outputs, data_y)  # Hint: use criterion as a callable
     43
     44             # Backward and optimize

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\modules\loss.py in forward(self, input, target)
    214     def forward(self, input: Tensor, target: Tensor) -> Tensor:
    215         assert self.weight is None or isinstance(self.weight, Tensor)
--> 216         return F.nll_loss(input, target, weight=self.weight, ignore_index=self.ignore_index, reduction=self.reduction)
    217
    218

C:\ProgramData\Anaconda3\lib\site-packages\torch\nn\functional.py in nll_loss(input, target, weight, size_average, ignore_index, reduce, reduction)
   2383         )
   2384     if dim == 2:
-> 2385         ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
   2386     elif dim == 4:
   2387         ret = torch._C._nn.nll_loss2d(input, target, weight, _Reduction.get_enum(reduction), ignore_index)

RuntimeError: 1D target tensor expected, multi-target not supported


title: What is the alternative of CUDA GPU for model training with CPU support?

body:
I dont have CUDA enabled GPU but I have i7 processor and 16GB Ram 1 GB amd graphics card

i want to disable that option and need to train a model with CPU support itself

mycodes are

parser = argparse.ArgumentParser()
parser.add_argument("--gpu", dest='gpu', type=str, default='0',                       help='Set CUDA_VISIBLE_DEVICES environment variable, optional')
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
params = vars(args)

how can i change into cpu version


title: Is there a way to write a custom BCE loss in pytorch?

body:
I am writing a custom BCE in pytorch but in some cases it returns -inf and nan most cases. Which is due to the log function.

bce_loss=y_true*torch.log2(y_pred) +(one_torch-y_true)*torch.log2(one_torch-y_pred)

Is there a way to rewrite this? Note y_pred is a sigmoid output which is between 0 and 1.


title: How do I serialize an NLP classification PyTorch model

body:
I am attempting to use a new NLP model within the PyTorch android demo app Demo App Git however I am struggling to serialize the model so that it works with Android.

The demonstration given by PyTorch is as follows for a Resnet model:

model = torchvision.models.resnet18(pretrained=True)
model.eval()
example = torch.rand(1, 3, 224, 224)
traced_script_module = torch.jit.trace(model, example)
traced_script_module.save("app/src/main/assets/model.pt")

However I am not sure what to use for the 'example' input with my NLP model.

The model that I am using from a fastai tutorial and the python is linked here: model

Here is the Python used to create my model (using the Fastai library). It is the same as in the model link above, but in a simplified form.

from fastai.text import *
path = untar_data('http://files.fast.ai/data/examples/imdb_sample')
path.ls()
#: [PosixPath('/storage/imdb_sample/texts.csv')]
data_lm = TextDataBunch.from_csv(path, 'texts.csv')
data = (TextList.from_csv(path, 'texts.csv', cols='text')
                .split_from_df(col=2)
                .label_from_df(cols=0)
                .databunch())
bs=48
path = untar_data('https://s3.amazonaws.com/fast-ai-nlp/imdb')
data_lm = (TextList.from_folder(path)
            .filter_by_folder(include=['train', 'test', 'unsup'])
            .split_by_rand_pct(0.1)
            .label_for_lm()
            .databunch(bs=bs))
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3)
learn.fit_one_cycle(1, 1e-2, moms=(0.8,0.7))
learn.unfreeze()
learn.fit_one_cycle(10, 1e-3, moms=(0.8,0.7))
learn.save_encoder('fine_tuned_enc')
path = untar_data('https://s3.amazonaws.com/fast-ai-nlp/imdb')
data_clas = (TextList.from_folder(path, vocab=data_lm.vocab)
             .split_by_folder(valid='test')
             .label_from_folder(classes=['neg', 'pos'])
             .databunch(bs=bs))
learn = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5)
learn.load_encoder('fine_tuned_enc')
learn.fit_one_cycle(1, 2e-2, moms=(0.8,0.7))
learn.freeze_to(-2)
learn.fit_one_cycle(1, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))
learn.freeze_to(-3)
learn.fit_one_cycle(1, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))
learn.unfreeze()
learn.fit_one_cycle(2, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))


title: Pytorch: looking for a function that let me to manually set learning rates for specific epochs intervals

body: For example, set lr = 0.01 for the first 100 epochs, lr = 0.001 from epoch 101 to epoch 1000, lr = 0.0005 for epoch 1001-4000. Basically my learning rate plan is not letting it decay exponentially with a fixed number of steps. I know it can be achieved by self-defined functions, just curious if there are already developed functions to do that.


title: Pytorch: size mismatch error although the sizes of the matrices do match (m1: [256 x 200], m2: [256 x 200])

body:
I trying to do transfer learning by pre training (Self supervised learning) a model on rotation (0, 90, 180, dn 270 degrees: 4 labels) on unlabelled data.
Here is the model:

class RotNet1(nn.Module):
    def __init__(self):
        keep_prob = 0.9
        super(RotNet1, self).__init__()
        self.layer1 = nn.Sequential(nn.Conv2d(in_channels = 3,
                                                  out_channels = 80,
                                                  kernel_size = 7,
                                                  stride = 1,
                                                  padding = 0),
                                        nn.ReLU(),
                                        nn.MaxPool2d(kernel_size = 2,
                      stride = 2,
                      padding = 1),
                      nn.Dropout(p=1 - keep_prob)
                      )
        self.bn1 = nn.BatchNorm2d(num_features = 80)
        self.dropout1 = nn.Dropout2d(p=0.02)
        self.layer2 = nn.Sequential(nn.Conv2d(in_channels = 80,
                                                  out_channels = 128,
                                                  kernel_size = 3,
                                                  stride = 1,
                                                  padding = 1),
                                        nn.ReLU(),
                                        nn.MaxPool2d(kernel_size = 2,
                      stride = 2,
                      padding = 1),
                      nn.Dropout(p=1 - keep_prob)
                      )
        self.bn2 = nn.BatchNorm2d(num_features = 128)
        self.layer3 = nn.Sequential(nn.Conv2d(in_channels = 128,
                                                  out_channels = 256,
                                                  kernel_size = 3,
                                                  stride = 1,
                                                  padding = 0),
                                        nn.ReLU(),
                                        nn.MaxPool2d(kernel_size = 2,
                      stride = 2,
                      padding = 1),
                      nn.Dropout(p=1 - keep_prob)
                      )
        self.bn3 = nn.BatchNorm2d(num_features = 256)
        self.layer4 = nn.Sequential(nn.Conv2d(in_channels = 256,
                                                  out_channels = 512,
                                                  kernel_size = 3,
                                                  stride = 1,
                                                  padding = 0),
                                        nn.ReLU(),
                                        nn.MaxPool2d(kernel_size = 2,
                      stride = 2,
                      padding = 1),
                      nn.Dropout(p=1 - keep_prob)
                      )
        self.bn4 = nn.BatchNorm2d(num_features = 512)
        self.layer5 = nn.Sequential(nn.Conv2d(in_channels = 512,
                                                  out_channels = 512,
                                                  kernel_size = 3,
                                                  stride = 1,
                                                  padding = 0),
                                        nn.ReLU(),
                                        nn.MaxPool2d(kernel_size = 2,
                      stride = 2,
                      padding = 1),
                      nn.Dropout(p=1 - keep_prob)
                      )
        self.bn5 = nn.BatchNorm2d(num_features = 512)
        self.drop_out = nn.Dropout()
        self.fc1 = nn.Linear(512* 2 * 2, 200)
        self.fc2 = nn.Linear(200, 4)
        #self.fc3 = nn.Linear(200, 100)

    def forward(self, input):
        out = self.layer1(input)
        out = self.bn1(out)
        out = self.dropout1(out)
        out = self.layer2(out)
        out = self.bn2(out)
        out = self.layer3(out)
        out = self.bn3(out)
        out = self.layer4(out)
        out = self.bn4(out)
        out = self.layer5(out)
        out = self.bn5(out)
        out = out.reshape(out.size(0), -1)
        out = self.drop_out(out)
        out = self.fc1(out)
        out = self.fc2(out)
        #out = self.fc3(out)
        return out

I trained this model on those 4 labels and names the model model_ssl. I then copied the model and changed the number the last fully connected layer from 4 to 200 (which is the number of labels in the labelled training and validation set where the number of example is restricted:

model_a = copy.copy(model_ssl)
#model_a.classifier
num_classes = 200
model_a.fc2 = nn.Linear(256,num_classes).cuda()

model_a.to(device)
loss_fn = torch.nn.CrossEntropyLoss()
n_epochs_a = 20
learning_rate_a = 0.01
alpha_a = 1e-5
momentum_a = 0.9
optimizer = torch.optim.SGD(model_a.parameters(),
                            momentum = momentum_a,
                            nesterov=True,
                            weight_decay = alpha_a,
                            lr=learning_rate_a)
train_losses_a, val_losses_a, train_acc_a, val_acc_a = train(model_a,
                                                             train_dataloader_sl,
                                                             val_dataloader_sl,
                                                             optimizer,
                                                             n_epochs_a,
                                                             loss_fn)

Here is the error message:

---------------------------------------------------------------------------

RuntimeError                              Traceback (most recent call last)

<ipython-input-27-f6f362ba8c53> in <module>()
     15                                                              optimizer,
     16                                                              n_epochs_a,
---> 17                                                              loss_fn)

6 frames

<ipython-input-23-df58f17c5135> in train(model, train_dataloader, val_dataloader, optimizer, n_epochs, loss_function)
     57     for epoch in range(n_epochs):
     58         model.train()
---> 59         train_loss, train_accuracy = train_epoch(model, train_dataloader, optimizer, loss_fn)
     60         model.eval()
     61         val_loss, val_accuracy = evaluate(model, val_dataloader, loss_fn)

<ipython-input-23-df58f17c5135> in train_epoch(model, train_dataloader, optimizer, loss_fn)
     10         labels = labels.to(device=device, dtype=torch.int64)
     11         # Run predictions
---> 12         output = model(images)
     13         # Set gradients to zero
     14         optimizer.zero_grad()

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

<ipython-input-11-2cd851b6d8e4> in forward(self, input)
     85         out = self.drop_out(out)
     86         out = self.fc1(out)
---> 87         out = self.fc2(out)
     88         #out = self.fc3(out)
     89         return out

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    548             result = self._slow_forward(*input, **kwargs)
    549         else:
--> 550             result = self.forward(*input, **kwargs)
    551         for hook in self._forward_hooks.values():
    552             hook_result = hook(self, input, result)

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py in forward(self, input)
     85
     86     def forward(self, input):
---> 87         return F.linear(input, self.weight, self.bias)
     88
     89     def extra_repr(self):

/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
   1608     if input.dim() == 2 and bias is not None:
   1609         # fused op is marginally faster
-> 1610         ret = torch.addmm(bias, input, weight.t())
   1611     else:
   1612         output = input.matmul(weight.t())

RuntimeError: size mismatch, m1: [256 x 200], m2: [256 x 200] at /pytorch/aten/src/THC/generic/THCTensorMathBlas.cu:283

The size of the matrices m1 and m2 seems to match but there is still that error message. What should I do?


title: Neural network, why do we sum the bias along axis 0 (np.sum(..., axis=0)) when doing batch gradient descent?

body:
I'm having a lot of troubles understanding why we have to do np.sum(..., axis=0) when we are computing the gradients of bias w.r.t. loss in a Neural Net.

I'm studying with a book and there's this nice 2-layer NN diagram:

now, if I want to know by how much I have to update (for example) B_2, I just do:

gradient of bias wrt loss = ∂(L)/∂(b_2) = ∂(L)/∂(P) * ∂(P)/∂(b_2) = -(Y - P) * np.ones_like(b_2)

Then just bias -= LR * (gradient of bias wrt loss).

And that's it, no? Where is calculus saying that we have to sum along the axis 0?


title: Why does higher need to deep copy the parameters of the base model to create a functional model?

body:
I found this line of code in the higher library:

self.param_groups = _copy.deepcopy(other.param_groups)

and I don't understand why that's needed.

If anything I think it's harmful as I've outline here. You can go to the issue to see my reasons but the gist is this:

Wouldn't having that deep copy mean the (outer loop) optimizer would be computing the gradients with respect to parameters no present in the computation graph? Since:

the parameters of the differentiable/inner optimizer are a deep copy compared to the initial parameters/weights
the outer optimizer (e.g. Adam) would have the original/initial parameters, so the gradient of these should always be zero.
That is the only explanation that I can think of to explain my issues in the past (gradients being zero unexpectedly) however it seems the higher MAML tutorial works, which should go against my theory. If my theory is right at the end of the inner loop of MAML and when the outer optimizer (usually Adam) computes the gradients, they should be zero (which I have observed sometimes). But I assume they are NOT zero, otherwise that tutorial wouldn't work.

So I am inquiring about the need to use deep copy when creating inner optimizers. What is its purpose and why is it not causing the issues I describe in the original MAML tutorial in higher. How is it that the deep copy doesn't break the forward pass and thus the whole computation of gradient wrt the initialization that the outer optimizer would use?

I think at the core of my confusion is that I don't understand why we need to do the deepcopy in the first place. Without all the other code (that seems convoluted to me) we even risk that the initialization we might want to train with the outer optimizer might not train, since the outer/meta optimizer has a pointer to the params of the original model and not a copy of the deep copy the inner optimizer could have had.

Why would the developers go through all that by adding substantial code that seems to have high risks?

Related question on how the copying of the initial parameters happens in higher: What does the copy_initial_weights documentation mean in the higher library for Pytorch?


title: Subset of K-Fold Dataset with PyTorch

body:
I have spent all day trying to read everywhere about implementing K-Fold cross-validation in PyTorch but can't get it to work. I have a custom subset written for my dataset but that does not work either... The code having issues is this:
    for epoch in range(training_configuration.epochs_count):
        mean = [0.5768, 0.4622, 0.3460]
        std = [0.2383, 0.2464, 0.2465]

        if data_augmentation:
            train_transforms = data_augmentation_transforms(mean, std)
            print('Data Augmentation: On')
        else:
            train_transforms = image_common_transforms(mean, std)
            print('Data Augmentation: Off')

        test_transforms = image_common_transforms(mean, std)

        dataset = KenyanFood13Dataset(
            root_dir = train_config.root_dir,
            img_path = train_config.img_path,
            csv_file = train_config.csv_path,
            transform = train_transforms)

        kf = KFold(n_splits=5, shuffle=True)
        for i, (train_index, test_index) in enumerate(kf.split(dataset)):
            train = torch.utils.data.Subset(dataset, train_index)
            test = torch.utils.data.Subset(dataset, test_index)

            train_loader = DataLoader(train, batch_size=train_config.batch_size, shuffle=True, num_workers=train_config.num_workers)
            test_loader = DataLoader(test, batch_size=train_config.batch_size, shuffle=False, num_workers=train_config.num_workers)

            print('Data: New fold successfully')

            init_val_loss, init_val_accuracy = validate(training_configuration, model, test_loader)
            print(f'Initial Validation Loss: {init_val_loss:.6f}, Initial Validation Accuracy: {init_val_accuracy*100:.3f}%\n')

            # --------- THIS IS WHERE THE ERROR IS -------------
            train_loss, train_acc = train(
                train_config=training_configuration,
                model=model,
                optimizer=optimizer,
                train_loader=train_loader,
                epoch_idx=epoch)

Traceback:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-19-c606f47fba53> in <module>
     34     tb_writer,
     35     scheduler=scheduler,
---> 36     data_augmentation=True)

<ipython-input-18-05bc5c268921> in main(model, optimizer, tb_writer, scheduler, training_configuration, data_augmentation)
     65                 optimizer=optimizer,
     66                 train_loader=train_loader,
---> 67                 epoch_idx=epoch)
     68
     69             epoch_train_loss = np.append(epoch_train_loss, [train_loss])

TypeError: 'Subset' object is not callable

Any thoughts on how I can implement this?


title: using ImagedataGenerator in pytorch module

body: Can I use imagedatagenerator  in keras to train a pytorch cnn module?? If not is there any other keyword which does the same function in pytorch?


title: torch.nn.fucntional.interpolate(): Parameters Settings

body:
I'm using torch.nn.functional.interpolate() to resize an image.
Firstly I use transforms.ToTensor() to transform an image into a tensor, which have size of (3, 252, 252), (252, 252) is the size of the imported image. What I want to do is to create a tensor with size (3, 504, 504) with interpolate() function.
I set the para scale_factor=2, but it returned a (3, 252, 504) tensor. Then I set it like scale_factor=(1,2,2) and received an error of dimensional conflict like this:
size shape must match input shape. Input is 1D, size is 3
So what way should I do to set the parameters in order to receive (3, 504, 504) tensor?


title: PyTorch RuntimeError: mat1 and mat2 shapes cannot be multiplied

body:
Purely for learning, I'd like to get the following code to work, without a DataLoader. I use Huggingface transformers regularly yet I struggle with PyTorch dimensions all the time so I have started with some simple projects from the book "Deep Learning with PyTorch."  One of the problems from the book suggested using a wine quality dataset on a super simple linear model.  I have toiled with the dimensions of the data, which I think is the source of my error:
RuntimeError: mat1 and mat2 shapes cannot be multiplied (3919x1 and 11x100)
Data is available here
import csv
from collections import OrderedDict

import numpy as np
import torch
import torch.optim as optim

import torch.nn as nn

wine_path = "winequality-white.csv"
wine_quality_numpy = np.loadtxt(wine_path, dtype=np.float32, delimiter=";",
                         skiprows=1)

col_list = next(csv.reader(open(wine_path), delimiter=';'))

wineq = torch.from_numpy(wine_quality_numpy)

# print(wineq.shape, wineq.dtype)

data = wineq[:, :-1]
target = wineq[:, -1]
target = target.unsqueeze(1)

n_samples = wine_quality_numpy.shape[0]
n_val = int(0.2 * n_samples)

shuffled_indices = torch.randperm(n_samples)

train_indices = shuffled_indices[:-n_val]
val_indices = shuffled_indices[-n_val:]

target_train = target[train_indices]
data_train = data[train_indices]

target_val = target[val_indices]
data_val = data[val_indices]

seq_model = nn.Sequential(OrderedDict([
    ('hidden_linear', nn.Linear(11, 100)),
    ('hidden_activation', nn.Tanh()),
    ('output_linear', nn.Linear(100, 7))
]))

def training_loop(n_epochs, optimizer, model, loss_fn, target_train, target_val,
                  data_train, data_val):
    for epoch in range(1, n_epochs + 1):
        t_p_train = model(target_train) # <1>
        loss_train = loss_fn(t_p_train, data_train)

        t_p_val = model(t_u_val) # <1>
        loss_val = loss_fn(t_p_val, data_val)

        optimizer.zero_grad()
        loss_train.backward() # <2>
        optimizer.step()

        if epoch == 1 or epoch % 1000 == 0:
            print(f"Epoch {epoch}, Training loss {loss_train.item():.4f},"
                  f" Validation loss {loss_val.item():.4f}")

optimizer = optim.SGD(seq_model.parameters(), lr=1e-3) # <1>

training_loop(
    n_epochs = 5000,
    optimizer = optimizer,
    model = seq_model,
    loss_fn = nn.MSELoss(),
    target_train = target_train,
    target_val = target_val,
    data_train = data_train,
    data_val = data_val)

Thank you!