title: The input to the CNN of Conv1D

body:
I'm working in the field of machine learning.
For the stronger Network, I'm going to adopt the techniques concerning Conv1D.
The input data is an one-dimension list data so I just would've thought that Conv1D is the best choice.
What would happen if the input size is (1, 740)? Would it be okay the input channel is 1?
I mean,I have a feeling that the (1, 740) tensor's conv1D output should be the same with that of a simple Linear networks.
Of course I'll also include other conv1d layer, like below.
self.conv1 = torch.nn.Conv1d(in_channels=1, out_channels=64, kernel_size=5)
self.conv2 = torch.nn.Conv1d(in_channels=64,out_channels=64, kernel_size=5)
self.conv3 = torch.nn.Conv1d(in_channels=64, out_channels=64, kernel_size=5)
self.conv4 = torch.nn.Conv1d(in_channels=64, out_channels=64, kernel_size=5)

Would it make sense when an input channel is 1?
Thanks in advance. :)


title: RuntimeError: input.size(-1) must be equal to input_size. Expected 28, got 0

body:
Here are my code by using Pysft
class Arguments:
        def __init__(self):
            # self.cuda = False
            self.no_cuda = True
            self.seed = 1
            self.batch_size = 50
            self.test_batch_size = 1000
            self.epochs = 10
            self.lr = 0.01
            self.momentum = 0.5
            self.log_interval = 10

hook = sy.TorchHook(torch)
bob = sy.VirtualWorker(hook, id="bob")
alice = sy.VirtualWorker(hook, id="alice")

Here is my LSTM model, in can run successfully by only use pytorch, but it can't run with pysyft
 class Model(torch.nn.Module):
        def __init__(self):
            super(Model, self).__init__()
            self.rnn = torch.nn.RNN(input_size=28,
            hidden_size=16,
            num_layers=2,
            batch_first=True,
            bidirectional=True)
            self.fc = torch.nn.Linear(32, 10)
        def forward(self, x):
            print(np.shape(x))
            x = x.squeeze()
            x, _ = self.rnn(x)
            x = self.fc(x[:, -1, :])
            return x.view(-1, 10)

def train(args, model, device, federated_train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(federated_train_loader):
        model.send(data.location) # <-- NEW: send the model to the right location
        data, target = data.to(device), target.to(device)
        # data, target = data.cuda(), target.cuda()
        optimizer.zero_grad()
        output = model(data.to(device))
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        model.get() # <-- NEW: get the model back
        if batch_idx % args.log_interval == 0:
            loss = loss.get() # <-- NEW: get the loss back
            losses.append(loss.item())
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * args.batch_size, len(federated_train_loader) * args.batch_size,
                100. * batch_idx / len(federated_train_loader), loss.item()))

When I use Pysyft to run my LSTM model,there is a mistakes.But if I use my model without Pysyft,it an run scuccessfully.I don't know how to resolve it?
import torch
import matplotlib.pyplot as plt
from torchvision import datasets, transforms
import torch.nn.functional as F
import time
import numpy as np
import syft as sy

class Arguments:
    def __init__(self):
        self.cuda = False
        self.no_cuda = True
        self.seed = 1
        self.batch_size = 50
        self.test_batch_size = 1000
        self.epochs = 10
        self.lr = 0.01
        self.momentum = 0.5
        self.log_interval = 10

hook = sy.TorchHook(torch)  # <-- NEW: hook PyTorch ie add extra functionalities to support Federated Learning
bob = sy.VirtualWorker(hook, id="bob")  # <-- NEW: define remote worker bob
alice = sy.VirtualWorker(hook, id="alice")  # <-- NEW: and alice

class Model(torch.nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.rnn = torch.nn.RNN(input_size=28,
        hidden_size=16,
        num_layers=2,
        batch_first=True,
        bidirectional=True)
        self.fc = torch.nn.Linear(32, 10)
    def forward(self, x):
        print(np.shape(x))
        x = x.squeeze()
        x, _ = self.rnn(x)
        x = self.fc(x[:, -1, :])
        return x.view(-1, 10)

def train(args, model, device, federated_train_loader, optimizer, epoch):
    model.train()
    for batch_idx, (data, target) in enumerate(federated_train_loader): # <-- now it is a distributed dataset
        model.send(data.location) # <-- NEW: send the model to the right location
        data, target = data.to(device), target.to(device)
        optimizer.zero_grad()
        output = model(data.to(device))
        loss = F.nll_loss(output, target)
        loss.backward()
        optimizer.step()
        model.get() # <-- NEW: get the model back
        if batch_idx % args.log_interval == 0:
            loss = loss.get() # <-- NEW: get the loss back
            losses.append(loss.item())
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * args.batch_size, len(federated_train_loader) * args.batch_size,
                100. * batch_idx / len(federated_train_loader), loss.item()))

if __name__ == '__main__':

    args = Arguments()

    use_cuda = not args.no_cuda and torch.cuda.is_available()

    torch.manual_seed(args.seed)

    device = torch.device("cuda" if use_cuda else "cpu")

    kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}

    losses = []

    federated_train_loader = sy.FederatedDataLoader(
        datasets.MNIST('../data', train=True, download=True,
                       transform=transforms.Compose([
                           transforms.ToTensor(),
                           transforms.Normalize((0.1307,), (0.3081,))
                       ]))
        .federate((bob, alice)),  # <-- NEW: we distribute the dataset across all the workers, it's now a FederatedDataset
        batch_size=args.batch_size, shuffle=True, **kwargs)
    test_loader = torch.utils.data.DataLoader(
        datasets.MNIST('../data', train=False, transform=transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])),
        batch_size=args.test_batch_size, shuffle=True, **kwargs)
    model = Model().to(device)
    optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)

    t = time.time()
    for epoch in range(1, args.epochs + 1):
        train(args, model, device, federated_train_loader, optimizer, epoch)

        test(args, model, device, test_loader)
    plt.plot(range(0,160),losses,marker='o')
    plt.xlabel("iterator")
    plt.ylabel("loss")
    plt.show()
    total_time = time.time() - t
    print(total_time)

Here are the whole codes


title: Fastai export/load_learner: why it is another model after reloading it

body:
I have trained a resnet34 learner
tfms = get_transforms(do_flip=False)
learn = cnn_learner(data, models.resnet34, metrics=[accuracy])
learn.lr_find()
learn.recorder.plot(suggestion=True)
learn.fit_one_cycle(10, min_grad_lr)
learn.recorder.min_grad_lr
learn.export(mypath+"Resnet34.pkl")

then I calculated accuracy_score, precision_score, f1_score, I got: 0.77, 0.78, 0.77
after that I loaded the resnet model with:
learn = load_learner(path=mypath, file="Resnet34.pkl")

after that, I calculated accuracy_score, precision_score, f1_score (as before, on exaclty the same validation set), and surprisingly I got lower values: 0.58, 0.61, 0.59.
I have torch 1.5.1+cpu, torchvision 0.6.1+cpu, fastai 1.0.61
Is there any bug/error somewhere?


title: How can I get data for a quiver plot in torch?

body:
I have some function z(x, y) and I would like to generate a quiver plot (a 2D plot of the gradients). Something like this:

In order to do it, I have to run gradient over a linear mesh and adjust data to the format that matplotlib.quiver does.

A naive way is to iterate forward and backward in a loop:

for i in range(10):
    for j in range(10):
        x = torch.tensor(1. * i, requires_grad=True)
        y = torch.tensor(1. * j, requires_grad=True)
        z = x ** 2 + y ** 2
        z.backward()
        print(x.grad, y.grad)

This is obviously very inefficient. There are some examples on how to generate a linear mesh from x, y but I would need later change the mesh back to the format of the forward formula, get vectors of gradient and put them back, etc..

A simple example in numpy would be:

import matplotlib.pyplot as plt
n = 25
x_range = np.linspace(-25, 25, n)
y_range = np.linspace(-25, 25, n)
X, Y = np.meshgrid(x_range, y_range)
Z = X**2 + Y**2
U, V = 2*X, 2*Y
plt.quiver(X, Y, U, V, Z, alpha=.9)

What would be the standard way of doing this with pytorch? Are there some simple examples available?


title: How does Tensorflow MobileNetV1 SSD Coco implementation work

body:
I'm trying to convert the Tensorflow ssd_mobilenet_v1_coco model to a PyTorch model in an efficient way, so I got all the tensorflow layers and I mapped them into the layers of a predefined MobileNetV1_SSD class.

The class was taken from https://github.com/qfgaohao/pytorch-ssd and was referred to the standard MobileNetV1 SSD architecture defined in the official paper. The problem is that when I mapped the layers from TF to PyTorch I got some layer dimension mismatches like if there were some changes from the initial architecture.
Adapting the class in according to the downloaded model I got this:

extras = ModuleList([
    Sequential(
        Conv2d(in_channels=1024, out_channels=256, kernel_size=1),
        ReLU(),
        Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1),
        ReLU()
    ),
    Sequential(
        Conv2d(in_channels=512, out_channels=128,kernel_size=1),
        ReLU(),
        Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1),
        ReLU()
    ),
    Sequential(
        Conv2d(in_channels=256, out_channels=128, kernel_size=1),
        ReLU(),
        Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1),
        ReLU()
    ),
    Sequential(
        Conv2d(in_channels=256, out_channels=64, kernel_size=1),
        ReLU(),
        Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1),
        ReLU()
    )
])

regression_headers = ModuleList([
    Conv2d(in_channels=512, out_channels=3 * 4, kernel_size=1, padding=1),
    Conv2d(in_channels=1024, out_channels=6 * 4, kernel_size=1, padding=1),
    Conv2d(in_channels=512, out_channels=6 * 4, kernel_size=1, padding=1),
    Conv2d(in_channels=256, out_channels=6 * 4, kernel_size=1, padding=1),
    Conv2d(in_channels=256, out_channels=6 * 4, kernel_size=1, padding=1),
    Conv2d(in_channels=128, out_channels=6 * 4, kernel_size=1, padding=1),
])

classification_headers = ModuleList([
    Conv2d(in_channels=512, out_channels=3 * num_classes, kernel_size=1, padding=1),
    Conv2d(in_channels=1024, out_channels=6 * num_classes, kernel_size=1, padding=1),
    Conv2d(in_channels=512, out_channels=6 * num_classes, kernel_size=1, padding=1),
    Conv2d(in_channels=256, out_channels=6 * num_classes, kernel_size=1, padding=1),
    Conv2d(in_channels=256, out_channels=6 * num_classes, kernel_size=1, padding=1),
    Conv2d(in_channels=128, out_channels=6 * num_classes, kernel_size=1, padding=1),
])

OK: The translation between models give me no errors

FAIL: I just have a problem when I have to evaluate the PyTorch model because kernel sizes and some in_channels and out_channels in the extras/classification/regression do not match the initial architecture specifications

  RuntimeError: The size of tensor a (2781) must match the size of tensor b (3000) at non-singleton dimension 1

QUESTION
Does somebody knows something about how Tensorflow developers implemented this model? So how did they manage the reduction of output in the first layer (from 6 * num_classes to 3 * num_classes) when choosing the anchors?


title: Do I need to match the representation of my output layer and the target labels in PyTorch Nllloss?

body:
I'm trying to solve a classification problem with 3 possible outputs: 0, 1, or 2.
My output layer finally outputs a vector of probabilities for each label, say [0.3,0.4,0.3]

My loss function is defined such:

loss = criterion(output_batch, label_batch) #criterion = nn.NLLLoss()

Now my question has to do with the outputs and labels not matching in the way the store data. The output is in the form of a size=3 probability vector (adding to 1 using soft max), and my target labels are simple scalars.

I can convert my labels to vectors when the loss function is calculated but I'm not sure if this is necessary

0 ==> [1,0,0]
1 ==> [0,1,0]
2 ==> [0,0,1]

Can someone please shed light on this issue? Thanks!


title: How to loop over every value in Pytorch tensor in C++?

body:
I am trying to do something like [0 if i < 1 else 1] for PyTorch tensor in C++.
I tried to use tensor.accessor(), but it seem like it require you to know the dimension beforehand. While I want to pass it dynamically.

Is there anyway I could do this in C++ for Pytorch?


title: How to convert Conv1d into Conv2d? [PyTorch]

body:
Is it possible with PyTorch to use Conv2d to perform a Conv1d? The question may seem weird, but I need to use a tool that is not compatible with conv1d, but it works with conv2d.

What if I have Conv1d(in,out, kernel_size=3, stride=stride, padding=1, bias=False)?
May unsqueeze help me?

I have the same problem with AvgPool1d (-> AvgPool2d) and MaxPool1d (-> MaxPool2d).


title: Trouble installing Torch on Google App Engine

body:
I've built a machine learning api that uses Torch as the ML framework. When I upload the code to Googe App Engine it runs out of memory.
After some debugging I found out that the issue it the installation of Torch.

  I'm using Torch 1.5.0 and python 3.7.4

So how do I fix this error? Maybe I can change something i app.yaml?

Error message:

Step #1 - "builder": OSError: [Errno 12] Cannot allocate memory
Step #1 - "builder":     self.pid = os.fork()
Step #1 - "builder":   File "/usr/lib/python2.7/subprocess.py", line 938, in _execute_child
Step #1 - "builder":     errread, errwrite)
Step #1 - "builder":   File "/usr/lib/python2.7/subprocess.py", line 394, in __init__
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__/ftl/python/layer_builder.py", line 346, in _python_version
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__/ftl/python/layer_builder.py", line     332, in GetCacheKeyRaw
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__/ftl/python/layer_builder.py", line 109, in GetCacheKeyRaw
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__/ftl/common/single_layer_image.py", line 60, in GetCacheKey
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__/ftl/python/layer_builder.py", line 153, in BuildLayer
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__/ftl/python/builder.py", line 114, in Build
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__.py", line 54, in main
Step #1 - "builder":   File "/usr/local/bin/ftl.par/__main__.py", line 65, in <module>
Step #1 - "builder":     exec code in run_globals
Step #1 - "builder":   File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
Step #1 - "builder":     "__main__", fname, loader, pkg_name)
Step #1 - "builder":   File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
Step #1 - "builder": Traceback (most recent call last):

And again this error message didn't appear when i didn't include torch in my requirements.txt

to reproduce:

app.yaml

runtime: python37
resources:
    memory_gb: 16
    disk_size_gb: 10

requirements.txt

gunicorn==20.0.4
aniso8601==8.0.0
beautifulsoup4==4.9.0
boto3==1.13.3
botocore==1.16.3
bs4==0.0.1
certifi==2020.4.5.1
chardet==3.0.4
click==7.1.2
colorama==0.4.3
docutils==0.15.2
filelock==3.0.12
Flask==1.1.2
Flask-RESTful==0.3.8
googletrans==2.4.0
idna==2.9
itsdangerous==1.1.0
Jinja2==2.11.2
jmespath==0.9.5
joblib==0.14.1
MarkupSafe==1.1.1
numpy==1.18.4
protobuf==3.11.3
python-dateutil==2.8.1
pytz==2020.1
regex==2020.4.4
requests==2.23.0
s3transfer==0.3.3
sacremoses==0.0.43
sentencepiece==0.1.86
six==1.14.0
soupsieve==2.0
tokenizers==0.5.2
tqdm==4.46.0
transformers==2.8.0
urllib3==1.25.9
Werkzeug==1.0.1

main.py

import flask
from flask import Flask, request
from flask_restful import Api, Resource

app = Flask(__name__)
api = Api(app)

production = False

import json

# Import api code

# Create main api 'view'
class main_api(Resource):

    def get(self):
        question = request.args.get('question')

        # Run the script
        # But not necessary for the minimum working test

        return {
            'question': question,
            # 'results': results_from_script,
        }

# Adds resource
api.add_resource(main_api, '/')

# Starts the api
if __name__ == '__main__':
    host = '127.0.0.1'
    port = 8080
    app.run(host=host, port=port, debug=not production)


title: Sampling from gaussian distribution

body:
My question is very specific. Given a k dimensional Gaussian distribution with mean and standard deviation, say I wish to sample 10 points from this distribution. But the 10 samples should be very different from each other. For example, I do not wish to sample 5 of those very close to the mean (By very close, we may assume for this example within 1 sigma) which may happen if I do random sampling. Let us also add an additional constraint that all the drawn samples should be at least 1 sigma away from each other. Is there a known way to sample in this fashion methodically? Is there any such module in PyTorch which can do so?

Sorry if this thought is ill posed but I am trying to understand if such a thing is possible.


title: How to extract keypoints with confidence score from a pytorch output tensor?

body:
I am using a pretrained model for pose estimation with Pytorch:
https://github.com/microsoft/human-pose-estimation.pytorch

I am using the model to detect persons and their 17 keypoints in an image but I am not sure how I should convert the output tensor to a format of pixel coordinates for each keypoint with a confidence score for each person.

This is the code for the prediction:

model.eval()
img_t = ToTensor()(img).unsqueeze(0)
out = model(img_t)
out.shape

The output shape is [1, 16, 64, 56]. I am a bit lost. Any help would be appreciated.


title: Pytorch Custom Loss Function does not converge to zero

body:
I am new in Pytorch and try to implement a custom loss function which is mentinoned in a paper, Deep Multi-Similarity Hashing for Multi-label Image Retrieval.
The loss function:
def pairWiseLoss(binaryCodes,labels):

    lambdaValue = 0.62

    numSample = len(binaryCodes)
    halfNumSample = numSample // 2
    binary1 = binaryCodes[:halfNumSample]
    binary2 = binaryCodes[halfNumSample:]

    label1 = labels[:halfNumSample]
    label2 = labels[halfNumSample:]

    similarity = torch.diagonal(torch.mm(label1.data.float(), label2.data.float().t()))
    maskCommonLabel = similarity.gt(0.0)
    maskNoCommonLabel = similarity.eq(0.0)

    tc = torch.exp(-similarity) * lambdaValue * 4 * args.bits
    m = 2 * args.bits
    hammingDistance = torch.diagonal( torch.cdist(binary1,binary2, p = 0) )
    zeros = torch.zeros_like(hammingDistance)

    loss = torch.sum(torch.masked_select(0.5 * torch.max(hammingDistance - tc , zeros ),maskCommonLabel  )  ) +  torch.sum(torch.masked_select(0.5 * torch.max( m - hammingDistance, zeros), maskNoCommonLabel ) )

    return loss

The training block:
 def train(trainloader, model, optimizer, epoch, use_cuda, train_writer,gpuDisabled):
        label_train = []
        name_train = []
        generated_codes = []

        lossTracker = MetricTracker()

        model.train()

        for idx, data in enumerate(tqdm(trainloader, desc="training")):

            numSample = data["bands10"].size(0)

            if gpuDisabled :
                bands = torch.cat((data["bands10"], data["bands20"],data["bands60"]), dim=1).to(torch.device("cpu"))
                labels = data["label"].to(torch.device("cpu"))
            else:
                bands = torch.cat((data["bands10"], data["bands20"],data["bands60"]), dim=1).to(torch.device("cuda"))
                labels = data["label"].to(torch.device("cuda"))

            optimizer.zero_grad()
            logits = model(bands)
            loss = pairWiseLoss(logits, labels)

            loss.backward()
            optimizer.step()

            lossTracker.update(loss.item(), numSample)

            generated_codes += list(logits)
            label_train += list(labels)
            name_train += list(data['patchName'])

        train_writer.add_scalar("loss", lossTracker.avg, epoch)

        print('Train loss: {:.6f}'.format(lossTracker.avg))
        return (generated_codes,label_train,name_train)

And, the model:
class ResNet50PairWise(nn.Module):
    def __init__(self, bits = 16):
        super().__init__()

        resnet = models.resnet50(pretrained=False)

        self.conv1 = nn.Conv2d(12, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
        self.encoder = nn.Sequential(
            self.conv1,
            resnet.bn1,
            resnet.relu,
            resnet.maxpool,
            resnet.layer1,
            resnet.layer2,
            resnet.layer3,
            resnet.layer4,
            resnet.avgpool
        )
        self.FC = nn.Linear(2048, bits)

        self.apply(weights_init_kaiming)
        self.apply(fc_init_weights)

    def forward(self, x):
        x = self.encoder(x)
        x = x.view(x.size(0), -1)

        logits = self.FC(x)
        sign = torch.sign(logits)
        binary_out = torch.relu(sign)

        return binary_out

However, the loss function does not converge to zero. It starts with big value and changes around that value. Where is the problem? How can I solve it?
I changed the learning rate but it does not affect.
Thanks in advance


title: Changing the pytorch weight decay value prevents the validation loss and training loss from decreasing

body: I'm trying to regularize my model with pytorch optimizer using the weight_decay parameter. When the weight_decay value is equal to 0 (which is the default vallue), the training loss and validation loss decrease. But when I try setting the weight_decay to different values (eg. 0.0001, 0.001, 0.01, 0.1...) the validation loss and training loss remain the same during all the epochs (0.70). What could be the problem?


title: rolling statistics in numpy or pytroch

body:
I have a tensors data of sensors, each tensor is of shape (4,1500)
This is 1500 timepoints and for each time point I have 4 features.
I want to "smooth" the sequences with rolling average or other rolling statistics. The end goal is to try to improve an lstm autoencoder with rolling statistics instead of the long raw sequence.
I am familiar with rolling windows of pandas and currently I am doing this:
#tensor shape:
 data.shape
 (4,1500)

 #convert data to numpy array and then to dataframe and perform rolling mean
 rolled_data=pd.DataFrame(data.numpy().swapaxes(1,0)).rolling(10).mean()[::10]
 rolled_data.shape
 (150, 4)

 # convert back the dataframe to tensor
 tensor_rolled_data=torch.Tensor(rolled_data.to_numpy().swapaxes(1,0))
 tensor_rolled_data.shape
 torch.Size([4, 150])

my question is- is there a better way to do it? a function in numpy/torch that can do rolling statistics in a cleaner or more efficient way?


title: colab : model = torch.hub.load('pytorch/fairseq', 'roberta.large') error

body:
I tried the code below.
import torch

model = torch.hub.load('pytorch/fairseq', 'roberta.large')

I got the error below.

I searched tutorial colab for RoBERTa already, however, it does not work either.
https://colab.research.google.com/github/pytorch/pytorch.github.io/blob/master/assets/hub/pytorch_fairseq_roberta.ipynb
Does any one have save issue? or have any solution for this?


title: No module named 'parse_config' while tryhing to load checkpoint in PyTorch

body:
I have a checkpoint file saved after training a model in Pytorch. I have to inspect it in a different module so I tried to load the checkpoint using the following code.
map_location = lambda storage, loc: storage
checkpoint = torch.load("model.pt", map_location=map_location)

But it is raising ModuleNotFoundError issue, which I couldn't find a way to resolve.
The error traceback  :
Traceback (most recent call last):
  File "main.py", line 11, in <module>
    model = loadmodel(hook_feature)
  File "/home/../model_loader.py", line 21, in loadmodel
    checkpoint = torch.load(settings.MODEL_FILE, map_location=map_location)
  File "/home/../.conda/envs/envreporting/lib/python3.6/site-packages/torch/serialization.py", line 584, in load
    return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
  File "/home/../.conda/envs/envreporting/lib/python3.6/site-packages/torch/serialization.py", line 842, in _load
    result = unpickler.load()
ModuleNotFoundError: No module named 'parse_config'

I couldn't find an already existing issue relevant to this one.


title: Cant install PyTorch using pip on Windows

body:
Hi after pasting this command to console: pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio===0.7.2 -f https://download.pytorch.org/whl/torch_stable.html (as shown on PyTorch website) I get this error:
C:\Users\m.povilaika>pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio===0.7.2 -f https://download.pytorch.org/whl/torch_stable.html

Looking in links: https://download.pytorch.org/whl/torch_stable.html

Collecting torch==1.7.1+cpu

  Using cached https://download.pytorch.org/whl/cpu/torch-1.7.1%2Bcpu-cp39-cp39-win_amd64.whl (184.2 MB)

Collecting torchvision==0.8.2+cpu

  Using cached https://download.pytorch.org/whl/cpu/torchvision-0.8.2%2Bcpu-cp39-cp39-win_amd64.whl (804 kB)

Collecting torchaudio===0.7.2

  Using cached https://download.pytorch.org/whl/torchaudio-0.7.2-cp39-none-win_amd64.whl (103 kB)

Requirement already satisfied: numpy in c:\users\m.povilaika\appdata\local\packages\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\localcache\local-packages\python39\site-packages (from torch==1.7.1+cpu) (1.19.4)

Requirement already satisfied: typing-extensions in c:\users\m.povilaika\appdata\local\packages\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\localcache\local-packages\python39\site-packages (from torch==1.7.1+cpu) (3.7.4.3)

Requirement already satisfied: pillow>=4.1.1 in c:\users\m.povilaika\appdata\local\packages\pythonsoftwarefoundation.python.3.9_qbz5n2kfra8p0\localcache\local-packages\python39\site-packages (from torchvision==0.8.2+cpu) (8.1.0)

Installing collected packages: torch, torchvision, torchaudio

ERROR: Could not install packages due to an EnvironmentError: [Errno 2] No such file or directory: 'C:\\Users\\m.povilaika\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python39\\site-packages\\caffe2\\python\\serialized_test\\data\\operator_test\\collect_and_distribute_fpn_rpn_proposals_op_test.test_collect_and_dist.zip'

Please help, I don't usually use windows, not sure what to do about the path error.
Thanks in advance.


I have already installed the module "art" yesterday, but when I try to run my code, it happens:
Traceback (most recent call last):
  File "D:/Desktop/captcha/src1/adv_ex.py", line 10, in <module>
    from art.attacks.evasion import FastGradientMethod
ModuleNotFoundError: No module named 'art.attacks'

I try to search it on Internet, but there is little information about this error.
I am sure that the module "art" (version 5.1) is already installed.
I hope you can help me. Many thanks.
Here is the full code:
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision.transforms as transforms
import torch
import os
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt
from art.attacks.evasion import FastGradientMethod
from art.attacks.evasion import BasicIterativeMethod
from art.estimators.classification import PyTorchClassifier
from art.utils import load_mnist

from model import Net
from model import CaptchaData
from model import DataLoader
from model import vec2text

# Step1: Load the original dataset
transform = transforms.Compose([transforms.ToTensor()]) # 不做数据增强和标准化了
test_data = CaptchaData('./testset/', transform=transform)
test_data_loader = DataLoader(test_data, batch_size=128, num_workers=0, shuffle=True, drop_last=True)

#Step 2: Load the model
net = Net()

#更换设备
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('current device:' , device)
print("right")
net.to(device)

#Define the loss function and optimizer
criterion = nn.MultiLabelSoftMarginLoss()
criterion.requires_grad = True #loss function
optimizer = torch.optim.Adam(net.parameters(), lr=0.001)

#Load the model
model_path = './module_build/model.pth'

checkpoint = torch.load(model_path)
net.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
net.eval()

# Step 3: Create the ART classifier for attacking
classifier = PyTorchClassifier(
    model=net,
    clip_values=(0.0,255.0),
    loss=criterion,
    optimizer=optimizer,
    input_shape=(3,140,44),
    nb_classes=222,
)

# Step 4: Define the eval function for evaulating the result
captcha_list = list('0123456789abcdefghijklmnopqrstuvwxcyz_')
captcha_length =6

def calculat_acc(output, target):
    output, target = output.view(-1,len(captcha_list)),target.view(-1,len(captcha_list))
    output = nn.functional.softmax(output, dim=1)
    output = torch.argmax(output,dim=1)
    target = torch.argmax(target,dim=1)
    output,target = output.view(-1, captcha_length),target.view(-1,captcha_length)
    c=0
    for i, j in zip(target,output):
        if torch.equal(i,j):
            c += 1
    acc = c / output.size()[0]* 100
    return acc

# Step 6: craft attack with FGSM and show the adversarial picture
acc,i =0,0
# with torch.no_grad( ) :
for inputs, labels in test_data_loader:

    # Before
    plt.figure()
    plt.imshow(inputs[124].permute(1,2,0))

    attack = FastGradientMethod(estimator=classifier, eps=0.1)

    inputs_adv = attack.generate(x=inputs)
    inputs_adv = torch.as_tensor(inputs_adv)
    print(vec2text(labels[124].view(6,-1)))
    # After
    plt.figure()
    plt.imshow(inputs_adv[124].permute(1,2,0))

    for i in range(128):
        result =transforms.ToPILImage()(inputs_adv[i])
        result.save("./input_adv/"+vec2text(labels[i].view(6,-1))+ ".jpg")
    outputs = net(inputs_adv)
    print(vec2text(outputs[124].view(6,-1)))
    acc += calculat_acc(outputs, labels)
    i += 1
    break
print ('Accuracy: %.3f %%' % (acc/i))

'''
# Step 6: craft attack with BIM and show the adversarial picture
acc,i =0,0
# with torch.no_grad( ) :
for inputs, labels in test_data_loader:

    # Before
    plt.figure()
    plt.imshow(inputs[124]. permute(1,2,0))

    attack = BasicIterativeMethod(estimator=classifier, eps=0.1, eps_step=0.01)

    inputs_adv = attack.generate(x=inputs)
    inputs_adv = torch.as_tensor(inputs_adv)

    # After
    plt.figure()
    plt.imshow(inputs_adv[124].permute(1,2,0))

    outputs = net(inputs_adv)
    acc += calculat_acc(outputs, labels)
    i += 1
    break
print ('Accuracy: %.3f %%' % (acc/i))
'''

#Step 7: crop the image
for inputs, labels in test_data_loader:
    # Before
    plt.figure()
    plt.imshows(inputs[124].permute(1,2,0))

    #Transform the tensor to image for further operation
    image = transforms.ToPILImage()(inputs[124])

    # crop the left part
    image_left = image.crop((28,0,140,44))
    plt.figure()
    plt.imshow(np.asarray(image_left))
    attack = FastGradientMethod(estimator = classifier,eps=0.1)

    inputs_adv = attack.generate(x = inputs)
    inputs_adv = torch.as_tensor(inputs_adv)
    #After
    plt.figure()
    plt.imshow(inputs_adv[124].permute(1,2,0))
    #Crop the one part of image.
    image = transforms.ToPILImage()(inputs_adv[124])
    (left, upper, right, lower) = (0,0,28,44)
    image_crop = image.crop((left, upper, right,lower))
    plt.figure()
    plt.imshow(np.asarray(image_crop))

    #combine the final image.
    dst = Image.new('RGB',(image_crop.width + image_left.width, image_crop.height))
    dst.paste(image_crop,(0, 0))
    dst.paste(image_left, (image_crop.width, 0))

    dst.save("./input_adv/"+ str(i) +".jpg")
    # Show the final result
    plt.figure()
    plt.imshow(np.asarray(dst))
    # Test it on pre-trained model
    outputs = net(inputs_adv)
    break

# save cropped image
for inputs, labels in test_data_loader:
    for i in range(128):

        # Transform the tensor to image for further operation
        image = transforms.ToPILImage()(inputs[i])

        #crop the left part
        image_left =image.crop((28,8,140,44))

        attack = FastGradientMethod(estimator=classifier, eps=0.1)

        inputs_adv = attack.generate(x=inputs)
        inputs_adv = torch.as_tensor(inputs_adv)
        #crop the one part of image.
        image = transforms.ToPILImage()(inputs_adv[i])
        (left, upper, right, lower) = (0,8,28,44)
        image_crop = image.crop((left,upper,right,lower))
        # combine the final image.
        dst = Image.new('RGB',(image_crop.width + image_left.width, image_crop.height))
        dst.paste(image_crop, (0,0))
        dst.paste(image_left,(image_crop.width,0))

        # Save the final results
        dst.save("./input_adv_cropped/"+vec2text(labels[i].view(6,-1))+ ".jpg")
    break

# Test accuracy on cropped images
test_data_cropped = CaptchaData('./input_adv_cropped/', transform=transform)
test_data_loader_cropped = DataLoader(test_data_cropped , batch_size=8,  num_workers=0,shuffle=True,drop_last=True)

acc,i=0, 0
with torch.no_grad():
    for inputs, labels in test_data_loader_cropped:
        outputs = net(inputs)
        for j in range(8):
            plt.figure()
            print(vec2text(labels[j].view(6.-1)))
            print(vec2text(outputs[j].vieu( 6.-1)))
            plt.imshow(inputs[j].permute(1,2,0))
            print("\n")
        acc += calculat_acc(outputs, labels)
        i +=1
        break
print('Accuracy: %.3f %%' %(acc/i))


title: Pytorch copying inexact value of numpy floating point number

body:
I'm converting a floating point number (or numpy array) to Pytorch tensor and it seems to be copying the inexact value to the tensor. The error comes in the 8th significant digit and afterwards. This is significant (no-pun intended) for my work as I deal with chaotic dynamics which is very sensitive towards the slight change in the initial conditions.

I'm already using torch.set_printoptions(precision=16) to print 16 significant digits.

np_x = state
print(np_x)
x = torch.tensor(np_x,requires_grad=True,dtype=torch.float32)
print(x.data[0])

and the output is :

0.7575408585008059
tensor(0.7575408816337585)

It would be helpful to know what is going wrong or how it could be resolved ?


title: Size dismatch when use multi-cuda(Dataparallel) in pytorch

body:
I'm using a custom model like this
class SimpleNN(nn.Module):
    def __init__(self, vectors_size, features_size, hidden_size=15, dropout_rate=0.1):
        super(SimpleNN, self).__init__()

        self.vectors_size = vectors_size
        self.features_size = features_size
        self.hidden_size = hidden_size
        self.dropout_rate = dropout_rate

        self.vectors_hidden = nn.Sequential(
            nn.Dropout(self.dropout_rate),
            nn.Linear(vectors_size, vectors_size//2),
            nn.Tanh(),
            nn.Linear(vectors_size//2, features_size),
            nn.Tanh()
        )
        self.hidden = nn.Sequential(
            nn.Linear(features_size*2, hidden_size),
            nn.ReLU(),
        )

        self.output = nn.Linear(hidden_size, 2)

    def forward(self, pairs, features):
        """
        features: (n_samples, features_size)
        """
        vectors = pairs2vectors(train_pub, pairs).to(device)
        embedding_features = self.vectors_hidden(vectors)
        combined_features = torch.cat([features, embedding_features], dim=1)
        return self.output(self.hidden(combined_features))

This model works well when i use only one cuda, but after 'DataParallel' used like below, it always tell me the size of features and embedding_features are not match, i find that the n_samples shape of features doesn't follow my expectation just like another batch data, i dont know why and how to solve this problem.
    if torch.cuda.device_count() > 1:
            print("Let's use", torch.cuda.device_count(), "GPUs!")
            model = nn.DataParallel(model)

BTW, here is the pic of error message is Error message
Actually, for parameters pairs and features in forward method, their sizes are (batch_size, pair) and (batch_size, features_size), and in my train function, code is below:
for batch_pairs, batch_data, batch_labels in tqdm(batch_iter(train_pairs, train_data, train_labels, train_batch_size), desc='Epoch'):
            train_iter += 1

            optimizer.zero_grad()

            batch_size = len(data)
            cum_examples += batch_size
            pred_labels = model(batch_pairs, batch_data)

            # Loss use torch.nn.CrossEntropyLoss
            loss_func = torch.nn.CrossEntropyLoss(weight=weight)
            loss = loss_func(pred_labels, batch_labels)

            # Backpropagation
            loss.backward()

            # Gradient clip
            grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad)

            # Update gradient
            optimizer.step()

I think my batch_pairs and batch_features have already with the batch_size, are they?
This is the official doc of torch.nn.DataParallel, it says my inputs will be chunked into serveral parts to be dealt with, in my situation it seems only features_size param is chunked but pairs not, maybe it's not a torch.Tensor instance? i'll have more trys and i think this mechanism is not reasonable.
I tried transform pairs into vectors outside and pass it to forward method but it still doesn't work.


I have data type problem implementing LSTM with Pytorch. Based on similar problems I tried to change the format of Input, h and c to ShortTensor as you can see but I still get the same error:

  RuntimeError: Expected object of scalar type Short but got scalar type
  Float for argument #2 'mat2' in call to _th_mm

class data(Dataset):
    def __init__(self, samples=10000, number=30):
        self.x = torch.from_numpy(np.matrix
                        (np.random.random_integers(0,9,samples*number).reshape(samples, number)))
        self.y = torch.from_numpy(np.zeros((samples))).type(torch.ShortTensor)
        for index, row in enumerate(self.x):
            self.y[index] = 1 if torch.sum(row) >= 130 else 0

class LSTM(nn.Module):
    def __init__(self,i_size, h_size, n_layer, batch_size = 30 ):
        super().__init__()
        self.lstm = nn.LSTM(input_size=i_size, hidden_size=h_size, num_layers=n_layer)
        self.h = torch.randn(n_layer, batch_size, h_size).type(torch.ShortTensor)
        self.c = torch.randn(n_layer, batch_size, h_size).type(torch.ShortTensor)
        self.hidden = (self.h, self.c)
        self.linear = nn.Linear(n_layer, 1)

    def forward(self,x):
        out, hidden = self.lstm(x.type(torch.ShortTensor), self.hidden)
        out = nn.Softmax(self.linear(out.short()))
        return out

data_set = data()
train_data = data_set.x[0:8000, :, None]
train_label = data_set.y[0:8000]
test_data = data_set.x[8000:, : , None]
test_label = data_set.y[8000:]

input_size = 1
hidden_size = 30
layer_num = 200
model_LSTM = LSTM(input_size, hidden_size, layer_num)
#model_LSTM.cuda()
y_ = model_LSTM(train_data)


title: How to perform text classification inference using the huggingface transformer's library

body:
To Whom It May Concern,

After training a binary classification model via examples/run_glue.py, I have a the following files.

merges.txt
tokenizer_config.json
config.json
pytorch_model.bin
training_args.bin
special_tokens_map.json
vocab.json

I can evaluate this model via the same script and the --do_eval arg; however, I would like to use this model to do inference not evaluation, as I don't have the labels for the unseen data I will be feeding into it. What is the best way to accomplish this using the main library? I currently use fast-bert. However, I'm transitioning this into a production pipeline and consequently can't stray from the main library anymore.

Thanks in advance!

Chris


title: Trouble building Swift macOS app with LibTorch dependency

body:
I built a mobile (iOS) app using LibTorch recently, with this tutorial as a guide: https://medium.com/flawless-app-stories/on-device-machine-learning-with-swiftui-and-pytorch-mobile-aa0dcec5d881. That project uses CocoaPods for the LibTorch dependency, so it was pretty straightforward, but now I'm trying to build a macOS version of the app and I honestly don't understand how to build and link to LibTorch for macOS (the pod seems to be iOS-only). I work in Swift, and have experience with Obj-C, Java, and Python, but not much with C/C++, so the whole process of building and linking C/C++ is pretty foreign to me.

I've gone through a LibTorch on macOS tutorial for C++ (here: https://medium.com/@albertsundjaja/installing-pytorch-c-api-d52c722f47ec), building a basic C++ "example-app" with a libtorch dependency. That works fine, but I'm not sure how that knowledge translates over to building a Swift app, with a LibTorch dependency, in Xcode.

I've also tried following the guide here: https://github.com/rystylee/ofxLibTorch, but have a ton of Undefined symbols for architecture x86_64: errors. I'm trying to link to the LibTorch downloaded from here: https://pytorch.org/get-started/locally/

Any help, advice, or links to other guides greatly appreciated.
I'm on Xcode 11.4.1 and Catalina (which has its own set of problems, from what I've read).

UPDATE: Going a little deeper down this rabbit hole, I wind up hitting this: Catalina C++: Using <cmath> headers yield error: no member named 'signbit' in the global namespace, but none of the "solutions" seem to help (specifically, I'm seeing the cmath problems that are mentioned a bunch of times in related threads).


title: Detectron2 add more classes or fine tune existing class in my model_final.pth

body:
Im using "COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml" and gave a custom dataset that holds picture of orange, apple, and pineapple. Now my model is ready and is able to predict all 3 fruits. Let's say now I need to add more fruits or add more data for pineapple (which I didn't feed enough to the system already).

Can I load the model and start the training with a new dataset(on top of weights which it already has)? and how good the result will be?

will it affect the previous predictions? Since orange and apple are predicted correctly and I don't want to mess that up. I'm going to send in pictures of pineapple only in this second training and also planning to add new classes like Banana or Watermelon in later training.

Is this the right way to do it? or Let me know if we have better options


title: How to get the confidence score of all class for a bounding box while using pretrained object detector in pytorch

body:
for a pretrained object detection model in pytorch and for each bounding box predicted by the model how to get the confidence score for each of the 80 COCO classes for that bounding box?
I have put the code I am using for object detection using pretrained fasterRCNN Resnet-50 FPN model
img = Image.open(img_path) # Load the image
transform = transforms.Compose([transforms.ToTensor()]) # Defing PyTorch Transform
img = transform(img) # Apply the transform to the image
pred = model([img.cuda()]) # Pass the image to the model
pred_class = [COCO_INSTANCE_CATEGORY_NAMES[i] for i in list(pred[0]['labels'].cpu().numpy())] # Get the Prediction Score
pred_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(pred[0]['boxes'].cpu().detach().numpy())] # Bounding boxes
pred_score = list(pred[0]['scores'].cpu().detach().numpy())

pred only provides all possible bounding boxes and the best possible class for the bounding box but how to get the confidence score for all possible classes for one bounding box?
Any help will be highly appreciated.


title: pytorch can't resolve and autocomplete in vscode

body:
here is what happening
I install the extension called Visual Studio Intellicode and pylint to autocomplete the code, but when
import torch
import numpy

It says unresolved import 'torch', and vscode didn't autocomplete the code of torch's modules like nn, but the code(e.g nn.Conv2d) can still work and get the right output. And the error only appears when import torch, numpy and pandas is alright.


title: pytorch concat layers with sequential

body:
So I used nn.Sequential several times before but now I'm experiencing a weird error where one layer will send 3 outputs to the next one instead of two (like expected)
I have defined my layer as :
class BidirectionalGRU(nn.Module):
    def __init__(self, rnn_dim, hidden_size, dropout=0.0, batch_first=False):
        super(BidirectionalGRU, self).__init__()

        self.BiGRU = nn.GRU(
            input_size=rnn_dim,
            hidden_size=hidden_size,
            num_layers=1,
            batch_first=batch_first,
            bidirectional=True,
        )

    def forward(self, x, hidden):
        x, hidden = self.BiGRU(x)
        return x, hidden

And I use it in my code here:
class Listener(nn.Module):
    def __init__(
        self, input_feature_dim_listener, hidden_size_listener, num_layers_listener
    ):
        super(Listener, self).__init__()
        assert num_layers_listener >= 1, "Listener should have at least 1 layer"
        self.hidden_size = hidden_size_listener
        self.gru_1 = nn.Sequential(
            BidirectionalGRU(
                rnn_dim=input_feature_dim_listener,
                hidden_size=hidden_size_listener,
                batch_first=True,
            ),
            BidirectionalGRU(
                rnn_dim=hidden_size_listener * 2,
                hidden_size=hidden_size_listener,
                batch_first=True,
            ),
            BidirectionalGRU(
                rnn_dim=hidden_size_listener * 2,
                hidden_size=hidden_size_listener,
                batch_first=True,
            ),
            BidirectionalGRU(
                rnn_dim=hidden_size_listener * 2,
                hidden_size=hidden_size_listener,
                batch_first=True,
            ),
        )

    def initHidden(self):
        return torch.zeros([2, 8, 512])

    def forward(self, x):
        x = x.squeeze().permute(0, 2, 1)
        fake_hidden = self.initHidden()
        output, hidden = self.gru_1(x, fake_hidden)
        #output, hidden = self.gru_2(output, hidden)
        #output, hidden = self.gru_3(output, hidden)
        #output, hidden = self.gru_4(output, hidden)
        return output, hidden

This DOESNT WORK and throws me an error:
    las(spectrograms, spectrograms, 0.5)
  File "/Users/jaime/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "caca.py", line 116, in forward
    listener_feature, hidden = self.listener(batch_data)
  File "/Users/jaime/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "caca.py", line 84, in forward
    output, hidden = self.gru_1(x, fake_hidden)
  File "/Users/jaime/anaconda3/envs/torch/lib/python3.7/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
TypeError: forward() takes 2 positional arguments but 3 were given

But If i define every layer manually instead of using nn.Sequential and pass the output,hidden myself then it works:
class Listener(nn.Module):
    def __init__(
        self, input_feature_dim_listener, hidden_size_listener, num_layers_listener
    ):
        super(Listener, self).__init__()
        assert num_layers_listener >= 1, "Listener should have at least 1 layer"
        self.hidden_size = hidden_size_listener
        self.gru_1 = BidirectionalGRU(
            rnn_dim=input_feature_dim_listener,
            hidden_size=hidden_size_listener,
            batch_first=True,
        )
        self.gru_2 = BidirectionalGRU(
            rnn_dim=hidden_size_listener * 2,
            hidden_size=hidden_size_listener,
            batch_first=True,
        )
        self.gru_3 = BidirectionalGRU(
            rnn_dim=hidden_size_listener * 2,
            hidden_size=hidden_size_listener,
            batch_first=True,
        )
        self.gru_4 = BidirectionalGRU(
            rnn_dim=hidden_size_listener * 2,
            hidden_size=hidden_size_listener,
            batch_first=True,
        )
    def initHidden(self):
        return torch.zeros([2, 8, 512])

    def forward(self, x):
        x = x.squeeze().permute(0, 2, 1)
        fake_hidden = self.initHidden()
        output, hidden = self.gru_1(x, fake_hidden)
        output, hidden = self.gru_2(output, hidden)
        output, hidden = self.gru_3(output, hidden)
        output, hidden = self.gru_4(output, hidden)
        return output, hidden

This works perfectly. I would like to make my code create multiple layers based on an argument and using nn.Sequential with a for loop inside would allow it.


title: How can I chunk a PyTorch tensor into a specified bucket size with overlap?

body:
Specifically, I have a tensor of shape: torch.Size([1, 16])
I want to bucket this into 7 buckets (of 4 each). Example:
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]
should become:
[[1, 2, 3, 4],
 [3, 4, 5, 6],
 [5, 6, 7, 8],
 [7, 8, 9, 10],
 [9, 10, 11, 12],
 [11, 12, 13, 14],
 [13, 14, 15, 16],
]

How can I achieve this with PyTorch?


title: Error when trying to train FasterRCNN with custom backbone on GRAYSCALE images

body:
I am following instructions from https://pytorch.org/tutorials/intermediate/torchvision_tutorial.html#putting-everything-together tutorial in order to create object detector for 1 class on GRAYSCALE images.
Here is my code (note that I am using DenseNet as a BACKBONE - pretrained model by me on my own dataset):
device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

num_classes = 2 # 1 class + background

model = torch.load(os.path.join(patch_classifier_model_dir, "densenet121.pt"))

backbone = model.features

backbone.out_channels = 1024

anchor_generator = AnchorGenerator(sizes=((32, 64, 128, 256, 512),),

                                   aspect_ratios=((0.5, 1.0, 2.0),))

roi_pooler = torchvision.ops.MultiScaleRoIAlign(featmap_names=[0],

                                                output_size=7,

                                                sampling_ratio=2)

# put the pieces together inside a FasterRCNN model

model = FasterRCNN(backbone,

                   num_classes=2,

                   rpn_anchor_generator=anchor_generator,

                   box_roi_pool=roi_pooler)

# move model to the right device

model.to(device)

optimizer = torch.optim.SGD(model.parameters(), lr=0.005,

                            momentum=0.9, weight_decay=0.0005)

# and a learning rate scheduler which decreases the learning rate by

# 10x every 3 epochs

lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,

                                               step_size=3,

                                               gamma=0.1)

This is the error that I am running into:
RuntimeError: Given groups=1, weight of size [64, 1, 7, 7], expected input[2, 3, 1344, 800] to have 1 channels, but got 3 channels instead
Based on FasterRCNN architecture, I assume problem is in the transform component because it tries to normalize images that are initially grayscale, and not RGB:
FasterRCNN(
  (transform): GeneralizedRCNNTransform(
      Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
      Resize(min_size=(800,), max_size=1333, mode='bilinear')
  )
  (backbone): Sequential(
    (conv0): Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
    (norm0): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (relu0): ReLU(inplace=True)
    (pool0): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
    (denseblock1): _DenseBlock(
      (denselayer1): _DenseLayer(
        (norm1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu1): ReLU(inplace=True)
        (conv1): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (relu2): ReLU(inplace=True)
        (conv2): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      )

      ...............

    (norm5): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  )
  (rpn): RegionProposalNetwork(
    (anchor_generator): AnchorGenerator()
    (head): RPNHead(
      (conv): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
      (cls_logits): Conv2d(1024, 15, kernel_size=(1, 1), stride=(1, 1))
      (bbox_pred): Conv2d(1024, 60, kernel_size=(1, 1), stride=(1, 1))
    )
  )
  (roi_heads): RoIHeads(
    (box_roi_pool): MultiScaleRoIAlign()
    (box_head): TwoMLPHead(
      (fc6): Linear(in_features=50176, out_features=1024, bias=True)
      (fc7): Linear(in_features=1024, out_features=1024, bias=True)
    )
    (box_predictor): FastRCNNPredictor(
      (cls_score): Linear(in_features=1024, out_features=2, bias=True)
      (bbox_pred): Linear(in_features=1024, out_features=8, bias=True)
    )
  )
)

Am I correct? If so, how do I resolve this issue? Is there a STANDARD PRACTICE on dealing with grayscale images and FasterRCNN?
Thanks in advance! Really appreciate it!


title: BigBird, or Sparse self-attention: How to implement a sparse matrix?

body:
This question is related to the new paper: Big Bird: Transformers for Longer Sequences. Mainly, about the implementation of the Sparse Attention (that is specified in the Supplemental material, part D). Currently, I am trying to implement it in PyTorch.
They suggest a new way to speed up the computation by blocking the original query and key matrices (see, below)
When you do the matrix multiplaciton in the step (b), you end up with something like that:
.
So I was wondering: how would you go from that representation (image above) to a sparse matrix (using PyTorch, see below)? In the paper, they just say: "simply reshape the result", and I do not know any easy ways to do so (especially, when I have multiple blocks in different positions (see step (c) on the first image).

RESOLUTION:
Huggingface has an implementation of BigBird in pytorch.


title: "[ERROR] OSError: [Errno 30] Read-only file system" when using simpletransformers on lambda docker

body:
I am getting the following error when testing my lambda function on AWS using a Docker image. The model is a simpletransformers classification model. The testing worked just fine locally.
[ERROR] OSError: [Errno 30] Read-only file system: '/home/sbx_user1051'
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/imp.py", line 235, in load_module
    return load_source(name, filename, file)
  File "/usr/local/lib/python3.6/imp.py", line 172, in load_source
    module = _load(spec)
  File "<frozen importlib._bootstrap>", line 684, in _load
  File "<frozen importlib._bootstrap>", line 665, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 678, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "app.py", line 28, in <module>
    model = torch.load(PATH)
  File "/function/torch/serialization.py", line 594, in load
    return _load(opened_zipfile, map_location, pickle_module, **pickle_load_args)
  File "/function/torch/serialization.py", line 853, in _load
    result = unpickler.load()
  File "/function/simpletransformers/classification/__init__.py", line 1, in <module>
    from simpletransformers.classification.classification_model import ClassificationModel
  File "/function/simpletransformers/classification/classification_model.py", line 37, in <module>
    from transformers.optimization import (
  File "/function/transformers/__init__.py", line 218, in <module>
    from .integrations import (  # isort:skip
  File "/function/transformers/integrations.py", line 43, in <module>
    import wandb
  File "/function/wandb/__init__.py", line 118, in <module>
    api = InternalApi()
  File "/function/wandb/apis/internal.py", line 17, in __init__
    self.api = InternalApi(*args, **kwargs)
  File "/function/wandb/sdk/internal/internal_api.py", line 82, in __init__
    load_settings=load_settings, root_dir=self.default_settings.get("root_dir")
  File "/function/wandb/old/settings.py", line 25, in __init__
    self._global_settings.read([Settings._global_path()])
  File "/function/wandb/old/settings.py", line 105, in _global_path
    util.mkdir_exists_ok(config_dir)
  File "/function/wandb/util.py", line 641, in mkdir_exists_ok
    os.makedirs(path)
  File "/usr/local/lib/python3.6/os.py", line 210, in makedirs
    makedirs(head, mode, exist_ok)
  File "/usr/local/lib/python3.6/os.py", line 210, in makedirs
    makedirs(head, mode, exist_ok)
  File "/usr/local/lib/python3.6/os.py", line 220, in makedirs
    mkdir(name, mode)

I think simpletransformers is trying to write to os. Is there any workaround?


title: How to get torch::Tensor shape

body:
If we << a torch::Tensor
#include <torch/script.h>
int main()
{
    torch::Tensor input_torch = torch::zeros({2, 3, 4});
    std::cout << input_torch << std::endl;
    return 0;
}

we see
(1,.,.) =
  0  0  0  0
  0  0  0  0
  0  0  0  0

(2,.,.) =
  0  0  0  0
  0  0  0  0
  0  0  0  0
[ CPUFloatType{2,3,4} ]

How to get the tensor shape (that 2,3,4)? I searched https://pytorch.org/cppdocs/api/classat_1_1_tensor.html?highlight=tensor for an API call but couldn't find one. And I searched for the operator<< overload code, and also couldn't find it.


title: conflict installing pytorch and torchvision using conda install

body:
I am trying to install specific versions of pytorch and torchvision but everytime i try install it with conda I face this Conflict Regarding python and cudatoolkit versions
`newstudent@snail-System-Product-Name:~$ conda install pytorch==1.2.0 torchvision==0.4.1 cudatoolkit=11.0 -c pytorch`

`Collecting package metadata (current_repodata.json): done
Solving environment: failed with initial frozen solve. Retrying with flexible solve.
Solving environment: failed with repodata from current_repodata.json, will retry with next repodata source.`

`Collecting package metadata (repodata.json): done
Solving environment: failed with initial frozen solve. Retrying with flexible solve.`

`Solving environment: \`

`Found conflicts! Looking for incompatible packages.
This can take several minutes.  Press CTRL-C to abort.
failed`

UnsatisfiableError: The following specifications were found
to be incompatible with the existing python installation in your environment:

Specifications:

  - torchvision==0.4.1 -> python[version='>=2.7,<2.8.0a0|>=3.5,<3.6.0a0|>=3.7,<3.8.0a0|>=3.6,<3.7.0a0']

Your python: python=3.8

If python is on the left-most side of the chain, that's the version you've asked for.
When python appears to the right, that indicates that the thing on the left is somehow
not available for the python version you are constrained to. Note that conda will not
change your python version to a different minor version unless you explicitly specify
that.

The following specifications were found to be incompatible with each other:

Output in format: Requested package -> Available versions

Package _libgcc_mutex conflicts for:
python=3.8 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]
cudatoolkit=11.0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]
pytorch==1.2.0 -> libgcc-ng[version='>=7.3.0'] -> _libgcc_mutex=[build=main]

Package libgcc-ng conflicts for:
pytorch==1.2.0 -> cffi -> libgcc-ng[version='>=7.2.0']
pytorch==1.2.0 -> libgcc-ng[version='>=7.3.0']
cudatoolkit=11.0 -> libgcc-ng[version='>=7.3.0']
python=3.8 -> libgcc-ng[version='>=7.3.0']
torchvision==0.4.1 -> numpy[version='>=1.11'] -> libgcc-ng[version='>=7.2.0|>=7.3.0']
python=3.8 -> zlib[version='>=1.2.11,<1.3.0a0'] -> libgcc-ng[version='>=7.2.0']`

Anyone can help with this?


title: Where is torch::autograd::profiler?

body:
I am using libtorch, c++ frontend, downloaded from the main page of https://pytorch.org/ and trying to profile layer by layer.

I've learn that in python i can use torch.autograd.profiler.profile. And i've read some website, including https://github.com/pytorch/pytorch/pull/16580/files#diff-81d29c29bbf9fb05edfa7fd70d189374 and https://caffe2.ai/doxygen-c/html/structtorch_1_1autograd_1_1profiler_1_1_record_profile.html

But when i use CLion to construct my code, use torch::autograd::profiler::RecordProfile , it report an error that

  no member named 'profiler' in namespace 'torch::autograd

I turned to https://pytorch.org/cppdocs/api/library_root.html, and cannot find something about profiler, so how should i do to use the profiler in my project?


title: Data transformation cropping with Normal Distribution using Pytorch

body:
I running a model on unsupervised learning for images. Before the images go in for learning I am performing various data augmentations, such as rotate, random erasing, flips, etc. However, I would like to also crop my images based on their Normal/Gaussian Distribution. I am also using the external library called Augmentator which also does not have cropping based on the normal distribution. Its basically just random center or just simply random.

All my transforms go into a list like this:

transforms_ = [
transforms.Grayscale(num_output_channels=1),
# transforms.Resize(int(img_height * 1.12), Image.BICUBIC),
transforms.RandomCrop(img_height, img_width), #this has to be based on normal distribution of the image
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
# transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
# transforms.Normalize((0.5,), (0.5,)), # greyscale normalisation
]

Any help greatly appreciated. I found no external libraries that do this so If i have to implement it on my own I need a way to integrate into Pytorch so any guidance would be helpful.


title: installing py-torch in anaconda , got error

body:
C:\WINDOWS\system32>python -m pip install torch
Collecting torch
  Using cached https://files.pythonhosted.org/packages/f8/02/880b468bd382dc79896eaecbeb8ce95e9c4b99a24902874a2cef0b562cea/torch-0.1.2.post2.tar.gz
Requirement already satisfied: pyyaml in c:\programdata\anaconda3\lib\site-packages (from torch) (5.1)
Building wheels for collected packages: torch
  Building wheel for torch (setup.py) ... error
  ERROR: Command errored out with exit status 1:
   command: 'C:\ProgramData\Anaconda3\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"'; __file__='"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' bdist_wheel -d 'C:\Users\vinayak\AppData\Local\Temp\pip-wheel-_w68wcvp' --python-tag cp37
       cwd: C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\
  Complete output (30 lines):
  running bdist_wheel
  running build
  running build_deps
  Traceback (most recent call last):
    File "<string>", line 1, in <module>
    File "C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\setup.py", line 265, in <module>
      description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
    File "C:\ProgramData\Anaconda3\lib\site-packages\setuptools\__init__.py", line 145, in setup
      return distutils.core.setup(**attrs)
    File "C:\ProgramData\Anaconda3\lib\distutils\core.py", line 148, in setup
      dist.run_commands()
    File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 966, in run_commands
      self.run_command(cmd)
    File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\ProgramData\Anaconda3\lib\site-packages\wheel\bdist_wheel.py", line 192, in run
      self.run_command('build')
    File "C:\ProgramData\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
      self.distribution.run_command(command)
    File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\ProgramData\Anaconda3\lib\distutils\command\build.py", line 135, in run
      self.run_command(cmd_name)
    File "C:\ProgramData\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
      self.distribution.run_command(command)
    File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 985, in run_command
      cmd_obj.run()
    File "C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\setup.py", line 51, in run
      from tools.nnwrap import generate_wrappers as generate_nn_wrappers
  ModuleNotFoundError: No module named 'tools.nnwrap'
  ----------------------------------------
  ERROR: Failed building wheel for torch
  Running setup.py clean for torch
  ERROR: Command errored out with exit status 1:
   command: 'C:\ProgramData\Anaconda3\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"'; __file__='"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' clean --all
       cwd: C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch
  Complete output (2 lines):
  running clean
  error: [Errno 2] No such file or directory: '.gitignore'
  ----------------------------------------
  ERROR: Failed cleaning build dir for torch
Failed to build torch
Installing collected packages: torch
    Running setup.py install for torch ... error
    ERROR: Command errored out with exit status 1:
     command: 'C:\ProgramData\Anaconda3\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"'; __file__='"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\vinayak\AppData\Local\Temp\pip-record-x966ekmg\install-record.txt' --single-version-externally-managed --compile
         cwd: C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\
    Complete output (23 lines):
    running install
    running build_deps
    Traceback (most recent call last):
      File "<string>", line 1, in <module>
      File "C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\setup.py", line 265, in <module>
        description="Tensors and Dynamic neural networks in Python with strong GPU acceleration",
      File "C:\ProgramData\Anaconda3\lib\site-packages\setuptools\__init__.py", line 145, in setup
        return distutils.core.setup(**attrs)
      File "C:\ProgramData\Anaconda3\lib\distutils\core.py", line 148, in setup
        dist.run_commands()
      File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 966, in run_commands
        self.run_command(cmd)
      File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 985, in run_command
        cmd_obj.run()
      File "C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\setup.py", line 99, in run
        self.run_command('build_deps')
      File "C:\ProgramData\Anaconda3\lib\distutils\cmd.py", line 313, in run_command
        self.distribution.run_command(command)
      File "C:\ProgramData\Anaconda3\lib\distutils\dist.py", line 985, in run_command
        cmd_obj.run()
      File "C:\Users\vinayak\AppData\Local\Temp\pip-install-67llvrv3\torch\setup.py", line 51, in run
        from tools.nnwrap import generate_wrappers as generate_nn_wrappers
    ModuleNotFoundError: No module named 'tools.nnwrap'
    ----------------------------------------
ERROR: Command errored out with exit status 1: 'C:\ProgramData\Anaconda3\python.exe' -u -c 'import sys, setuptools, tokenize; sys.argv[0] = '"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"'; __file__='"'"'C:\\Users\\vinayak\\AppData\\Local\\Temp\\pip-install-67llvrv3\\torch\\setup.py'"'"';f=getattr(tokenize, '"'"'open'"'"', open)(__file__);code=f.read().replace('"'"'\r\n'"'"', '"'"'\n'"'"');f.close();exec(compile(code, __file__, '"'"'exec'"'"'))' install --record 'C:\Users\vinayak\AppData\Local\Temp\pip-record-x966ekmg\install-record.txt' --single-version-externally-managed --compile Check the logs for full command output.


title: Given groups=1, weight of size [256, 256, 3, 3], expected input[16, 128, 32, 32] to have 256 channels, but got 128 channels instead

body:
I am having Resnet9 Model here and I am feeding images of size 36464 and then I am getting error like this...
In the last line of my code, I am passing my model to CUDA device.
Here is my Model:
def conv_block(in_channels, out_channels, pool=False):
layers = [nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1),
          nn.BatchNorm2d(out_channels),
          nn.ReLU(inplace=True)]
if pool: layers.append(nn.MaxPool2d(2))
return nn.Sequential(*layers)

class ResNet9(ImageClassificationBase):
def __init__(self, in_channels, out_ch):
    super().__init__()

    self.conv1 = conv_block(in_channels, 64) # 128 * 64 * 64
    self.conv2 = conv_block(64, 128, pool=True) # 256 * 32 * 32
    self.res1 = nn.Sequential(conv_block(128, 128), conv_block(256, 256))

    self.conv3 = conv_block(128, 256, pool=True) # 256 * 16 * 16
    self.conv4 = conv_block(256, 512, pool=True) # 512 * 8 * 8
    self.res2 = nn.Sequential(conv_block(512, 512), conv_block(512,512))

    self.classifier = nn.Sequential(nn.MaxPool2d(8),
                                    nn.Flatten(),
                                    nn.Linear(512, out_ch))

def forward(self, xb):
    out = self.conv1(xb)
    out = self.conv2(out)
    out = self.res1(out) + out
    out = self.conv3(out)
    out = self.conv4(out)
    out = self.res2(out) + out
    out = self.classifier(out)
    return out

model = to_device(ResNet9(3, 2), device)


title: Why does requires_grad turns from true to false when doing torch.nn.conv2d operation?

body:
I have Unet network which takes in MRI images of the brain, where the goal is to segment white substance in the brain. The images has the shape 256x256x183 (reshaped to 183x256x256) (FLAIR and T1 images). The problem I am having is that before sending the input to the Unet network, I have requires_grad=True on my pytorch tensor, but after one torch.nn.conv2d operation the requires_grad=False. This is a huge problem since the gradient will not update and learn.
from collections import OrderedDict

import torch
import torch.nn as nn

class UNet(nn.Module):

    def __init__(self, in_channels=3, out_channels=1, init_features=32):
        super(UNet, self).__init__()

        features = init_features
        self.encoder1 = UNet._block(in_channels, features, name="enc1")
        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)
        self.encoder2 = UNet._block(features, features * 2, name="enc2")
        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)
        self.encoder3 = UNet._block(features * 2, features * 4, name="enc3")
        self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)
        self.encoder4 = UNet._block(features * 4, features * 8, name="enc4")
        self.pool4 = nn.MaxPool2d(kernel_size=2, stride=2)

        self.bottleneck = UNet._block(features * 8, features * 16, name="bottleneck")

        self.upconv4 = nn.ConvTranspose2d(
            features * 16, features * 8, kernel_size=2, stride=2
        )
        self.decoder4 = UNet._block((features * 8) * 2, features * 8, name="dec4")
        self.upconv3 = nn.ConvTranspose2d(
            features * 8, features * 4, kernel_size=2, stride=2
        )
        self.decoder3 = UNet._block((features * 4) * 2, features * 4, name="dec3")
        self.upconv2 = nn.ConvTranspose2d(
            features * 4, features * 2, kernel_size=2, stride=2
        )
        self.decoder2 = UNet._block((features * 2) * 2, features * 2, name="dec2")
        self.upconv1 = nn.ConvTranspose2d(
            features * 2, features, kernel_size=2, stride=2
        )
        self.decoder1 = UNet._block(features * 2, features, name="dec1")

        self.conv = nn.Conv2d(
            in_channels=features, out_channels=out_channels, kernel_size=1
        )

    def forward(self, x):

        print(x.requires_grad) #<---- here it is true
        enc1 = self.encoder1(x)#<---- where the problem happens

        print(enc1.requires_grad) #<---- here it is false
        enc2 = self.encoder2(self.pool1(enc1))
        print(enc2.requires_grad)
        enc3 = self.encoder3(self.pool2(enc2))
        print(enc3.requires_grad)
        enc4 = self.encoder4(self.pool3(enc3))
        print(enc4.requires_grad)

        bottleneck = self.bottleneck(self.pool4(enc4))
        print(bottleneck.requires_grad)

        dec4 = self.upconv4(bottleneck)
        print(dec4.requires_grad)
        dec4 = torch.cat((dec4, enc4), dim=1)
        print(dec4.requires_grad)
        dec4 = self.decoder4(dec4)
        print(dec4.requires_grad)
        dec3 = self.upconv3(dec4)
        print(dec3.requires_grad)
        dec3 = torch.cat((dec3, enc3), dim=1)
        print(dec3.requires_grad)
        dec3 = self.decoder3(dec3)
        print(dec3.requires_grad)
        dec2 = self.upconv2(dec3)
        print(dec2.requires_grad)
        dec2 = torch.cat((dec2, enc2), dim=1)
        print(dec2.requires_grad)
        dec2 = self.decoder2(dec2)
        print(dec2.requires_grad)
        dec1 = self.upconv1(dec2)
        print(dec1.requires_grad)
        dec1 = torch.cat((dec1, enc1), dim=1)
        print(dec1.requires_grad)
        dec1 = self.decoder1(dec1)
        print(dec1.requires_grad)
        print("going out")
        return torch.sigmoid(self.conv(dec1))

    @staticmethod
    def _block(in_channels, features, name):
        return nn.Sequential(
            OrderedDict(
                [
                    (
                        name + "conv1",
                        nn.Conv2d(
                            in_channels=in_channels,
                            out_channels=features,
                            kernel_size=3,
                            padding=1,
                            bias=False,
                        ),
                    ),
                    (name + "norm1", nn.BatchNorm2d(num_features=features)),
                    (name + "relu1", nn.ReLU(inplace=True)),
                    (
                        name + "conv2",
                        nn.Conv2d(
                            in_channels=features,
                            out_channels=features,
                            kernel_size=3,
                            padding=1,
                            bias=False,
                        ),
                    ),
                    (name + "norm2", nn.BatchNorm2d(num_features=features)),
                    (name + "relu2", nn.ReLU(inplace=True)),
                ]
            )
        )

Edit:
This is the training code
class run_network:
def __init__(self, eta, epoch, batch_size, train_file_path, validation_file_path, shuffle_after_epoch = True):
    self.eta = eta
    self.epoch = epoch
    self.batch_size = batch_size
    self.train_file_path = train_file_path
    self.validation_file_path = validation_file_path
    self.shuffle_after_epoch = shuffle_after_epoch

def __call__(self, is_train = False):

    device = torch.device("cpu" if not torch.cuda.is_available() else torch.cuda())
    unet = torch.hub.load('mateuszbuda/brain-segmentation-pytorch', 'unet',
    in_channels=3, out_channels=1, init_features=32, pretrained=True)
    unet.to(device)
    unet = unet.double()

    optimizer = optim.Adam(unet.parameters(), lr=self.eta)
    dsc_loss = DiceLoss()

    Load_training   = NiftiLoader(self.train_file_path)
    Load_validation = NiftiLoader(self.validation_file_path)

    mean_flair, mean_t1, std_flair, std_t1 = Load_training.average_mean_and_std(20, 79,99)

    total_mean = [mean_flair, mean_t1]
    total_std = [std_flair, std_t1]

    loss_train = []
    loss_validation = []

    for current_epoch in tqdm(range(self.epoch)):
        for phase in ["train", "validation"]:

            if phase == "train":
                mini_batch = Load_training.create_batch(self.batch_size, self.shuffle_after_epoch)
                unet.train()
                print("her22")

            if phase == "validation":
                print("her")
                mini_batch = Load_validation.create_batch(self.batch_size, self.shuffle_after_epoch)
                unet.eval()

            dim1, dim2, dim3 = mini_batch.shape

            for iteration in range(1):
                if phase == "train":
                    current_batch = Load_training.Load_Image_batch(mini_batch, iteration)
                    image_batch = Load_training.image_zero_mean_normalizer(current_batch)
                if phase == "validation":
                    current_batch = Load_validation.Load_Image_batch(mini_batch, iteration)
                    image_batch = Load_training.image_zero_mean_normalizer(current_batch, False, mean_list, std_list)

                image_dim0, image_dim1, image_dim2, image_dim3, image_dim4 = image_batch.shape
                image_batch = image_batch.reshape((
                                                    image_dim0,
                                                    image_dim1*image_dim2,
                                                    image_dim3,
                                                    image_dim4
                                                    ))

                image_batch = np.swapaxes(image_batch, 0,1)
                image_batch = torch.as_tensor(image_batch)#.requires_grad_(True) #, requires_grad=True)
                image_batch = image_batch.to(device)
                print(image_batch.requires_grad)
                optimizer.zero_grad()

                with torch.set_grad_enabled(is_train == "train"):
                    for j in range(0, 10, 1):
                        # [183*5, 3, 256, 256] -> [12, 3, 256, 256]
                        # ANTALL ITERASJONER: (183*5/12) -> en chunk

                        input_image = image_batch[j:j+2,0:3,:,:]
                        print(input_image.requires_grad)
                        print("går inn")
                        y_predicted = unet(input_image)

                        print(y_predicted.requires_grad)
                        print(image_batch[j:j+2,3,:,:].requires_grad)
                        loss = dsc_loss(y_predicted.squeeze(1), image_batch[j:j+2,3,:,:])
                        print(loss.requires_grad)

                        if phase == "train":
                            loss_train.append(loss.item())

                            loss.backward()
                            print(loss.item())
                            exit()
                            optimizer.step()
                            print(loss.item())
                            exit()
                        if phase == "validation":
                            loss_validation.append(loss.item())

Number of iteration and print statement are for experimenting what the cause could be.


title: How do I create a shear matrix for PyTorch's F.affine_grid & F.grid_sample?

body:
I need to create a shear matrix that is autograd compatible, works on B,C,H,W tensors, and takes input values (possibly generated randomly) for the shear values. How can I generate the shear matrix for this?
import torch
import torch.nn.functional as F
import torchvision.transforms as transforms
from PIL import Image

# Load image
def preprocess_simple(image_name, image_size):
    Loader = transforms.Compose([transforms.Resize(image_size), transforms.ToTensor()])
    image = Image.open(image_name).convert('RGB')
    return Loader(image).unsqueeze(0)

# Save image
def deprocess_simple(output_tensor, output_name):
    output_tensor.clamp_(0, 1)
    Image2PIL = transforms.ToPILImage()
    image = Image2PIL(output_tensor.squeeze(0))
    image.save(output_name)

def get_shear_mat(theta):
    ...
    return shear_mat

def shear_img(x, theta, dtype):
    shear_mat = get_shear_mat(theta)
    grid = F.affine_grid(shear_mat , x.size()).type(dtype)
    x = F.grid_sample(x, grid)
    return x

# Shear tensor
test_input = # Test image
shear_values = (3,4) # Example values
sheared_tensor = shear_img(test_input, shear_values)


title: BERT Based CNN - Convolution and Maxpooling

body:
I’m trying to fine-tune a pre-trainted BERT model (huggingface transformers) by inserting a CNN layer.
In this model, the outputs of all transformer encoders are used, not only the output of the latest
transformer encoder.
So that the output vectors of each transformer encoder are concatenated, and a matrix is produced:
The convolutional operation is performed with a window of size (3, hidden size of BERT which is 768 in BERT_base model) and the maximum value is generated for each transformer encoder by
applying max pooling on the convolution output.
By concatenating these values, a vector is generated which is given as input to a fully connected network. By applying softmax on the input, the classification operation is performed.

My problem is that I can’t seem to find the right arguments to perform the convolution and the maxpooling on that matrix.
With batch size = 32, there are 13 layers of Transformer encoders, each one get as an input [64, 768] of encoded tokenized text and outputs an encoding of the same dimensions. (64 is the max-length in tokenization)
I want to perform convolution on each transformer’s output matrix ([64,768]) separately, then perform maxpooling on that convolution’s output. So, I should get a max-value per each transformer, and these max values are inserted into the neural network.
My code is:
class BERT_Arch(nn.Module):

    def __init__(self, bert):
        super(BERT_Arch, self).__init__()
        self.bert = BertModel.from_pretrained('bert-base-uncased')
        self.conv = nn.Conv2d(in_channels=13, out_channels=13, kernel_size= (3, 768), padding=True)
        self.relu = nn.ReLU()
        self.pool = nn.MaxPool1d(kernel_size=768, stride=1)
        self.dropout = nn.Dropout(0.1)
        self.fc = nn.Linear(9118464, 3)
        self.flat = nn.Flatten()
        self.softmax = nn.LogSoftmax(dim=1)

    def forward(self, sent_id, mask):
        _, _, all_layers = self.bert(sent_id, attention_mask=mask, output_hidden_states=True)
        # all_layers  = [32, 13, 64, 768]
        x = torch.cat(all_layers, 0) # x= [416, 64, 768]
        x = self.conv(x)
        x = self.relu(x)
        x = self.pool(x)
        x = self.flat(x)
        x = self.fc(x)
        return self.softmax(x)

I keep getting an error saying that the convolution method expected a certain dimensions as input but got a different one.
<generator object BERT_Arch.forward.<locals>.<genexpr> at 0x7fbeffc2d200>
torch.Size([416, 64, 768])
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-12-3a2c2cd7c02d> in <module>()
    362
    363         # train model
--> 364         train_loss, _ = train()
    365
    366         # evaluate model

5 frames
<ipython-input-12-3a2c2cd7c02d> in train()
    148
    149         # get model predictions for the current batch
--> 150         preds = model(sent_id, mask)
    151
    152         # compute the loss between actual and predicted values

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

<ipython-input-12-3a2c2cd7c02d> in forward(self, sent_id, mask)
     42         x = torch.cat(all_layers, 0) # torch.Size([13, 32, 64, 768])
     43         print(x.shape)
---> 44         x = self.conv(x)
     45         x = self.relu(x)
     46         x = self.pool(x)

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
    725             result = self._slow_forward(*input, **kwargs)
    726         else:
--> 727             result = self.forward(*input, **kwargs)
    728         for hook in itertools.chain(
    729                 _global_forward_hooks.values(),

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py in forward(self, input)
    421
    422     def forward(self, input: Tensor) -> Tensor:
--> 423         return self._conv_forward(input, self.weight)
    424
    425 class Conv3d(_ConvNd):

/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py in _conv_forward(self, input, weight)
    418                             _pair(0), self.dilation, self.groups)
    419         return F.conv2d(input, weight, self.bias, self.stride,
--> 420                         self.padding, self.dilation, self.groups)
    421
    422     def forward(self, input: Tensor) -> Tensor:

RuntimeError: Expected 4-dimensional input for 4-dimensional weight [13, 13, 3, 768], but got 3-dimensional input of size [416, 64, 768] instead

I tried different values for the convolution method arguments, I still got a similar error.
And sometimes an error saying that the maxpooling output size is too small:
Given input size: (64x62x1). Calculated output size: (64x31x0). Output size is too small

and sometimes this error (after changing the arguments of the cnn layer):
RuntimeError: Given groups=1, weight of size [32, 32, 3, 3], expected input[13, 4, 64, 768] to have 32 channels, but got 4 channels instead

or
Expected input batch_size (X) to match target batch_size (Y)

How can I do this?
I would be grateful for any help on how to do this CNN layer correctly.


title: Can you use a different image size during transfer learning?

body:
I have made a switch from TensorFlow to PyTorch recently. I use a famous Github repo for training on EfficientNets. I wrote the model initiation class as follows:
class CustomEfficientNet(nn.Module):
    def __init__(self, config: type, pretrained: bool=True):
        super().__init__()
        self.config = config
        self.model = geffnet.create_model(
            model_name='EfficientNetB5',
            pretrained=pretrained)
        n_features = self.model.classifier.in_features
        self.model.classifier = nn.Linear(n_features, num_classes=5)

    def forward(self, input_neurons):
        output_predictions = self.model(input_neurons)
        return output_predictions

In addition, in my transforms, I tend to useResize(img_size = 512, img_size=512) for my training on certain image classification tasks (Mostly Kaggle Competitions). So the question here is, the official input size for EfficientNetB5 is 456x456, but I used 512x512 or even 256x256 and get very decent results. Is this normal? Or did I miss out the source code where the author will resize into the native resolution for you?
PS: This seems to be the norm in all the PyTorch Tutorials I saw on Kaggle. My full code can be seen here in this notebook here; I like to not leave logic gaps and therefore this question popped up.


title: Training yolov5 causes a CUDNN_STATUS_NOT_INITIALIZED error

body:
I'm following this guide without changing anything. I'm using an aws server with deep learning ami: Deep Learning AMI (Ubuntu 18.04) Version 40.0
I've tried to change my custom dataset to the coco dataset and to a small subset of the custom one.
batch size doesn't seems to matter, CUDA and other drivers seems to work.
The exception is thrown when the batch starts the training process. This is the full stack trace:
Logging results to runs/train/exp66
Starting training for 5 epochs...

     Epoch   gpu_mem       box       obj       cls     total   targets  img_size
  0%|                                                                                | 0/22 [00:00<?, ?it/s]
Traceback (most recent call last):
  File "train.py", line 533, in <module>
    train(hyp, opt, device, tb_writer, wandb)
  File "train.py", line 298, in train
    pred = model(imgs)  # forward
  File "/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ubuntu/yolov5/models/yolo.py", line 121, in forward
    return self.forward_once(x, profile)  # single-scale inference, train
  File "/home/ubuntu/yolov5/models/yolo.py", line 137, in forward_once
    x = m(x)  # run
  File "/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ubuntu/yolov5/models/common.py", line 113, in forward
    return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))
  File "/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/ubuntu/yolov5/models/common.py", line 38, in forward
    return self.act(self.bn(self.conv(x)))
  File "/usr/local/lib/python3.8/site-packages/torch/nn/modules/module.py", line 889, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 399, in forward
    return self._conv_forward(input, self.weight, self.bias)
  File "/usr/local/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 395, in _conv_forward
    return F.conv2d(input, weight, bias, self.stride,
RuntimeError: cuDNN error: CUDNN_STATUS_NOT_INITIALIZED


title: RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED when calling `cublasSgemm( handle, opa, opb, m, n, k, &alpha, a, lda, b, ldb, &beta, c, ldc

body:
I am using ubuntu 20.04, torch==1.7.1
I am building a Deep reinforcement learning agent using pytorch. Everything works fine until today where while training outputs error as stated above. Weird thing is that this happens few days back and got fixed when I restarted my computer. This time restarting did not fix it.
Another weird thing is when run my script in terminal it outputs RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED when calling cublasSgemm( handle, opa, opb, m, n, k, &alpha, a, lda, b, ldb, &beta, c, ldc
however when I use debugging in pycharm it outputs different error: RuntimeError: CUDA error: device-side assert triggered The error seems to arise when I pass input into first fully connect layer.
  File "MYPATH/fc.py", line 33, in forward
    x = F.elu(self.fc1(inputs))
  File "MYPATH/python3.8/site-packages/torch/nn/functional.py", line 1227, in elu
    result = torch._C._nn.elu(input, alpha)

And as I was writing this question running my script on terminal outputs RuntimeError: CUDA error: device-side assert triggered


title: Implementing a tutorial for Tensorflow and SaaK Transform: Handwritten Digits Recognition

body:
I am learning to work with Tensorflow. I have installed most of the libraries.

# load libs
import torch
import argparse
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
import numpy as np
from data.datasets import MNIST
import torch.utils.data as data_utils
from sklearn.decomposition import PCA
import torch.nn.functional as F
from torch.autograd import Variable

But I am facing an error like this:

ImportError Traceback (most recent call last)
<ipython-input-8-257b7af364e1> in <module>()
  5 import matplotlib.pyplot as plt
  6 import numpy as np
----> 7 from data.datasets import MNIST
  8 import torch.utils.data as data_utils
  9 from sklearn.decomposition import PCA

ImportError: No module named data.datasets

I have installed the "dataset" library using the following command:

sudo -H pip install dataset

But I am still seeing this error. I am using python 2.7.
I am pretty new to python. Can anyone help me in identifying the missing thing. Thanks in advance.


title: Error when implementing RBF kernel bandwidth differentiation in Pytorch

body:
I'm implementing an RBF network by using some beginer examples from Pytorch Website. I have a problem when implementing the kernel bandwidth differentiation for the network. Also, Iwould like to know whether my attempt ti implement the idea is fine. This is a code sample to reproduce the issue. Thanks

# -*- coding: utf-8 -*-
import torch
from torch.autograd import Variable

def kernel_product(x,y, mode = "gaussian", s = 1.):
    x_i = x.unsqueeze(1)
    y_j = y.unsqueeze(0)
    xmy = ((x_i-y_j)**2).sum(2)

    if   mode == "gaussian" : K = torch.exp( - xmy/s**2) )
    elif mode == "laplace"  : K = torch.exp( - torch.sqrt(xmy + (s**2)))
    elif mode == "energy"   : K = torch.pow(   xmy + (s**2), -.25 )

    return torch.t(K)

class MyReLU(torch.autograd.Function):
    """
    We can implement our own custom autograd Functions by subclassing
    torch.autograd.Function and implementing the forward and backward passes
    which operate on Tensors.
    """

    @staticmethod
    def forward(ctx, input):
        """
        In the forward pass we receive a Tensor containing the input and return
        a Tensor containing the output. ctx is a context object that can be used
        to stash information for backward computation. You can cache arbitrary
        objects for use in the backward pass using the ctx.save_for_backward method.
        """
        ctx.save_for_backward(input)
        return input.clamp(min=0)

    @staticmethod
    def backward(ctx, grad_output):
        """
        In the backward pass we receive a Tensor containing the gradient of the loss
        with respect to the output, and we need to compute the gradient of the loss
        with respect to the input.
        """
        input, = ctx.saved_tensors
        grad_input = grad_output.clone()
        grad_input[input < 0] = 0
        return grad_input

dtype = torch.cuda.FloatTensor
N, D_in, H, D_out = 64, 1000, 100, 10

# Create random Tensors to hold input and outputs, and wrap them in Variables.
x = Variable(torch.randn(N, D_in).type(dtype), requires_grad=False)
y = Variable(torch.randn(N, D_out).type(dtype), requires_grad=False)

# Create random Tensors for weights, and wrap them in Variables.
w1 = Variable(torch.randn(H, D_in).type(dtype), requires_grad=True)
w2 = Variable(torch.randn(H, D_out).type(dtype), requires_grad=True)

# I've created this scalar variable (the kernel bandwidth)
s = Variable(torch.randn(1).type(dtype), requires_grad=True)

learning_rate = 1e-6
for t in range(500):
    # To apply our Function, we use Function.apply method. We alias this as 'relu'.
    relu = MyReLU.apply

    # Forward pass: compute predicted y using operations on Variables; we compute
    # ReLU using our custom autograd operation.
#    y_pred = relu(x.mm(w1)).mm(w2)
    y_pred = relu(kernel_product(w1, x, s)).mm(w2)

    # Compute and print loss
    loss = (y_pred - y).pow(2).sum()
    print(t, loss.data[0])

    # Use autograd to compute the backward pass.
    loss.backward()

    # Update weights using gradient descent
    w1.data -= learning_rate * w1.grad.data
    w2.data -= learning_rate * w2.grad.data

    # Manually zero the gradients after updating weights
    w1.grad.data.zero_()
    w2.grad.data.zero_()

However I get this error, which dissapears when I simply use a fixed scalar in the default input parameter of kernel_product():

RuntimeError: eq() received an invalid combination of arguments - got (str), but expected one of:
 * (float other)
      didn't match because some of the arguments have invalid types: (str)
 * (Variable other)
      didn't match because some of the arguments have invalid types: (str)


title: How to apply conditions for rows in a tensor where there is boolean values

body:
I have the following tensor:

predictions = torch.tensor([[ True, False, False],
                            [False, False,  True],
                            [False,  True,  True],
                            [ True, False, False]])

I applied conditions along the axis like below.

new_pred= []

if predictions == ([True,False,False]):
       new_pred = torch.Tensor(0)
if predictions == ([False,False,True]):
       new_pred = torch.Tensor(2)
if predictions == ([False,True,True]):
       new_pred = torch.Tensor(2)

So I want the final output (new_pred) to be:
tensor([0, 2, 2, 0])

But I am getting a blank [] for the new_pred tensor. I think my logic must be flawed since nothing is getting stored in the new_pred. Can someone help me write this logic accurately?


title: How to convert a matrix of torch.tensor to a larger tensor?

body:
I meet a problem to convert a python matrix of torch.tensor to a torch.tensor

For example, M is an (n,m) matrix, with each element M[i][j] is a torch.tensor with same size (p, q, r, ...). How to convert python list of list M to a torch.tensor with size (n,m,p,q,r,...)
e.g.

M = []
for i in range(5):
    row = []
    for j in range(10):
        row.append(torch.rand(3,4))
    M.append(row)

How to convert above M to a torch.tensor with size (5,10,3,4).


title: Reshape rows into groups of columns

body:
I have a number of row vectors which I would like to batch as column vectors and use as input for Conv1d. As an example I'd like to reshape the tensor x into y i.e. making two groups of two column vectors.
# size = [4, 3]
x = torch.tensor([
    [0,  1,  2],
    [3,  4,  5],
    [6,  7,  8],
    [9, 10, 11]
])

# size = [2, 3, 2]
y = torch.tensor([
    [[0,  3],
     [1,  4],
     [2,  5]],
    [[6,  9],
     [7, 10],
     [8, 11]]
])

Is there a way to do this with just reshape and similar functions? The only way I can think of is using loops and copying into a new tensor.