title: Pytorch input tensor size with wrong dimension Conv1D

body:
def train(epoch):
      model.train()
      train_loss = 0

  for batch_idx, (data, _) in enumerate(train_loader):
    data = data[None, :, :]
    print(data.size())    # something seems to change between here

    data = data.to(device)
    optimizer.zero_grad()
    recon_batch, mu, logvar = model(data) # and here???

    loss = loss_function(recon_batch, data, mu, logvar)
    loss.backward()
    train_loss += loss.item()

    optimizer.step()

    if batch_idx % 1000 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                100. * batch_idx / len(train_loader),
                loss.item() / len(data)))

  print('====> Epoch: {} Average loss: {:.4f}'.format(epoch, train_loss / len(train_loader.dataset)))

for epoch in range(1, 4):
        train(epoch)

This is very strange looking at the training loop it does recognize that the size is [1,1,1998] but then something changes after it is sent to the device?
    torch.Size([1, 1, 1998])
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-138-70cca679f91a> in <module>()
     27
     28 for epoch in range(1, 4):
---> 29         train(epoch)

5 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/conv.py in forward(self, input)
    255                             _single(0), self.dilation, self.groups)
    256         return F.conv1d(input, self.weight, self.bias, self.stride,
--> 257                         self.padding, self.dilation, self.groups)
    258
    259

RuntimeError: Expected 3-dimensional input for 3-dimensional weight [12, 1, 1], but got 2-dimensional input of size [1, 1998] instead

Also here is my model (I recognize there is likely a couple of other issues here but I am asking about the tensor size not registering)
class VAE(nn.Module):
  def __init__(self):
    super(VAE, self).__init__()

    self.conv1 = nn.Conv1d( 1,12, kernel_size=1,stride=5,padding=0)
    self.conv1_drop = nn.Dropout2d()
    self.pool1 = nn.MaxPool1d(kernel_size=3, stride=2)

    self.fc21 = nn.Linear(198, 1)
    self.fc22 = nn.Linear(198, 1)

    self.fc3 = nn.Linear(1, 198)
    self.fc4 = nn.Linear(198, 1998)

  def encode(self, x):
    h1 = self.conv1(x)
    h1 = self.conv1_drop(h1)
    h1 = self.pool1(h1)
    h1 = F.relu(h1)
    h1 = h1.view(1, -1) # 1 is the batch size
    return self.fc21(h1), self.fc22(h1)

  def reparameterize(self, mu, logvar):
    std = torch.exp(0.5*logvar)
    eps = torch.rand_like(std)
    return mu + eps*std

  def decode(self, z):
    h3 = F.relu(self.fc3(z))
    return torch.sigmoid(self.fc4(h3))

  def forward(self, x):
    mu, logvar = self.encode(x.view(-1, 1998))
    z = self.reparameterize(mu, logvar)
    return self.decode(z), mu, logvar

So why doesn't Pytorch keep the dimensions after reshaping and would that be the correct tensor size if it did?


title: PyTorch Replacing For-Loop Linears with Grouped Conv1d

body:
So I have this simple model:
class ModelSerial(nn.Module):
    def __init__(self, in_features, hidden, out_features):
        super().__init__()
        blocks = [
            nn.Sequential(
                nn.Linear(in_features, hidden),
                nn.ReLU(inplace=True),

                nn.Linear(hidden, hidden),
                nn.ReLU(inplace=True),

                nn.Linear(hidden, out_features),
            )
            for _ in range(out_features)
        ]
        self.blocks = nn.ModuleList(blocks)

    def forward(self, x):
        out = torch.cat([b(x) for b in self.blocks], dim=-1)
        return out

Notice how each sequential block's output is not chained in the forward pass. Rather, each is executed on the input separately, and the results are simply concatenated. As one would image, it runs extremely slow, especially as the number of output features increases or the depth of the network increases. This is due the Python for-loop. I had considered trying torch script jit, but that wasn't an option for this particular engagement. So next best thing? Try and parallelize it with grouped convolutions. Here's my attempt:
class ModelParallel(nn.Module):
    def __init__(self, in_features, hidden, out_features):
        super().__init__()

        self.out_features = out_features

        self.block = nn.Sequential(
            nn.Conv1d(
                in_features * self.out_features,
                hidden * self.out_features,
                kernel_size=1,
                groups=self.out_features
            ),
            nn.ReLU(inplace=True),

            nn.Conv1d(
                hidden * self.out_features,
                hidden * self.out_features,
                kernel_size=1,
                groups=self.out_features
            ),
            nn.ReLU(inplace=True),

            nn.Conv1d(
                hidden * self.out_features,
                out_features,
                kernel_size=1,
                groups=self.out_features
            )
        )

    def forward(self, x):
        # torch.Size([B, 150])
        bs = x.shape[0]

        # torch.Size([B, 150, 1])
        x = x[:, :, None]

        # torch.Size([B, 150, 50])
        x = x.expand(bs, x.shape[1], self.out_features)

        # torch.Size([B, 7500, 1]) because conv1d works on [B C L]
        x = x.reshape(bs, -1, 1)

        # torch.Size([B, 50, 1])
        x = self.block(x)

        # torch.Size([B, 50])
        x = x.squeeze(dim=-1)

        return x

The model compiles. Input size and output size are producing what is expected. But it doesn't train at all. Not even close. Validation and Train loss are completely out of sync, it's a mess.
I believe my lack of understanding how groups are computed are what are messing things up. So I even tried re-arranging the setup as follows in case the groups are 'interlaced' as opposed to 'chunked':
########
# Change:

# torch.Size([B, 150, 1])
x = x[:, :, None]

# torch.Size([B, 150, 50])
x = x.expand(bs, x.shape[1], self.out_features)

########
# Into:

# torch.Size([B, 1, 150])
x = x[:, None, :]

# torch.Size([B, 50, 150])
x = x.expand(bs, self.out_features, x.shape[2])

But unfortunately, that still didn't work (train). Epoch time was a hell of a lot faster though, as desired. My example tests:
ms = ModelSerial(
    in_features=150,
    hidden=10,
    out_features=50
)

mp = ModelSerial(
    in_features=150,
    hidden=10,
    out_features=50
)

I wouldn't expect the results to be exact, I mean, linear and conv are using different weight initialization schemes even if deterministic is set. But I would at least expect the parallel model to train. Any guidance on how I can architect this problem to execute in parallel? Thank you!!


title: Pytorch and Scikit-Learn - Labels starting from 0 or 1

body:
For both Pytorch and Scikit-Learn, is there any requirement for labels for start from 0 all the way to 1, 2, 3, number of classes? Or can I have labels start from 1, for example?
This is just a matter of having to apply a label encoder manually to the dataset or not.


title: Tweak positional encodings shape (DETR model) to support batchsize > 1

body:
Referencing from this notebook, and would like to scale this to support batch size > 1, as it state on in the comments Only batch size 1 supported.. I'm having trouble tweaking the pos statement inside the forward(). How to go about doing this? Any tips will be very helpful too.
This is the original code taken from the notebook](https://colab.research.google.com/github/facebookresearch/detr/blob/colab/notebooks/detr_demo.ipynb#scrollTo=h91rsIPl7tVl):
import torch
from torch import nn
from torchvision.models import resnet50
import torchvision.transforms as T
torch.set_grad_enabled(False);

class DETRdemo(nn.Module):
    """
    Demo DETR implementation.

    Demo implementation of DETR in minimal number of lines, with the
    following differences wrt DETR in the paper:
    * learned positional encoding (instead of sine)
    * positional encoding is passed at input (instead of attention)
    * fc bbox predictor (instead of MLP)
    The model achieves ~40 AP on COCO val5k and runs at ~28 FPS on Tesla V100.
    Only batch size 1 supported.
    """
    def __init__(self, num_classes, hidden_dim=256, nheads=8,
                 num_encoder_layers=6, num_decoder_layers=6):
        super().__init__()

        # create ResNet-50 backbone
        self.backbone = resnet50()
        del self.backbone.fc

        # create conversion layer
        self.conv = nn.Conv2d(2048, hidden_dim, 1)

        # create a default PyTorch transformer
        self.transformer = nn.Transformer(
            hidden_dim, nheads, num_encoder_layers, num_decoder_layers)

        # prediction heads, one extra class for predicting non-empty slots
        # note that in baseline DETR linear_bbox layer is 3-layer MLP
        self.linear_class = nn.Linear(hidden_dim, num_classes + 1)
        self.linear_bbox = nn.Linear(hidden_dim, 4)

        # output positional encodings (object queries)
        self.query_pos = nn.Parameter(torch.rand(100, hidden_dim))

        # spatial positional encodings
        # note that in baseline DETR we use sine positional encodings
        self.row_embed = nn.Parameter(torch.rand(50, hidden_dim // 2))
        self.col_embed = nn.Parameter(torch.rand(50, hidden_dim // 2))

    def forward(self, inputs):
        # propagate inputs through ResNet-50 up to avg-pool layer
        x = self.backbone.conv1(inputs)
        x = self.backbone.bn1(x)
        x = self.backbone.relu(x)
        x = self.backbone.maxpool(x)

        x = self.backbone.layer1(x)
        x = self.backbone.layer2(x)
        x = self.backbone.layer3(x)
        x = self.backbone.layer4(x)

        # convert from 2048 to 256 feature planes for the transformer
        h = self.conv(x)

        # construct positional encodings
        H, W = h.shape[-2:]
        pos = torch.cat([ ## <--- trouble scaling `pos` to support batch size > 1
            self.col_embed[:W].unsqueeze(0).repeat(H, 1, 1),
            self.row_embed[:H].unsqueeze(1).repeat(1, W, 1),
        ], dim=-1).flatten(0, 1).unsqueeze(1)

        # propagate through the transformer
        h = self.transformer(pos + 0.1 * h.flatten(2).permute(2, 0, 1),
                             self.query_pos.unsqueeze(1)).transpose(0, 1)

        # finally project transformer outputs to class labels and bounding boxes
        return {'pred_logits': self.linear_class(h),
                'pred_boxes': self.linear_bbox(h).sigmoid()}

For development:
model = DETRdemo(num_classes=10)
x = torch.ones((1,3,128,128)) # <-- this is ok
y = model(x)

model = DETRdemo(num_classes=10)
x = torch.ones((2,3,128,128)) # <-- this is NOT ok
y = model(x)


title: RNN(GRU) initial memory(hidden state) when testing the real data

body:
I try to use the Pytorch to learn the RNN(GRU).
I already known the forward process.
But I am confuse about testing real data after I finish training process.
Suppose I have a min-batch data which dimension is
(40, 6, 15)=(seq_len, batch_size, word_vec_size).
Suppose define a single-direction GRU layer with hidden state size is 7 and layer is 1(no stack).

Do forward, I need to initialize a hidden state.
My question is when I test a real data, how do I determine initial hidden state?
If randomly determine, input same data, it probably get the different output.
I don't known my concept is right or not.
Thank you.


title: Calling .backward() function for two different neural networks but getting retain_graph=True error

body:
I have an Actor Critic neural network where the Actor is its own class and the Critic is its own class with its own neural network and .forward() function. I then am creating an object of each of these classes in a larger Model class. My setup is as follows:
self.actor = Actor().to(device)
self.actor_opt = optim.Adam(self.actor.parameters(), lr=lr)
self.critic = Critic().to(device)
self.critic_opt = optim.Adam(self.critic.parameters(), lr=lr)

I then calculate two different loss functions and want to update each neural network separately. For the critic:
loss_critic = F.smooth_l1_loss(value, expected)
self.critic_opt.zero_grad()
loss_critic.backward()
self.critic_opt.step()

and for the actor:
loss_actor = -self.critic(state, action)
self.actor_opt.zero_grad()
loss_actor.backward()
self.actor_opt.step()

However, when doing this, I get the following error:
RuntimeError: Trying to backward through the graph a second time, but the saved
intermediate results have already been freed. Specify retain_graph=True when
calling backward the first time.

When reading up on this, I understood that I only need to retain_graph=True when calling backward twice on the same network, and in most cases this is not good to set to True as I will run out of GPU. Moreover, when I comment out one of the .backward() functions, the error goes away, leading me to believe that for some reason the code is thinking that both backward() functions are being called on the same neural network, even though I think I am doing it separately. What could be the reason for this? Is there a way to specify for which neural network I am calling the backward function on?
Edit:
For reference, the optimize() function in this code here https://github.com/wudongming97/PyTorch-DDPG/blob/master/train.py uses backward() twice with no issue (I've cloned the repo and tested it). I'd like my code to operate similarly where I backprop through critic and actor separately.


title: How to fix the RuntimeError: builtin cannot be used as a value (with a dict) when creating checkpoints from jit in pytorch?

body:
I want to compile arbitrary nn modules using jit.
In my code I compare my value to a dict and it throws an error.
        """
        if type(json_data) is dict:
                              ~~~~ <--- HERE

To Reproduce
Simple, any code that has a comparison with a dict:
class Node(object):
    def __init__(self):
        pass

    @classmethod
    def from_json(cls, json_data):
        if type(json_data) is dict:
            node_data = next(iter(json_data))
            assert type(json_data[node_data]) is list
            node_children = [cls.from_json(child) for child in json_data[node_data]]
            return Node(node_data, node_children)
        else:
            return Node(json_data)

Expected behavior
Jit makes my checkpoint.
Environment

PyTorch / torchvision Version (e.g., 1.0 / 0.4.0): 1.7.1
OS (e.g., Linux): mac os x
How you installed PyTorch / torchvision (conda, pip, source): conda
Build command you used (if compiling from source): conda
Python version: 3.8
CUDA/cuDNN version: CPU
GPU models and configuration: CPU
Any other relevant information: CPU

Additional context
Compiling arbitrary custom nn modules to jit
error:
/Users/brando/anaconda3/envs/coq_gym/bin/python /Applications/PyCharm.app/Contents/plugins/python/helpers/pydev/pydevd.py --cmd-line --multiproc --qt-support=auto --client 127.0.0.1 --port 59213 --file /Users/brando/ML4Coq/playground/running_pytorch_ocaml/treenn2jit_ckpt.py
Connected to pydev debugger (build 203.7148.72)
1.7.1
Traceback (most recent call last):
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/site-packages/torch/jit/_recursive.py", line 680, in compile_unbound_method
    create_methods_and_properties_from_stubs(concrete_type, (stub,), ())
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/site-packages/torch/jit/_recursive.py", line 304, in create_methods_and_properties_from_stubs
    concrete_type._create_methods_and_properties(property_defs, property_rcbs, method_defs, method_rcbs, method_defaults)
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/site-packages/torch/jit/annotations.py", line 330, in try_ann_to_type
    torch.jit._script._recursive_compile_class(ann, loc)
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/site-packages/torch/jit/_script.py", line 1056, in _recursive_compile_class
    _compile_and_register_class(obj, rcb, _qual_name)
  File "/Users/brando/anaconda3/envs/coq_gym/lib/python3.7/site-packages/torch/jit/_script.py", line 64, in _compile_and_register_class
    torch._C._jit_script_class_compile(qualified_name, ast, defaults, rcb)
RuntimeError:
builtin cannot be used as a value:
  File "/Users/brando/ML4Coq/ml4coq-proj/embeddings_zoo/extract_tactic_from_lasse_data.py", line 56
            term = string
        """
        if type(json_data) is dict:
                              ~~~~ <--- HERE
            node_data = next(iter(json_data))
            assert type(json_data[node_data]) is list
'Node.from_json' is being compiled since it was called from '__torch__.embeddings_zoo.extract_tactic_from_lasse_data.Node'

related:

https://github.com/pytorch/vision/issues/3392
https://github.com/pytorch/vision/issues/1675


title: Embedding Layer in Python: How to Use Correctly with Torchsummary?

body:
This is a minimally working/reproducible example:
import torch
import torch.nn as nn
from torchsummary import summary

class Network(nn.Module):
    def __init__(self, channels_img, features_d, num_classes, img_size):
        super(Network, self).__init__()
        self.img_size = img_size
        self.disc = nn.Conv2d(
            in_channels = channels_img + 1,
            out_channels = features_d,
            kernel_size = (4,4)
        )

        # ConditionalGan:
        self.embed = nn.Embedding(
            num_embeddings = num_classes,
            embedding_dim = img_size * img_size
        )

   def forward(self, x, labels):
        embedding = self.embed(labels).view(labels.shape[0], 1, self.img_size, self.img_size)
        x = torch.cat([x, embedding], dim = 1)
        return self.disc(x)

# device:
device = torch.device("cpu")

# hyperparameter:
batch_size = 64

# Initialize model:
model = Network(
    channels_img = 1,
    features_d = 16,
    num_classes = 10,
    img_size = 28).to(device)

# Print model summary:
summary(
    model,
    input_size = [(1, 28, 28), (1, 28, 28)], # MNIST
    batch_size = batch_size
)

The error message I get is (for the line with summary(...)):
Expected tensor for argument #1 'indices' to have scalar type Long; but got torch.cuda.FloatTensor instead (while checking arguments for embedding)
I saw in this post, that .to(torch.int64) is supposed to help, but I honestly don't know where to write it.
Thank you!


title: Understanding custom policies in stable-baselines3

body:
I was trying to understand the policy networks in stable-baselines3 from this doc page.

As explained in this example, to specify custom CNN feature extractor, we extend BaseFeaturesExtractor class and specify it in policy_kwarg.features_extractor_class with first param CnnPolicy:
model = PPO("CnnPolicy", "BreakoutNoFrameskip-v4", policy_kwargs=policy_kwargs)

Q1. Can we follow same approach for custom MLP feature extractor?

As explained in this example, to specify custom MLP feature extractor, we extend ActorCriticPolicy class and override _build_mlp_extractor() and pass it as first param:
class CustomActorCriticPolicy(ActorCriticPolicy): ...
model = PPO(CustomActorCriticPolicy, "CartPole-v1", verbose=1)

Q2. Can we follow same approach for custom CNN feature extractor?

I feel either we can have CNN extractor or MLP extractor. So it makes no sense to pass MlpPolicy as first param to model and then specify CNN feature extractor in policy_kwarg.features_extractor_class as in this example. This result in following policy (containing both features_extractor and mlp_extractor), which I feel is incorrect:
ActorCriticPolicy(
  (features_extractor): Net(
    (conv1): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1))
    (conv2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1))
    (fc3): Linear(in_features=384, out_features=512, bias=True)
  )
  (mlp_extractor): MlpExtractor(
    (shared_net): Sequential(
      (0): Linear(in_features=512, out_features=64, bias=True)
      (1): ReLU()
    )
    (policy_net): Sequential(
      (0): Linear(in_features=64, out_features=32, bias=True)
      (1): ReLU()
      (2): Linear(in_features=32, out_features=16, bias=True)
      (3): ReLU()
    )
    (value_net): Sequential(
      (0): Linear(in_features=64, out_features=32, bias=True)
      (1): ReLU()
      (2): Linear(in_features=32, out_features=16, bias=True)
      (3): ReLU()
    )
  )
  (action_net): Linear(in_features=16, out_features=7, bias=True)
  (value_net): Linear(in_features=16, out_features=1, bias=True)
)

Q3. Am I correct with this understanding? If yes, then is one of the MLP or CNN feature extractor ignored?


I'm new on PyTorch and I'm trying to code with it
so I have a function called OHwich tack a number and return a victor like this
  def OH(x,end=10,l=12):
    x = T.LongTensor([[x]])
    end = T.LongTensor([[end]])
    one_hot_x = T.FloatTensor(1,l)
    one_hot_end = T.FloatTensor(1,l)
    first=one_hot_x.zero_().scatter_(1,x,1)
    second=one_hot_end.zero_().scatter_(1,end,1)
    vector=T.cat((one_hot_x,one_hot_end),dim=1)
    return vector
OH(0)
output:
tensor([[1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 1., 0.]])

now I have a NN that takes this output and return number but this warning always appear in my compiling
online.act(OH(obs))
output:
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:17: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).
4

I tried to to use online.act(OH(obs).clone().detach()) but it give me the same warning
and the code works fine and give good results but I need to understand this warning
Edit
the following is my NN that has the act function
class Network(nn.Module):
  def __init__(self,lr,n_action,input_dim):
    super(Network,self).__init__()
    self.f1=nn.Linear(input_dim,128)
    self.f2=nn.Linear(128,64)
    self.f3=nn.Linear(64,32)
    self.f4=nn.Linear(32,n_action)
    #self.optimizer=optim.Adam(self.parameters(),lr=lr)
    #self.loss=nn.MSELoss()
    self.device=T.device('cuda' if T.cuda.is_available() else 'cpu')
    self.to(self.device)

  def forward(self,x):
    x=F.relu(self.f1(x))
    x=F.relu(self.f2(x))
    x=F.relu(self.f3(x))
    x=self.f4(x)
    return x

  def act(self,obs):
    state=T.tensor(obs).to(device)
    actions=self.forward(state)
    action=T.argmax(actions).item()

    return action


title: Will installing Nvidia CUDA ver10 break CUDA ver9 on the same machine?

body:
I am currently using pytorch and tensorflow with cuda ver9.0. I am pondering whether to install the latest cuda version 10. Will installing cuda v10 break cuda v9? Can both co-exist on the same desktop PC? Is it advisable to uninstall cuda v9 after installing cuda v10 or is it better to leave both versions installed?

I am using Windows 10.


title: Where do I get Visual Studio 15.4 that I need for pyTorch 0.4.1

body:
I have a problem with the following dependency chain:

MedicalDetectionToolkit (MDT) needs pyTorch 0.4.1 since in pyTorch 1.0 torch.utils.ffi is depricated
pyTorch 0.4.1 needs CUDA 9.0 (does not work with 10.0)
CUDA 9.0 does not work with Visual Studio > 15.4
Visual Studio 15.4 is not available anywhere, the MSVS installer will install 15.9

The Visual Studio Installer allows to install the "Compiler Tools 15.4" side-by.side, but MDT uses torch.utils.ffi at some point, which just uses the 15.9.

So

How do I install Visual Studio 15.4?

Or

How do I tell torch.utils.ffi to use the 15.4 compiler?

(MedicalDetectionToolkit will not be updated by the author. I may try it, but would like to have a working version first.)


title: Should I transpose a Tensor when feeding it into a CNN

body: I am using a custom dataset with images of different sizes in the Lab format (Lightness, a, b) which are feed into a CNN. The input layer has 3 in-channels and so my idea was to split all 3 channels (L, a, b) and feed those into the network. Next I was wondering if each tensor needs to be transposed? My doubt is that it would lose its dimensions which are variable from image to image and I would not be able to reconstruct the image in the end. Any thoughts or ideas how I should normalize the image?


title: How do I extract indices of non-equivalent entries between two tensors?

body:
I have a tensor with N predictions of N objects' classes, and I have another tensor with the real N target objects' classes. I would like to pull out the tensor indices where my classifier predictions are wrong.

Consider the two following tensors defined as:

import torch
predictions = torch.tensor([ [0], [1], [1], [0], [0], [1] ])
target      = torch.tensor([ [0], [0], [1], [1], [0], [1] ])

I want to find some function where I can pass these two vectors and have returned a list like index_diff = [1, 3]. Does this function exist? My current thoughts are to cast both of these vectors to numpy arrays and then loop through N times and compare each entry at each index, but this seemed a little circuitous to me. Is there an alternative?


Why doesn't torch.autograd compute the gradient in this case?

import torch
x = torch.tensor([1., 2., ], requires_grad=True)
y = torch.tensor([x[0] + x[1], x[1] - x[0], ], requires_grad=True)
z = y[0] + y[1]
z.backward()
x.grad

Output is a blank line (None). The same occurs for x[0].grad. Why?

PS: In retrospect, I realize the motivation for making y a tensor with requires_grad was so I could examine its own gradient. I learned that one must use retain_grad for that here: Why does autograd not produce gradient for intermediate variables?


title: Implemented train function in PyTorch

body:
I'm building models of neural networks for some experiments. I use PyTorch and each time I train a model I use the following code:

def train_and_evaluate(net, optimizer, criterion):
    start_time = time.time()
    train_losses, test_losses, train_acc, test_acc = [], [], [], []
    # net.double()

    for epoch in range(num_epochs):
        epoch_start_time = time.time()
        running_loss_train, running_loss_test = 0.0, 0.0
        total, correct, test_total, test_correct = 0, 0, 0, 0

        # Train mode
        net.train()

        #Loop batches
        for i, (images, labels) in enumerate(train_loader):

            if use_cuda and torch.cuda.is_available():
                images = images.cuda()
                labels = labels.cuda()

            optimizer.zero_grad()
            outputs = net(images.detach())
            # import pdb; pdb.set_trace()
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            running_loss_train += loss.item()

            # Calculate training accuracy for epoch
            _, predicted = torch.max(outputs.data, 1) # Get th
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

        # Calculate test loss and accuracy for epoch
        net.eval()
        with torch.no_grad():
            for images, labels in test_loader:

                if use_cuda and torch.cuda.is_available():
                    images = images.cuda()
                    labels = labels.cuda()

                outputs = net(images)
                test_loss = criterion(outputs, labels)
                running_loss_test += test_loss.item()

                _, predicted = torch.max(outputs.data, 1)
                test_total += labels.size(0)
                test_correct += (predicted == labels).sum().item()

            if (epoch + 1) % 10 == 0 or epoch == 0:
                print(f'Epoch [{epoch+1:02d}/{num_epochs}]\tTime: {time.time() - epoch_start_time:.2f}\tTrain Loss: {(running_loss_train / len(train_loader)):.4f}\tTrain Acc: {(correct / total):.0%}\tTest Loss: {running_loss_test / len(test_loader):.4f}\tTest Accuracy: {(test_correct / test_total):.0%}'.expandtabs(4))

            train_losses.append(running_loss_train / len(train_loader))
            test_losses.append(running_loss_test / len(test_loader))
            train_acc.append(correct / total)
            test_acc.append(test_correct / test_total)

    print(f'Training time: {(time.time() - start_time)/60:5.2f} min.')
    return train_losses, test_losses, train_acc, test_acc

I simply copy and paste it from other projects and I find it redundant to use that same code each time, it is useful almost in every model I test and usually looks quite similar. I wonder if there is a shortcut or some module with implementation of this function to reduce number of code lines and make it more readable. Something like:

model = NeuralNetwork()
train_losses, test_losses, train_acc, test_acc = train(model,
                                                       epochs=50,
                                                       verbose=True,
                                                       cuda=True,
                                                       optimizer=optimizer,
                                                       criterion=criterion)

That I could just pass the parameters for training and not explicitly 'write' the code evey time.


title: What is vectorised way of doing this operation in pytorch instead of two FOR loops

body:
Hello,
I have a tensor 'A' in Pytorch of dimesnsions Batch x Channel x Height x Width. I want to reshape it into 'B' such that dimesnions H and W are increased by 'r' and channels reduced by a factor of 'r^2'. For 'r'=2, the illustration is shown in figure attached.
In the figure if 'B' had 4 channels then first 4 channels, with 4th channel having border color violet, then first four channels in 'A' will be peack/skin colored pixels with border colors red, green, blue and violet and remaining channels adjusted accordingly.

I know the 'pack' and 'unpack' each can be done with 2 for loops. But that takes more time. There shoud be vectorised way in PyTorch to do switch between 'A' and 'B' with just reshape and pemutation commands. Can someone help me on that?

In this example, the batch size is set to 1. But if batch dimension is more I would like the operations shown in the figure to operate individually on each batch entry.

Can someone please help me with a generic code to switch between A and B in the vectorised way in PyTorch. Better if that also works when Batch size is more than 1.

Please note the two operations cannot be done with already implemented nn.PixelShuffle

Thanks a lot.


title: displaying weights & activations in cnn & unet

body:
I am programming in Python using Tensorflow, Numpy and Matplotlib and wanted to see how to visualize weights learnt and also activations in a diagram of my CNN.

The basic flow of my algorithm (which I would like represented in my diagram) is as follows:

Input image
Binarize the image
Run image through CNN (basic conv2d with relu/maxpool)
Run it through many examples of input data.

I would like to visualize the weights learnt.

I would also like to have an unlabelled image that I want to test against and also visualize the weights for while testing.

Is there a way to visualize using images instead of trying to use TensorBoard?


title: Does running HuggingFace Transformers with different sequence length on TPU causes XLA re-compilation every time?

body:
Would running the running HuggingFace Transformers with different sequence length on TPU cause new computation graph to be made each time ? And thus causing XLA re-compilation every time ?

Also, while training , does that mean all batches should be padded to the over-all maximum length in the entire dataset ? If I use the trainer, will it do it automatically for me ?

Or does the ParallelLoader of PyTorch/XLA do it ?


title: Pytorch: Recover network with customized VGG model that was saved improperly

body:
I am currently doing work with customizing the forward method for models. I was using some tutorial code that ran VGG. I did a few runs with the baseline model and it seemed to work fine. Afterwards, I replaced the forward method for the VGG using:

net.forward = types.MethodType(forward_vgg_new, net)

Unfortunately, the way that the tutorial code saves the models is:

            state = {
                    'net':net,
                    'acc':acc,
                    'epoch':epoch,
            }
...
            torch.save(state, ...)

While This worked for the original tutorial code, loading no longer works for my custom models as I get:

  AttributeError: 'VGG' object has no attribute 'forward_vgg_new'

I have since read from the documentation that it is better for me to save the model's state_dict:

            state = {
                    'net':net.state_dict(),
                    'acc':acc,
                    'epoch':epoch,
            }
...
            torch.save(state, ...)

While I will change the code for future runs, I was wondering if it was possible to salvage the models I have already trained. I naively already tried to import the VGG class and add my forward_vgg_new method to it:

setattr(VGG, 'forward_vgg_new', forward_vgg_new)

before calling torch.load, but it doesn't work.


title: Torch LSTMCell Method

body:
self.conv1 = nn.Conv2d(num_inputs, 32, 3, stride=2, padding=1)
self.conv2 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.conv3 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.conv4 = nn.Conv2d(32, 32, 3, stride=2, padding=1)
self.lstm = nn.LSTMCell(32 * 3 * 3, 256)

Can somebody provide the formula of the calculation lstm input (32 * 3 * 3). Where is this 3 comes from.


title: Training deep adversarial neural network in pytorch

body:
I am trying to implement domain adversarial neural network in PyTorch. I made data set and data loader as shown below:
``import h5py as h5
  from torch.utils import dataclass MyDataset(data.Dataset):
  def __init__(self, root, transform=None):
    self.root = h5py.File(root, 'r')
    self.labels = self.root.get('train').get('targets')[()]
    self.data = self.root.get('train').get('inputs')[()]
    self.transform = transform
  def __getitem__(self, index):
    datum = self.data[index]
    if self.transform is not None:
    datum = self.transform(datum)
    return datum, self.labels[index]
  def __len__(self):
    return len(self.labels)
  def close(self):
  self.root.close()``

Then i trained my network using the code available on github as:
n_epochs = 10
for epoch_idx in range(n_epochs):
   print(f'Epoch {epoch_idx+1:04d} / {n_epochs:04d}', end='\n=================\n')
dl_source_iter = iter(dl_source)
dl_target_iter = iter(dl_target)
for batch_idx in range(max_batches):
    optimizer.zero_grad()
    #i=0
    #while i<max_batches:
    # Training progress and GRL lambda
    p = float(batch_idx+epoch_idx * max_batches) / (n_epochs * max_batches)
    # Calculating lambda
    grl_lambda = 2. / (1. + np.exp(-10 * p)) - 1
    # Train on source domain
    #taking images and labels from source domain
    inputs, targets = (dl_source_iter)
    # generate source domain labels
    targets = torch.zeros(batch_size, dtype=torch.long)
    #Feeding model images and lambda parameter
   # Getting Prediction for the class and domain
   class_pred, domain_pred = model(X_s, grl_lambda)
   #Calculating class (labels) loss for the source domain
   loss_s_label = loss_fn_class(class_pred, y_s)
   #Calculating domain loss for the source data
   loss_s_domain = loss_fn_domain(domain_pred, y_s_domain)
   # Train on target domain
   #Ignoring the labels for the target domain
   X_t, _ = next(dl_target_iter)
   #Getting domain labeels for target
    y_t_domain = torch.ones(len(X_t), dtype=torch.long)
    # Getting domain predictions for the target data
   _, domain_pred = model(X_t, grl_lambda)
    #Calculating the domain loss for target data
    loss_t_domain = loss_fn_domain(domain_pred, y_t_domain)
    # Calculating total loss
    loss = loss_t_domain + loss_s_domain + loss_s_label
    loss.backward()
    optimizer.step()
    print(f'[{batch_idx+1}/{max_batches}] ' f'class_loss: {loss_s_label.item():.4f} ' f's_domain_loss:
   {loss_s_domain.item():.4f} ' f't_domain_loss: {loss_t_domain.item():.4f} ' f'grl_lambda:
   {grl_lambda:.3f} '
         )

But i am getting  a strange error like:
)
But i am getting  a strange error like:
ValueError                                Traceback (most recent call last)
 in ()
  ValueError                                Traceback (most recent call last)
 <ipython-input-210-eb295f5d5052> in <module>()
 18         # Train on source domain
 19         #taking images and labels from source domain
---> 20         inputs, targets = (dl_source_iter)
 21         # generate source domain labels
 22         targets = torch.zeros(batch_size, dtype=torch.long)

 6 frames
 /usr/local/lib/python3.6/dist-packages/torchvision/transforms/functional.py in to_tensor(pic)

 44     if _is_numpy(pic) and not _is_numpy_image(pic):
---> 45         raise ValueError('pic should be 2/3 dimensional. Got {}
dimensions.'.format(pic.ndim))
 46
 47     if isinstance(pic, np.ndarray):

ValueError: pic should be 2/3 dimensional. Got 1 dimensions.
Please guide me, how i can remove this error?.
Regards


title: Change the step number of displayed images on Pytorch Tensorboard

body:
I am running tensorboard on a pytorch lightning_logs folder containing some results. However, in the images tab, the batch of images produced are always at step=0 (see attached screenshot). How do I view images produced at a later step on tensorboard? I tried to modify the global_step variable in the image functions in env/lib/python3.7/site-packages/torch/utils/tensorboard/writer.py  but this doesn't do anything. I have also attached the command I have run and the contents of the lightning_log folder, as I am unsure if the writer.py file I previously mentioned is the one being accessed.
Command: tensorboard --logdir experiments/lightning_logs/version_8744369/
Contents of lightning_logs/version_8744369/: events.out.tfevents.1594223100.gl1019.arc-ts.umich.edu.150337.0,
events.out.tfevents.1594223139.gl1019.arc-ts.umich.edu.150338.0,
hparams.yaml
tensorboard output screenshot


title: Output of a deep neural network in Pytorch

body:
I have a neural network and this is the function for it:
def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity):
    # Build a feedforward neural network. outputs are the logits
    layers = []
    for j in range(len(sizes)-1):
        act = activation if j < len(sizes)-2 else output_activation
        layers += [nn.Linear(sizes[j], sizes[j+1]), act()]
    return nn.Sequential(*layers)

and sizes=[10, 20]
My input is a tensor in the shape of [100,1,10] and with this network, I get a tensor as my output in the shape of [100,20]. What I would like to add is two more layers so that instead of a matrix as the output, I get a vector of size 20. Is this possible considering that the input includes a batch of images?


title: Covnert a List of Tensors to a Tensor

body:
I have a list of tensors like this:
[tensor(-2.9222, grad_fn=<SqueezeBackward1>), tensor(-2.8192, grad_fn=<SqueezeBackward1>), tensor(-3.1894, grad_fn=<SqueezeBackward1>), tensor(-2.9048, grad_fn=<SqueezeBackward1>)]

I want it to be in this form:
tensor([-0.5575, -0.9004, -0.8491,  ..., -0.7345, -0.6729, -0.7553],
   grad_fn=<SqueezeBackward1>)

How can I do this?
I appreciate the help.


title: Unittest the pytorch forward function

body:
I want to unittest the overridden forward function of my Network modell in Pytorch. So I loaded my model (pretrained from Zoo) with the setUp method, loaded a seed and created some random batch. In my method testForward I tested the result of forward against shape and numel, but I also want to check a specific value which a apears to be 0. I wasn't shure about that so checked my params in setUp also, which appears not to be 0.
import unittest
import torch
from SemanticSegmentation.models.fcn8 import FCN8

class TestFCN8(unittest.TestCase):

    def setUp(self):
        self.model = FCN8(8, pretrained=True)
        torch.manual_seed(0)
        self.x = torch.rand((4, 3, 45, 45))
        for param in self.model.parameters():
            print(param.data)

    def testForward(self):
        self.assertEqual(self.model.forward(self.x).shape.numel(), 64800)
        self.assertEqual(str(self.model.forward(self.x).shape), 'torch.Size([4, 8, 45, 45])')
        print(self.model.named_parameters)

if __name__ == "__main__":
    unittest.main()

So my Question is: the sahpe of the forward return tensor is what I expect but why is this tensor completly zero? I expected at least a few values.
The imported modell is based on an VGG16 network with upscoring the after ConvLayer 4 , 8 and 16. If needed I could also present the modell code.


title: Pytorch: Inference on Specific Layers of a Model

body:
I’m building a flexible module that implements Monte Carlo Dropout for Bayesian Inference on any kind of model.
(Implementing MC dropout by adding dropout to the fc layers during test time and inference for multiple times to get distribution of outputs)
To save time, I want to divide the model into two parts:

No dropout implemented so run only once
Layers which are after the first dropout (first fc layer) therefore will run multiple times.

I can't use nn.Sequential() because many of the models I work with have functionality in forward (like x.permute()).
Do you have any idea how to inference on only specific layers without changing the model itself or the forward() code?


title: Restoring mask to image

body:
My PyTorch model outputs a segmented image with values (0,1,2) for each one of the three classes. During the preparation of the set, I mapped black to 0, red to 1 and white to 2. I have two questions:

How can I show what each class represents? for example take a look at the image:

I am currently using the following method to show each class:
     output = net(input)

     input = input.cpu().squeeze()
     input = transforms.ToPILImage()(input)

     probs = F.softmax(output, dim=1)
     probs = probs.squeeze(0)

     full_mask = probs.squeeze().cpu().numpy()

     fig, (ax0, ax1, ax2, ax3, ax4) = plt.subplots(1, 5, figsize=(20,10), sharey=True)

     ax0.set_title('Input Image')
     ax1.set_title('Background Class')
     ax2.set_title('Neuron Class')
     ax3.set_title('Dendrite Class')
     ax4.set_title('Predicted Mask')

     ax0.imshow(input)
     ax1.imshow(full_mask[0, :, :].squeeze())
     ax2.imshow(full_mask[1, :, :].squeeze())
     ax3.imshow(full_mask[2, :, :].squeeze())

     full_mask = np.argmax(full_mask, 0)
     img = mask_to_image(full_mask)

But there appears to be shared pixels between the classes, is there a better way to show this (I want the first image to only of the background class, the the second only of the neuron class and the third only of the dendrite class)?
2.My second question is about generating a black, red and white image from the mask, currently the mask is of shape (512,512) and has the following values:
[[0 0 0 ... 0 0 0]
 [0 0 0 ... 2 0 0]
 [0 0 0 ... 2 2 0]
 ...
 [2 1 2 ... 2 2 2]
 [2 1 2 ... 2 2 2]
 [0 2 0 ... 2 2 2]]

And the results look like this:

Since I am using this code to convert to image:
def mask_to_image(mask):
   return Image.fromarray((mask).astype(np.uint8))


title: I get a tensor of 600 values instead of 3 values for mean and std of train_loader in PyTorch

body:
I am trying to Normalize my images data and for that I need to find the mean and std for train_loader.
mean = 0.0
std = 0.0
nb_samples = 0.0
for data in train_loader:
    images, landmarks = data["image"], data["landmarks"]
    batch_samples = images.size(0)
    images_data = images.view(batch_samples, images.size(1), -1)
    mean +=  torch.Tensor.float(images_data).mean(2).sum(0)
    std += torch.Tensor.float(images_data).std(2).sum(0)
    ###mean += images_data.mean(2).sum(0)
    ###std += images_data.std(2).sum(0)
    nb_samples += batch_samples

mean /= nb_samples
std /= nb_samples

the mean and std here are each a torch.Size([600])
When I tried (almost) same code on dataloader, it worked as expected:
# code from https://discuss.pytorch.org/t/about-normalization-using-pre-trained-vgg16-networks/23560/6?u=mona_jalal
mean = 0.0
std = 0.0
nb_samples = 0.0
for data in dataloader:
    images, landmarks = data["image"], data["landmarks"]
    batch_samples = images.size(0)

    images_data = images.view(batch_samples, images.size(1), -1)
    mean += images_data.mean(2).sum(0)
    std += images_data.std(2).sum(0)
    nb_samples += batch_samples

mean /= nb_samples
std /= nb_samples

and I got:
mean is: tensor([0.4192, 0.4195, 0.4195], dtype=torch.float64), std is: tensor([0.1182, 0.1184, 0.1186], dtype=torch.float64)
So my dataloader is:
class MothLandmarksDataset(Dataset):
    """Face Landmarks dataset."""

    def __init__(self, csv_file, root_dir, transform=None):
        """
        Args:
            csv_file (string): Path to the csv file with annotations.
            root_dir (string): Directory with all the images.
            transform (callable, optional): Optional transform to be applied
                on a sample.
        """
        self.landmarks_frame = pd.read_csv(csv_file)
        self.root_dir = root_dir
        self.transform = transform

    def __len__(self):
        return len(self.landmarks_frame)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        img_name = os.path.join(self.root_dir, self.landmarks_frame.iloc[idx, 0])
        image = io.imread(img_name)
        landmarks = self.landmarks_frame.iloc[idx, 1:]
        landmarks = np.array([landmarks])
        landmarks = landmarks.astype('float').reshape(-1, 2)
        sample = {'image': image, 'landmarks': landmarks}

        if self.transform:
            sample = self.transform(sample)

        return sample

transformed_dataset = MothLandmarksDataset(csv_file='moth_gt.csv',
                                           root_dir='.',
                                           transform=transforms.Compose(
                                               [
                                               Rescale(256),
                                               RandomCrop(224),

                                               ToTensor()
                                               ]
                                                                        )
                                           )

dataloader = DataLoader(transformed_dataset, batch_size=3,
                        shuffle=True, num_workers=4)

and train_loader is:
# Device configuration
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
seed = 42
np.random.seed(seed)
torch.manual_seed(seed)

# split the dataset into validation and test sets
len_valid_set = int(0.1*len(dataset))
len_train_set = len(dataset) - len_valid_set

print("The length of Train set is {}".format(len_train_set))
print("The length of Test set is {}".format(len_valid_set))

train_dataset , valid_dataset,  = torch.utils.data.random_split(dataset , [len_train_set, len_valid_set])

# shuffle and batch the datasets
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=8, shuffle=True, num_workers=4)
test_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=8, shuffle=True, num_workers=4)

Please let me know if more information is needed.
I basically need to get 3 values for mean of train_loader and 3 values for std of train_loader to use as args for Normalize.
images_data in dataloader is torch.Size([3, 3, 50176]) inside the loop and images_data in train_loader is torch.Size([8, 600, 2400])


title: How can I make my deeplearning chatbot use the correct path to my dataset?

body:
I am trying to get the code from a deeplearning chatbot to work. This chatbot uses pytorch and the dataset from Cornell movie corpus. But the code can't seem to find the path to the dataset, and I don't know how to code it in. This is the source for the deeplearning chatbot code  [https://colab.research.google.com/github/pytorch/tutorials/blob/gh-pages/_downloads/chatbot_tutorial.ipynb][1]
This is as far as I've gotten with it.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import torch
from torch.jit import script, trace
import torch.nn as nn
from torch import optim
import torch.nn.functional as F
import csv
import random
import re
import os
import unicodedata
import codecs
from io import open
import itertools
import math

USE_CUDA = torch.cuda.is_available()
device = torch.device("cuda" if USE_CUDA else "cpu")

corpus_name = "cornell movie-dialogs corpus"
corpus = os.path.join("data", corpus_name)

def printLines(file, n=10):
    with open(file, 'rb') as datafile:
        lines = datafile.readlines()
    for line in lines[:n]:
        print(line)

printLines(os.path.join(corpus, "movie_lines.txt"))

And this is my error log
D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py:26: SyntaxWarning: 'str' object is not callable; perhaps you missed a comma?
  corpus = "D:\Documents\Python\intents\cornell_movie_dialogs_corpus.zip\cornell movie-dialogs corpus\\"("data", corpus_name)
Traceback (most recent call last):
  File "D:\Documents\Python\python pycharm files\pythonProject4\3.9 Chatbot.py", line 26, in <module>
    corpus = "D:\Documents\Python\intents\cornell_movie_dialogs_corpus.zip\cornell movie-dialogs corpus\\"("data", corpus_name)
TypeError: 'str' object is not callable

I hope there is a solution that doesn't alter the source code too much, but any tips or help are welcome.


title: how to complete a sentence using Character-based Language Model with AllenNLP

body:
i'm reading a Character-based Language Model with AllenNLP, it contains a function generate,
def generate(self) -> Tuple[List[Token], torch.tensor]:

    start_symbol_idx = self.vocab.get_token_index(START_SYMBOL, 'tokens')
   # print(start_symbol_idx)
    end_symbol_idx = self.vocab.get_token_index(END_SYMBOL, 'tokens')
    padding_symbol_idx = self.vocab.get_token_index(DEFAULT_PADDING_TOKEN, 'tokens')

    log_likelihood = 0.
    words = []
    state = (torch.zeros(1, 1, self.hidden_size), torch.zeros(1, 1, self.hidden_size))

    word_idx = start_symbol_idx

    for i in range(self.max_len):
        tokens = torch.tensor([[word_idx]])

        embeddings = self.embedder({'tokens': tokens})
        output, state = self.rnn._module(embeddings, state)
        output = self.hidden2out(output)

        log_prob = torch.log_softmax(output[0, 0], dim=0)

        dist = torch.exp(log_prob)

        word_idx = start_symbol_idx

        while word_idx in {start_symbol_idx, padding_symbol_idx}:
            word_idx = torch.multinomial(
                dist, num_samples=1, replacement=False).item()

        log_likelihood += log_prob[word_idx]

        if word_idx == end_symbol_idx:
            break

        token = Token(text=self.vocab.get_token_from_index(word_idx, 'tokens'))
        words.append(token)

   return words, log_likelihood

This function every time returns a sentence of 80 characters. Is there a way to adapt this function in order to complete a sentence, example : when i write "i'm going now to" i got something.


title: Install Pytorch for only inference time

body:
I have already make some research but could not find any useful information. Is it possible to install something like mini pytorch just for load a pre-trained model and call prediction method?
Entire PyTorch library size is too big, so I would like to prevent on this issue.
Does anyone have idea? Thanks in advance.


title: PoseWarping: How to vectorize this for loop (z-buffer)

body:
I'm trying to warp a frame from view1 to view2 using ground truth depth map, pose information, and camera matrix. I've been able to remove most of the for-loops and vectorize it, except one for-loop. When warping, multiple pixels in view1 may get mapped to a single location in view2, due to occlusions. In this case, I need to pick the pixel with the lowest depth value (foreground object). I'm not able to vectorize this part of the code. Any help to vectorize this for loop is appreciated.
Context:I'm trying to warp an image into a new view, given ground truth pose, depth, and camera matrix. After computing warped locations, I'm rounding them off. Any suggestions to implement inverse bilinear interpolation are also welcome. My images are of full HD resolution. Hence it is taking a lot of time to warp the frames to the new view. If I can vectorize, I'm planning to convert the code to TensorFlow or PyTorch and run it on a GPU. Any other suggestions to speed up warping, or existing implementations are also welcome.
 Code:
def warp_frame_04(frame1: numpy.ndarray, depth: numpy.ndarray, intrinsic: numpy.ndarray, transformation1: numpy.ndarray,
                  transformation2: numpy.ndarray, convert_to_uint: bool = True, verbose_log: bool = True):
    """
    Vectorized Forward warping. Nearest Neighbor.
    Offset requirement of warp_frame_03() overcome.
    mask: 1 if pixel found, 0 if no pixel found
    Drawback: Nearest neighbor, collision resolving not vectorized
    """
    height, width, _ = frame1.shape
    assert depth.shape == (height, width)
    transformation = numpy.matmul(transformation2, numpy.linalg.inv(transformation1))

    y1d = numpy.array(range(height))
    x1d = numpy.array(range(width))
    x2d, y2d = numpy.meshgrid(x1d, y1d)
    ones_2d = numpy.ones(shape=(height, width))
    ones_4d = ones_2d[:, :, None, None]
    pos_vectors_homo = numpy.stack([x2d, y2d, ones_2d], axis=2)[:, :, :, None]

    intrinsic_inv = numpy.linalg.inv(intrinsic)
    intrinsic_4d = intrinsic[None, None]
    intrinsic_inv_4d = intrinsic_inv[None, None]
    depth_4d = depth[:, :, None, None]
    trans_4d = transformation[None, None]

    unnormalized_pos = numpy.matmul(intrinsic_inv_4d, pos_vectors_homo)
    world_points = depth_4d * unnormalized_pos
    world_points_homo = numpy.concatenate([world_points, ones_4d], axis=2)
    trans_world_homo = numpy.matmul(trans_4d, world_points_homo)
    trans_world = trans_world_homo[:, :, :3]
    trans_norm_points = numpy.matmul(intrinsic_4d, trans_world)
    trans_pos = trans_norm_points[:, :, :2, 0] / trans_norm_points[:, :, 2:3, 0]
    trans_pos_int = numpy.round(trans_pos).astype('int')

    # Solve occlusions
    a = trans_pos_int.reshape(-1, 2)
    d = depth.ravel()
    b = numpy.unique(a, axis=0, return_index=True, return_counts=True)
    collision_indices = b[1][b[2] >= 2]  # Unique indices which are involved in collision
    for c1 in tqdm(collision_indices, disable=not verbose_log):
        cl = a[c1].copy()  # Collision Location
        ci = numpy.where((a[:, 0] == cl[0]) & (a[:, 1] == cl[1]))[0]  # Colliding Indices: Indices colliding for cl
        cci = ci[numpy.argmin(d[ci])]  # Closest Collision Index: Index of the nearest point among ci
        a[ci] = [-1, -1]
        a[cci] = cl
    trans_pos_solved = a.reshape(height, width, 2)

    # Offset both axes by 1 and set any out of frame motion to edge. Then crop 1-pixel thick edge
    trans_pos_offset = trans_pos_solved + 1
    trans_pos_offset[:, :, 0] = numpy.clip(trans_pos_offset[:, :, 0], a_min=0, a_max=width + 1)
    trans_pos_offset[:, :, 1] = numpy.clip(trans_pos_offset[:, :, 1], a_min=0, a_max=height + 1)

    warped_image = numpy.ones(shape=(height + 2, width + 2, 3)) * numpy.nan
    warped_image[trans_pos_offset[:, :, 1], trans_pos_offset[:, :, 0]] = frame1
    cropped_warped_image = warped_image[1:-1, 1:-1]
    mask = numpy.isfinite(cropped_warped_image)
    cropped_warped_image[~mask] = 0
    if convert_to_uint:
        final_warped_image = cropped_warped_image.astype('uint8')
    else:
        final_warped_image = cropped_warped_image
    mask = mask[:, :, 0]
    return final_warped_image, mask

 Code Explanation

I'm using the equations[1,2] to get pixel locations in view2
 Once I have the pixel locations, I need to figure out if there are any occlusions and if so, I have to pick the foreground pixels.
 `b = numpy.unique(a, axis=0, return_index=True, return_counts=True)` gives me unique locations.
 If multiple pixels from view1 map to a single pixel in view2 (collision), `return_counts` will give a value greater than 1.
 `collision_indices = b[1][b[2] >= 2]` gives indices which are involved in collision. Note that this gives only one index per collision.
 For each of such collision points, `ci = numpy.where((a[:, 0] == cl[0]) & (a[:, 1] == cl[1]))[0]` provides indices of all pixels from view1 which map to the same point in view2.
 `cci = ci[numpy.argmin(d[ci])]` gives the pixel index with lowest depth value.
 `a[ci] = [-1, -1]` and `a[cci] = cl` maps all other background pixels to location (-1,-1) which is out of frame and hence will be ignored.

[1] https://i.stack.imgur.com/s1D9t.png
[2] https://dsp.stackexchange.com/q/69890/32876


title: I am not able to load the .npy file which was generated in another system with different GPU serial number

body:
I have trained a model in a system with GPU device having serial number 2 and saved the training loss as .npy file. Now when I am loading it in another system having only one GPU with serial number 0, by giving command as follows:
Q=np.load('/home/rishabh/6DOF_Shape/TrainLoss1.npy',allow_pickle=True)
print(Q)
I am getting the error as follows:
RuntimeError: Attempting to deserialize object on CUDA device 2 but torch.cuda.device_count() is 1. Please use torch.load with map_location to map your storages to an existing device.
Then I wrote the following code:
device = torch.device("cuda:0")
S = torch.from_numpy(np.load('/home/rishabh/6DOF_Shape/TrainLoss1.npy',allow_pickle=True))
S = S.to(device)
A = torch.load(S,map_location=torch.device('cuda:0'))
But still I am getting the same error where I have initialized S.
Please help me to rectify this issue.


title: The training output of a simple pytorch model is different with evaluating

body:
I'm confused about why the training output is different from evaluating. I printed out the parameters after training and during evaluating to make sure they were the same. why the output is different?
The python version is 3.6, PyTorch is 1.4, Cuda version is 10.0.
The codes are the following. Are there any mistakes in the codes? Please point out the problem, thank you very much!
import torch
import torch.nn as nn
import random
import numpy as np

class Classifier(nn.Module):
    def __init__(self):
        super(Classifier, self).__init__()
        self.linear = nn.Linear(10, 2)

    def forward(self, x):
        output = self.linear(x)

        return output

if __name__ == "__main__":
    random.seed(2020)
    np.random.seed(2020)
    torch.manual_seed(2020)
    torch.cuda.manual_seed_all(2020)
    torch.backends.cudnn.deterministic = True

    model = Classifier().to('cuda')

    optimizer = torch.optim.Adam(model.parameters(), lr=3e-4)
    criterion = nn.CrossEntropyLoss()

    optimizer.zero_grad()
    model.zero_grad()

    data = torch.randn((4, 10)).to('cuda')
    label = torch.tensor([1, 0, 1, 0]).long().to('cuda')
    print('data', data)
    output = model(data)
    print('output', output)
    print('*'*20)

    loss = criterion(output, label)
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()
    model.zero_grad()

    for p in model.parameters():
        print('p', p)
    print('*' * 20)

    model.eval()
    with torch.no_grad():
        output = model(data)

    print('data', data)
    print('output', output)
    print('*' * 20)

    for p in model.parameters():
        print('p', p)
    print('*' * 20)


title: Why do I get test loss 0 and accuracy 1.0 constant for all epochs?

body:
I get test loss 0 and accuracy 1.0 for all epochs.
Calculate test loss with test loss = test loss / len(testloader)
and for accuracy = accuracy / len(testloader)
This can be seen below in the code
.....................................................................
.....................................................................
Screenshot:

Code :
device  = torch.device("cuda" if torch.cuda.is_available() else "cpu")

model = models.densenet121(pretrained=True)

for params in model.parameters():
  params.requires_grad = False

model.classifier = nn.Sequential(OrderedDict([
                                          ('fc1',nn.Linear(1024,256)),
                                          ('relu',nn.ReLU()),
                                          ('dropout',nn.Dropout(0.2)),
                                          ('fc2',nn.Linear(256,2)),
                                          ('output',nn.LogSoftmax(dim=1))]))

criterion = nn.NLLLoss()
optimizer = optim.Adam(model.classifier.parameters(),lr = 0.003)

model.to(device)
epochs = 50
steps = 0
running_loss = 0
print_every = 5

for epoch in range(epochs):
  for inputs, labels in trainloader:
    steps +=1

    inputs,labels = inputs.to(device),labels.to(device)

    optimizer.zero_grad()

    logps = model.forward(inputs)
    loss = criterion(logps,labels)
    loss.backward()
    optimizer.step()

    running_loss += loss.item()

    if steps % print_every == 0:
      test_loss = 0
      accuracy = 0

      model.eval()

      with torch.no_grad():
        for inputs,labels in testloader:
          inputs,labels = inputs.to(device),labels.to(device)

          logps = model.forward(inputs)
          batch_loss = criterion(logps,labels)

          test_loss += batch_loss.item()

          #calculate accuracy
          ps = torch.exp(logps)
          top_p,top_class = ps.topk(1,dim=1)
          equals = top_class == labels.view(*top_class.shape)
          accuracy += torch.mean(equals.type(torch.FloatTensor)).item()

      print(f'Epoch : {epoch+1}/{epochs}..'
            f'Training loss : {running_loss/print_every:.3f}...'
            f'Test loss: {test_loss/len(testloader):.3f}.. '
            f'Test accuracy: {accuracy/len(testloader):.3f}')

      running_loss = 0
      model.train()

My dataset folder structure:


title: How to sort a tensor by first dimension in libtorch in C++?

body:
I have the following 2D tensor and I would like to sort by the first dimension:
torch::Tensor tensor = torch::randn({3, 4});


title: Pytorch how to reshape/reduce the number of filters without altering the shape of the individual filters

body:
With a 3D tensor of shape (number of filters, height, width), how can one reduce the number of filters with a reshape which keeps the original filters together as whole blocks?
Assume the new size has dimensions chosen such that a whole number of the original filters can fit side by side in one of the new filters. So an original size of (4, 2, 2) can be reshaped to (2, 2, 4).
A visual explanation of the side by side reshape where you see the standard reshape will alter the individual filter shapes:

I have tried various pytorch functions such as gather and select_index but not found a way to get to the end result in a general manner (i.e. works for different numbers of filters and different filter sizes).
I think it would be easier to rearrange the tensor values after performing the reshape but could not get a tensor of the pytorch reshaped form:
[[[1,2,3,4],
  [5,6,7,8]],

 [[9,10,11,12],
  [13,14,15,16]]]

to:
[[[1,2,5,6],
  [3,4,7,8]],

 [[9,10,13,14],
  [11,12,15,16]]]

for completeness, the original tensor before reshaping:
[[[1,2],
  [3,4]],

 [[5,6],
  [7,8]],

 [[9,10],
  [11,12]],

 [[13,14],
  [15,16]]]


title: Running two different independent PyTorch programs on a single GPU

body:
I have a single NVIDIA GPU which has a memory of 16GB. I have to run two different (and independent; meaning, two different problems: one is a vision type task, another is NLP task) Python programs. The codes are written using PyTorch and both the codes can use GPU.
I have tested that program 1 takes roughly 5GB of GPU memory, and the rest is free. If I run the two programs, will it hamper the model performance or will it cause any process conflicts?
Linked question; but it does not necessarily mean PyTorch codes


title: How to feed a model output as another model's input in pytorch-lightning?

body:
I have some pytorch-lightning model I have already trained, using
...
checkpoint_callback = ModelCheckpoint(dirpath=lightning_checkpoints_path)
trainer = Trainer(deterministic=config.is_deterministic,
                  max_epochs=config.num_epochs,
                  default_root_dir=lightning_checkpoints_path,
                  logger=logger,
                  callbacks=[checkpoint_callback],
                  gpus=1
                  )
trainer.fit(model, datamodule=datamodule)

I now want to train another model that goes over the same datamodule, but this time each data point input to the new model should be the output from the already trained model, running in eval mode.
I couldn't find anything similar in the docs, and doing the same in plain Pytorch is very cumbersome which is why I went to lightning in the first place.

I want to do this to create a semi-supervised learning framework consisting of a VAE and a classifier.
This is explained here.


Please I am working on AdultDataset for a classification task
I found out: from dataset import AdultDataset
is giving the error below:
ImportError: cannot import name 'AdultDataset' from 'dataset'

Import Relevant Libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

import torch
import torch.nn as nn

import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import DataLoader

from dataset import AdultDataset

So when I am trying to create a 3 layer Feed Forward Neural network
using pytorch that takes as input the a dataset entries and classifies
if the individuals gains more or less than 50K (i.e., the fnlwgt label)
starting with the code below I get an error
train_dataset = AdultDataset(X_train, y_train)
test_dataset = AdultDataset(X_test, y_test)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-49-4ab31bdb6258> in <module>
      1 # Using norm_D01
      2
----> 3 train_dataset = AdultDataset(X_train, y_train)
      4 test_dataset = AdultDataset(X_test, y_test)
      5

NameError: name 'AdultDataset' is not defined


title: How to install Pytorch and Blender in one Docker image?

body:
I would like to utilize a git repository combining a computer graphic software named Blender 2.90 version and PyTorch GPU version 1.50 together. My question is how I can create a Dockerfile to have both of them in one Docker image?
This is the dockfile for Blender 2.90.
This is the Pytorch docker image I want to pull.


title: How to prepare data and perform training of autoencoder?

body:
I have a dataset of images that looks like this:
array([[[[0.35980392, 0.26078431, 0.14313725],
     [0.38137255, 0.26470588, 0.15196078],
     [0.51960784, 0.3745098 , 0.26176471],
     ...,
     [0.34313725, 0.22352941, 0.15      ],
     [0.30784314, 0.2254902 , 0.15686275],
     [0.28823529, 0.22843137, 0.16862745]],

    [[0.38627451, 0.28235294, 0.16764706],
     [0.45098039, 0.32843137, 0.21666667],
     [0.62254902, 0.47254902, 0.36470588],
     ...,
     [0.34607843, 0.22745098, 0.15490196],
     [0.30686275, 0.2245098 , 0.15588235],
     [0.27843137, 0.21960784, 0.16176471]],

    [[0.41568627, 0.30098039, 0.18431373],
     [0.51862745, 0.38529412, 0.27352941],
     [0.67745098, 0.52058824, 0.40980392],
     ...,
     [0.34901961, 0.22941176, 0.15588235],
     [0.29901961, 0.21666667, 0.14901961],
     [0.26078431, 0.20098039, 0.14313725]],

    ...,

This is how i download it:
data, attrs = fetch_dataset()

This is how fetch_dataset() function works:
import numpy as np
import os
from skimage.transform import resize
import skimage.io
import pandas as pd

def fetch_dataset(attrs_name = "lfw_attributes.txt",
                  images_name = "lfw-deepfunneled",
                  dx=80,dy=80,
                  dimx=45,dimy=45
):

#download if not exists
if not os.path.exists(images_name):
    print("images not found, donwloading...")
    os.system("wget http://vis-www.cs.umass.edu/lfw/lfw-deepfunneled.tgz -O tmp.tgz")
    print("extracting...")
    os.system("tar xvzf tmp.tgz && rm tmp.tgz")
    print("done")
    assert os.path.exists(images_name)

if not os.path.exists(attrs_name):
    print("attributes not found, downloading...")
    os.system("wget http://www.cs.columbia.edu/CAVE/databases/pubfig/download/%s" % attrs_name)
    print("done")

#read attrs
df_attrs = pd.read_csv("lfw_attributes.txt",sep='\t',skiprows=1,)
df_attrs = pd.DataFrame(df_attrs.iloc[:,:-1].values, columns = df_attrs.columns[1:])

#read photos
photo_ids = []
for dirpath, dirnames, filenames in os.walk(images_name):
    for fname in filenames:
        if fname.endswith(".jpg"):
            fpath = os.path.join(dirpath,fname)
            photo_id = fname[:-4].replace('_',' ').split()
            person_id = ' '.join(photo_id[:-1])
            photo_number = int(photo_id[-1])
            photo_ids.append({'person':person_id,'imagenum':photo_number,'photo_path':fpath})

photo_ids = pd.DataFrame(photo_ids)
# print(photo_ids)
#mass-merge
#(photos now have same order as attributes)
df = pd.merge(df_attrs,photo_ids,on=('person','imagenum'))

assert len(df)==len(df_attrs),"lost some data when merging dataframes"

# print(df.shape)
#image preprocessing
all_photos =df['photo_path'].apply(skimage.io.imread)\
                            .apply(lambda img:img[dy:-dy,dx:-dx])\
                            .apply(lambda img: resize(img,[dimx,dimy]))

all_photos = np.stack(all_photos.values)#.astype('uint8')
all_attrs = df.drop(["photo_path","person","imagenum"],axis=1)

return all_photos,all_attrs

Next I'm trying to convert my dataset to Tensor:
import torch
import torchvision.transforms
from torchvision import transforms as transforms

class MyDataset(torch.utils.data.Dataset):
def __init__(self, data):
        self.dataset = data

def __getitem__(self, idx):
    sample = self.dataset[idx]
    data, label = sample[0], sample[1]

    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])
    return transform(data), torch.tensor(label)

def __len__(self):
    return len(self.dataset)

Then:
from sklearn.model_selection import train_test_split

train, val = train_test_split(data, test_size=0.25, random_state=42)

train_dataset = MyDataset(train)
val_dataset = MyDataset(val)

Autoencoder class:
from copy import deepcopy

class Autoencoder(nn.Module):
def __init__(self):
    #<определите архитектуры encoder и decoder>
      super(Autoencoder,self).__init__()

      self.encoder = nn.Sequential(
        nn.Conv2d(3, 6, kernel_size=5),
        nn.ReLU(True),
        nn.Conv2d(6,16,kernel_size=5),
        nn.ReLU(True))
      self.decoder = nn.Sequential(
        nn.Conv2d(16, 6, kernel_size=5),
        nn.ReLU(True),
        nn.Conv2d(6,3,kernel_size=5),
        nn.ReLU(True))

def forward(self, x):

    latent_code = self.encoder(x)
    reconstruction = self.decoder(latent_code)

    return reconstruction, latent_code

Then:
criterion = nn.BCELoss()

autoencoder = Autoencoder()

optimizer = torch.optim.Adam(autoencoder.parameters())

The training itself:
#<тут Ваш код тренировки автоэнкодера>
num_epochs = 5
for epoch in range(num_epochs):
for x in train_dataset:
    img, _ = x
    img = Variable(img).cpu()

    output = autoencoder(img)
    loss = criterion(output, img)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
print('epoch [{}/{}], loss:{:.4f}'.format(epoch+1, num_epochs, loss.data()))

And I get this error:
RuntimeError: output with shape [1, 45, 3] doesn't match the broadcast shape [3, 45, 3]

I think I'm making mistakes somewhere in preparing my data. Could someome please explain me how to do it properly in my case? Thanks for any help.


title: How to convert Adjacency matrix to edge_list in Torch?

body: How to use from_scipy_sparse_matrix() in pytorch-geometric to convert an adjacency matrix in Tensor to edge_list in Tensor ?


title: How to Integrate a Python model into an app that works on IOS and Android

body:
I am a beginner in developing mobile apps, I have a python model with me, and I am trying to build a mobile app using flutter or any other frameworks like react-native. Here the problem is I don't know How to integrate a python ML model into an app that works both on Android and IOS. There is a  path like converting a model into  TensorFlow lite  and import this file in android studios gradle and use it in-app
My primary concerns are

There are many files in my python ML model and I don't know how and what to convert into TensorFlow lite .tflite

The model mainly revolves around a pre-built standard model and a .py file that analyses the pre-built model, I don't know what to convert into (.tflite)

Is a single .tflite sufficient to handle the whole app (as many YouTubers have just used a single .tflite file in their app development).


title: Switching from CPU to GPU in Jupyter notebook

body:
%%time
%run ./DeepRecommender/data_utils/netflix_data_convert.py training_set NF_DATA

I am trying to run these commands in a cell in Jupyter Notebook. But I would like to make this process run on my GPU (Nvidia GTX 1050Ti 4GB-laptop) since it is computationally very expensive. I have installed CUDA Toolkit properly, I have verified it.
My question differs from the How to make Jupyter Notebook to run on GPU?.
How do I use .cuda() / .to() or something else??
Because I'm running that python from the Notebook cell itself, what should I do  now?
netflix_data_convert.py
# Copyright (c) 2017 NVIDIA Corporation
from os import listdir, path, makedirs
import random
import sys
import time
import datetime

def print_stats(data):
  total_ratings = 0
  print("STATS")
  for user in data:
    total_ratings += len(data[user])
  print("Total Ratings: {}".format(total_ratings))
  print("Total User count: {}".format(len(data.keys())))

def save_data_to_file(data, filename):
  with open(filename, 'w') as out:
    for userId in data:
      for record in data[userId]:
        out.write("{}\t{}\t{}\n".format(userId, record[0], record[1]))

def create_NETFLIX_data_timesplit(all_data,
                                  train_min,
                                  train_max,
                                  test_min,
                                  test_max):
  """
  Creates time-based split of NETFLIX data into train, and (validation, test)
  :param all_data:
  :param train_min:
  :param train_max:
  :param test_min:
  :param test_max:
  :return:
  """
  train_min_ts = time.mktime(datetime.datetime.strptime(train_min,"%Y-%m-%d").timetuple())
  train_max_ts = time.mktime(datetime.datetime.strptime(train_max, "%Y-%m-%d").timetuple())
  test_min_ts = time.mktime(datetime.datetime.strptime(test_min, "%Y-%m-%d").timetuple())
  test_max_ts = time.mktime(datetime.datetime.strptime(test_max, "%Y-%m-%d").timetuple())

  training_data = dict()
  validation_data = dict()
  test_data = dict()

  train_set_items = set()

  for userId, userRatings in all_data.items():
    time_sorted_ratings = sorted(userRatings, key=lambda x: x[2])  # sort by timestamp
    for rating_item in time_sorted_ratings:
      if rating_item[2] >= train_min_ts and rating_item[2] <= train_max_ts:
        if not userId in training_data:
          training_data[userId] = []
        training_data[userId].append(rating_item)
        train_set_items.add(rating_item[0]) # keep track of items from training set
      elif rating_item[2] >= test_min_ts and rating_item[2] <= test_max_ts:
        if not userId in training_data: # only include users seen in the training set
          continue
        p = random.random()
        if p <=0.5:
          if not userId in validation_data:
            validation_data[userId] = []
          validation_data[userId].append(rating_item)
        else:
          if not userId in test_data:
            test_data[userId] = []
          test_data[userId].append(rating_item)

  # remove items not not seen in training set
  for userId, userRatings in test_data.items():
    test_data[userId] = [rating for rating in userRatings if rating[0] in train_set_items]
  for userId, userRatings in validation_data.items():
    validation_data[userId] = [rating for rating in userRatings if rating[0] in train_set_items]

  return training_data, validation_data, test_data

def main(args):
  # create necessary folders:
  for output_dir in [
    "Netflix/N3M_TRAIN", "Netflix/N3M_VALID", "Netflix/N3M_TEST", "Netflix/N6M_TRAIN",
    "Netflix/N6M_VALID", "Netflix/N6M_TEST", "Netflix/N1Y_TRAIN", "Netflix/N1Y_VALID",
    "Netflix/N1Y_TEST", "Netflix/NF_TRAIN", "Netflix/NF_VALID", "Netflix/NF_TEST"]:
    makedirs(output_dir, exist_ok=True)

  user2id_map = dict()
  item2id_map = dict()
  userId = 0
  itemId = 0
  all_data = dict()

  folder = args[1]
  out_folder = args[2]

  text_files = [path.join(folder, f)
                for f in listdir(folder)
                if path.isfile(path.join(folder, f)) and ('.txt' in f)]

  for text_file in text_files:
    with open(text_file, 'r') as f:
      print("Processing: {}".format(text_file))
      lines = f.readlines()
      item = int(lines[0][:-2]) # remove newline and :
      if not item in item2id_map:
        item2id_map[item] = itemId
        itemId += 1

      for rating in lines[1:]:
        parts = rating.strip().split(",")
        user = int(parts[0])
        if not user in user2id_map:
          user2id_map[user] = userId
          userId += 1
        rating = float(parts[1])
        ts = int(time.mktime(datetime.datetime.strptime(parts[2],"%Y-%m-%d").timetuple()))
        if user2id_map[user] not in all_data:
          all_data[user2id_map[user]] = []
        all_data[user2id_map[user]].append((item2id_map[item], rating, ts))

  print("STATS FOR ALL INPUT DATA")
  print_stats(all_data)

  # Netflix full
  (nf_train, nf_valid, nf_test) = create_NETFLIX_data_timesplit(all_data,
                                                                "1999-12-01",
                                                                "2005-11-30",
                                                                "2005-12-01",
                                                                "2005-12-31")
  print("Netflix full train")
  print_stats(nf_train)
  save_data_to_file(nf_train, out_folder + "/NF_TRAIN/nf.train.txt")
  print("Netflix full valid")
  print_stats(nf_valid)
  save_data_to_file(nf_valid, out_folder + "/NF_VALID/nf.valid.txt")
  print("Netflix full test")
  print_stats(nf_test)
  save_data_to_file(nf_test, out_folder + "/NF_TEST/nf.test.txt")

  (n3m_train, n3m_valid, n3m_test) = create_NETFLIX_data_timesplit(all_data,
                                                                   "2005-09-01",
                                                                   "2005-11-30",
                                                                   "2005-12-01",
                                                                   "2005-12-31")
  print("Netflix 3m train")
  print_stats(n3m_train)
  save_data_to_file(n3m_train, out_folder+"/N3M_TRAIN/n3m.train.txt")
  print("Netflix 3m valid")
  print_stats(n3m_valid)
  save_data_to_file(n3m_valid, out_folder + "/N3M_VALID/n3m.valid.txt")
  print("Netflix 3m test")
  print_stats(n3m_test)
  save_data_to_file(n3m_test, out_folder + "/N3M_TEST/n3m.test.txt")

  (n6m_train, n6m_valid, n6m_test) = create_NETFLIX_data_timesplit(all_data,
                                                                   "2005-06-01",
                                                                   "2005-11-30",
                                                                   "2005-12-01",
                                                                   "2005-12-31")
  print("Netflix 6m train")
  print_stats(n6m_train)
  save_data_to_file(n6m_train, out_folder+"/N6M_TRAIN/n6m.train.txt")
  print("Netflix 6m valid")
  print_stats(n6m_valid)
  save_data_to_file(n6m_valid, out_folder + "/N6M_VALID/n6m.valid.txt")
  print("Netflix 6m test")
  print_stats(n6m_test)
  save_data_to_file(n6m_test, out_folder + "/N6M_TEST/n6m.test.txt")

  # Netflix 1 year
  (n1y_train, n1y_valid, n1y_test) = create_NETFLIX_data_timesplit(all_data,
                                                                   "2004-06-01",
                                                                   "2005-05-31",
                                                                   "2005-06-01",
                                                                   "2005-06-30")
  print("Netflix 1y train")
  print_stats(n1y_train)
  save_data_to_file(n1y_train, out_folder + "/N1Y_TRAIN/n1y.train.txt")
  print("Netflix 1y valid")
  print_stats(n1y_valid)
  save_data_to_file(n1y_valid, out_folder + "/N1Y_VALID/n1y.valid.txt")
  print("Netflix 1y test")
  print_stats(n1y_test)
  save_data_to_file(n1y_test, out_folder + "/N1Y_TEST/n1y.test.txt")

if __name__ == "__main__":
    main(sys.argv)

training_set (this is the folder under the root directory of DeepRecommender folder, training_set contains the dataset which I got from Netflix Dataset)
Will this statement do something in favor of what I'm expecting?
torch.cuda.set_device('cuda:0')

cuda:0 refers to my GPU.


title: How to read the predicted label of a Neural Network with Cross Entropy Loss?

body:
I am using a neural network to predict the quality of the Red Wine dataset, available on UCI machine Learning, using Pytorch, and Cross-Entropy Loss as the loss function.
This is my code:
input_size = len(input_columns)
hidden_size = 12
output_size = 6 #because there are 6 classes

#Loss function
loss_fn = F.cross_entropy

class WineQuality(nn.Module):
    def __init__(self):
        super().__init__()
        # input to hidden layer
        self.linear1 = nn.Linear(input_size, hidden_size)
        # hidden layer and output
        self.linear2 = nn.Linear(hidden_size, output_size)

    def forward(self, xb):
        out = self.linear1(xb)
        out = F.relu(out)
        out = self.linear2(out)
        return out

    def training_step(self, batch):
        inputs, targets = batch
        # Generate predictions
        out = self(inputs)
        # Calcuate loss
        loss = loss_fn(out,torch.argmax(targets, dim=1))
        return loss

    def validation_step(self, batch):
        inputs, targets = batch
        # Generate predictions
        out = self(inputs)
        # Calculate loss
        loss = loss_fn(out, torch.argmax(targets, dim=1))
        return {'val_loss': loss.detach()}

    def validation_epoch_end(self, outputs):
        batch_losses = [x['val_loss'] for x in outputs]
        epoch_loss = torch.stack(batch_losses).mean()   # Combine losses
        return {'val_loss': epoch_loss.item()}

    def epoch_end(self, epoch, result, num_epochs):
        # Print result every 100th epoch
        if (epoch+1) % 100 == 0 or epoch == num_epochs-1:
            print("Epoch [{}], val_loss: {:.4f}".format(epoch+1, result['val_loss']))

model = WineQuality()

def evaluate(model, val_loader):
    outputs = [model.validation_step(batch) for batch in val_loader]
    return model.validation_epoch_end(outputs)

def fit(epochs, lr, model, train_loader, val_loader, opt_func=torch.optim.SGD):
    history = []
    optimizer = opt_func(model.parameters(), lr)
    for epoch in range(epochs):
        # Training Phase
        for batch in train_loader:
            loss = model.training_step(batch)
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
        # Validation phase
        result = evaluate(model, val_loader)
        model.epoch_end(epoch, result, epochs)
        history.append(result)
    return history

loss_value = evaluate(model, valid_dl)

#model=WineQuality()
epochs = 1000
lr = 1e-5
history = fit(epochs, lr, model, train_loader, val_loader)

I can see that the model is good and that the loss decreases. The problem is when I have to do a prediction on an example:
def predict_single(input, target, model):
    inputs = input.unsqueeze(0)
    predictions = model(inputs)
    prediction = predictions[0].detach()
    print("Input:", input)
    print("Target:", target)
    print("Prediction:", prediction)
    return prediction

input, target = val_df[1]
prediction = predict_single(input, target, model)

This returns:
Input: tensor([0.8705, 0.3900, 2.1000, 0.0650, 4.1206, 3.3000, 0.5300, 0.2610])
Target: tensor([6.])
Prediction: tensor([ 3.6465,  0.2800, -0.4561, -1.6733, -0.6519, -0.1650])

I want to see what are associated these logits, in the sense that I know that the highest logit is associated with the predicted class, but I want to see that class. I also applied softmax to rescale these values in a probability:
prediction = F.softmax(prediction)
print(prediction)
output = model(input.unsqueeze(0))
_,pred = output.max(1)
print(pred)

And the output is the following:
tensor([0.3296, 0.1361, 0.1339, 0.1324, 0.1335, 0.1346])
tensor([0])

I don't know what is that tensor([0]). I expect my predicted label, a value like 6.1 if the target is 6. But I am not able to obtain this.


title: quantization vector with numpy/pytorch

body:
I have created quantized values as: {0.0, 0.5, 1.0} and would like to do quantization for a vector based on the set. For instance, given vector [0.1, 0.1, 0.9, 0.8, 0.6, 0.6] would be transferred into [0.0, 0.0, 1.0, 1.0, 0.5, 0.5].
Please guide me fastest way using python/numpy/pytorch. Thank you very much!


title: Understanding Jacobian tensor gradients in pytorch

body:
I was going through official pytorch tut, where it explains tensor gradients and Jacobian products as follows:

Instead of computing the Jacobian matrix itself, PyTorch allows you to compute Jacobian Product  for a given input vector v=(v1…vm). This is achieved by calling backward with v as an argument:

inp = torch.eye(5, requires_grad=True)
out = (inp+1).pow(2)
out.backward(torch.ones_like(inp), retain_graph=True)
print("First call\n", inp.grad)
out.backward(torch.ones_like(inp), retain_graph=True)
print("\nSecond call\n", inp.grad)
inp.grad.zero_()
out.backward(torch.ones_like(inp), retain_graph=True)
print("\nCall after zeroing gradients\n", inp.grad)

Ouptut:
First call
 tensor([[4., 2., 2., 2., 2.],
        [2., 4., 2., 2., 2.],
        [2., 2., 4., 2., 2.],
        [2., 2., 2., 4., 2.],
        [2., 2., 2., 2., 4.]])

Second call
 tensor([[8., 4., 4., 4., 4.],
        [4., 8., 4., 4., 4.],
        [4., 4., 8., 4., 4.],
        [4., 4., 4., 8., 4.],
        [4., 4., 4., 4., 8.]])

Call after zeroing gradients
 tensor([[4., 2., 2., 2., 2.],
        [2., 4., 2., 2., 2.],
        [2., 2., 4., 2., 2.],
        [2., 2., 2., 4., 2.],
        [2., 2., 2., 2., 4.]])

Though I get what is Jacobian matrix is, I didnt get how is this Jacobian product is calculated.
Here, are different tensors I tried to print out to get understanding:
>>> out
tensor([[4., 1., 1., 1., 1.],
        [1., 4., 1., 1., 1.],
        [1., 1., 4., 1., 1.],
        [1., 1., 1., 4., 1.],
        [1., 1., 1., 1., 4.]], grad_fn=<PowBackward0>)
>>> torch.eye(5)
tensor([[1., 0., 0., 0., 0.],
        [0., 1., 0., 0., 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 0., 1., 0.],
        [0., 0., 0., 0., 1.]])
>>> torch.ones_like(inp)
tensor([[1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1.],
        [1., 1., 1., 1., 1.]])
>>> inp
tensor([[1., 0., 0., 0., 0.],
        [0., 1., 0., 0., 0.],
        [0., 0., 1., 0., 0.],
        [0., 0., 0., 1., 0.],
        [0., 0., 0., 0., 1.]], requires_grad=True)

But I didnt get how the tuts output is calculated. Can someone explain a bit of Jacobian matrix with calculations done in this example?


title: Where is one suppose to call torch.distributed.destroy_process_group() in Pytorch?

body:
I noticed that the docs do not have that function. Thus, it's unclear where one should be calling that. Does one have to:

call it at the end of each worker code (i.e. inside of mp.spawn)
or call it outside of mp.spawn i.e. by the main process

Note there is a gitissue requesting to put this function on the docs: https://github.com/pytorch/pytorch/issues/48203
this is an example of what 2 means:
def test_setup():
    print('test_setup')
    if torch.cuda.is_available():
        world_size = torch.cuda.device_count()
    else:
        world_size = 4
    master_port = find_free_port()
    mp.spawn(setup_process, args=(world_size, master_port), nprocs=world_size)
    dist.destroy_process_group()
    print('successful test_setup!')


title: Flask app serving GPT2 on Google Cloud Run not persisting downloaded files?

body:
I have a Flask app running on Google Cloud Run, which needs to download a large model (GPT-2 from huggingface). This takes a while to download, so I am trying to set up so that it only downloads on deployment and then just serves this up for subsequent visits. That is I have the following code in a script that is imported by my main flask app app.py:
import torch
# from transformers import GPT2Tokenizer, GPT2LMHeadModel
from transformers import AutoTokenizer, AutoModelWithLMHead
# Disable gradient calculation - Useful for inference
torch.set_grad_enabled(False)

# Check if gpu or cpu
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Load tokenizer and model
try:
    tokenizer = AutoTokenizer.from_pretrained("./gpt2-xl")
    model = AutoModelWithLMHead.from_pretrained("./gpt2-xl")
except Exception as e:

    print('no model found! Downloading....')

    AutoTokenizer.from_pretrained('gpt2').save_pretrained('./gpt2-xl')
    AutoModelWithLMHead.from_pretrained('gpt2').save_pretrained('./gpt2-xl')
    tokenizer = AutoTokenizer.from_pretrained("./gpt2-xl")
    model = AutoModelWithLMHead.from_pretrained("./gpt2-xl")

model = model.to(device)

This basically tries to load the the downloaded model, and if that fails it downloads a new copy of the model. I have autoscaling set to a minimum of 1 which I thought would mean something would always be running and therefore the downloaded file would persist even after activity. But it keeps having to redownload the model which freezes up the app when some people try to use it. I am trying to recreate something like this app https://text-generator-gpt2-app-6q7gvhilqq-lz.a.run.app/ which does not appear to have the same load time issue . In the flask app itself I have the following:
@app.route('/')
@cross_origin()
def index():
    prompt = wp[random.randint(0, len(wp)-1)]
    res = generate(prompt, size=75)
    generated = res.split(prompt)[-1] + '\n \n...TO BE CONTINUED'
    #generated = prompt
    return flask.render_template('main.html', prompt = prompt, output = generated)

if __name__ == "__main__":
    app.run(host='0.0.0.0',
            debug=True,
            port=PORT)

But it seems to redownload the models every few hours...how can I avoid having the app re-downloading the models and the app freezing for those who want to try it?


title: How to classify all words in a sentence with a context?

body:
I have the names of the companies (in Russian). The name can contain abbreviations, words with capital letters, words with lowercase letters, and mixed words. The model is trained according to the principle: At the input, the name is given in upper case, at the output - in the "correct" version. For example:
ОТДЕЛЕНИЕ СТД РФ (ВТО) - СТД РЕСПУБЛИКИ АДЫГЕЯ -> Отделение СТД РФ (ВТО) - СТД Республики Адыгея

ОБЩЕСТВО С ОГРАНИЧЕННОЙ ОТВЕТСТВЕННОСТЬЮ "СИГМА" -> Общество с ограниченной ответственностью "СИГМА"

"ЭЛЕКТРОПРОФСОЮЗ" РБ ОО - "ВЭП" -> "Электропрофсоюз" РБ ОО - "ВЭП"

And then I have to predict sentences that I don't know answers for.
I tried to make a character-based seq2seq model with attention and bidirectional GRU, but with all the hyperparameters I tried, it seems to be underfitted. It generates the beginning of phrases quite well, but at the end it breaks down.
Now I think I need to work with word tokens. But I do not know if there are methods for classifying words from a single text in a context.
I want to tokenize a sentence, and assign a property to each word, what it is: uppercase, lowercase, or with a capital first letter. Also I need to do something with mixed words like "МосГосПаравоз" (or like "McDonald's" which not only starts with a capital letter, but also contains it inside.).
Perhaps you need a completely different approach. I will be happy to accept your help
UPD. English interpretation of examples:
"SIGMA" LIMITED LIABILITY COMPANY -> "SIGMA" Limited liability company
"SIGMA" LLT -> "SIGMA" LLT
PJSC "GAZPROM"-> PJSC "Gazprom"
STATE BUDGETARY EDUCATIONAL INSTITUTION OF THE CITY OF MOSCOW "LYCEUM NO. 1568" -> State budgetary Educational institution of the city of Moscow "Lyceum No. 1568"


title: Are the random transforms applied at each epoch in my Pytorch convolutional neural net? (data augmentation)

body:
I'm new to Pytorch and I try to make a convolutional neural net to classify a set of images (personal iris recognition problem). My issue is that I have a small number of images (10 classes and 20 images per class). I tried to make data augmentation (random transforms for every epoch) but I'm not sure that these are applied at each epoch as I entended. Here's my code. If anyone can confirm that I'm doing it right or if it's not ok, is there a way to make the transforms inside the loop?
from torch import utils, nn, optim, no_grad
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from ConvNet import ConvNet
from ImagesDataset import ImagesDataset, AddGaussianNoise

DATABASE_PATH = "C://Users//Maria//Downloads//ees//CASIA-IrisV2"
MODEL_PATH = "entire_model.pt"
dataArray = []

# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# data augmentation by applying some transforms randomly for every batch
transform = transforms.Compose([transforms.RandomCrop(5), transforms.RandomHorizontalFlip(p=0.1),
                                transforms.ColorJitter(brightness=0.1, contrast=0.2, saturation=0, hue=0),
                                AddGaussianNoise(0.1, 0.05), transforms.ToTensor()])
dataset = ImagesDataset(csv_file="generate_csv//generate_csv_correctly_detected.csv", root_dir=DATABASE_PATH, transform=transforms.ToTensor())
num_epochs = 300
num_classes = 10
batch_size = 100
learning_rate = 0.01

# the dataset is partitioned in 5 subsets to perform cross validation
sum_percents = 0
data_set = utils.data.random_split(dataset, [40, 40, 40, 40, 40])

for i in range(5):
    test_set = data_set[i]
    train_set = []
    for j in range(5):
        if j != i:
            train_set += data_set[j]

    train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True)
    test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=True)

    model = ConvNet(0).to(device)

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    # Train the model
    total_step = len(train_loader)
    loss_list = []
    acc_list = []

    # delete contents of loss1 file
    file = open("loss1.txt", "r+")
    file.truncate(0)
    file.close()

    for epoch in range(num_epochs):
        print("Epoch: " + str(epoch))
        for i, (images, labels) in enumerate(train_loader):
            # Run the forward pass
            images = images.to(device)
            labels = labels.to(device)
            outputs = model(images)
            loss = criterion(outputs, labels)
            # set the gradients to zero
            optimizer.zero_grad()
            # compute gradients
            loss.backward()
            # update the parameters
            optimizer.step()

            # Track the accuracy
            total = labels.size(0)
            _, predicted = torch.max(outputs.data, 1)
            correct = (predicted == labels).sum().item()
            acc_list.append(correct / total)

    # Save
    torch.save(model, MODEL_PATH)

    # Test the model
    model.eval()
    with no_grad():
        correct = 0
        total = 0
        for images, labels in test_loader:
            outputs = model(images)
            _, predicted = torch.max(outputs.data, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

        print('Test Accuracy of the model on the 132 test images: {} %'.format((correct / total) * 100))
    sum_percents += (correct / total) * 100
print('Average accuracy is {}%'.format((sum_percents/5)))