title: How to generate n-dimensional dome, sampled uniformly in numpy?

body:
For the 2d case, generating k n dimensional arcs, I have the following working code. For those who don't know torch, just think of it as numpy.
    def _create_data_clusters(self):
        self._clusters = [torch.zeros((self._cluster_size, self._data_dim)) for _ in range(self._n_clusters)]
        centers_radius = self._centers_radius
        for i, c in enumerate(self._clusters):
            r, x, y = 3.0, centers_radius * np.cos(i * np.pi * 2 / self._n_clusters), centers_radius * np.sin(
                i * np.pi * 2 / self._n_clusters)
            cluster_length = 1.1
            cluster_start = i * 2 * np.pi / self._n_clusters
            cluster_end = cluster_length * (i + 1) * 2 * np.pi / self._n_clusters
            cluster_inds = torch.linspace(start=cluster_start, end=cluster_end, steps=self._cluster_size,
                                          dtype=torch.float)
            c[:, 0] = r * torch.sin(cluster_inds) + y
            c[:, 1] = r * torch.cos(cluster_inds) + x

creates

I want to generalize for n dimensions.
How to do that?
I thought about sampling a gaussian, but this would not be exact.
Surely there is some magic formula for this.

Full runnable code for completeness
from torch.utils.data import Dataset
import matplotlib.pyplot as plt
import torch
import numpy as np
from Testing.Research.config.ConfigProvider import ConfigProvider

class ClustersDataset(Dataset):
    __default_dataset = None
    __default_dataset_train = None
    __default_dataset_val = None
    __default_dataset_test = None

    def __init__(self, cluster_size: int, noise_factor: float = 0, transform=None, n_clusters=2, centers_radius=4.0, data_dim=2):
        super(ClustersDataset, self).__init__()
        self._cluster_size = cluster_size
        self._noise_factor = noise_factor
        self._n_clusters = n_clusters
        self._centers_radius = centers_radius
        self._transform = transform
        self._size = self._cluster_size * self._n_clusters
        self._data_dim = data_dim

        self._create_data_clusters()
        self._combine_clusters_to_array()
        self._normalize_data()
        self._add_noise()

        self._plot()
        pass

    @staticmethod
    def clusters_dataset_by_config(transform=None):
        if ClustersDataset.__default_dataset is not None:
            return \
                ClustersDataset.__default_dataset, \
                ClustersDataset.__default_dataset_train, \
                ClustersDataset.__default_dataset_val, \
                ClustersDataset.__default_dataset_test
        config = ConfigProvider.get_config()
        default_dataset = ClustersDataset(
            cluster_size=config.cluster_size,
            noise_factor=config.noise_factor,
            transform=transform,
            n_clusters=config.n_clusters,
            centers_radius=config.centers_radius,
        )

        train_size = int(config.train_size * len(default_dataset))
        val_size = int(config.val_size * len(default_dataset))
        test_size = len(default_dataset) - train_size - val_size
        train_dataset, val_dataset, test_dataset = \
            torch.utils.data.random_split(default_dataset, [train_size, val_size, test_size])

        ClustersDataset.__default_dataset = default_dataset
        ClustersDataset.__default_dataset_train = train_dataset
        ClustersDataset.__default_dataset_val = val_dataset
        ClustersDataset.__default_dataset_test = test_dataset

        return default_dataset, train_dataset, val_dataset, test_dataset

    def _create_data_clusters(self):
        self._clusters = [torch.zeros((self._cluster_size, self._data_dim)) for _ in range(self._n_clusters)]
        centers_radius = self._centers_radius
        for i, c in enumerate(self._clusters):
            r, x, y = 3.0, centers_radius * np.cos(i * np.pi * 2 / self._n_clusters), centers_radius * np.sin(
                i * np.pi * 2 / self._n_clusters)
            cluster_length = 1.1
            cluster_start = i * 2 * np.pi / self._n_clusters
            cluster_end = cluster_length * (i + 1) * 2 * np.pi / self._n_clusters
            cluster_inds = torch.linspace(start=cluster_start, end=cluster_end, steps=self._cluster_size,
                                          dtype=torch.float)
            c[:, 0] = r * torch.sin(cluster_inds) + y
            c[:, 1] = r * torch.cos(cluster_inds) + x

    def _plot(self):
        plt.figure()
        plt.scatter(self._noisy_values[:, 0], self._noisy_values[:, 1], s=1, color='b', label="noisy_values")
        plt.scatter(self._values[:, 0], self._values[:, 1], s=1, color='r', label="values")
        plt.legend(loc="upper left")
        plt.show()

    def _combine_clusters_to_array(self):
        size = self._size
        self._values = torch.zeros(size, 2)
        self._labels = torch.zeros(size, dtype=torch.long)
        for i, c in enumerate(self._clusters):
            self._values[i * self._cluster_size: (i + 1) * self._cluster_size, :] = self._clusters[i]
            self._labels[i * self._cluster_size: (i + 1) * self._cluster_size] = i

    def _add_noise(self):
        size = self._size

        mean = torch.zeros(size, 2)
        std = torch.ones(size, 2)
        noise = torch.normal(mean, std)
        self._noisy_values = torch.zeros(size, 2)
        self._noisy_values[:] = self._values
        self._noisy_values = self._noisy_values + noise * self._noise_factor

    def _normalize_data(self):
        values_min, values_max = torch.min(self._values), torch.max(self._values)
        self._values = (self._values - values_min) / (values_max - values_min)
        self._values = self._values * 2 - 1

    def __len__(self):
        return self._size  # number of samples in the dataset

    # def __getitem__(self, index):
    #     item = self._values[index, :]
    #     noisy_item = self._noisy_values[index, :]
    #     # if self._transform is not None:
    #     #     noisy_item = self._transform(item)
    #     return item, noisy_item, self._labels[index]

    # def __getitem__(self, index):
    #     item = self._values[index, :]
    #     noisy_item = self._noisy_values[index, :]
    #     # if self._transform is not None:
    #     #     noisy_item = self._transform(item)
    #     return item, self._labels[index]

    def __getitem__(self, index):
        item = self._values[index, :]
        noisy_item = self._noisy_values[index, :]
        # if self._transform is not None:
        #     noisy_item = self._transform(item)
        return (item, noisy_item), self._labels[index]

    @property
    def values(self):
        return self._values

    @property
    def noisy_values(self):
        return self._noisy_values

_default_dataset, _train_dataset, _val_dataset, _test_dataset = \
                ClustersDataset.clusters_dataset_by_config()


title: High %wa from top command while running PyTorch

body:
Everything was fine yesterday, but I noticed high dataloading time this morning.
How do I go about trying to figure out how to fix this?
You can see the screenshots below.
top:

process state:

iotop:

iostat:


title: Specified procedure couldn't be found

body:
I have installed Pytorch but when I run it , it gives the following error:
OSError: [WinError 127] The specified procedure could not be found. Error loading "c:\users\fatima.arshad\appdata\local\continuum\anaconda2\envs\web_scraping\lib\site-packages\torch\lib\caffe2.dll" or one of its dependencies.

I have only tried and imported it like this:
import torchvision


title: What's the best way of handling the GAN training output?

body:
Supervising the training of GANs usually involves outputting not only metrics, but also images at a certain interval of epochs. My application also involves printing tables. I use jupyter notebooks, but just printing it all on the notebooks makes each notebook for each experiment way too large (+100 MB), and the internet browser gets slow and crashes often because of that.
I suppose the usual practice would be to save the image outputs somewhere else (either with tensorboard or just plain image files), but that would not be ideal for me because I like to observe each image together with the text/table output relative to it's epoch. It would be even better if I could save the entire training output into a single file, so I could just scroll down through it, observing each epoch output with text/table/image, just like the jupyter notebook output. Is there a way of implementing this? Or perhaps a better way that I'm not considering?
Thank you and sorry if this question is somehow inadequate. If so, let me know and I'll delete it.


title: How can I choose specific output and calculate loss for the that output in pytorch?

body:
I am using a multilingual BERT model from hugging face, and doing some additional pretraining using MLM.
However, it is a bit different. I want to get mean value of sequence of masks and calculate the loss.
For example,
**model = AutoModel.from_pretrained('bert-base-multilingual-cased')
input = [MASK][MASK][MASK] is a great guy.
label = [PERSON] is a great guy.**

I want to train the model using the loss between mean value of three [MASK] tokens and a [PERSON] token


title: Using Kmeans to find false positives and negatives, how to filter with cluster

body:
I have a dataset that I want to filter to find false positives and negatives.
The data is similar to below,

My current approach is as follows.
Looking at the distribution of datasets where an event has happened,
And then the same thing with a point at 0.
By using kmeans cluster, as below,
Hi everyone,
I am trying to use K-Means clustering to filter out error data.
this is the code that feeds into the error data,
start_tot = time.time()
#group data
df_cluster_noneEvent = df_cluster[df_cluster.event==1]

testthis = df_cluster_noneEvent.groupby('date')['feature1','feature2','feature3','feature4','feature5','feature6','feature7','feature8'].std()

model = cluster.MiniBatchKMeans(n_clusters=3)
model.fit(testthis.fillna(0))
testthis['kmeans_4'] = model.predict(testthis.fillna(0))
end_tot = time.time()

testthis.head()

I can see where there a clumps of data for each event type (using mean for none event, STD for event).
How do I use the clusters given to filter the dataset?
Using
model.cluster_centers_

I get an array with the returned cluster centers.
What I'm trying to work out is:

How can I apply these kmeans clusters to filter the dataset I have currently?

How could I take the average distribution for each type of event, and then remove outliers from it?


title: Exporting torch.nn.EmbeddingBag to ONNX

body: Have a model which includes a torch.nn.EmbeddingBag layer as its first layer, and I would like to export it to ONNX. Is there a way to provide a dummy input to the torch.onnx.export function such that the exported model will behave similarly to the PyTorch model? It does not seem to work with an arbitrary dummy input.


title: Why does a working torch setup suddenly raise a numpy.core.multiarray import error?

body:
I had a working setup for a project of mine which uses pytorch. It suddenly started raising a numpy.core.multiarray import error even though everything ran just fine before that.
I will paste the according code parts below.
python3
Python 3.8.6 (default, Jan 12 2021, 09:26:00)
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import torch
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "*blurred out*/.pyenv/versions/deeplearning/lib/python3.8/site-packages/torch/__init__.py", line 136, in <module>
    from torch._C import *
ImportError: numpy.core.multiarray failed to import
>>>

python3
Python 3.8.6 (default, Jan 12 2021, 09:26:00)
[GCC 9.3.0] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import numpy.core.multiarray
Traceback (most recent call last):
  File "_bit_generator.pyx", line 40, in numpy.random._bit_generator
  File "*blurred out*/.pyenv/versions/3.8.6/lib/python3.8/secrets.py", line 20, in <module>
    from random import SystemRandom
ImportError: cannot import name 'SystemRandom' from 'random' (*blurred out*)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "*blurred out*/.pyenv/versions/maml/lib/python3.8/site-packages/numpy/__init__.py", line 152, in <module>
    from . import random
  File "*blurred out*/.pyenv/versions/maml/lib/python3.8/site-packages/numpy/random/__init__.py", line 181, in <module>
    from . import _pickle
  File "*blurred out*/.pyenv/versions/maml/lib/python3.8/site-packages/numpy/random/_pickle.py", line 1, in <module>
    from .mtrand import RandomState
  File "mtrand.pyx", line 1, in init numpy.random.mtrand
  File "_bit_generator.pyx", line 43, in init numpy.random._bit_generator
ImportError: cannot import name SystemRandom


title: In pytorch, Multi-gpus don't work, How can I solve this?

body:
I have a question about working with multi-GPUs.
I use 4 GPU 1080ti. I want to use them for deeplearning.
I ran the data parallelism example in the PyTorch tutorial, but it doesn't work. I also reinstalled CUDA (10.2, 11.0) and reinstalled the OS. (I used linux mint and reinstalled both 19 and 20.) But the result was always the same.
On nvidia-smi it looks like this:

The terminal running Python is in this state:


title: How to implement an epoch-dependent parameter inside a neural network in Pytorch?

body:
For example, let's say I want to use Softmax with temperature after a layer and I also want to decrease the temperature after 5 epochs.
for epoch in range(EPOCHS):
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
    inputs, labels = data[0].to(torch_device), data[1].to(torch_device)
    optimizer.zero_grad()
    outputs = my_model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()
    my_model.adjust_softmax_temperature(epoch)
    # ...

How should I implement the adjust_softmax_temperature method in order to make my code to work?


title: slice tensor of tensors using boolean tensor

body:
Having two tensors :inputs_tokens is a batch of 20x300 of token ids
and seq_A is my model output with size of [20, 300, 512] (512 vector for each of the tokens in the batch)
seq_A.size()
Out[1]: torch.Size([20, 300, 512])

inputs_tokens.size()
torch.Size([20, 300])

I would like to get only the vectors of the token 101 (CLS) as follow:
cls_tokens = (inputs_tokens == 101)
cls_tokens
Out[4]:
tensor([[ True, False, False,  ..., False, False, False],
       [ True, False, False,  ..., False, False, False],
       [ True, False, False,  ..., False, False, False], ...

How do I slice seq_A to get only the vectors which are true in cls_tokens for each batch?
when I do
seq_A[cls_tokens].size()
Out[7]: torch.Size([278, 512])

but I still need it to bee in the size of [20 x N x 512 ] (otherwise I don't know to which sample it belongs)


title: CondaHTTPError while pytorch installation

body:
I have installed Anaconda 4.2 on Windows 10. When I try to run the following command to install PyTorch.
conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch

I get the following error:
(base) C:\Users\DELL>conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch
Collecting package metadata (current_repodata.json): failed

CondaHTTPError: HTTP 000 CONNECTION FAILED for url <https://conda.anaconda.org/pytorch/win-64/current_repodata.json>
Elapsed: -

An HTTP error occurred when trying to retrieve this URL.
HTTP errors are often intermittent, and a simple retry will get you on your way.
'https://conda.anaconda.org/pytorch/win-64'

Can anyone help me resolve the issue?


title: Jupyter notebook: Widget Javascript not detected after enable of widgetsnbextension (Pytorch tutorial)

body:
I'm trying to run the notebook autograd_tutorial.ipynb from the Pytorch "A 60 minute blitz" tutorial (https://pytorch.org/tutorials/beginner/blitz/autograd_tutorial.html). However, when I run the code:
import torch, torchvision
model = torchvision.models.resnet18(pretrained=True)

I get the error:
Widget Javascript not detected.  It may not be installed or enabled properly.

I have ipywidgets installed with version 6 and widgetsnbextension with version 2. I also activated widgetsnbextension with the command:
sudo jupyter nbextension enable --py widgetsnbextension

However, I still get the exactly same error. What am I missing?


title: PyTorch: Why is my dataset class giving index out of range errors?

body:
I am trying to figure out why my data set is giving out of range index errors.
Consider this torch data set:
# prepare torch data set
class MSRH5Processor(torch.utils.data.Dataset):
    def __init__(self, type, shard=False, **args):
        # init configurable string
        self.type = type
        # init shard for sampling large ds if specified
        self.shard = shard
        # set seed if given
        self.seed = args
        # set loc
        self.file_path = 'C:\\data\\h5py_embeds\\'
        # set file paths
        self.val_embed_path = self.file_path + 'msr_dev_bert_embeds.h5'

        # if true, initialize the dev data
        if self.type == 'dev':
            # embeds are shaped: [layers, tokens, features]
            self.embeddings = h5py.File(self.val_embed_path, 'r')["embeds"]

    def __len__(self):
        return len(self.embeddings)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        if self.type == 'dev':
            sample = {'embeddings': self.embeddings[idx]}
            return sample

# load dataset
processor = MSRH5Processor(type='dev', shard=False)
# check length
len(processor)  # 22425

# iterate over the samples
count = 0
for step, batch in enumerate(processor):
    count += 1
# error: Index (22425) out of range (0-22424)

with h5py.File('C:\\w266\\h5py_embeds\\msr_dev_bert_embeds.h5', 'r') as f:
    print(f['embeds'].attrs['last_index'])  # 22425
    print(f['embeds'].shape)  # (22425, 128, 768)
    print(len(f['embeds']))  # 22425

If I manually change the data set length to 100 or 22424, I will still get the same error. What is telling PyTorch to look for index 22425?
If I were to make a CSV data set, with 1000 observations (where len = 1000), it would stop entering indices into the __getitem__() method at 999 and not 1000.
Edit:
It seems to be an issue with just the Dataset class and H5py files. If I use a torch dataloader, it will run to the natural length of my data set. Although, I would love to know what Torch is doing to get this figure for my H5 files that is causing it to behave differently than say a CSV.


title: AWS EC2 causing issue with Streamlit ML App

body:
It's strange because on my local machine, this issue doesn't happen, and my app works fine.
However, when I run the app on an AWS EC2 instance, it gives me an error regarding a matplotlib import. Below the matplotlib import, I have matplotlib.use('TkAgg'). When the code is like this, the Streamlit app gives me this error (only on the EC2 instance):
ImportError: Cannot load backend 'TkAgg' which requires the 'tk' interactive framework, as 'headless' is currently running
Traceback:
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/streamlit/script_runner.py", line 332, in _run_script
    exec(code, module.__dict__)
File "/home/ubuntu/extremely_unnecessary/app.py", line 16, in <module>
    matplotlib.use('TkAgg')
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/matplotlib/__init__.py", line 1171, in use
    plt.switch_backend(name)
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/matplotlib/pyplot.py", line 287, in switch_backend
    newbackend, required_framework, current_framework))

After doing some research, I tried changing the offending line to matplotlib.use('agg'). When I do this, the app works properly, however none of the models except one work when selected.
The app is hosted here: http://54.193.229.139:8501/ The way it works is you upload an image, then select a pretrained model from the drop down menu to apply "style transfer" to the image you uploaded.
For some bizarre reason, the 12th model in the list (chicken-strawberries-market-069_10000.pth) works, but none of the other models do. Again, this only happens on the EC2 instance - even when I use matplotlib.use('agg'), all the models work when running the streamlit app locally.
I also tried using some other variations including matplotlib.use('GTK3Agg') and matplotlib.use('WebAgg'), which give me various other error messages.
Does anyone know how to fix this so I can get all the models working on the EC2 instance?
Edit: I've started receiving a new error message, I'm working on trying to change the code now. I use CUDA through my GPU, apparently I have to make some cpu-bound changes so it'll work on the ubuntu server. Not sure why the chicken strawberries model works though...
RuntimeError: Attempting to deserialize object on a CUDA device but torch.cuda.is_available() is False. If you are running on a CPU-only machine, please use torch.load with map_location=torch.device('cpu') to map your storages to the CPU.
Traceback:
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/streamlit/script_runner.py", line 332, in _run_script
    exec(code, module.__dict__)
File "/home/ubuntu/extremely_unnecessary/app.py", line 91, in <module>
    main()
File "/home/ubuntu/extremely_unnecessary/app.py", line 54, in main
    transformer.load_state_dict(torch.load(checkpoint))
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/torch/serialization.py", line 595, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/torch/serialization.py", line 774, in _legacy_load
    result = unpickler.load()
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/torch/serialization.py", line 730, in persistent_load
    deserialized_objects[root_key] = restore_location(obj, location)
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/torch/serialization.py", line 175, in default_restore_location
    result = fn(storage, location)
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/torch/serialization.py", line 151, in _cuda_deserialize
    device = validate_cuda_device(location)
File "/home/ubuntu/anaconda3/envs/streamlit/lib/python3.6/site-packages/torch/serialization.py", line 135, in validate_cuda_device
    raise RuntimeError('Attempting to deserialize object on a CUDA '

The code for the app:
import matplotlib.pyplot as plt
from PIL import Image
from torchvision.utils import save_image
import tqdm
import streamlit as st
from models import TransformerNet
from utils import *
import torch
import numpy as np
from torch.autograd import Variable
import argparse
import tkinter as tk
import os
import cv2
import matplotlib
matplotlib.use('agg')

def main():

    uploaded_file = st.file_uploader(
        "Choose an image", type=['jpg', 'png', 'webm', 'mp4', 'gif', 'jpeg'])
    if uploaded_file is not None:
        st.image(uploaded_file, width=200)

    folder = os.path.abspath(os.getcwd())
    folder = folder + '/models'

    fnames = []

    for basename in os.listdir(folder):
        print(basename)
        fname = os.path.join(folder, basename)

        if fname.endswith('.pth'):
            fnames.append(fname)

    checkpoint = st.selectbox('Select a pretrained model', fnames)

    os.makedirs("images/outputs", exist_ok=True)

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    # device = torch.device("cpu")
    transform = style_transform()

    # Define model and load model checkpoint
    transformer = TransformerNet().to(device)
    transformer.load_state_dict(torch.load(checkpoint))
    transformer.eval()

    # Prepare input
    image_tensor = Variable(transform(Image.open(
        uploaded_file).convert('RGB'))).to(device)
    image_tensor = image_tensor.unsqueeze(0)

    # Stylize image
    with torch.no_grad():
        stylized_image = denormalize(transformer(image_tensor)).cpu()

    fn = str(np.random.randint(0, 100)) + 'image.jpg'
    save_image(stylized_image, f"images/outputs/stylized-{fn}")

    st.image(f"images/outputs/stylized-{fn}")

if __name__ == "__main__":
    main()


title: Converting element-wise matrix operations into matrix multiplications?

body:
In order to utilize my tensor cores, I need to write matrix multiplications (correct me if I'm wrong). Unfortunately, my solution isn't naturally in matmul form. But maybe it can be converted into it.
Here is one iteration of my computation graph:
down = scent.roll(1, 0) * obstaclesMask * topWallMask;
up = scent.roll(-1, 0) * obstaclesMask * bottomWallMask;
right = scent.roll(1, 1) * obstaclesMask * leftWallMask;
left = scent.roll(-1, 1) * obstaclesMask * rightWallMask;
scent = scent + ((down - scent) + (up - scent) + (right - scent) + (left - scent)) * diffusionRate;
scent = torch::max(scent, entities);

(This is libtorch, but pytorch would look very similar). All of these are NxN matrices except diffusionRate, which is a scalar. I'm thinking down/up/right/left - or some intermediate matrices - can be stacked and matmuled, but I'm not sure. Thoughts?


title: Concatenate along z axis PyTorch Dataloader

body:
I have a multispectral dataset with 9 bands. Since the data is very large, I split each band to 256 x 256 samples. So I have 16 such samples for each band, and I saved them into different folders. Now how can I concatenate each sample of 9 bands?
For examples, I want to concatenate the 1st sample from the 1st band data with 1st sample of 2nd band, with 1st sample of 3rd band so on, till the 9th band. Then 2nd sample of 1st, 2nd ... 9th band. so on till 16th sample.


title: Modify trainable affinity matrix (parameter) in Pytorch to calculate Laplacian Matrix

body:
I have a problem in which part of the loss function depends on a trainable affinity matrix W and pairwise differences between elements of the output, which looks like this: \sum_{ij} w_ij * (y_i - y_j), where w_ij represents the affinity between position i and j. The previous equation can be written in terms of the Laplacian matrix: y*L*y^T where L =  D - W and D, the degree matrix, is a diagonal matrix so that D_ii = \sum_j w_ij (each element of the diagonal is the sum of the elements of the corresponding row in the affinity matrix).
The thing is that I want to learn W but it is easier to use L instead, so I created a module to calculate my loss that would look like this:
class CRFlayer(nn.Module, ABC):
    def __init__(self, nodes=25):
        super(CRFlayer, self).__init__()
        self.nodes = nodes
        self.W = nn.Parameter(zeros(nodes, nodes))
        self.W.requires_grad = True

    def forward(self, Y, H):
        # Calculate Degree matrix using W
        D = "???"
        # Calculate Laplacian matrix using W
        L = D - self.W
        # Calculate loss based on L, the output Y, and other inputs
        loss = ...
        return loss

My question is that if it's possible to do that. In particular, I don't know how to code the degree matrix D and I don't know if the gradients will be correctly calculated over W after that. Do you have any recommendations?
Thanks!


title: Decoder upsample size

body:
I have a simple autoencoder which looks like this:
class Autoencoder(nn.Module):
    def __init__(self):
        super(Autoencoder, self).__init__()

        self.channels = [3, 8, 16, 32]

        self.encoder = nn.Sequential(
                # layer 1
                nn.Conv2d(self.channels[0], self.channels[1], 3, 1, 1),
                nn.BatchNorm2d(self.channels[1]),
                nn.ReLU(inplace=True),
                nn.MaxPool2d(kernel_size=4, stride=4),
                # layer 2
                nn.Conv2d(self.channels[1], self.channels[2], 3, 1, 1),
                nn.BatchNorm2d(self.channels[2]),
                nn.ReLU(inplace=True),
                nn.MaxPool2d(kernel_size=4, stride=4),
                # layer 3
                nn.Conv2d(self.channels[2], self.channels[3], 3, 1, 1),
                nn.BatchNorm2d(self.channels[3]),
                nn.ReLU(inplace=True),
                nn.MaxPool2d(kernel_size=2, stride=2)
            )

        self.decoder = nn.Sequential(
                nn.Conv2d(self.channels[3], self.channels[2], 3, 1, 1),
                nn.ReLU(inplace=True),
                nn.Upsample(size=(15, 20), mode='bilinear', align_corners=False),

                nn.Conv2d(self.channels[2], self.channels[1], 3, 1, 1),
                nn.ReLU(inplace=True),
                nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False),

                nn.Conv2d(self.channels[1], 1, 1),
                nn.ReLU(inplace=True),
                nn.Upsample(scale_factor=4, mode='bilinear', align_corners=False)
            )

    def forward(self, x):
        h = self.encoder(x)
        y = self.decoder(h)
        return y

Can someone clarify if the nn.Upsample(size=(15, 20)... in the Decoder has something to do with restoring the images to their original dimensions? My input images(torch tensors) are of size 240*320 and the network is supposed to restore the input tensor to its original size.
Also if I add a fourth layer to my encoder with 64 channels and a 4th layer to my decoder (similar to the layers above), what should the upsampling size be in the first layere of the decode?


title: Loading PyTorch model in multiprocessing Process flashes cmd

body:
Problem:
When I open pytorch model (read as load state_dict from disk) in subprocess it pops up cmd window for couple ms which causes other programs to loose focus - annoying when working on something else etc.
I have traced the cause to 2 lines, both causing it in some circumstances and managed to reproduce it for one (the second one is when doing model.to(device))
main.py
model_path = 'testing\\agent\\model_test.pth'

# create model
from testing.agent.torch_model import Net
import torch
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # I have cuda available
m = Net()
m.to(device)
# save it
torch.save(m.state_dict(), model_path)

# open it in subprocess
from testing.agent.AgentOpenSim_Process import Open_Agent_Sim
p = Open_Agent_Sim(p=model_path, msgLogger=None)
p.start()

torch_model.py
(source pydocs: https://pytorch.org/tutorials/recipes/recipes/save_load_across_devices.html)
import torch.nn as nn
import torch.functional as F

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

AgentOpenSim_Process.py
from multiprocessing import Queue, Process
import os, time, torch

from testing.agent.torch_model import Net

class Open_Agent_Sim(Process):
    def __init__(self, p:str, **kwargs):
        super(Process, self).__init__(daemon=True)
        self.path = p
        self._msgLogger = kwargs['msgLogger'] if kwargs['msgLogger'] is not None else Queue()
        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    def __print(self, msg: str, verbosity):
        # personal tool for debugging multiprocessing (it sends messages to Queue and main process
        # reads them inside thread and prints into console... and yes i know about multiprocessing
        # logger, its used aswell)
        self._msgLogger.put(('Open_Agent_Sim: '+msg, verbosity))

    def run(self):
        self.__pid = os.getpid()
        try:
            self.__print('opening model',0)
            self.init_agent()
            self.__print('opening model - done',0)
        except Exception as e:
            # solved by custom exception wrapper
            pass
        else:
            self.__print('has ended',0)
            return

    def init_agent(self):
        # init instance
        self.__print('0', 0)
        m = Net()
        time.sleep(2)
        # load state dict
        self.__print('1', 0)
        m.load_state_dict(torch.load(self.path, map_location=self.device)) # ----> This line pops up cmd
        time.sleep(2)
        # set to device
        self.__print('2', 0)
        try:
            m.to(self.device)
            time.sleep(2)
            self.__print('3', 0)
        except RuntimeError as e:
            self.__print(str(e), 0)

When visually debugging those cmd pops up, its always in step 1 (m.load_state_dict(torch.load(self.path, map_location=self.device)))
I have tried something like disabling console output which didnt work.
import contextlib
with contextlib.redirect_stdout(None):
    ...

The if __name__=='__main__': makes no difference and also this is all part of heavy multiprocessing in some lower subprocess


title: How to modify my python multi-process program？

body:
I try to process the image files in order from the directory /base_path. The /base_path directory contains 78 folders, named 00001-00078. In these 78 files, each has 1000 folders, each named 0001-1000, and each directory contains three pictures, named im1.png, im2.png, im3.png, and the requirement is through func( ) to generate the optical flow ft1.flo of img2->img1 and the optical flow of im3->im1 in the /output directory. Therefore, the dataset is very large. I have tried to use multi-process methods to speed up. The program has been running without errors, but there is no output file, and the CPU utilization rate is also very low. I guess my multi-process part is written incorrectly. Can anyone help me, thank you very much! Below is part of my code.
def func(list):
    #os.system('cp /code/pytorch-liteflownet/network-default.pytorch /root/.cache/torch/checkpoints/') #copy pretrained-model
    img2 = cv2.imread(list[0])
    img1 = cv2.imread(list[1])
    img3 = cv2.imread(list[2])
    tenFirst = torch.FloatTensor(numpy.ascontiguousarray(img2.transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0)))
    tenSecond = torch.FloatTensor(numpy.ascontiguousarray(img1.transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0)))
    tenThird = torch.FloatTensor(numpy.ascontiguousarray(img3.transpose(2, 0, 1).astype(numpy.float32) * (1.0 / 255.0)))
    tenOutput = estimate(tenFirst, tenSecond)
    tenOutput2 = estimate(tenFirst, tenThird)
    objOutput = open(list[3], 'wb')
    objOutput2 = open(list[4], 'wb')

    numpy.array([ 80, 73, 69, 72 ], numpy.uint8).tofile(objOutput)
    numpy.array([ tenOutput.shape[2], tenOutput.shape[1] ], numpy.int32).tofile(objOutput)
    numpy.array(tenOutput.numpy().transpose(1, 2, 0), numpy.float32).tofile(objOutput)

    numpy.array([ 80, 73, 69, 72 ], numpy.uint8).tofile(objOutput2)
    numpy.array([ tenOutput.shape[2], tenOutput.shape[1] ], numpy.int32).tofile(objOutput2)
    numpy.array(tenOutput.numpy().transpose(1, 2, 0), numpy.float32).tofile(objOutput2)
    objOutput.close()
    objOutput2.close()

if __name__ == '__main__':
    base_path = '/data/dachunkai/vimeo_triplet/sequences/'
    datalist = []
    for i in range(1, 79):
        for j in range(1, 1001):
            path_in = base_path + str(i).rjust(5, '0') + '/' + str(j).rjust(4, '0')
            path_out = '/output' + '/data' + '/' + str(i).rjust(5, '0') + '/' + str(j).rjust(4, '0')

            arguments_strFirst = path_in + '/im2.png'
            arguments_strSecond = path_in + '/im1.png'
            arguments_strThird = path_in + '/im3.png'
            arguments_strOut = path_out + '/ft0.flo'
            arguments_strOut2 = path_out + '/ft1.flo'
            datalist.append([arguments_strFirst, arguments_strSecond, arguments_strThird, arguments_strOut, arguments_strOut2])

    cores = cpu_count()
    print('cpu_count', cores)
    pool = Pool(cores)
    pool.map(func, datalist)
    pool.close()
    pool.join()


title: Initialize weight in pytorch neural net

body:
I have created this  neural net:
class _netD(nn.Module):
    def __init__(self, num_classes=1, nc=1, ndf=64):
        super(_netD, self).__init__()
        self.num_classes = num_classes
        # nc is number of channels
        # num_classes is number of classes
        # ndf is the number of output channel at the first layer
        self.main = nn.Sequential(
            # input is (nc) x 28 x 28
            # conv2D(in_channels, out_channels, kernelsize, stride, padding)
            nn.Conv2d(nc, ndf , 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),
            # state size. (ndf) x 14 x 14
            nn.Conv2d(ndf, ndf * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 2),
            nn.LeakyReLU(0.2, inplace=True),
            # state size. (ndf*2) x 7 x 7
            nn.Conv2d(ndf * 2, ndf * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 4),
            nn.LeakyReLU(0.2, inplace=True),
            # state size. (ndf*4) x 3 x 3
            nn.Conv2d(ndf * 4, ndf * 8, 3, 2, 1, bias=False),
            nn.BatchNorm2d(ndf * 8),
            nn.LeakyReLU(0.2, inplace=True),
            # state size. (ndf*8) x 2 x 2
            nn.Conv2d(ndf * 8, num_classes, 2, 1, 0, bias=False),
            # out size = batch x num_classes x 1 x 1
        )

        if self.num_classes == 1:
            self.main.add_module('prob', nn.Sigmoid())
            # output = probability
        else:
            pass
            # output = scores

    def forward(self, input):

        output = self.main(input)
        return output.view(input.size(0), self.num_classes).squeeze(1)

I want to loop through the different layers and apply a weight initialization depending on the type of layer. I am trying to do the following:
D = _netD()

for name, param in D.named_parameters():
    if type(param) == nn.Conv2d:
      param.weight.normal_(...)

But that is not working. Can you please help me?
Thanks


title: Dead Kernel after running torchvision.utils.make_grid(images)

body:
To make it simple, I am following this tutorial provided by PyTorch to create a CNN.
However, it appears that when I'm running this particular code block with this respective line:
# show images
imshow(torchvision.utils.make_grid(images))

It somehow kills the kernel. Which confuses me because it is only a simple function.
I have also went through numerous SO posts that are related to PyTorch but all of them are not related to torchvision. The most similar issue I found was posted 5 months ago without answer.
Please let me know if I need to add on any information to make this question clearer.


title: How can I initialize an at::Scalar?

body:
I was trying to invoke a PyTorch C++ API function called linspace. When I tried to write it like Python style, I wrote:
torch::Tensor x = torch::linspace(-1, 1, 100);

and this code got a segmentation fault:
[1]    6038 segmentation fault  ./example-app

I read the C++ docs in pytorch.org, and it gave the format of linspace:
at::Tensor torch::linspace(const at::Scalar &start, const at::Scalar &end, c10::optional<int64_t> steps = c10::nullopt, at::TensorOptions options = {})

Here I didn't know what exactly at::Scalar means, so I searched it in the docs about at namespace, but I didn't find any class called Scalar.
However, I did find many functions in the at namespace that have a parameter of Scalar type.
This is the docs about linspace function, and this is the docs about at namespace.


title: Finding function/class definitions in PyTorch

body:
I want to find out where certain classes and functions are defined within PyTorch (and other libraries).
Unfortunately, the following doesn't work:
import inspect
import torch

inspect.getsource(torch.tensor)

It throws the following error:
TypeError: module, class, method, function, traceback, frame, or code object was expected, got builtin_function_or_method

What's more, within PyCharm, I usually do 'gd' (in vim mode) to find a function/class definition, but this doesn't work either for PyTorch.
Please help me understand what is the problem here, and more importantly, how I can find these definitions in general.


title: sampler argument in DataLoader of Pytorch

body:
While using Pytorch's DataLoader utility, in sampler what is the purpose of RandomIdentitySampler? And in RandomIdentitySampler there is an argument instances. Does instances depends upon number of workers? If there is are 4 workers then should there be 4 instances as well?
Following is the chunk of code:
c_dataloaders = DataLoader(Preprocessor(cluster_dataset.train_set,
                                        root=cluster_dataset.images_dir,
                                        transform=train_transformer),
                                        batch_size=args.batch_size_stage2,
                                        num_workers=args.workers,
                                        sampler=RandomIdentitySampler(cluster_dataset.train_set,
                                        args.batch_size_stage2,
                                        args.instances)


title: Large .tif image dataset - How do I convert it to a usable dataset for a CNN using Tensorflow with labels in a separate CSV? Python 3

body:
I have been trying to find a way to convert my 270,000 .tif images and their corresponding labels into a usable Tensorflow dataset for a few days now and I have decided to post here for help.
What I have currently done is just taken a subset of the images and converted them to numpy arrays and then put them into a list. I also did this with the labels. So I now have a pandas dataframe that has two columns; id (the images file name) and the label (a 1 or a 0) and a list of numpy arrays. Below is the code that I use to do that.
from PIL import Image
import matplotlib.pyplot as plt
import glob
import tensorflow as tf
import pandas as pd
import re

path = "./training_images/*"

img_names = []
img_arr_list = []
for file in glob.glob(path):
    img = Image.open(file)
    img_arr = np.asarray(img).astype("float32")
    img_arr_list.append(img_arr)
    name = re.split(' |/|\\\\' , file)
    name = name[2][:-4]
    img_names.append(name)

labels = pd.read_csv("train_labels.csv")

# Convert list of file names to a pandas dataframe and change the 0 field to "id" to match the field
# name in labels
name_df = pd.DataFrame(img_names)
name_df = name_df.rename(columns={0:"id"})

# inner join to get only the images names and their labels that exist in the image subset
labels_df = pd.merge(name_df,labels,on='id',how='inner')

From here, I have tried to convert my list of numpy arrays as well is it's corresponding labels into a Tensorflow dataset that I can then preprocess and then feed into a CNN that I will later create. When I execute the below code.....
train_dataset = tf.data.Dataset.from_tensor_slices((img_arr_list, labels_df))

.... I get this error..
ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type int).

I believe I am doing this incorrectly and need to instead be doing something where I iterate over my list of numpy arrays and my labels_df to create a tensor during each loop in the for loop and then append them to a Tensorflow dataset. Below is some pseudo-code that I have thought of using.
for row in [0,len(img_arr_list)]:
    tensor = tf.data.Dataset.from_tensor_slices((img_arr_list[row], labels_df.iloc[row]))
    <append tensor to tensorflow dataset>

Any help would be appreciated. I am also open to using a pytorch-based approach.


title: Streamlit : TypeError: a bytes-like object is required, not 'Tensor'

body:
I am working on a style transfer task, my model returns a tensor. recently I was saving that image using torchvision.utils
torchvision.utils.save_image(genarated_image, result_path)

now I have passed the same image to streamlit.
def image_input():
content_file = st.sidebar.file_uploader("Choose a Content Image", type=["png", "jpg", "jpeg"])
if content_file is not None:
    content = Image.open(content_file)
    content = np.array(content)  # pil to cv
    content = cv2.cvtColor(content, cv2.COLOR_RGB2BGR)
else:
    st.warning("Upload an Image OR Untick the Upload Button)")
    st.stop()

WIDTH = st.sidebar.select_slider('QUALITY (May reduce the speed)', list(range(150, 501, 50)), value=200)
content = imutils.resize(content, width=WIDTH)
generated = genarate_image(content)
st.sidebar.image(content, width=300, channels='BGR')
st.image(generated, channels='BGR', clamp=True)

But now streamlit giving me this error.

TypeError: a bytes-like object is required, not 'Tensor'

is there a way to convert tensor into a "bytes-like object" ?


title: How to create a TabularDataset using the first 1000 instances from training set?

body:
I am doing a text classification task using RNN in pytorch. I have a training dataset and testing dataset. For my task, I need to train my model using the first 1000 instances of the training dataset. Can I create a TabularDataset using the first 1000 training instances?
I want to create a tabularDataset for the first 1000 instances. Is there any way to do so?
Below, I have created a tabularDataset using the entire training and testing data.
train_data,  test_data = TabularDataset.splits(
   path='cola_public/for_torch_text', train='in_domain_train.tsv',
    path='/content/drive/MyDrive', train='/content/drive/MyDrive/axcs_train.csv',
    test='/content/drive/MyDrive/axcs_test.csv', format='csv',skip_header=True,
    fields=tv_datafields)


title: How to assign particular value to net.parameters() using torch

body:
This is my network; I have 2 CNN layers and 1 FC layer.
I've already trained CNN filters for particular domain, so I'm trying to train the FC layer only.
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()

        self.layer1 = nn.Sequential(
            nn.Conv2d(in_channels = 1, out_channels = 1, kernel_size = (2,2), stride = 1, padding = 1, bias = False),
            nn.AvgPool2d(kernel_size = (2,2), stride = (2,2)),
            )
        self.layer2 = nn.Sequential(
            nn.Conv2d(in_channels = 1, out_channels = 1, kernel_size = (2,2), stride = 1, padding = 1, bias = False),
            nn.AvgPool2d(kernel_size = (2,2), stride = (2,2)),
            )
        self.fc = nn.Linear(1200, 1)
        self.dropout = nn.Dropout(p = 0.10)
        self.sigmoid = nn.Sigmoid()

    def forward(self, x):
        out1 = self.layer1(x)
        out2 = self.layer2(out1)
        out3 = out2.reshape(out2.size(0), -1)
        output = self.sigmoid(self.dropout((self.fc(out3))))
        return output

And I am trying to assign pre-trained values of parameters in this network.
params = list(Net().parameters())
params

Output:

[Parameter containing:
 tensor([[[[ 0.2240,  0.2135],
           [-0.2901,  0.4827]]]], requires_grad=True),
 Parameter containing:
 tensor([[[[-0.0363, -0.2801],
           [ 0.0853, -0.0217]]]], requires_grad=True),
 Parameter containing:
 tensor([[-0.0155, -0.0073, -0.0065,  ...,  0.0012, -0.0213, -0.0287]],
        requires_grad=True),
 Parameter containing:
 tensor([-0.0010], requires_grad=True)]

params[0][0][0][0][0] = -0.2454

RuntimeError: a view of a leaf Variable that requires grad is being used in an in-place operation.

Is there any specific way to assign values to network's parameters?


title: NumPy array with largest value on diagonal and other values shuffled

body:
I am trying to create a square NumPy (or PyTorch, since PyTorch code can be turned into NumPy with minimal effort) matrix which has the following property: given a set of values, the diagonal elements in each row have the largest value and the other values are randomly shuffled for the other positions.
For example, if I have [1, 2, 3, 4], a possible desired output is:
[[4, 3, 1, 2],
 [1, 4, 3, 2],
 [2, 1, 4, 3],
 [2, 3, 1, 4]]

There can be (several) other possible outputs, as long as the diagonal elements are the largest value (4 in this case) and the off-diagonal elements in each row contain the other values but shuffled.
A hacky/inefficient way of doing this could be first creating a square matrix (4x4) of zeros and putting the largest value (4) in all the diagonal positions, and then traversing the matrix row by row, where for each row i, populate the elements except index i with shuffled remaining values (shuffled versions of [1, 2, 3]). This would be very slow as the matrix size increases. Is there a cleaner/faster/Pythonic way of doing it? Thank you.


title: Pytorch - Custom Dataset out of Range

body:
im new to Pytorch and DL in genereal, so i hope this is the right place to ask questions like this.
I wanted to create my first Dataset, but my Dataset always runs out of bound. This problem should be easiest to show with the codes and outputs.
class DataframeDataset(torch.utils.data.Dataset):
    """Load Pytorch Dataset from Dataframe

    """

    def __init__(self, data_frame, input_key, target_key, transform=None, features=None):
        self.data_frame = data_frame
        self.input_key = input_key
        self.target_key = target_key
        self.inputs = self.data_frame[input_key]
        self.targets = self.data_frame[target_key]
        self.transform = transform
        self.features = [input_key, target_key] if features is None else features
        self.len = len(self.inputs)

    def __len__(self):
        return self.len

    def __str__(self):
        return str(self.info())

    def info(self):
        info = {
            'features': self.features,
            'num_rows': len(self)
        }
        return "Dataset("+ str(info) + ")"

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx = idx.tolist()

        data = {
            self.input_key: self.inputs[idx],
            self.target_key: self.targets[idx]
        }

        if self.transform:
            return self.transform(data)

        return data

    @staticmethod
    def collate_fn(input_key, output_key):
        def __call__(batch):
            speeches = [data[input_key] for data in batch]
            sentences = [data[output_key] for data in batch]
            return speeches, sentences

        return __call__

with some mook data:
    data = [("x1", "y2", "A3"), ("x1", "y2", "b3"), ("x1", "y2", "c3"), ("x1", "y2", "d3")]
    df = pd.DataFrame(data, columns=['input', 'target', 'random'])
    print(df.head())

  input target random
0    x1     y2     A3
1    x1     y2     b3
2    x1     y2     c3
3    x1     y2     d3

    ds = DataframeDataset(data_frame=df, input_key="input", target_key="target", transform=None)
    print("Len:", len(ds))
    print("Ds", ds)
    print(ds[0])

Len: 4
Ds Dataset({'features': ['input', 'target'], 'num_rows': 4})
{'input': 'x1', 'target': 'y2'}

So the basic functions seem to work. However, if I want to iterate over the data with a foreach loop, then unfortunately the loop does not know the boundaries. So I get a key-error, because the torch accesses indicies outside the boundary.
    for idx, data in enumerate(ds):
        print(idx,"->",data)

0 -> {'input': 'x1', 'target': 'y2'}
1 -> {'input': 'x1', 'target': 'y2'}
2 -> {'input': 'x1', 'target': 'y2'}
3 -> {'input': 'x1', 'target': 'y2'}

Traceback (most recent call last):
  File "/home/warmachine/.local/lib/python3.8/site-packages/pandas/core/indexes/range.py", line 351, in get_loc
    return self._range.index(new_key)
ValueError: 4 is not in range`

If i do something like
    for idx in range(0, len(ds)):
        data = ds[idx]
        print(idx, "->", data)

it works, but i need to be able to use the for-each Style, so that i can use this Dataset within the Trainer of Hugging Face.
Ty in advcanded


title: NVIDIA Driver Not found during Nvidia + Cuda - Docker Image build

body:
I am trying to create a GPU microservice using Nvidia cuda Base image, but during the docker build, I am facing Driver not found issue, can someone point out what is missing here?
DockerFile:
FROM nvidia/cuda:10.1-devel

# Install some basic utilities
    RUN apt-get update && apt-get install -y \
        curl \
        ca-certificates \
        sudo \
        git \
        bzip2 \
        libx11-6 \
     && rm -rf /var/lib/apt/lists/*

ENV CONDA_AUTO_UPDATE_CONDA=false
ENV PATH=/home/user/miniconda/bin:$PATH
RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py37_4.8.2-Linux-x86_64.sh \
 && chmod +x ~/miniconda.sh \
 && ~/miniconda.sh -b -p ~/miniconda \
 && rm ~/miniconda.sh \
 && conda install -y python==3.7 \
 && conda clean -ya

ENV PATH="/usr/local/cuda-10.1/bin:$PATH"
ENV LD_LIBRARY_PATH="/usr/local/cuda-10.1/lib64:$LD_LIBRARY_PATH"
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
ENV NVIDIA_VISIBLE_DEVICES=all
ENV FORCE_CUDA="1"

RUN conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.1 -c pytorch

RUN pip install -v -e .

Error:
"/home/user/miniconda/lib/python3.7/site-packages/torch/utils/cpp_extension.py", line 1013, in _get_cuda_arch_flags
capability = torch.cuda.get_device_capability()
File "/home/user/miniconda/lib/python3.7/site-packages/torch/cuda/__init__.py", line 320, in get_device_capability
    prop = get_device_properties(device)

File "/home/user/miniconda/lib/python3.7/site-packages/torch/cuda/__init__.py", line 325, in get_device_properties
  _lazy_init()  # will define _get_device_properties and _CudaDeviceProperties
File "/home/user/miniconda/lib/python3.7/site-packages/torch/cuda/__init__.py", line 196, in _lazy_init
  _check_driver()

File "/home/user/miniconda/lib/python3.7/site-packages/torch/cuda/__init__.py", line 101, in _check_driver
  http://www.nvidia.com/Download/index.aspx""")
    AssertionError:
    Found no NVIDIA driver on your system. Please check that you
    have an NVIDIA GPU and installed a driver from
    http://www.nvidia.com/Download/index.aspx

The issues happens during execution of last step in docker file.
I tried using multiple Nvidia base docker images, but didn't help much. (cuda:10.1-base-ubuntu18.04, cuda:10.1-runtime-ubuntu18.04)
Any pointers appreciated.


title: Import TextLMDataBunch from Fastai

body:
I am following this tutorial to build a NLP sentiment analysis model.
from fastai.text import *

This is the only import specified that includes fastai.
Unfortunately the TextLMDataBunch is undefined.
What import should I used to have this class avaialable?
I have already tried:
from fastai.text.data import TextLMDataBunch

But apparently fastai.text.data is not even a package.


title: How to accelerate batch-size data from memory when using dataloader

body:
I am trying to use dataloader for training. The dataset is 150G, which are all .npz files. Due to the limitation of memory size, only one sample is read at a time from the disk. The following is part of the code.
class VimeoDataset(Dataset):
def __init__(self, mode, batch_size=32, num_workers = 8, num_gpus = 4):
    self.batch_size = batch_size
    self.num_workers = num_workers
    self.num_gpus = num_gpus
    self.mode = mode
    self.load_data()
    self.h = 256
    self.w = 448
    xx = np.arange(0, self.w).reshape(1,-1).repeat(self.h,0)
    yy = np.arange(0, self.h).reshape(-1,1).repeat(self.w,1)
    self.grid = np.stack((xx,yy),2).copy()
    self.npzs=[]

    count = self.batch_size * self.num_workers * self.num_gpus
    if self.mode == 'train':
        filelist = glob('/data/vimeoFlow2/dataset/train/*.npz')
        self.npzs = [filelist[i:i + count] for i in range(0, len(filelist), count)]
    else:
        filelist = glob('/data/vimeoFlow2/dataset/val/*.npz')
        self.npzs = [filelist[i:i + count] for i in range(0, len(filelist), count)]

def __len__(self):
    return len(self.npzs)

def load_data(self, index):
    self.data = []
    self.flow_data = []

    for i in range(len(self.npzs[index])):
        f = np.load(self.npzs[index][i])
        self.data.append(f['i0i1gt'])
        if self.mode == 'train':
            self.flow_data.append(f['ft0ft1'])
        else:
            self.flow_data.append(np.zeros((256, 448, 4)))

def getimg(self, index):
    data = self.meta_data[index]
    img0 = data[0:3].transpose(1, 2, 0)
    img1 = data[3:6].transpose(1, 2, 0)
    gt = data[6:9].transpose(1, 2, 0)
    flow_gt = (self.flow_data[index]).transpose(1, 2, 0)
    return img0, gt, img1, flow_gt

def __getitem__(self, index):
    img0, gt, img1, flow_gt = self.getimg(index)

dataset = VimeoDataset(mode = 'train',  batch_size=32, num_workers = 8, num_gpus = 4)
sampler = DistributedSampler(dataset)
train_data = DataLoader(dataset, batch_size=args.batch_size, pin_memory=True, num_workers=args.num_workers, drop_last=True, sampler=sampler)
dataset_val = VimeoDataset(mode = 'val',  batch_size=32, num_workers = 8, num_gpus = 4)
val_data = DataLoader(dataset_val, batch_size=args.batch_size, pin_memory=True, num_workers=args.num_workers)

However, reading data from the disk one by one causes the dataloader to be very time-consuming. So I want to improve this program, first load the amount of data of num_gpus×num_workers×batch_size into the memory, then read the data from the memory with __getitem__, and finally replace the data in the memory after each iteration. But I still don’t know how to achieve it. I have tried my idea as in the code above. I don't konw how to allocate the load_data function parameters.


title: Related to SubsetRandomSampler

body:
I am using SubsetRandomSampler for splitting a classification dataset into test and validation. Can we split the dataset for each class.
import numpy as np
import torch
from torchvision import transforms
from torch.utils.data.sampler import SubsetRandomSampler

train_transforms = transforms.Compose([transforms.ToTensor(),
                                       transforms.Normalize([0.485, 0.456, 0.406],
                                                            [0.229, 0.224, 0.225])])
dataset = datasets.ImageFolder( '/data/images/train', transform=train_transforms )

validation_split = .2
shuffle_dataset = True
random_seed= 42
batch_size = 20

dataset_size = len(dataset) #4996
indices = list(range(dataset_size))
split = int(np.floor(validation_split * dataset_size))

if shuffle_dataset :
    np.random.seed(random_seed)
    np.random.shuffle(indices)
train_indices, val_indices = indices[split:], indices[:split]

train_sampler = SubsetRandomSampler(train_indices)
valid_sampler = SubsetRandomSampler(val_indices)

train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler)
validation_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=valid_sampler)


title: How to convert probability to angle degree in a head-pose estimation problem?

body:
I reused code from others to make head-pose prediction in Euler angles. The author trained a classification network that returns bin classification results for the three angles, i.e. yaw, roll, pitch. The number of bins is 66. They somehow convert the probabilities to the corresponding angle, as written from line 150 to 152 here. Could someone help to explain the formula?
These are the relevant lines of code in the above file:
[56]  model = hopenet.Hopenet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], 66) # a variant of ResNet50
[80]  idx_tensor = [idx for idx in xrange(66)]
[81]  idx_tensor = torch.FloatTensor(idx_tensor).cuda(gpu)
[144] yaw, pitch, roll = model(img)
[146] yaw_predicted = F.softmax(yaw)
[150] yaw_predicted = torch.sum(yaw_predicted.data[0] * idx_tensor) * 3 - 99


title: How to merge sequential integer values into intervals in PyTorch?

body:
I have a 1D array:
[3, 4, 5, 6, 7, 20, 31, 32, 33, 34]

which I want to turn into a 2D interval array by merging every consecutive values into intervals:
[
  [ 3,  7],
  [20, 20],
  [32, 34]
]

What would be a decent, possibly GPU-friendly, way to do it?


title: Implementing Backprop for custom loss functions

body:
I have a neural network Network that has a vector output. Instead of using a typical loss function, I would like to implement my own loss function that is a method in some class. This looks something like:
class whatever:
    def __init__(self, network, optimizer):
         self.network = network
         self.optimizer = optimizer

    def cost_function(relevant_data):
        ...implementation of cost function with respect to output of network and relevant_data...

    def train(self, epochs, other_params):
        ...part I'm having trouble with...

The main thing I'm concerned with is about taking gradients. Since I'm taking my own custom loss function, do I need to implement my own gradient with respect to the cost function?
Once I do the math, I realize that if the cost is J, then the gradient of J is a fairly simple function in terms of the gradient of the final layer of the Network. I.e, it looks something like: Equation link.
If I used some traditional loss function like CrossEntropy, my backprocess would look like:
objective = nn.CrossEntropyLoss()
for epochs:
    optimizer.zero_grad()
    output = Network(input)
    loss = objective(output, data)
    loss.backward()
    optimizer.step()

But how do we do this in my case? My guess is something like:
for epochs:
     optimizer.zero_grad()
     output = Network(input)
     loss = cost_function(output, data)
     #And here is where the problem comes in
     loss.backward()
     optimizer.step()

loss.backward() as I understand it, takes the gradients of the loss function with respect to the parameters. But can I still invoke it while using my own loss function (presumably the program doesn't know what the gradient equation is). Do I have to implement another method/subroutine to find the gradients as well?
Which brings me to my other question: if I do want to implement gradient calculation for my loss function, I also need the gradient of the neural network parameters. How do I obtain those? Is there a function for that?


title: What is the substitute for numpy.random.choice statement for tensors?

body:
I am trying to implement a learning algorithm that has the following block of code to select the action with a probability distribution and using np.random.choice :
states.append(state)
probs = self.policy.forward(Variable(torch.from_numpy(state).float().unsqueeze(0)))
highest_prob_action = np.random.choice(self.num_actions, p=np.squeeze(probs.numpy()))
log_prob = torch.log(probs.squeeze(0)[highest_prob_action])
action_l.append(highest_prob_action)

I tried to use torch.multinomial as follows :
 highest_prob_action = np.random.choice(torch.multinomial(probs, self.num_actions).squeeze(0))

But it returns the error :
    130             states.append(state)
    131             probs = self.policy.forward(Variable(torch.from_numpy(state).float().unsqueeze(0)))
--> 132             highest_prob_action = np.random.choice(torch.multinomial(probs, self.num_actions).squeeze(0))
    133 #             highest_prob_action = np.random.choice(self.num_actions, p=np.squeeze(probs.numpy()))
    134             log_prob = torch.log(probs.squeeze(0)[highest_prob_action])

RuntimeError: probability tensor contains either `inf`, `nan` or element < 0


title: COLAB and pytorch : RuntimeError: DataLoader worker is killed by signal: Killed

body:
I am trying to run fastdetph with the suggested line of code for the evaluation :
!python3 main.py --evaluate mobilenet-nnconv5dw-skipadd-pruned.pth.tar

in the terminal I read this:

Namespace(cuda=True, data='nyudepthv2', evaluate='mobilenet-nnconv5dw-skipadd-pruned.pth.tar', gpu='0', modality='rgb', print_freq=50, workers=16)

But after a while I receive this error:

RuntimeError: DataLoader worker (pid 677) is killed by signal:
Killed. Details are lost due to multiprocessing. Rerunning with
num_workers=0 may give better error trace.

So, I tried with --workers=0 and the code interrupt without any error but with this line

^C

Why this happens?
I also tried with --workers=1 but the error of the dataloader is the same. I tried both colab gpu and cpu


title: Nearly Constant training and validation accuracy

body:
I’m new to pytorch and my problem may be a little naive
I’m training a pretrained VGG16 network on my dataset which it’s size is near 33000 images in 8 classes with labels [1,2,…,8] and my classes are imbalanced. my problem is that during training, validation and training accuracy is low and doesn’t increase, is there any problem in my code?
if not, what do you suggest to improve training?
'''
import torch
import time
import torch.nn as nn
import numpy as np
from sklearn.model_selection import train_test_split
from torch.optim import Adam
import cv2
import torchvision.models as models
from classify_dataset import Classification_dataset
from torchvision import transforms

transform = transforms.Compose([transforms.Resize((224,224)),
                                    transforms.RandomHorizontalFlip(p=0.5),
                                    transforms.RandomVerticalFlip(p=0.5),
                                    transforms.RandomRotation(degrees=45),
                                    transforms.ToTensor(),
                                    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
                                    ])
dataset = Classification_dataset(root_dir=r'//home/arisa/Desktop/Hamid/IQA/Hamid_Dataset',
                                     csv_file=r'/home/arisa/Desktop/Hamid/IQA/new_label.csv',transform=transform)

target = dataset.labels - 1

train_indices, test_indices = train_test_split(np.arange(target.shape[0]), stratify=target)
test_dataset = torch.utils.data.Subset(dataset, indices=test_indices)
train_dataset = torch.utils.data.Subset(dataset, indices=train_indices)

class_sample_count = np.array([len(np.where(target[train_indices] == t)[0]) for t in np.unique(target)])
weight = 1. / class_sample_count
samples_weight = np.array([weight[t] for t in target[train_indices]])
samples_weight = torch.from_numpy(samples_weight)
samples_weight = samples_weight.double()

sampler = torch.utils.data.WeightedRandomSampler(samples_weight, len(samples_weight), replacement = True)

train_loader = torch.utils.data.DataLoader(train_dataset,
                                           batch_size=64,
                                           sampler=sampler)
test_loader = torch.utils.data.DataLoader(test_dataset,
                                           batch_size=64,
                                           shuffle=False)
for param in model.parameters():
     param.requires_grad = False

num_ftrs = model.classifier[0].in_features
model.classifier = nn.Linear(num_ftrs,8)

optimizer = Adam(model.parameters(), lr = 0.0001 )
criterion = nn.CrossEntropyLoss()
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.01)

path = '/home/arisa/Desktop/Hamid/IQA/'
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print(device)
def train_model(model, train_loader,valid_loader, optimizer, criterion, scheduler=None, num_epochs=10 ):

        min_valid_loss = np.inf
        model.train()
        start = time.time()
        TrainLoss = []
        model = model.to(device)
        for epoch in range(num_epochs):
            total = 0
            correct = 0
            train_loss = 0
            #lr_scheduler.step()
            print('Epoch {}/{}'.format(epoch+1, num_epochs))
            print('-' * 10)

            train_loss = 0.0
            for x,y in train_loader:
                x = x.to(device)
                #print(y.shape)
                y = y.view(y.shape[0],).to(device)
                y = y.to(device)
                y -= 1
                out = model(x)

                loss = criterion(out, y)
                optimizer.zero_grad()
                loss.backward()

                TrainLoss.append(loss.item()* y.shape[0])
                train_loss += loss.item() * y.shape[0]
                _,predicted = torch.max(out.data,1)
                total += y.size(0)
                correct += (predicted == y).sum().item()
                optimizer.step()
                lr_scheduler.step()
            accuracy = 100*correct/total
            valid_loss = 0.0
            val_loss = []
            model.eval()
            val_correct = 0
            val_total = 0

            with torch.no_grad():

                for x_val, y_val in test_loader:
                    x_val = x_val.to(device)
                    y_val = y_val.view(y_val.shape[0],).to(device)
                    y_val -= 1
                    target = model(x_val)
                    loss = criterion(target, y_val)
                    valid_loss += loss.item() * y_val.shape[0]

                    _,predicted = torch.max(target.data,1)
                    val_total += y_val.size(0)
                    val_correct += (predicted == y_val).sum().item()

                    val_loss.append(loss.item()* y_val.shape[0])
                val_acc = 100*val_correct / val_total

                print(f'Epoch {epoch + 1} \t\t Training Loss: {train_loss / len(train_loader)} \t\t Validation Loss: {valid_loss / len(test_loader)} \t\t Train Acc:{accuracy} \t\t Validation Acc:{val_acc}')
                if min_valid_loss > (valid_loss / len(test_loader)):
                    print(f'Validation Loss Decreased({min_valid_loss:.6f}--->{valid_loss / len(test_loader):.6f}) \t Saving The Model')
                    min_valid_loss = valid_loss / len(test_loader)
                    state = {'state_dict': model.state_dict(),'optimizer': optimizer.state_dict(),}
                    torch.save(state,'/home/arisa/Desktop/Hamid/IQA/checkpoint.t7')

        end = time.time()
        print('TRAIN TIME:')
        print('%.2gs'%(end-start))

    train_model(model=model, train_loader=train_loader, optimizer=optimizer, criterion=criterion, valid_loader= test_loader,num_epochs=500  )

Thanks in advance
here is the result of 15 epoch
Epoch 1/500
----------
Epoch 1          Training Loss: 205.63448420514916       Validation Loss: 233.89266112356475         Train Acc:39.36360386127994         Validation Acc:24.142040038131555
Epoch 2/500
----------
Epoch 2          Training Loss: 199.05699240435197       Validation Loss: 235.08799531243065         Train Acc:41.90998291820601         Validation Acc:24.27311725452812
Epoch 3/500
----------
Epoch 3          Training Loss: 199.15626737127448       Validation Loss: 236.00033430619672         Train Acc:41.1035633416756          Validation Acc:23.677311725452814
Epoch 4/500
----------
Epoch 4          Training Loss: 199.02581041173886       Validation Loss: 233.60767459869385         Train Acc:41.86628530568466         Validation Acc:24.606768350810295
Epoch 5/500
----------
Epoch 5          Training Loss: 198.61493769454472       Validation Loss: 233.7503859202067          Train Acc:41.53656695665991         Validation Acc:25.0
Epoch 6/500
----------
Epoch 6          Training Loss: 198.71323942956585       Validation Loss: 234.17176149830675         Train Acc:41.639852222619474        Validation Acc:25.369399428026693
Epoch 7/500
----------
Epoch 7          Training Loss: 199.9395153770592        Validation Loss: 234.1744423635078          Train Acc:40.98041552456998         Validation Acc:24.84509056244042
Epoch 8/500
----------
Epoch 8          Training Loss: 199.3533399020355        Validation Loss: 235.4645173188412          Train Acc:41.26643626107337         Validation Acc:24.165872259294567
Epoch 9/500
----------
Epoch 9          Training Loss: 199.6451746921249        Validation Loss: 233.33387595956975         Train Acc:40.96452548365312         Validation Acc:24.59485224022879
Epoch 10/500
----------
Epoch 10         Training Loss: 197.9305159737011        Validation Loss: 233.76405122063377         Train Acc:41.8782028363723          Validation Acc:24.6186844613918
Epoch 11/500
----------
Epoch 11         Training Loss: 199.33247244055502       Validation Loss: 234.41085289463854         Train Acc:41.59218209986891         Validation Acc:25.119161105815063
Epoch 12/500
----------
Epoch 12         Training Loss: 199.87399289874256       Validation Loss: 234.23621463775635         Train Acc:41.028085647320545        Validation Acc:24.49952335557674
Epoch 13/500
----------
Epoch 13         Training Loss: 198.85540591944292       Validation Loss: 234.33149099349976         Train Acc:41.206848607635166        Validation Acc:24.857006673021925
Epoch 14/500
----------
Epoch 14         Training Loss: 199.92641723337513       Validation Loss: 233.37722391070741         Train Acc:41.15520597465539         Validation Acc:24.988083889418494
Epoch 15/500
----------
Epoch 15         Training Loss: 197.82172771698328       Validation Loss: 234.4943131533536          Train Acc:41.69943987605768         Validation Acc:24.380362249761678


title: SpeechBrain: dataio_prepare function with csv

body:
I am currently following the ASRfromScratch tutorial but I am trying to make it work with the Fluent Speech Dataset https://fluent.ai/fluent-speech-commands-a-dataset-for-spoken-language-understanding-research/. I was able to go through the Tokenizer section and the Language Model section with no problem but I am struggling with the SpeechRecognizer section. I modified the dataio_prepare function as such, but I am not sure if it is the correct approach:
    """This function prepares the datasets to be used in the brain class.
    It also defines the data processing pipeline through user-defined functions.

    Arguments
    ---------
    hparams : dict
        This dictionary is loaded from the `train.yaml` file, and it includes
        all the hyperparameters needed for dataset construction and loading.

    Returns
    -------
    datasets : dict
        Dictionary containing "train", "valid", and "test" keys that correspond
        to the DynamicItemDataset objects.
    """
    # Define audio pipeline. In this case, we simply read the path contained
    # in the variable wav with the audio reader.
    @sb.utils.data_pipeline.takes("path")
    @sb.utils.data_pipeline.provides("sig")
    def audio_pipeline(path):
        """Load the audio signal. This is done on the CPU in the `collate_fn`."""
        sig = sb.dataio.dataio.read_audio('../fluent_speech_commands_dataset/' + path)
        return sig

    # Define text processing pipeline. We start from the raw text and then
    # encode it using the tokenizer. The tokens with BOS are used for feeding
    # decoder during training, the tokens with EOS for computing the cost function.
    # The tokens without BOS or EOS is for computing CTC loss.
    @sb.utils.data_pipeline.takes("transcription")
    @sb.utils.data_pipeline.provides(
        "words", "tokens_list", "tokens_bos", "tokens_eos", "tokens"
    )
    def text_pipeline(transcription):
        """Processes the transcriptions to generate proper labels"""
        yield transcription
        tokens_list = hparams["tokenizer"].encode_as_ids(transcription)
        yield tokens_list
        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
        yield tokens_bos
        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
        yield tokens_eos
        tokens = torch.LongTensor(tokens_list)
        yield tokens

    # Define datasets from json data manifest file
    # Define datasets sorted by ascending lengths for efficiency
    datasets = {}
    data_folder = hparams["data_folder"]
    for dataset in ["train", "valid", "test"]:
        datasets[dataset] = sb.dataio.dataset.DynamicItemDataset.from_csv(
            csv_path = hparams[f"{dataset}_annotation"],
            replacements={"data_root": data_folder},
            dynamic_items=[audio_pipeline, text_pipeline],
            output_keys=[
                "id",
                "sig",
                "words",
                "tokens_bos",
                "tokens_eos",
                "tokens",
            ],
            )
        hparams[f"{dataset}_dataloader_opts"]["shuffle"] = False

    # Sorting training data with ascending order makes the code  much
    # faster  because we minimize zero-padding. In most of the cases, this
    # does not harm the performance.
    if hparams["sorting"] == "ascending":
        datasets["train"] = datasets["train"].filtered_sorted(sort_key="length")
        hparams["train_dataloader_opts"]["shuffle"] = False

    elif hparams["sorting"] == "descending":
        datasets["train"] = datasets["train"].filtered_sorted(
            sort_key="length", reverse=True
        )
        hparams["train_dataloader_opts"]["shuffle"] = False

    elif hparams["sorting"] == "random":
        hparams["train_dataloader_opts"]["shuffle"] = True
        pass

    else:
        raise NotImplementedError(
            "sorting must be random, ascending or descending"
        )
    return datasets

To clarify, the .csv files look like that :
ID,path,speakerId,transcription,action,object,location
0,wavs/speakers/2BqVo8kVB2Skwgyb/0a3129c0-4474-11e9-a9a5-5dbec3b8816a.wav,2BqVo8kVB2Skwgyb,Change language,change language,none,none
1,wavs/speakers/2BqVo8kVB2Skwgyb/0ee42a80-4474-11e9-a9a5-5dbec3b8816a.wav,2BqVo8kVB2Skwgyb,Resume,activate,music,none
2,wavs/speakers/2BqVo8kVB2Skwgyb/144d5be0-4474-11e9-a9a5-5dbec3b8816a.wav,2BqVo8kVB2Skwgyb,Turn the lights on,activate,lights,none
3,wavs/speakers/2BqVo8kVB2Skwgyb/1811b6e0-4474-11e9-a9a5-5dbec3b8816a.wav,2BqVo8kVB2Skwgyb,Switch on the lights,activate,lights,none

Also I removed the lines corresponding to the prenaining phase because I don't know how to make them work with my own dataset.
run_on_main(hparams["pretrainer"].collect_files)
hparams["pretrainer"].load_collected(device=run_opts["device"])

My problem is that the fit model phase keeps getting stuck at the first data to process and I don't know how to fix it :
(Polette) aurelienmarchal@aurelienmarchal-X556UQ:~/Stage/Polette/speech_recognizer$ python3 train.py train.yaml --batch_size=2
../noise/rirs_noises.zip exists. Skipping download
speechbrain.core - Beginning experiment!
speechbrain.core - Experiment folder: results/CRDNN_BPE_960h_LM/42
speechbrain.core - Info: ckpt_interval_minutes arg from hparam file is used
speechbrain.core - 171.8M trainable parameters in ASR
speechbrain.utils.checkpoints - Would load a checkpoint here, but none found yet.
speechbrain.utils.epoch_loop - Going into epoch 1
  0%|                                                                                | 0/11566 [00:00<?, ?it/s]
speechbrain.core - Exception:
Traceback (most recent call last):
  File "train.py", line 452, in <module>
    asr_brain.fit(
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/speechbrain/core.py", line 1011, in fit
    for batch in t:
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/tqdm/std.py", line 1133, in __iter__
    for obj in iterable:
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 517, in __next__
    data = self._next_data()
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/torch/utils/data/dataloader.py", line 557, in _next_data
    data = self._dataset_fetcher.fetch(index)  # may raise StopIteration
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py", line 47, in fetch
    return self.collate_fn(data)
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/speechbrain/dataio/batch.py", line 125, in __init__
    padded = PaddedData(*padding_func(values, **padding_kwargs))
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/speechbrain/utils/data_utils.py", line 415, in batch_pad_right
    padded, valid_percent = pad_right_to(
  File "/home/aurelienmarchal/.local/lib/python3.8/site-packages/speechbrain/utils/data_utils.py", line 353, in pad_right_to
    valid_vals.append(tensor.shape[j] / target_shape[j])
ZeroDivisionError: division by zero

Also, my train.yaml looks like that :
# ############################################################################
# Model: E2E ASR with attention-based ASR
# Encoder: CRDNN
# Decoder: GRU + beamsearch + RNNLM
# Tokens: 500 BPE
# losses: CTC+ NLL
# Training: mini-librispeech
# Pre-Training: librispeech 960h
# Authors:  Ju-Chieh Chou, Mirco Ravanelli, Abdel Heba, Peter Plantinga, Samuele Cornell 2020
# # ############################################################################

# Seed needs to be set at top of yaml, before objects with parameters are instantiated
seed: 42
__set_seed: !apply:torch.manual_seed [!ref <seed>]

# If you plan to train a system on an HPC cluster with a big dataset,
# we strongly suggest doing the following:
# 1- Compress the dataset in a single tar or zip file.
# 2- Copy your dataset locally (i.e., the local disk of the computing node).
# 3- Uncompress the dataset in the local folder.
# 4- Set data_folder with the local path
# Reading data from the local disk of the compute node (e.g. $SLURM_TMPDIR with SLURM-based clusters) is very important.
# It allows you to read the data much faster without slowing down the shared filesystem.

data_folder: ../fluent_speech_commands_dataset # In this case, data will be automatically downloaded here.
data_folder_rirs: ../noise # noise/ris dataset will automatically be downloaded here
output_folder: !ref results/CRDNN_BPE_960h_LM/<seed>
wer_file: !ref <output_folder>/wer.txt
save_folder: !ref <output_folder>/save
train_log: !ref <output_folder>/train_log.txt

# Language model (LM) pretraining
# NB: To avoid mismatch, the speech recognizer must be trained with the same
# tokenizer used for LM training. Here, we download everything from the
# speechbrain HuggingFace repository. However, a local path pointing to a
# directory containing the lm.ckpt and tokenizer.ckpt may also be specified
# instead. E.g if you want to use your own LM / tokenizer.
pretrained_path: ../language_model/results/RNNLM/save/CKPT+2021-05-12+15-27-08+00/

# Path where data manifest files will be stored. The data manifest files are created by the
# data preparation script
train_annotation: ../fluent_speech_commands_dataset/data/train_data.csv
valid_annotation: ../fluent_speech_commands_dataset/data/valid_data.csv
test_annotation: ../fluent_speech_commands_dataset/data/test_data.csv

# The train logger writes training statistics to a file, as well as stdout.
train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
    save_file: !ref <train_log>

# Training parameters
number_of_epochs: 15
number_of_ctc_epochs: 5
batch_size: 8
lr: 1.0
ctc_weight: 0.5
sorting: random
ckpt_interval_minutes: 15 # save checkpoint every N min
label_smoothing: 0.1

# Dataloader options
train_dataloader_opts:
    batch_size: !ref <batch_size>

valid_dataloader_opts:
    batch_size: !ref <batch_size>

test_dataloader_opts:
    batch_size: !ref <batch_size>

# Feature parameters
sample_rate: 16000
n_fft: 400
n_mels: 40

# Model parameters
activation: !name:torch.nn.LeakyReLU
dropout: 0.15
cnn_blocks: 2
cnn_channels: (128, 256)
inter_layer_pooling_size: (2, 2)
cnn_kernelsize: (3, 3)
time_pooling_size: 4
rnn_class: !name:speechbrain.nnet.RNN.LSTM
rnn_layers: 4
rnn_neurons: 1024
rnn_bidirectional: True
dnn_blocks: 2
dnn_neurons: 512
emb_size: 128
dec_neurons: 1024
output_neurons: 500  # Number of tokens (same as LM)
blank_index: 0
bos_index: 0
eos_index: 0
unk_index: 0

# Decoding parameters
min_decode_ratio: 0.0
max_decode_ratio: 1.0
valid_beam_size: 8
test_beam_size: 80
eos_threshold: 1.5
using_max_attn_shift: True
max_attn_shift: 240
lm_weight: 0.50
ctc_weight_decode: 0.0
coverage_penalty: 1.5
temperature: 1.25
temperature_lm: 1.25

# The first object passed to the Brain class is this "Epoch Counter"
# which is saved by the Checkpointer so that training can be resumed
# if it gets interrupted at any point.
epoch_counter: !new:speechbrain.utils.epoch_loop.EpochCounter
    limit: !ref <number_of_epochs>

# Feature extraction
compute_features: !new:speechbrain.lobes.features.Fbank
    sample_rate: !ref <sample_rate>
    n_fft: !ref <n_fft>
    n_mels: !ref <n_mels>

# Feature normalization (mean and std)
normalize: !new:speechbrain.processing.features.InputNormalization
    norm_type: global

# Added noise and reverb come from OpenRIR dataset, automatically
# downloaded and prepared with this Environmental Corruption class.
env_corrupt: !new:speechbrain.lobes.augment.EnvCorrupt
    openrir_folder: !ref <data_folder_rirs>
    babble_prob: 0.0
    reverb_prob: 0.0
    noise_prob: 1.0
    noise_snr_low: 0
    noise_snr_high: 15

# Adds speech change + time and frequency dropouts (time-domain implementation).
augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
    sample_rate: !ref <sample_rate>
    speeds: [95, 100, 105]

# The CRDNN model is an encoder that combines CNNs, RNNs, and DNNs.
encoder: !new:speechbrain.lobes.models.CRDNN.CRDNN
    input_shape: [null, null, !ref <n_mels>]
    activation: !ref <activation>
    dropout: !ref <dropout>
    cnn_blocks: !ref <cnn_blocks>
    cnn_channels: !ref <cnn_channels>
    cnn_kernelsize: !ref <cnn_kernelsize>
    inter_layer_pooling_size: !ref <inter_layer_pooling_size>
    time_pooling: True
    using_2d_pooling: False
    time_pooling_size: !ref <time_pooling_size>
    rnn_class: !ref <rnn_class>
    rnn_layers: !ref <rnn_layers>
    rnn_neurons: !ref <rnn_neurons>
    rnn_bidirectional: !ref <rnn_bidirectional>
    rnn_re_init: True
    dnn_blocks: !ref <dnn_blocks>
    dnn_neurons: !ref <dnn_neurons>
    use_rnnp: False

# Embedding (from indexes to an embedding space of dimension emb_size).
embedding: !new:speechbrain.nnet.embedding.Embedding
    num_embeddings: !ref <output_neurons>
    embedding_dim: !ref <emb_size>

# Attention-based RNN decoder.
decoder: !new:speechbrain.nnet.RNN.AttentionalRNNDecoder
    enc_dim: !ref <dnn_neurons>
    input_size: !ref <emb_size>
    rnn_type: gru
    attn_type: location
    hidden_size: !ref <dec_neurons>
    attn_dim: 1024
    num_layers: 1
    scaling: 1.0
    channels: 10
    kernel_size: 100
    re_init: True
    dropout: !ref <dropout>

# Linear transformation on the top of the encoder.
ctc_lin: !new:speechbrain.nnet.linear.Linear
    input_size: !ref <dnn_neurons>
    n_neurons: !ref <output_neurons>

# Linear transformation on the top of the decoder.
seq_lin: !new:speechbrain.nnet.linear.Linear
    input_size: !ref <dec_neurons>
    n_neurons: !ref <output_neurons>

# Final softmax (for log posteriors computation).
log_softmax: !new:speechbrain.nnet.activations.Softmax
    apply_log: True

# Cost definition for the CTC part.
ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
    blank_index: !ref <blank_index>

# Tokenizer initialization
tokenizer: !new:sentencepiece.SentencePieceProcessor

# Objects in "modules" dict will have their parameters moved to the correct
# device, as well as having train()/eval() called on them by the Brain class
modules:
    encoder: !ref <encoder>
    embedding: !ref <embedding>
    decoder: !ref <decoder>
    ctc_lin: !ref <ctc_lin>
    seq_lin: !ref <seq_lin>
    normalize: !ref <normalize>
    env_corrupt: !ref <env_corrupt>
    lm_model: !ref <lm_model>

# Gathering all the submodels in a single model object.
model: !new:torch.nn.ModuleList
    - - !ref <encoder>
      - !ref <embedding>
      - !ref <decoder>
      - !ref <ctc_lin>
      - !ref <seq_lin>

# This is the RNNLM that is used according to the Huggingface repository
# NB: It has to match the pre-trained RNNLM!!
lm_model: !new:speechbrain.lobes.models.RNNLM.RNNLM
    output_neurons: !ref <output_neurons>
    embedding_dim: !ref <emb_size>
    activation: !name:torch.nn.LeakyReLU
    dropout: 0.0
    rnn_layers: 2
    rnn_neurons: 2048
    dnn_blocks: 1
    dnn_neurons: 512
    return_hidden: True  # For inference

# Beamsearch is applied on the top of the decoder. If the language model is
# given, a language model is applied (with a weight specified in lm_weight).
# If ctc_weight is set, the decoder uses CTC + attention beamsearch. This
# improves the performance, but slows down decoding. For a description of
# the other parameters, please see the speechbrain.decoders.S2SRNNBeamSearchLM.

# It makes sense to have a lighter search during validation. In this case,
# we don't use the LM and CTC probabilities during decoding.
valid_search: !new:speechbrain.decoders.S2SRNNBeamSearcher
    embedding: !ref <embedding>
    decoder: !ref <decoder>
    linear: !ref <seq_lin>
    ctc_linear: !ref <ctc_lin>
    bos_index: !ref <bos_index>
    eos_index: !ref <eos_index>
    blank_index: !ref <blank_index>
    min_decode_ratio: !ref <min_decode_ratio>
    max_decode_ratio: !ref <max_decode_ratio>
    beam_size: !ref <valid_beam_size>
    eos_threshold: !ref <eos_threshold>
    using_max_attn_shift: !ref <using_max_attn_shift>
    max_attn_shift: !ref <max_attn_shift>
    coverage_penalty: !ref <coverage_penalty>
    temperature: !ref <temperature>

# The final decoding on the test set can be more computationally demanding.
# In this case, we use the LM + CTC probabilities during decoding as well.
# Please, remove this part if you need a faster decoder.
test_search: !new:speechbrain.decoders.S2SRNNBeamSearchLM
    embedding: !ref <embedding>
    decoder: !ref <decoder>
    linear: !ref <seq_lin>
    ctc_linear: !ref <ctc_lin>
    language_model: !ref <lm_model>
    bos_index: !ref <bos_index>
    eos_index: !ref <eos_index>
    blank_index: !ref <blank_index>
    min_decode_ratio: !ref <min_decode_ratio>
    max_decode_ratio: !ref <max_decode_ratio>
    beam_size: !ref <test_beam_size>
    eos_threshold: !ref <eos_threshold>
    using_max_attn_shift: !ref <using_max_attn_shift>
    max_attn_shift: !ref <max_attn_shift>
    coverage_penalty: !ref <coverage_penalty>
    lm_weight: !ref <lm_weight>
    ctc_weight: !ref <ctc_weight_decode>
    temperature: !ref <temperature>
    temperature_lm: !ref <temperature_lm>

# This function manages learning rate annealing over the epochs.
# We here use the NewBoB algorithm, that anneals the learning rate if
# the improvements over two consecutive epochs is less than the defined
# threshold.
lr_annealing: !new:speechbrain.nnet.schedulers.NewBobScheduler
    initial_value: !ref <lr>
    improvement_threshold: 0.0025
    annealing_factor: 0.8
    patient: 0

# This optimizer will be constructed by the Brain class after all parameters
# are moved to the correct device. Then it will be added to the checkpointer.
opt_class: !name:torch.optim.Adadelta
    lr: !ref <lr>
    rho: 0.95
    eps: 1.e-8

# Functions that compute the statistics to track during the validation step.
error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats

cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
    split_tokens: True

# This object is used for saving the state of training both so that it
# can be resumed if it gets interrupted, and also so that the best checkpoint
# can be later loaded for evaluation or inference.
checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
    checkpoints_dir: !ref <save_folder>
    recoverables:
        model: !ref <model>
        scheduler: !ref <lr_annealing>
        normalizer: !ref <normalize>
        counter: !ref <epoch_counter>

# This object is used to pretrain the language model and the tokenizers
# (defined above). In this case, we also pretrain the ASR model (to make
# sure the model converges on a small amount of data)
#pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
#    collect_in: !ref <save_folder>
#    loadables:
#        lm: !ref <lm_model>
#        tokenizer: !ref <tokenizer>
#        model: !ref <model>
#    paths:
#        lm: !ref <pretrained_path>/lm.ckpt
#        tokenizer: !ref <pretrained_path>/tokenizer.ckpt
#        model: !ref <pretrained_path>/asr.ckpt

That's it ! If you have an idea on how to do it or if you are comfortable with the SpeechBrain library let my now !
Thank you for reading my post


title: Calculate Second Gradient with PyTorch

body:
In PyTorch, there are two ways of calculating second gradients. The first method is to use torch.autograd.grad function, and the other is to use backward function. I use the following examples to illustrate it:
Method 1:
x=torch.tensor([3.0], requires_grad=True)
y = torch.pow(x, 2)
grad_1 = torch.autograd.grad(y, x, create_graph=True)
print(grad_1[0].item())
grad_2 = torch.autograd.grad(grad_1[0], x)
print(grad_2)

The result makes sense for me, and the second gradient of the function is 2.
Method 2:
x=torch.tensor([3.0], requires_grad=True)
y = torch.pow(x, 2) # y=x**2
y.backward(retain_graph=True)
print(x.grad)

y.backward()
print(x.grad)

When calculating the first gradient, I use create_graph=True to make sure that we can use back prorogation method to calculate the second gradient. However, the result is is 12, which is wrong. I was wondering what's wrong with the second method?


title: Torchaudio C++ extension is not available - Python

body:
Having just started learning Torchaudio, I got this error. I fixed the first part but even after some deep internet surfing, I can't find a fix. I use Windows 10.
C:\Users\bala006\Miniconda39\lib\site-packages\torchaudio\extension\extension.py:13: UserWarning: torchaudio C++ extension is not available.
  warnings.warn('torchaudio C++ extension is not available.')

Thanks for any help !


title: Why is Pytorch out-of-memory for a little example?

body:
I am learning torch and my gpu info on ubuntu is below:
# nvidia-smi
Fri May 21 09:04:45 2021
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla V100-SXM2...  On   | 00000000:00:10.0 Off |                    0 |
| N/A   33C    P0    57W / 300W |   3176MiB / 16130MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
|   1  Tesla V100-SXM2...  On   | 00000000:00:11.0 Off |                    0 |
| N/A   33C    P0    57W / 300W |   2121MiB / 16130MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+
|   2  Tesla V100-SXM2...  On   | 00000000:00:12.0 Off |                    0 |
| N/A   55C    P0   273W / 300W |  16127MiB / 16130MiB |     97%      Default |
+-------------------------------+----------------------+----------------------+
|   3  Tesla V100-SXM2...  On   | 00000000:00:13.0 Off |                    0 |
| N/A   55C    P0   257W / 300W |  15514MiB / 16130MiB |     99%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
+-----------------------------------------------------------------------------+

And I am testing this little example:
https://pytorch.org/docs/stable/notes/cuda.html
And I got the out of memory error:
In [4]: cuda = torch.device('cuda')     # Default CUDA device
   ...: cuda0 = torch.device('cuda:0')
   ...: cuda2 = torch.device('cuda:2')  # GPU 2 (these are 0-indexed)
   ...:
   ...: x = torch.tensor([1., 2.], device=cuda0)
   ...: # x.device is device(type='cuda', index=0)
   ...: y = torch.tensor([1., 2.]).cuda()
   ...: # y.device is device(type='cuda', index=0)
   ...:
   ...: with torch.cuda.device(1):
   ...:     # allocates a tensor on GPU 1
   ...:     a = torch.tensor([1., 2.], device=cuda)
   ...:
   ...:     # transfers a tensor from CPU to GPU 1
   ...:     b = torch.tensor([1., 2.]).cuda()
   ...:     # a.device and b.device are device(type='cuda', index=1)
   ...:
   ...:     # You can also use ``Tensor.to`` to transfer a tensor:
   ...:     b2 = torch.tensor([1., 2.]).to(device=cuda)
   ...:     # b.device and b2.device are device(type='cuda', index=1)
   ...:
   ...:     c = a + b
   ...:     # c.device is device(type='cuda', index=1)
   ...:
   ...:     z = x + y
   ...:     # z.device is device(type='cuda', index=0)
   ...:
   ...:     # even within a context, you can specify the device
   ...:     # (or give a GPU index to the .cuda call)
   ...:     d = torch.randn(2, device=cuda2)
   ...:     e = torch.randn(2).to(cuda2)
   ...:     f = torch.randn(2).cuda(cuda2)
   ...:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-4-a8b7a1844cfa> in <module>
     28     # even within a context, you can specify the device
     29     # (or give a GPU index to the .cuda call)
---> 30     d = torch.randn(2, device=cuda2)
     31     e = torch.randn(2).to(cuda2)
     32     f = torch.randn(2).cuda(cuda2)

RuntimeError: CUDA error: out of memory

In [5]: print(torch.__version__)
1.7.1

The same code my mac works fine.