title: Import pytorch-encoding fails on docker build on macOS BigSur

body:
I have a docker image based on pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel image. It builds successfully on macOS Catalina but it fails to build on macOS Big Sur. It fails when running import encoding in python script (see stack trace below).
Seems like the issue is related to python deep learning libraries that are wrapping c++ implementation.
I don't want to manually fix c++ implementations and would prefer a more general solution.
I tried running the project without docker (on Big Sur machine itself) and it was successful. I tried building it on another machine with macOS Catalina and it still builds successfully.
I really need to build it in docker on Big Sur. Could somebody help with this ?
Details:

base docker image: pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel
docker version 3.2.2
Dockerfile, conda.yaml, preload_model.py and error trace attached

Dockerfile:
FROM pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel

RUN apt-get -y update && apt-get install -y --no-install-recommends \
         curl \
         ca-certificates \
    && rm -rf /var/lib/apt/lists/* && apt-get clean

ENV PATH="/miniconda/bin:$PATH"
RUN curl https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh >> miniconda.sh \
    && bash ./miniconda.sh -b -p /miniconda; rm ./miniconda.sh

WORKDIR /opt/app

COPY conda.yaml ./

RUN apt-get -y update && apt-get install -y build-essential cmake \
    && conda env update --prefix /miniconda --file conda.yaml \
    && conda clean -tipsy \
    && rm -rf /var/lib/apt/lists/* && apt-get clean \
    && rm -rf ~/.cache/pip

RUN apt-get -y update && apt-get install -y --no-install-recommends \
         libgl1 libglib2.0-0 \
    && rm -rf /var/lib/apt/lists/* && apt-get clean

COPY app/preload_model.py ./

RUN python preload_model.py

preload_model.py
import encoding #fails here

encoding.models.get_model('DeepLab_ResNeSt200_ADE', pretrained=True)

conda.yaml
channels:
  - defaults
  - pytorch
dependencies:
  - python=3.8
  - pytorch=1.6.0
  - cudatoolkit=10.1
  - scipy=1.5.2
  - Flask=1.1.2
  - gunicorn=20.0.4
  - torchvision=0.7.0
  - Pillow=7.2.0
  - requests=2.24.0
  - numpy=1.19.1
  - ca-certificates
  - certifi
  - pip=20.2.2
  - pip:
      - brotlipy==0.7.0
      - chardet==3.0.4
      - click==7.1.2
      - future==0.18.2
      - itsdangerous==1.1.0
      - Jinja2==2.11.2
      - nose==1.3.7
      - opencv-python==4.4.0.44
      - portalocker==2.0.0
      - six==1.15.0
      - torch-encoding==1.2.1
      - tqdm==4.50.0
      - Werkzeug==1.0.1
      - imagehash==4.2.0
      - Flask-Caching==1.9.0

Error:
> [10/17] RUN python preload_model.py:
#14 1.680 No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'
#14 69.42 Traceback (most recent call last):
#14 69.42   File "/miniconda/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1509, in _run_ninja_build
#14 69.42     subprocess.run(
#14 69.42   File "/miniconda/lib/python3.8/subprocess.py", line 516, in run
#14 69.42     raise CalledProcessError(retcode, process.args,
#14 69.42 subprocess.CalledProcessError: Command '['ninja', '-v']' returned non-zero exit status 1.
#14 69.42
#14 69.42 During handling of the above exception, another exception occurred:
#14 69.42
#14 69.42 Traceback (most recent call last):
#14 69.42   File "preload_model.py", line 1, in <module>
#14 69.42     import encoding
#14 69.42   File "/miniconda/lib/python3.8/site-packages/encoding/__init__.py", line 13, in <module>
#14 69.42     from . import nn, functions, parallel, utils, models, datasets, transforms
#14 69.42   File "/miniconda/lib/python3.8/site-packages/encoding/nn/__init__.py", line 12, in <module>
#14 69.42     from .encoding import *
#14 69.42   File "/miniconda/lib/python3.8/site-packages/encoding/nn/encoding.py", line 18, in <module>
#14 69.42     from ..functions import scaled_l2, aggregate, pairwise_cosine
#14 69.42   File "/miniconda/lib/python3.8/site-packages/encoding/functions/__init__.py", line 2, in <module>
#14 69.42     from .encoding import *
#14 69.42   File "/miniconda/lib/python3.8/site-packages/encoding/functions/encoding.py", line 14, in <module>
#14 69.42     from .. import lib
#14 69.42   File "/miniconda/lib/python3.8/site-packages/encoding/lib/__init__.py", line 9, in <module>
#14 69.42     cpu = load('enclib_cpu', [
#14 69.42   File "/miniconda/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 963, in load
#14 69.42     return _jit_compile(
#14 69.42   File "/miniconda/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1170, in _jit_compile
#14 69.42     _write_ninja_file_and_build_library(
#14 69.42   File "/miniconda/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1276, in _write_ninja_file_and_build_library
#14 69.42     _run_ninja_build(
#14 69.42   File "/miniconda/lib/python3.8/site-packages/torch/utils/cpp_extension.py", line 1529, in _run_ninja_build
#14 69.42     raise RuntimeError(message)
#14 69.42 RuntimeError: Error building extension 'enclib_cpu': [1/7] c++ -MMD -MF encoding_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/encoding_cpu.cpp -o encoding_cpu.o
#14 69.42 [2/7] c++ -MMD -MF rectify_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/rectify_cpu.cpp -o rectify_cpu.o
#14 69.42 FAILED: rectify_cpu.o
#14 69.42 c++ -MMD -MF rectify_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/rectify_cpu.cpp -o rectify_cpu.o
#14 69.42 c++: internal compiler error: Killed (program cc1plus)
#14 69.42 Please submit a full bug report,
#14 69.42 with preprocessed source if appropriate.
#14 69.42 See <file:///usr/share/doc/gcc-7/README.Bugs> for instructions.
#14 69.42 [3/7] c++ -MMD -MF nms_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/nms_cpu.cpp -o nms_cpu.o
#14 69.42 FAILED: nms_cpu.o
#14 69.42 c++ -MMD -MF nms_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/nms_cpu.cpp -o nms_cpu.o
#14 69.42 c++: internal compiler error: Killed (program cc1plus)
#14 69.42 Please submit a full bug report,
#14 69.42 with preprocessed source if appropriate.
#14 69.42 See <file:///usr/share/doc/gcc-7/README.Bugs> for instructions.
#14 69.42 [4/7] c++ -MMD -MF syncbn_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/syncbn_cpu.cpp -o syncbn_cpu.o
#14 69.42 FAILED: syncbn_cpu.o
#14 69.42 c++ -MMD -MF syncbn_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/syncbn_cpu.cpp -o syncbn_cpu.o
#14 69.42 c++: internal compiler error: Killed (program cc1plus)
#14 69.42 Please submit a full bug report,
#14 69.42 with preprocessed source if appropriate.
#14 69.42 See <file:///usr/share/doc/gcc-7/README.Bugs> for instructions.
#14 69.42 [5/7] c++ -MMD -MF roi_align_cpu.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/roi_align_cpu.cpp -o roi_align_cpu.o
#14 69.42 In file included from /miniconda/lib/python3.8/site-packages/torch/include/ATen/ATen.h:9:0,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/types.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader_options.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/base.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/stateful.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/all.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/extension.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/

.....

roi_align_cpu.cpp:1:
#14 69.42 /miniconda/lib/python3.8/site-packages/torch/include/ATen/core/TensorBody.h:354:7: note: declared here
#14 69.42    T * data() const {
#14 69.42        ^~~~
#14 69.42 In file included from /miniconda/lib/python3.8/site-packages/torch/include/ATen/ATen.h:9:0,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/types.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader_options.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/base.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/stateful.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/all.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/extension.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/roi_align_cpu.cpp:1:
#14 69.42 /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/roi_align_cpu.cpp:472:34: warning: ‘T* at::Tensor::data() const [with T = float]’ is deprecated: Tensor.data<T>() is deprecated. Please use Tensor.data_ptr<T>() instead. [-Wdeprecated-declarations]
#14 69.42        bottom_rois.data<scalar_t>(),
#14 69.42                                   ^
#14 69.42 In file included from /miniconda/lib/python3.8/site-packages/torch/include/ATen/Tensor.h:3:0,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/ATen/Context.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/ATen/ATen.h:5,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/types.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader_options.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/base.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader/stateful.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data/dataloader.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/data.h:3,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include/torch/all.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/torch/include/torch/extension.h:4,
#14 69.42                  from /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/roi_align_cpu.cpp:1:
#14 69.42 /miniconda/lib/python3.8/site-packages/torch/include/ATen/core/TensorBody.h:354:7: note: declared here
#14 69.42    T * data() const {
#14 69.42        ^~~~
#14 69.42 [6/7] c++ -MMD -MF operator.o.d -DTORCH_EXTENSION_NAME=enclib_cpu -DTORCH_API_INCLUDE_EXTENSION_H -isystem /miniconda/lib/python3.8/site-packages/torch/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/torch/csrc/api/include -isystem /miniconda/lib/python3.8/site-packages/torch/include/TH -isystem /miniconda/lib/python3.8/site-packages/torch/include/THC -isystem /miniconda/include/python3.8 -D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -std=c++14 -c /miniconda/lib/python3.8/site-packages/encoding/lib/cpu/operator.cpp -o operator.o
#14 69.42 ninja: build stopped: subcommand failed.
#14 69.42
------
executor failed running [/bin/sh -c python preload_model.py]: exit code: 1


title: Is there a way to save a fixed pytorch tensor in a pytorch nn module as a parameter?

body: So I have a fixed tensor, which are centroids that I pre-calculated. Essentially the centroids matrix is N classes by D dimensions. I want to save this in a Pytorch module at the end of training in such a way that when I save this model and load it this parameter will be there for me to use after loading the saved checkpoint. Also there should never be a gradient passing through this matrix.


title: Where can i find/generate the wav file from torch

body:
This code below generates an Audio output using IPython. I could only play this in a notebook(colab) and i want to export the wav file.
    if len(i) < 1: continue;
    print(i)
    if raw_input:
        if i[-1] != ";": i=i+";"
    else: i = ARPA(i)
    print(i)
    with torch.no_grad(): # save VRAM by not including gradients
        sequence = np.array(text_to_sequence(i, ['english_cleaners']))[None, :]
        sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long()
        mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)
        audio = waveglow.infer(mel_outputs_postnet, sigma=sigma); print(""); ipd.display(ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate))```


title: Splitting data into batches

body:
I currently have the following data:
f_map, inputs, s_bias = ml_dataset.dataset_for_s_bias()

where f_map is a tensor of matrices, inputs is a tensor of floats, and s_bias is a tensor of floats. The first two, f_map and inputs, are the inputs to my ML regression algorithm, and s_bias is the expected output. The reason there are two kinds of inputs is because f_map is processed using a CNN, and the CNN turns each matrix in f_map into a float, which is concatenated with the inputs tensor, and the resulting tensor is inputted into an MLP to get a prediction for s_bias.
I don’t know how to split f_map, inputs and s_bias into batches. I tried using a dataloader, but this causes me problems in my CNN which expects a tensor (I have an unsqueeze operation in my CNN, but dataloader is incompatible with unsqueeze). I need f_map, inputs, and s_bias to be split the same way and preserve the order. What is the best way to do this?


title: pytorch model can read succes on local envi(windows,pycharm) but fail on server(unix)

body:
[I can load model on local(window, pycharm) successfully, but it fail on server(unix) with the same code.][1]
Traceback (most recent call last):
  File "inference.py", line 113, in <module>
    main()
  File "inference.py", line 82, in main
    model.load_state_dict(torch.load(config.ESIM_MODEL_PATH))
  File "/home/ai-center/yangyitai/py38/lib/python3.8/site-packages/torch/serialization.py", line 587, in load
    with _open_zipfile_reader(opened_file) as opened_zipfile:
  File "/home/ai-center/yangyitai/py38/lib/python3.8/site-packages/torch/serialization.py", line 242, in __init__
    super(_open_zipfile_reader, self).__init__(torch._C.PyTorchFileReader(name_or_buffer))
RuntimeError: [enforce fail at inline_container.cc:145] . PytorchStreamReader failed reading zip archive: not a ZIP archive


title: RNN Input Dimension! Pytorch

body:
having hard time to input the data into RNN in pytorch..
The output dimension from my previous linear layer is (32,50) where 32 is batch size.
I want to feed this to RNN layer.
I tried reshaping my data in 2 forms: [32,1,50] and [32,50,1] both of the times I get error.
class trainer:
    def __init__():
       self.r = nn.RNN(input_size= 50, hidden_size=2, num_layers = 3, batch_first=True)

    def forward(self):
       previous_layer_output
       previous_layer_output.unsqueeze_(-1) Makes shape [32,50,1] # I also tried
     #previous_layer_output.unsqueeze_(1) makes shape [32,1,50]

       rnn = self.r(previous_layer_output )

I get this error : RuntimeError: input.size(-1) must be equal to input_size. Expected 50, got 1
If i do previous_layer_output.unsqueeze_(1) I get the below error:
ValueError: Expected target size (32, 50), got torch.Size([32])


title: How do I recalculate the gradient after changing the input? Pytorch

body:
I am trying to do something like this without redefining a = f(x,y):
a = f(x,y)
find gradient of a with respect to x
change x
find gradient of a with respect to x
find gradient of a with respect to y

I tried a partial example below but it just gives me an error. Does anyone know how I can do this without redefining the original function everytime?
>>> x = torch.tensor([2.], requires_grad=True)
>>> y = 10*x**2
>>> torch.autograd.grad(y,x, retain_graph=True)
(tensor([40.]),)
>>> x = torch.tensor([1.], requires_grad=True)
>>> torch.autograd.grad(y,x, retain_graph=True)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/Philip/anaconda3/lib/python3.7/site-packages/torch/autograd/__init__.py", line 157, in grad
    inputs, allow_unused)
RuntimeError: One of the differentiated Tensors appears to not have been used in the graph. Set allow_unused=True if this is the desired behavior.


title: Time series forecast pytorch Encoder Decoder (sequence to sequence model)

body:
I am bulding a time series forecasting moddel with input sequence length of 180 and target sequence length of 60.
input shape is [800,180,1] 800 datapoints, 180 features,
target shape is [800, 60,1] 800 datapoints 60 features
using dataloader the input and target are processed with batch size of 4..
input is reshaped to input shape [180,1,1]
targetshape [60,1]    per batch
#Encoder Decoder Model
MAX_OUTPUT = 60

class Forecast_EncoderDecoder(nn.Module):

    def __init__(self, input_size, hidden_size, output_size,dropout,verbose=False):
        super(Forecast_EncoderDecoder, self).__init__()

        self.hidden_size = hidden_size
        self.output_size = output_size
        self.dropout = nn.Dropout(dropout)
#         self.n_layers = n_layers

        self.encoder_rnn_cell = nn.GRU(input_size, hidden_size, dropout=dropout)

        self.decoder_rnn_cell = nn.GRU(output_size, hidden_size,dropout=dropout)

        self.h2o = nn.Linear(hidden_size, output_size)
        self.verbose = verbose

    def forward(self, input,max_output = MAX_OUTPUT, device = 'cpu', ground_truth = None):

        # encoder
        out, hidden = self.encoder_rnn_cell(input)

        if self.verbose:
            print('Encoder input', input.shape)
            print('Encoder output', out.shape)
            print('Encoder hidden', hidden.shape)

        # decoder
        decoder_state = hidden.reshape((1,1,self.hidden_size))
        decoder_input = torch.zeros(1,1, self.output_size).to(device)

        outputs = []

        if self.verbose:
            print('Decoder state', decoder_state.shape)
            print('Decoder input', decoder_input.shape)

        for i in range(max_output):
            out, decoder_state = self.decoder_rnn_cell(decoder_input, decoder_state)

            if self.verbose:
                print('Decoder intermediate output', out.shape)

            out = self.h2o(decoder_state)
            outputs.append(out.reshape(1))

            if self.verbose:
                print('Decoder output', out.shape)

            max_idx = out

            if not ground_truth is None:
                max_idx = ground_truth[i].reshape(1, 1, 1)
#                 max_idx = max_idx.to(torch.long)

            decoder_input = max_idx.detach()
#             print('decoder_input',decoder_input.shape)
        return outputs

Please help me, if my input/output shape and architecture is correct?


title: A2C Agent does not learn anything in a simple 2D grid world

body:
I am trying to implement a simple Advantage Actor Critic algorithm on a simple self-defined environment. My complete code is shown below. It is a 10x10 grid world environment where there is one end state with +100 reward and other end states with 0 reward. The agent can take a maximum of 15 steps with the actions (up, down, left and right). If the agent runs out on one edge of the field, he comes out on the opposite side. For each step on the field he gets +1 reward. The code for my Policy Value Network and the training code is shown in full below. I would expect the agent to learn over time where the +100 field is and steer specifically there so that the average reward of a trajectory converges to 100. I have shown one run as a result below. As you can see, the Episode Reward hardly/not at all increases, but stagnates at about 15 (which is the maximum number of steps on the field and he gets +1 reward for each step). Every now and then he hits the +100 field but that seems more random instead of him learning over time where the field is. I would be very grateful for any hints as to why my agent is not learning.
2d-world.py
import torch
import pandas as pd
import numpy as np
import torch.nn as nn
from torch.distributions.categorical import Categorical
from model import PolicyValueNetwork
from torch.utils.tensorboard import SummaryWriter
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
writer = SummaryWriter()

lr = 0.01
gamma = 0.99
episodes = 1000000
normalize_targets = True
n_steps = 15

class Environment():
    def __init__(self):
        field = {
                "0": [0, 1, 1, 1, 0, 1, 1, 1, 1, 1],
                "1": [1, 0, 1, 1, 1, 1, 1, 1, 0, 1],
                "2": [1, 1, 1, 0, 1, 1, 0, 1, 1, 1],
                "3": [1, 1, 1, 1, 0, 1, 1, 1, 0, 0],
                "4": [1, 0, 1, 1, 1, 1, 1, 1, 1, 1],
                "5": [1, 1, 1, 0, 1, 1, 1, 0, 1, 1],
                "6": [1, 0, 1, 1, 0, 1, 0, 1, 1, 0],
                "7": [1, 1, 1, 0, 1, 1, 1, 1, 1, 1],
                "8": [1, 1, 0, 1, 1, 0, 1, 100, 1, 1],
                "9": [1, 1, 1, 1, 1, 1, 1, 1, 1, 0]}
        env = pd.DataFrame(data=field).T
        self.rewards = env.values
        n_rows = self.rewards.shape[0]
        n_cols = self.rewards.shape[1]
        self.start_points = []
        for row in range(0, n_rows):
            for col in range(0, n_cols):
                if self.rewards[row][col] == 1:
                    self.start_points.append((col, row)) # (x, y)
        self.s = self.reset()

    def step(self, action): # action in [0, 1, 2, 3] means ["up", "down", "left", "right"]

        # First we calculate the possible new positions on the field
        x = self.s[0]
        y = self.s[1]
        nx = [x, x, 9 if x == 0 else x - 1, 0 if x == 9 else x + 1] # possible next x values
        ny = [9 if y == 0 else y - 1, 0 if y == 9 else y + 1, y, y] # possible next y values

        # Get next state and reward
        ns = (nx[action], ny[action])
        self.s = ns
        reward = self.rewards[ns[1]][ns[0]] # indexing is done like [y][x]

        # Calculate done signal
        done = reward == 100 or reward == 0

        return ns, reward, done

    def reset(self):
        i = np.random.choice(list(range(0, len(self.start_points))))
        self.s = self.start_points[i]
        return self.s

# Get cuda device
device = torch.device(0)

# Instantiate model
model = PolicyValueNetwork()
model.to(device)

# Optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
#optimizer = torch.optim.RMSprop(model.parameters())

# Loss function for value loss
loss_fn = nn.MSELoss(reduction="sum")

# Set environment
env = Environment()

for episode in range(0, episodes):

    state = env.reset()

    # Set up arrays for this episode
    values = []
    log_probs = []
    rewards = []
    states = []
    for i in range(0, n_steps):

        states.append(state)

        # Create tensor for current state
        state_tensor = torch.tensor(data=state).float().to(device)

        # Get policy and value from model
        probs, value = model(state_tensor)
        values.append(value)
        categorical = Categorical(probs=probs)

        # Sample an action
        a = categorical.sample()
        log_prob = categorical.log_prob(a)
        log_probs.append(log_prob)

        # Execute step in the environment
        state, reward, done = env.step(a.item())
        rewards.append(reward)

        if done:
            break

    # Calculate discounted returns
    state_tensor = torch.tensor(data=state).float().to(device)
    _, value = model(state_tensor)
    R = 0.0 if done else value.item()
    T = len(values)
    targets = np.empty(T, dtype=np.float32)
    for i in reversed(range(T)):
        targets[i] = R = rewards[i] + gamma * R

    # Normalize discounted returns alias targets
    if normalize_targets:
        targets = scaler.fit_transform(targets.reshape(-1,1))
    targets = torch.tensor(data=targets).to(device).squeeze()

    # Calculate advantages
    values = torch.stack(values).squeeze()
    advantages = targets - values

    # Calculate policy loss
    log_probs = torch.stack(log_probs).squeeze()
    policy_loss = torch.sum(-advantages.detach() * log_probs)

    # Value loss
    value_loss = loss_fn(values, targets)

    # Backpropagation
    optimizer.zero_grad()
    policy_loss.backward(retain_graph=True)
    value_loss.backward()
    optimizer.step()

    # Logging with tensorboard
    writer.add_scalar("Episode Reward", sum(rewards), episode)
    writer.add_scalar("Episode Policy Loss", policy_loss.item(), episode)
    writer.add_scalar("Episode Value Loss", value_loss.item(), episode)

model.py
import torch.nn as nn

# Neural network
class PolicyValueNetwork(nn.Module):
    def __init__(self):
        super(PolicyValueNetwork, self).__init__()
        self.lin1 = nn.Linear(in_features=2, out_features=128, bias=True)
        self.policy = nn.Linear(in_features=128, out_features=4, bias=True)
        self.value = nn.Linear(in_features=128, out_features=1, bias=True)
        self.relu = nn.ReLU()
        self.softmax = nn.Softmax(dim=0)

    def forward(self, inp):
        out1 = self.lin1(inp)
        relu1 = self.relu(out1)
        policy = self.policy(relu1)
        value = self.value(relu1)
        probs = self.softmax(policy)
        return probs, value

Results
Episode returns of approximately 45000 episodes.

Disclaimer
Since I wasn't quite sure where best to post the question, I also posted the same question on Cross Validated.


title: Decreasing number of nodes each layers in torch.nn.lstm

body:
Is there an easy way to decrease the number of nodes in each layer by a factor? I don't see this option on the documentation page, perhaps there is a similar function I can use though instead of manually defining each layer?
    self.lstm = nn.LSTM(
        input_size=input_size,
        hidden_size=hidden_size,
        num_layers=num_layers,
        batch_first=True,
        dropout=0.2,
    )  # lstm


title: How to sample similar vectors given a vector and cosine similarity in pytorch?

body:
I have this vector
>>> vec
tensor([[0.2677, 0.1158, 0.5954, 0.9210, 0.3622, 0.4081, 0.4477, 0.7930, 0.1161,
         0.5111, 0.2010, 0.3680, 0.1162, 0.1563, 0.4478, 0.9732, 0.7962, 0.0873,
         0.9793, 0.9382, 0.9468, 0.0851, 0.7601, 0.0322, 0.7553, 0.4025, 0.3627,
         0.5706, 0.3015, 0.1344, 0.8343, 0.8187, 0.4287, 0.5785, 0.9527, 0.1632,
         0.2890, 0.5411, 0.5319, 0.7163, 0.3166, 0.5717, 0.5018, 0.5368, 0.3321]])

using this vector I want to generate 15 vectors which have a cosine similarity greater than 80%.
How can I do this in pytorch?


title: What is the right way to generate long sequence using PyTorch-Transformers?

body:
I am trying to generate a long sequence of text using PyTorch-Transformers from a sample text. I am following this tutorial for this purpose. Because the original article only predicts one word from a given text, I modified that script to generate long sequence instead of one. This is the modified part of the code
# Encode a text inputs
text = """An examination can be defined as a detailed inspection or analysis
 of an object or person. For example, an engineer will examine a structure,
  like a bridge, to see if it is safe. A doctor may conduct"""

indexed_tokens = tokenizer.encode(text)

# Convert indexed tokens in a PyTorch tensor
tokens_tensor = torch.tensor([indexed_tokens])
seq_len = tokens_tensor.shape[1]
tokens_tensor = tokens_tensor.to('cuda')

with torch.no_grad():
    for i in range(50):
        outputs = model(tokens_tensor[:,-seq_len:])
        predictions = outputs[0]
        predicted_index = torch.argmax(predictions[0, -1, :])
        tokens_tensor = torch.cat((tokens_tensor,predicted_index.reshape(1,1)),1)

pred = tokens_tensor.detach().cpu().numpy().tolist()
predicted_text = tokenizer.decode(pred[0])
print(predicted_text)

Output

An examination can be defined as a detailed inspection or analysis
of an object or person. For example, an engineer will examine a
structure,   like a bridge, to see if it is safe. A doctor may conduct
an examination of a patient's body to see if it is safe.
The doctor may also examine a patient's body to see if it is safe. A
doctor may conduct an examination of a patient's body to see if it is
safe.

As you can see the generated text does not generates any unique text sequence but it generates the same sentence over and over again with minor changes.
How should we create long sequence using  PyTorch-Transformers?


title: Pytorch uses more memory on the first training batch

body:
On the very first call to the forward pass of my model there is far less memory available (200mb) than what is normally available (12gb).
Every forward pass after that seems to have the correct amount.
Code dump:
def __init__(self, config, num_labels=5, num_truncs = 1):
        super(BertMutSeq, self).__init__(config)
        self.num_labels = num_labels
        self.num_truncs = num_truncs
        self.bert = BertModel(config)
        self.dropout = torch.nn.Dropout(config.hidden_dropout_prob)
        self.classifier = torch.nn.Linear(config.hidden_size * self.num_truncs, num_labels)

def forward(self, input_ids, token_type_ids=None, attention_mask=None, labels=None):
        input_ids = input_ids.to(torch.cuda.current_device());
        attention_mask = attention_mask.to(torch.cuda.current_device());
        labels = labels.to(torch.cuda.current_device());
        concat_truncs = torch.Tensor(len(input_ids), self.num_truncs * self.config.hidden_size).cuda(torch.cuda.current_device())

        for i in range(self.num_truncs):
            pooled_output = self.bert(input_ids[:,i,:], token_type_ids, attention_mask[:,i,:], output_hidden_states = False)
            pooled_output = self.dropout(pooled_output[1])
            concat_truncs[:,i * self.config.hidden_size : (i + 1) * self.config.hidden_size] = pooled_output

        logits = self.classifier(concat_truncs)

        if labels is not None:
            labels = labels.type_as(logits)
            loss_fct = BCEWithLogitsLoss()
            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1, self.num_labels))
            return loss
        else:
            return logits

If I calculate the free memory in the forward pass by printing (with BERT-Medium) torch.cuda.memory_reserved(torch.cuda.current_device()) - torch.cuda.memory_allocated(torch.cuda.current_device()) for the first call to the forward pass I get the 57793536 (577 MB) of free memory:
The next time I call forward I get have 4805510144 (4.8 GB free). The third time it increases to 6095237120 (6.09 GB). It then stays around that size for the rest of training epoch.
I found a similar issue here (https://discuss.pytorch.org/t/larger-gpu-memory-usage-during-the-first-batch/58768), but didn't find much of a solution to my problem.
If I try to run my model with 1 sample per batch using BERT-base or BERT-Large I get a out of memory error. Some of my samples can be extremely long, with about 3,100 input tokens:
RuntimeError: CUDA out of memory. Tried to allocate 20.00 MiB (GPU 0; 11.93 GiB total capacity; 11.24 GiB already allocated; 8.06 MiB free; 11.24 GiB reserved in total by PyTorch)
And it says I have this much memory free 13175808 / 13 MB.
torch.cuda.empty_cache does nothing for me either.
Thank you.


title: Can I add new data on a Fine-Tuned Wav2Vec2 Model?

body:
I finetuned a Wav2Vec2 pretrained model with commonvoice dataset.
I created my custom dataset, can I add new data on a already FineTuned model?


title: Why mobilenet is slower than vgg?

body:
I tried both vgg16 and mobilenet v2 pretrained models on same image (input size = 224). The inference time of vgg16(6ms) is shorter than mobilenet v2(12ms). I thought mobilenet should be much faster than vgg. Does anyone have an idea about it? Thanks. Here is my code
import torch
import cv2
from torchvision import transforms
import time

if __name__ == "__main__":

    model_mobile = torch.hub.load('pytorch/vision:v0.9.0', 'mobilenet_v2', pretrained=True)
    model_mobile.eval()
    model_mobile.cuda()

    model_vgg = torch.hub.load('pytorch/vision:v0.9.0', 'vgg16', pretrained=True)
    model_vgg.eval()
    model_vgg.cuda()

    preprocess = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
    ])
    im = cv2.imread('1.jpg')
    im = preprocess(im)
    im = im.unsqueeze(0)
    im = im.cuda()

    result_mobile = model_mobile(im)

    t1 = time.time()
    result_mobile = model_mobile(im)
    t2 = time.time()
    result_vgg = model_vgg(im)
    t3= time.time()

    print("mobilenet time = {:.3f}".format(t2-t1))
    print("vgg time = {:.3f}".format(t3-t2))


title: Does pytorch register modules that are assigned to object properties?

body:
Say I have a module like:
class MyModule(nn.Module):
    def __init__(self):
        super().__init__()
        self.cnn = CNN(params)

And then I do:
module = MyModule()
module.cnn = CNN(some_other_params)

Is the replacement registered? Will there be any nasty side effects down the line?


title: What is the relation between a learning rate scheduler and an optimizer?

body:
If I have a model:
import torch
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module):
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(2, 20)
            self.fc2=nn.Linear(20, 20)
            self.out=nn.Linear(20, 4)

        def forward(self, x):
            x=self.fc1(x)
            x=self.fc2(x)
            x=self.out(x)
            return x

nx = net_x()

And then I'm defining my inputs, optimizer (with lr=0.1), scheduler (with base_lr=1e-3), and training:
r = torch.tensor([1.0,2.0])
optimizer = optim.Adam(nx.parameters(), lr = 0.1)
scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=1e-3, max_lr=0.1, step_size_up=1, mode="triangular2", cycle_momentum=False)

path = 'opt.pt'
for epoch in range(10):
    optimizer.zero_grad()
    net_predictions = nx(r)
    loss = torch.sum(torch.randint(0,10,(4,)) - net_predictions)
    loss.backward()
    optimizer.step()
    scheduler.step()
    print('loss:' , loss)

    #save state dict
    torch.save({    'epoch': epoch,
                    'net_x_state_dict': nx.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                    'scheduler': scheduler.state_dict(),
                    }, path)
#loading state dict
checkpoint = torch.load(path)
nx.load_state_dict(checkpoint['net_x_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
scheduler.load_state_dict(checkpoint['scheduler'])

The optimizer seems to take the learning rate of the scheduler
for g in optimizer.param_groups:
    print(g)
>>>
{'lr': 0.001, 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0, 'amsgrad': False, 'initial_lr': 0.001, 'params': [Parameter containing:

Does the learning rate scheduler overwrite the optimizer? How does it connect to it? Trying to understand the relation between them (i.e how they interact, etc.)


title: Implementing "Neural Assimilation" for my research project

body:
I am attempting to reproduce the Neural Assimilation model architecture described in this paper: https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7304751/. They are essentially trying to design an architecture that would be much faster than using a Kalman Filter by bypassing the correction step. Despite the authors providing some code in Keras, I have had some issues trying to implement this with PyTorch as I simply cannot see how they have built the model. (link to the code is here: https://drive.google.com/drive/folders/1TBg9ZfPyXiegoXbn2sZURPjPml8TWQIO)
From my understanding, we need two LSTMs with one using the model forecasted data and the other using the observational data. The tricky part is that each LSTM requires as input the hidden state and cell state from the other LSTM which I am having some difficulty trying to implement. Anyone have any ideas on how I could do this? So far I have only used simple LSTMs for many-to-many prediction.
This is a diagram of the architecture:

class Neural_Assimilation(nn.Module):
  def __init__(self, n_inputs : tuple, n_hidden, num_layers, n_outputs):
    super(Neural_Assimilation, self).__init__()
    self.D1 = n_inputs[0]
    self.D2 = n_inputs[1]
    self.M = n_hidden
    self.K = n_outputs
    self.L = num_layers

    self.rnn_f = nn.LSTM(
        input_size=self.D2,
        hidden_size=self.M,
        num_layers=self.L,
        batch_first=True)

    self.rnn_o = nn.LSTM(
        input_size=self.D1,
        hidden_size=self.M,
        num_layers=self.L,
        batch_first=True)

    self.fc = nn.Linear(self.M, self.K) # For BiDir LSTM need to multiply self.M x 2

  def forward(self, X_f, X_o):
    # initial hidden states

    h0_f = torch.zeros(self.L, X_f.size(0), self.M).to(device) # For BiDir LSTM need to multiply self.L x 2
    c0_f = torch.zeros(self.L, X_f.size(0), self.M).to(device) # For BiDir LSTM need to multiply self.L x 2
    out_f, (h_t_f, c_t_f) = self.rnn_f(X_f, (h0_f, c0_f))
    out_o, (h_t_o, c_t_o) = self.rnn_o(X_o, (h_t_f, c_t_f))

    # we want h(t) and c(t) at the final time step so we can feed this into the other RNN
    out_f = self.fc(out_f[:, -1, :])
    out_o = self.fc(out_o[:, -1, :])

    return out_f, out_o

My issue is I haven't been able to think of a good way to 'couple' the LSTMs so that each RNN has as input either the observation or model forecast, AS WELL AS the hidden and cell state of the other network
I would really appreciate your expertise!


title: YOLO and protobufs

body:
I'm building a video processing pipeline using RedisAI. The template I'm using loads a TinyYOLO protobuf, and then a PyTorch script:
print('Loading model - ', end='')
with open('models/tiny-yolo-voc.pb', 'rb') as f:
    model = f.read()
    res = conn.execute_command('AI.MODELSET', 'yolo:model', 'TF', args.device, 'INPUTS', 'input', 'OUTPUTS', 'output', model)
    print(res)

# Load the PyTorch post processing boxes script
print('Loading script - ', end='')
with open('yolo_boxes.py', 'rb') as f:
    script = f.read()
    res = conn.execute_command('AI.SCRIPTSET', 'yolo:script', args.device, script)
    print(res)

Then it runs a RedisAI model runner on every frame, produces some boxes and throws out the ones whose category is not 14 (people).
modelRunner = redisAI.createModelRunner('yolo:model')
    redisAI.modelRunnerAddInput(modelRunner, 'input', image_tensor)
    redisAI.modelRunnerAddOutput(modelRunner, 'output')
    model_replies = redisAI.modelRunnerRun(modelRunner)
    model_output = model_replies[0]
    prf.add('model')

# log('script')
# The model's output is processed with a PyTorch script for non maxima suppression
scriptRunner = redisAI.createScriptRunner('yolo:script', 'boxes_from_tf')
redisAI.scriptRunnerAddInput(scriptRunner, model_output)
redisAI.scriptRunnerAddOutput(scriptRunner)
script_reply = redisAI.scriptRunnerRun(scriptRunner)
prf.add('script')

# log('boxes')
# The script outputs bounding boxes
shape = redisAI.tensorGetDims(script_reply)
buf = redisAI.tensorGetDataAsBlob(script_reply)
boxes = np.frombuffer(buf, dtype=np.float32).reshape(shape)

# Iterate boxes to extract the people
ratio = float(IMG_SIZE) / max(pil_image.width, pil_image.height)  # ratio = old / new
pad_x = (IMG_SIZE - pil_image.width * ratio) / 2                  # Width padding
pad_y = (IMG_SIZE - pil_image.height * ratio) / 2                 # Height padding
boxes_out = []
people_count = 0
for box in boxes[0]:
    if box[4] == 0.0:  # Remove zero-confidence detections
        continue
    if box[-1] != 14:  # Ignore detections that aren't people
        continue
    people_count += 1

My question is, if I wanted to use YOLO4 or YOLO5, would I be able to take a trained model, produce a protobuf from it, and just drop it into this process? Would the categories be numbered differently?


title: "RuntimeError: The expanded size of the tensor (118) must match the existing size (135) at non-singleton dimension 1" Pytorch Machine Translation

body:
Using this German to English Pytorch Seq2Seq Machine Translatior, I have been trying to create Middle English to Present Day English Seq2Seq Machine Translatior using custom datasets. Other than creating the custom datasets, I virtually copied and pasted the German template and replaced the word German with me (Middle English) and English with pdf (Present Day English), but I have been getting two error messages when trying to train the model. When I run the German version of the code, it trains just fine. The following is my code and the two error messages.
My code:
# imports
import torch
import torch.nn as nn
import torch.optim as optim
from torchtext import data, datasets
from torchtext.data import Field, BucketIterator, TabularDataset
import numpy as np # other useful math-related libraries and modules
import spacy # this is where he gets his nlp datasets
import spacy.cli
import random
from utils2 import translate_sentence, bleu, save_checkpoint, load_checkpoint

spacy_eng = spacy.load('en') # loading up the English tokenizer
def tokenizer_eng(text): # English tokenizer function for the fields
  return [tok.text for tok in spacy_eng.tokenizer(text)]

# constructing the fields for pde and me
me = Field(sequential=True, use_vocab=True, tokenize=tokenizer_eng, lower=True, init_token='sos', eos_token='<eos>')
pde = Field(sequential=True, use_vocab=True, tokenize=tokenizer_eng, lower=True, init_token='sos', eos_token='<eos>')

fields = {'Middle English': ('me', me), 'Present Day English': ('pde', pde)}

train_data, validation_data, test_data = TabularDataset.splits(
    path='/Users/user/Desktop/Code/datasets',
    train='train.tsv',
    test='test.tsv',
    validation='validation.tsv',
    format='tsv',
    fields=fields
)

#print(test_data[0].__dict__.keys())
#print(validation_data[1].__dict__.values())

# Building the vocabulary
me.build_vocab(train_data)
pde.build_vocab(train_data)

# Building the seq2seq and encoder decoder models
class Encoder(nn.Module): # first LSTM
    def __init__(self, input_size, embedding_size, hidden_size, num_layers, p):
        super(Encoder, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.dropout = nn.Dropout(p)
        self.embedding = nn.Embedding(input_size, embedding_size)
        self.rnn = nn.LSTM(embedding_size, hidden_size, num_layers, dropout=p)
    def forward(self, x): # defines the vector of indices
        # x vector shape: (seq_length, N)
        embedding = self.dropout(self.embedding(x))
        # embedding vector shape: (seq_length, N, embedding_size)
        outputs, (hidden, cell) = self.rnn(embedding)
        return hidden, cell

class Decoder(nn.Module):
  def __init__(self, input_size, embedding_size, hidden_size, output_size, num_layers, p):
      # output_size should be the same as input_size, because the size of the vocabulary should change.
      super(Decoder, self).__init__()
      self.hidden_size = hidden_size
      self.num_layers = num_layers

      self.dropout = nn.Dropout(p)
      self.embedding = nn.Embedding(input_size, embedding_size)
      self.rnn = nn.LSTM(embedding_size, hidden_size, num_layers, dropout=p)
      self.fc = nn.Linear(hidden_size, output_size) # fc stands for fully connected

  def forward(self, x, hidden, cell):
      x = x.unsqueeze(0) #this adds another dimension

      embedding = self.dropout(self.embedding(x))
      # embedding shape: (1, N, embedding_size)

      outputs, (hidden, cell) = self.rnn(embedding, (hidden, cell))
      # outputs is what we think this next word should be
      # shape of the outputs: (1, N, hidden_size)

      predictions = self.fc(outputs)
      # shape of predictions: (1, N, length_of_vocab)
      # this will be sent to the loss function

      predictions = predictions.squeeze(0)

      return predictions, hidden, cell

class Seq2Seq(nn.Module): # combines the encoder and decoder
  def __init__(self, encoder, decoder):
      super(Seq2Seq, self).__init__()
      self.encoder = encoder
      self.decoder = decoder

  def forward(self, source, target, teacher_force_ratio=0.5):
      batch_size = source.shape[1]
      # batch_size should look like (trg_len, N)
      target_len = target.shape[0]
      target_vocab_size = len(pde.vocab) #make sure to change this to pde

      outputs = torch.zeros(target_len, batch_size, target_vocab_size).to(device)
      # predicts one word at a time, but each word predicts an entire batch and every prediction is a vector of the entire vocabulary size

      hidden, cell = self.encoder(source)

      x = target[0]

      for t in range(1, target_len):
          output, hidden, cell = self.decoder(x, hidden, cell)

          outputs[t] = output

          # the output will look like (N, english_vocab_size)
          best_guess = output.argmax(1)

          x = target[t] if random.random() < teacher_force_ratio else best_guess

      return outputs

# hyperparameters for the training model
num_epochs = 2
learning_rate = 3e-4
batch_size = 32 # batch size must be smaller than total amount of data

# model hyperparameters
save_model = True
load_model = False
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
input_size_encoder = len(me.vocab)
input_size_decoder = len(pde.vocab)
output_size = len(me.vocab)
encoder_embedding_size = 50
decoder_embedding_size = 50
hidden_size = 100
num_layers = 2
enc_dropout = 0.5
dec_dropout = 0.5

# writing iterators
train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
    (train_data, validation_data, test_data), # make sure these are in same order as declared variables
    batch_size=batch_size,
    sort_within_batch = True,
    sort_key = lambda x: len(x.me), # formerly x.src
    device=device
)

# running the encoder decoder models
encoder_net = Encoder(input_size_encoder, encoder_embedding_size, hidden_size, num_layers, enc_dropout).to(device)
decoder_net = Decoder(input_size_decoder, decoder_embedding_size, hidden_size, output_size, num_layers, dec_dropout).to(device)

model = Seq2Seq(encoder_net, decoder_net).to(device)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)

pad_idx = pde.vocab.stoi['<pad>']
criterion = nn.CrossEntropyLoss(ignore_index=pad_idx)

if load_model:
    load_checkpoint(torch.load("my_checkpoint2.pth.tar"), model, optimizer)

# This is an example sentence.
sentence = (
    "Is owr dyner dyght?"
)

for epoch in range(num_epochs):
    print(f'Epoch [{epoch} / {num_epochs}]')

    if save_model:
        checkpoint = {
            "state_dict": model.state_dict(),
            "optimizer": optimizer.state_dict(),
        }
        save_checkpoint(checkpoint)

    model.eval()

    translated_sentence = translate_sentence(
        model, sentence, me, pde, device, max_length=50
    )

    print(f"Translated example sentence: \n {translated_sentence}")

    model.train()

    for batch_idx, batch in enumerate(train_iterator):
        inp_data = batch.me.to(device)
        target = batch.pde.to(device)
        # this was formerly batch.src and batch.trg

        output = model(inp_data, target)
        # Output is of shape (trg_len, batch_size, output_dim) but Cross Entropy Loss
        # doesn't take input in that form. Here we can view it in a similar
        # way so we have output_words * batch_size that we want to send in into
        # our cost function, so we need to do some reshaping."
        output = output[1:].reshape(-1, output.shape[2])
        target = target[1:].reshape(-1)

        optimizer.zero_grad()
        loss = criterion(output, target)

        loss.backward()

        optimizer.step()

print("run complete")

Error one
I think this error primarily has to do with the output not being shaped as it's supposed to, but I don't know how to make it the right shape. I think the problem probably lies either in how I created the dataset, how I created the Seq2Seq2 class, or that very last for loop of the program.
Traceback (most recent call last):
  File "/Users/user/PycharmProjects/pythonProject/ME-PDE.py", line 250, in <module>
    output = model(inp_data, target)
  File "/Users/user/.conda/envs/pythonProject/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Users/user/PycharmProjects/pythonProject/ME-PDE.py", line 150, in forward
    outputs[t] = output
RuntimeError: The expanded size of the tensor (118) must match the existing size (135) at non-singleton dimension 1.  Target sizes: [20, 118].  Tensor sizes: [20, 135]

Error two
This error randomly comes up sometimes. I think it might because my hyperparameters are too large for my sample size, but I'm not sure why this error message only comes up instead of the first one sometimes.
Traceback (most recent call last):
  File "/Users/user/PycharmProjects/pythonProject/ME-PDE.py", line 235, in <module>
    translated_sentence = translate_sentence(
  File "/Users/user/PycharmProjects/pythonProject/utils2.py", line 53, in translate_sentence
    output, hidden, cell = model.decoder(previous_word, hidden, cell)
  File "/Users/user/.conda/envs/pythonProject/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Users/user/PycharmProjects/pythonProject/ME-PDE.py", line 105, in forward
    embedding = self.dropout(self.embedding(x))
  File "/Users/user/.conda/envs/pythonProject/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Users/user/.conda/envs/pythonProject/lib/python3.8/site-packages/torch/nn/modules/sparse.py", line 124, in forward
    return F.embedding(
  File "/Users/user/.conda/envs/pythonProject/lib/python3.8/site-packages/torch/nn/functional.py", line 1852, in embedding
    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
IndexError: index out of range in self

As of now, I only have 25 translations in test.tsv, 20 in train.tsv, and 5 in validation.tsv. I will get more, but I wanted the program to be able to successfully run first.
example from validation.tsv
Middle English  Present Day English
Than make I buter ferther on the day    Then I make butter later in the day.
Ye wold say, 'they be prowde!'  You would say, ‘they are proud!’
Whyll yow slepe fulle stylle,   While you sleep soundly,
Kype wylle owr chelderne and let them not wepe. Keep our children well and don’t let them weep.
Yet I have not a feyr word whan that I have done.   Yet I don’t get any kind words when I have done that.

Thank you so much for any help you can give!


title: PyTorch model doesn't seem to optimize

body:
I have the following code which runs just fine, but the tensors from the model itself always seem to be close to about .5, it just doesn't seem to go very far from there. I know that I definitely have enough epochs, so what's going wrong?
import torch
import torch.nn as nn
import torch.nn.functional as F
from tqdm import tqdm
import matplotlib.pyplot as plt
import os
import keras
import sys
devicet = 'cuda' if torch.cuda.is_available() else 'cpu'
device = torch.device(devicet)
if devicet == 'cpu':
  print ('Using CPU')
else:
  print ('Using GPU')
cuda0 = torch.device('cuda:0')
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.step1 = nn.Linear(5, 25)
        self.step2 = nn.Linear(25, 50)
        self.step3 = nn.Linear(50, 100)
        self.step4 = nn.Linear(100, 100)
        self.step5 = nn.Linear(100, 10)
        self.step6 = nn.Linear(10, 1)
    def forward(self, x):
      x = F.relu(x)
      x = self.step1(x)
      x = self.step2(x)
      x = self.step3(x)
      x = self.step4(x)
      x = self.step5(x)
      x = self.step6(x)
      return (x)
net = Net()
x = torch.rand(50,5)
y = torch.rand(50, 1)
x.to(devicet)
y.to(devicet)
learning_rate = 1e-4
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
loss_fn = torch.nn.MSELoss()
acc_list = []
for i in range(1000):
    y_pred = net(x)
    loss = loss_fn(y_pred, y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    acc_list.append(abs(net(x).detach().numpy()[0]-y.detach().numpy()[0]))
    sys.stdout.write("\rEpoch: {0}, Tensor Difference: {1}".format(len(acc_list), net(x).detach().numpy()[0]-y.detach().numpy()[0]))
    sys.stdout.flush()
    with torch.no_grad():
        for param in net.parameters():
            param -= learning_rate * param.grad
print ('\nFinished training in {} epochs.'.format(len(acc_list)))
plt.plot(range(len(acc_list)),acc_list)
plt.show()
print (net(x).detach().numpy()[0:5])
print (y.detach().numpy()[0:5])

Keep in mind that I'm fairly new to PyTorch and this is my first self designed model, not just the built in Sequential.


title: Train on top of a Torchscript model

body: I currently have a Torchscript model I load via torch.jit.load. I would like to take some data I have and train on top of these weights, however I cannot find out how to train a serialised torchscript model.


title: Delete and Reinitialize pertained BERT weights / parameters

body:
I tried to fine-tune BERT for a classification downstream task.
Now I loaded the model again and I run into the following warning:
Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertModel: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight']

This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).

[Screen Shot][1]
[1]: https://i.stack.imgur.com/YJZVc.png
I already deleted and reinstalled transformers==4.6.0 but nothing helped.
I thought maybe through the parameter "force_download=True" it might get the original weights back but nothing helped.
Shall I continue and ignore the warning? Is there a way to delete the model checkpoints such when the model is downloaded the weights are fixed again?
Thanks in advance!
Best,
Alex


title: RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target' in call to _thnn_nll_loss_forward

body:
I am trying Bert on the Tweeter dataset. I am encountered with the following error message.
# set initial loss to infinite
best_valid_loss = float('inf')

# empty lists to store training and validation loss of each epoch
train_losses=[]
valid_losses=[]

#for each epoch
for epoch in range(epochs):

    print('\n Epoch {:} / {:}'.format(epoch + 1, epochs))

#train model
train_loss, _ = train()

#evaluate model
valid_loss, _ = evaluate()

#save the best model
if valid_loss < best_valid_loss:
    best_valid_loss = valid_loss
    torch.save(model.state_dict(), 'saved_weights.pt')

# append training and validation loss
train_losses.append(train_loss)
valid_losses.append(valid_loss)

print(f'\nTraining Loss: {train_loss:.3f}')
print(f'Validation Loss: {valid_loss:.3f}')

It is a very long code. Searching for the issues led me to change .float() to long(). I already did that. Kindly suggest me the solution.
Very Important: The same code work perfectly well on another dataset (with same number of columns and same type of data) but is not working on tweets data. (Only difference is the size. Previous was having 5500 entries while the tweet dataset has 10000 entries)


title: While I am running the below code I'm getting error :

body:
code:
model = BertWSD.from_pretrained(model_dir)
tokenizer = BertTokenizer.from_pretrained(model_dir)
# add new special token
if '[TGT]' not in tokenizer.additional_special_tokens:
    tokenizer.add_special_tokens({'additional_special_tokens': ['[TGT]']})
    assert '[TGT]' in tokenizer.additional_special_tokens
    model.resize_token_embeddings(len(tokenizer))

model.to(DEVICE)
model.eval()

For some reason I keep getting a strange error whenever I run the code:
RuntimeError                              Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    625             try:
--> 626                 state_dict = torch.load(resolved_archive_file, map_location="cpu")
    627             except Exception:

3 frames
RuntimeError: unexpected EOF, expected 5298350 more bytes. The file might be corrupted.

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
/usr/local/lib/python3.7/dist-packages/transformers/modeling_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    627             except Exception:
    628                 raise OSError(
--> 629                     "Unable to load weights from pytorch checkpoint file. "
    630                     "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. "
    631                 )

OSError: Unable to load weights from pytorch checkpoint file. If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True.

Does anyone know why I'm getting this error?


title: Parallelization of torch.stack inside forwad pass

body:
Inside a forward pass of a Neural Network I have this snippet of code.
myNewTensor = th.stack([
                         self.my_function(inputA[i], inputB[i] for i in range(inputA.shape[0])
                                         )
                      ])

This is obviously embarrassingly parallel and there is no way to vectorize the function. Is there a way to parallelize this operation in pytorch while still being able to backpropagate through it?


title: Training loss not decreasing in detectron2

body:
I am using detectron2 for training custom model for detecting layout of documents which includes class like ['header', 'title', 'text', 'form', 'footer', 'table', 'list', 'figure']. Loss is not decreasing after training of more than 15000 iterations.
The data I have around 3000 samples in training.
My config file looks like-
CUDNN_BENCHMARK: false
DATALOADER:
  ASPECT_RATIO_GROUPING: true
  FILTER_EMPTY_ANNOTATIONS: true
  NUM_WORKERS: 4
  REPEAT_THRESHOLD: 0.0
  SAMPLER_TRAIN: TrainingSampler
DATASETS:
  PRECOMPUTED_PROPOSAL_TOPK_TEST: 1000
  PRECOMPUTED_PROPOSAL_TOPK_TRAIN: 2000
  PROPOSAL_FILES_TEST: []
  PROPOSAL_FILES_TRAIN: []
  TEST:
  - layout_valid
  TRAIN:
  - layout_train
GLOBAL:
  HACK: 1.0
INPUT:
  CROP:
    ENABLED: false
    SIZE:
    - 0.9
    - 0.9
    TYPE: relative_range
  FORMAT: BGR
  MASK_FORMAT: polygon
  MAX_SIZE_TEST: 1333
  MAX_SIZE_TRAIN: 1333
  MIN_SIZE_TEST: 800
  MIN_SIZE_TRAIN:
  - 640
  - 672
  - 704
  - 736
  - 768
  - 800
  MIN_SIZE_TRAIN_SAMPLING: choice
  RANDOM_FLIP: horizontal
MODEL:
  ANCHOR_GENERATOR:
    ANGLES:
    - - -90
      - 0
      - 90
    ASPECT_RATIOS:
    - - 0.5
      - 1.0
      - 2.0
    NAME: DefaultAnchorGenerator
    OFFSET: 0.0
    SIZES:
    - - 32
    - - 64
    - - 128
    - - 256
    - - 512
  BACKBONE:
    FREEZE_AT: 2
    NAME: build_resnet_fpn_backbone
  DEVICE: cuda
  FPN:
    FUSE_TYPE: sum
    IN_FEATURES:
    - res2
    - res3
    - res4
    - res5
    NORM: ''
    OUT_CHANNELS: 256
  KEYPOINT_ON: false
  LOAD_PROPOSALS: false
  MASK_ON: true
  META_ARCHITECTURE: GeneralizedRCNN
  PANOPTIC_FPN:
    COMBINE:
      ENABLED: true
      INSTANCES_CONFIDENCE_THRESH: 0.5
      OVERLAP_THRESH: 0.5
      STUFF_AREA_LIMIT: 4096
    INSTANCE_LOSS_WEIGHT: 1.0
  PIXEL_MEAN:
  - 103.53
  - 116.28
  - 123.675
  PIXEL_STD:
  - 57.375
  - 57.12
  - 58.395
  PROPOSAL_GENERATOR:
    MIN_SIZE: 0
    NAME: RPN
  RESNETS:
    DEFORM_MODULATED: false
    DEFORM_NUM_GROUPS: 1
    DEFORM_ON_PER_STAGE:
    - false
    - false
    - false
    - false
    DEPTH: 101
    NORM: FrozenBN
    NUM_GROUPS: 32
    OUT_FEATURES:
    - res2
    - res3
    - res4
    - res5
    RES2_OUT_CHANNELS: 256
    RES5_DILATION: 1
    STEM_OUT_CHANNELS: 64
    STRIDE_IN_1X1: false
    WIDTH_PER_GROUP: 8
  RETINANET:
    BBOX_REG_LOSS_TYPE: smooth_l1
    BBOX_REG_WEIGHTS: &id001
    - 1.0
    - 1.0
    - 1.0
    - 1.0
    FOCAL_LOSS_ALPHA: 0.25
    FOCAL_LOSS_GAMMA: 2.0
    IN_FEATURES:
    - p3
    - p4
    - p5
    - p6
    - p7
    IOU_LABELS:
    - 0
    - -1
    - 1
    IOU_THRESHOLDS:
    - 0.4
    - 0.5
    NMS_THRESH_TEST: 0.5
    NORM: ''
    NUM_CLASSES: 80
    NUM_CONVS: 4
    PRIOR_PROB: 0.01
    SCORE_THRESH_TEST: 0.05
    SMOOTH_L1_LOSS_BETA: 0.1
    TOPK_CANDIDATES_TEST: 1000
  ROI_BOX_CASCADE_HEAD:
    BBOX_REG_WEIGHTS:
    - - 10.0
      - 10.0
      - 5.0
      - 5.0
    - - 20.0
      - 20.0
      - 10.0
      - 10.0
    - - 30.0
      - 30.0
      - 15.0
      - 15.0
    IOUS:
    - 0.5
    - 0.6
    - 0.7
  ROI_BOX_HEAD:
    BBOX_REG_LOSS_TYPE: smooth_l1
    BBOX_REG_LOSS_WEIGHT: 1.0
    BBOX_REG_WEIGHTS:
    - 10.0
    - 10.0
    - 5.0
    - 5.0
    CLS_AGNOSTIC_BBOX_REG: false
    CONV_DIM: 256
    FC_DIM: 1024
    NAME: FastRCNNConvFCHead
    NORM: ''
    NUM_CONV: 0
    NUM_FC: 2
    POOLER_RESOLUTION: 7
    POOLER_SAMPLING_RATIO: 0
    POOLER_TYPE: ROIAlignV2
    SMOOTH_L1_BETA: 0.0
    TRAIN_ON_PRED_BOXES: false
  ROI_HEADS:
    BATCH_SIZE_PER_IMAGE: 512
    IN_FEATURES:
    - p2
    - p3
    - p4
    - p5
    IOU_LABELS:
    - 0
    - 1
    IOU_THRESHOLDS:
    - 0.5
    NAME: StandardROIHeads
    NMS_THRESH_TEST: 0.5
    NUM_CLASSES: 8
    POSITIVE_FRACTION: 0.25
    PROPOSAL_APPEND_GT: true
    SCORE_THRESH_TEST: 0.05
  ROI_KEYPOINT_HEAD:
    CONV_DIMS:
    - 512
    - 512
    - 512
    - 512
    - 512
    - 512
    - 512
    - 512
    LOSS_WEIGHT: 1.0
    MIN_KEYPOINTS_PER_IMAGE: 1
    NAME: KRCNNConvDeconvUpsampleHead
    NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS: true
    NUM_KEYPOINTS: 17
    POOLER_RESOLUTION: 14
    POOLER_SAMPLING_RATIO: 0
    POOLER_TYPE: ROIAlignV2
  ROI_MASK_HEAD:
    CLS_AGNOSTIC_MASK: false
    CONV_DIM: 256
    NAME: MaskRCNNConvUpsampleHead
    NORM: ''
    NUM_CONV: 4
    POOLER_RESOLUTION: 14
    POOLER_SAMPLING_RATIO: 0
    POOLER_TYPE: ROIAlignV2
  RPN:
    BATCH_SIZE_PER_IMAGE: 256
    BBOX_REG_LOSS_TYPE: smooth_l1
    BBOX_REG_LOSS_WEIGHT: 1.0
    BBOX_REG_WEIGHTS: *id001
    BOUNDARY_THRESH: -1
    CONV_DIMS:
    - -1
    HEAD_NAME: StandardRPNHead
    IN_FEATURES:
    - p2
    - p3
    - p4
    - p5
    - p6
    IOU_LABELS:
    - 0
    - -1
    - 1
    IOU_THRESHOLDS:
    - 0.3
    - 0.7
    LOSS_WEIGHT: 1.0
    NMS_THRESH: 0.7
    POSITIVE_FRACTION: 0.5
    POST_NMS_TOPK_TEST: 1000
    POST_NMS_TOPK_TRAIN: 1000
    PRE_NMS_TOPK_TEST: 1000
    PRE_NMS_TOPK_TRAIN: 2000
    SMOOTH_L1_BETA: 0.0
  SEM_SEG_HEAD:
    COMMON_STRIDE: 4
    CONVS_DIM: 128
    IGNORE_VALUE: 255
    IN_FEATURES:
    - p2
    - p3
    - p4
    - p5
    LOSS_WEIGHT: 1.0
    NAME: SemSegFPNHead
    NORM: GN
    NUM_CLASSES: 54
  WEIGHTS: /content/drive/MyDrive/layout/resnext101.pth
OUTPUT_DIR: ./output
SEED: -1
SOLVER:
  AMP:
    ENABLED: false
  BASE_LR: 8.0e-06
  BIAS_LR_FACTOR: 1.0
  CHECKPOINT_PERIOD: 2000
  CLIP_GRADIENTS:
    CLIP_TYPE: value
    CLIP_VALUE: 1.0
    ENABLED: false
    NORM_TYPE: 2.0
  GAMMA: 0.1
  IMS_PER_BATCH: 2
  LR_SCHEDULER_NAME: WarmupMultiStepLR
  MAX_ITER: 25000
  MOMENTUM: 0.9
  NESTEROV: false
  REFERENCE_WORLD_SIZE: 0
  STEPS:
  - 210000
  - 250000
  WARMUP_FACTOR: 0.001
  WARMUP_ITERS: 1000
  WARMUP_METHOD: linear
  WEIGHT_DECAY: 0.0001
  WEIGHT_DECAY_BIAS: 0.0001
  WEIGHT_DECAY_NORM: 0.0
TEST:
  AUG:
    ENABLED: false
    FLIP: true
    MAX_SIZE: 4000
    MIN_SIZES:
    - 400
    - 500
    - 600
    - 700
    - 800
    - 900
    - 1000
    - 1100
    - 1200
  DETECTIONS_PER_IMAGE: 100
  EVAL_PERIOD: 0
  EXPECTED_RESULTS: []
  KEYPOINT_OKS_SIGMAS: []
  PRECISE_BN:
    ENABLED: false
    NUM_ITER: 200
VERSION: 2
VIS_PERIOD: 0

How can I resolve this?


title: Solving Non linear PINN model

body:
I am solving 1D non-linear diffusion equation. I have successfully solved linear equation suing PINN model. For non-linear equation when i am training my network , it is showing Nan value after 1 iteration. I am using this piece of code for internal data.
def loss_initernal(self, x_train):
    g = x_train.clone()
    g.requires_grad = True
    u = self.forward(g)
    u_g = gradients(u, g)[0]
    u_x, u_t= u_g[:, [0]], u_g[:, [1]]
    D = 1.1*10**(-9)*(self.forward(g))**1.2

    u_xx = gradients(D, g)[0]
    u_xx = u_xx[:, [0]]
    pde = u_t - u_xx
    loss_pde = pde.pow(2).mean()
    print("D",D,"u_x",u,'loss', loss_pde)
    return loss_pde

def gradients(outputs, inputs):
         return torch.autograd.grad(outputs, inputs, grad_outputs=torch.ones_like(outputs), create_graph=True)

Pleas help me, i am unable to understand the issue here.


title: multi label problem with intermediate labels

body:
I am trying to create a model for the following problem
id   input (diagnoses)    elapsed_days    output (medication)
 1     [2,3,4]                 0             [3,4]
 1     [4,5,6]                 7             [1]
 1     [2,3]                  56             [6,3]
 2     [6,5,9,10]              0             [5,3,1]

Rather than a single label for the different codes over time, there are labels at each time period.
I am think that my arch would be  [input] -> [embedding for diagnoses] -> [append normalized elapsed days to embeddings]
-> [LSTM] -> [FFNs] -> [labels over time]
I am familiar with how to set this up if there were a single label per id.  Given there are labels for each row (i.e. multiple per id), should I be passing the hidden layers of the LSTM through the FFN and then assigning the labels?  I would really appreciate if somebody could point me to a reference/blog/github/anything for this kind of problem or suggest an alternative approach here.


title: pytorch : retain_graph=True error even though i add this

body:
I keep getting this error
"Trying to backward through the graph a second time, but the saved intermediate results have already been freed. Specify retain_graph=True when calling .backward() or autograd.grad() the first time."
in the beginning, it was without retain_graph=True and then I got the error so I add it to the backward but I am still getting the same error.
i read similar questions but nothing helped.
would like to get help!
trained_cnnfmnist_model=net

class CNNFMnist2(nn.Module):
    def __init__(self, trained_cnnfmnist_model):
        super(CNNFMnist2, self).__init__()
        self.trained_cnnfmnist_model = trained_cnnfmnist_model
        # now a few fully connected layers
        self.fc1 = nn.Linear(64, 32)
        self.fc2=  nn.Linear(32,16)
        self.fc3=  nn.Linear(16,10)

    def forward(self, x):
      x = self.trained_cnnfmnist_model(x)
      x = F.relu(self.fc1(x[0]))
      print(x.shape)
      # x = x.view(-1, self.num_flat_features(x))
      x = F.relu(self.fc2(x))
      x = self.fc3(x)
      return x

trainset = torchvision.datasets.FashionMNIST(root='./data', train=True,
                                        download=True, transform=transforms.ToTensor())

testset = torchvision.datasets.FashionMNIST(root='./data', train=False,
                                       download=True, transform=transforms.ToTensor())

trainloader = torch.utils.data.DataLoader(trainset, batch_size=4,
                                          shuffle=True)

testloader = torch.utils.data.DataLoader(testset, batch_size=4,
                                         shuffle=False)

net2 = CNNFMnist2(trained_cnnfmnist_model).cuda()
optimizer = torch.optim.SGD(net2.parameters(), lr=0.001, momentum=0.9)

for epoch in range(2):

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # get the inputs
        inputs, labels = data

        inputs = inputs.cuda() # -- For GPU
        labels = labels.cuda() # -- For GPU

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        output = net2(inputs)
        loss = criterion(outputs, labels)
        loss.backward(retain_graph=True)
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if (i+1) % 2000 == 0:
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')


title: How to solve unexpected status exception: : Error for Training job huggingface-pytorch-training

body:
I am getting unexpected status exception: : Error for Training job huggingface-pytorch-training. How can I get rid of it?
    hyperparameters={'epochs': 1,
                     'model_type': 'bert',
                     'model_name':'bert-base-cased',
                     'per_gpu_train_batch_size': 64,
                     'per_gpu_eval_batch_size': 64,
                     'save_steps': 150,
                     'logging_steps': 150
                     }
    huggingface_estimator = HuggingFace(entry_point='train.py',
                                source_dir='./scripts',
                                instance_type='ml.p3.2xlarge',
                                instance_count=1,
                                role=role,
                                framework_version='1.8.0',
                                transformers_version='4.4.2',
                                pytorch_version='1.6.0',
                                py_version='py36',
                                hyperparameters = hyperparameters)
    huggingface_estimator.fit({'train': 's3://grammar-poc/train', 'test': 's3://grammar-poc/test'})

UnexpectedStatusException: Error for Training job huggingface-pytorch-training-2021-05-27-16-20-59-984: Failed. Reason: AlgorithmError: ExecuteUserScriptError:
Command "/opt/conda/bin/python3.6 train.py --epochs 1 --logging_steps 150 --model_name bert-base-cased --model_type bert --output_dir s3://grammar-poc/sagemaker/bert-grammar-correction --per_gpu_eval_batch_size 64 --per_gpu_train_batch_size 64 --save_steps 150"
Traceback (most recent call last):
  File "train.py", line 49, in <module>
    train_dataset = load_from_disk(args.training_dir)
  File "/opt/conda/lib/python3.6/site-packages/datasets/load.py", line 786, in load_from_disk
    "Directory {} is neither a dataset directory nor a dataset dict directory.".format(dataset_path)
FileNotFoundError: Directory /opt/ml/input/data/train is neither a dataset directory nor a dataset dict directory.


title: How to spherical interpolate two unit vectors in Python Torch?

body:
I have two Torch tensors A and B representing two unit vectors in d-dimensional space (lying on the (d−1)-dimensional hypersphere), that i have obtained from the uniform sphere distribution using botorch.utils.sampling.sample_hypersphere(d).
Let's call the distance along the great circle between A and B as x. (Assume that B≠−A). For a given t, i want to obtain another vector on the unit sphere, whose distance along the great circle from A is tx and from B is (1−t)x.
How do i obtain such vector, preferably using existing tools in Torch / BoTorch?


title: Can’t train a multilabel classifier on a small dataset

body:
I have a dataset of 3 classes animals, landscapes, and buildings. It is only a 100 images per dataset and I am trying to train a classifier using ResNet34 + fastai but I'm running into several issues.
The first one is that I don't think my model is training properly. When I use the lr_finder, my validation loss is na, unless this is expected:

Then I run 10 epochs. It looks either like its learning well or its overfitting like hell.

Then I unfreeze to find another learning rate and my validation loss is still na:

And now when I train 10 epochs, the graph starts to oscillate so that tells me that it can't learn anymore. The accuracy looks good (unless its overfitting) so then I decide to use my validation set to see how my model is actually doing:

Because I have 300 images, 240 are in the train folder and 60 are in the test folder. My test images are labelled because I want to run an accuracy score on it. This is the cell that I run:
    path = '/content/dataset/'

    data_test = ImageList.from_folder(path).split_by_folder(train='train', valid='test').label_from_re(file_parse).transform(size=512).databunch().normalize(imagenet_stats)

    learn = cnn_learner(data, models.resnet50, metrics=[accuracy, top_1],callback_fns=ShowGraph)

    learn.load('stage-2')

And this is the error that I am met with:
    RuntimeError                              Traceback (most recent call last)
    <ipython-input-28-f1232eb8cf47> in <module>()
      4
      5 learn = cnn_learner(data, models.resnet50, metrics=[accuracy, top_1],callback_fns=ShowGraph)
    ----> 6 learn.load('stage-2')

    1 frames
    /usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py in load_state_dict(self, state_dict, strict)
    828         if len(error_msgs) > 0:
    829             raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format(
    --> 830                                self.__class__.__name__, "\n\t".join(error_msgs)))
    831         return _IncompatibleKeys(missing_keys, unexpected_keys)
    832

    RuntimeError: Error(s) in loading state_dict for Sequential:
        Missing key(s) in state_dict: "0.4.0.conv3.weight", "0.4.0.bn3.weight", "0.4.0.bn3.bias", "0.4.0.bn3.running_mean", "0.4.0.bn3.running_var", "0.4.0.downsample.0.weight", "0.4.0.downsample.1.weight", "0.4.0.downsample.1.bias", "0.4.0.downsample.1.running_mean", "0.4.0.downsample.1.running_var", "0.4.1.conv3.weight", "0.4.1.bn3.weight", "0.4.1.bn3.bias", "0.4.1.bn3.running_mean", "0.4.1.bn3.running_var", "0.4.2.conv3.weight", "0.4.2.bn3.weight", "0.4.2.bn3.bias", "0.4.2.bn3.running_mean", "0.4.2.bn3.running_var", "0.5.0.conv3.weight", "0.5.0.bn3.weight", "0.5.0.bn3.bias", "0.5.0.bn3.running_mean", "0.5.0.bn3.running_var", "0.5.1.conv3.weight", "0.5.1.bn3.weight", "0.5.1.bn3.bias", "0.5.1.bn3.running_mean", "0.5.1.bn3.running_var", "0.5.2.conv3.weight", "0.5.2.bn3.weight", "0.5.2.bn3.bias", "0.5.2.bn3.running_mean", "0.5.2.bn3.running_var", "0.5.3.conv3.weight", "0.5.3.bn3.weight", "0.5.3.bn3.bias", "0.5.3.bn3.running_mean", "0.5.3.bn3.running_var", "0.6.0.conv3.weight", "0.6.0.bn3.weight", "0.6.0.bn3.bias", "0.6.0.bn3.running_mean", "0.6.0.bn3.running_var", "0.6.1.conv3.weight", "0.6.1.bn3.weight", "0.6.1.bn3.bias", "0.6.1.bn3.running_mean", "0.6.1.bn3.running_var", "0.6.2.conv3.weight", "0.6.2.bn3.weight", "0.6.2.bn3.bias", "0.6.2.bn3.running_mean", "0.6.2.bn3.running_var", "0.6.3.conv3.weight", "0.6.3.bn3.weight", "0.6.3.bn3.bias", "0.6.3.bn3.running_mean", "0.6.3.bn3.running_var", "0.6.4.conv3.weight", "0.6.4.bn3.weight", "0.6.4.bn3.bias", "0.6.4.bn3.running_mean", "0.6....
        size mismatch for 0.4.0.conv1.weight: copying a param with shape torch.Size([64, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([64, 64, 1, 1]).
        size mismatch for 0.4.1.conv1.weight: copying a param with shape torch.Size([64, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([64, 256, 1, 1]).
        size mismatch for 0.4.2.conv1.weight: copying a param with shape torch.Size([64, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([64, 256, 1, 1]).
        size mismatch for 0.5.0.conv1.weight: copying a param with shape torch.Size([128, 64, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 256, 1, 1]).
        size mismatch for 0.5.0.downsample.0.weight: copying a param with shape torch.Size([128, 64, 1, 1]) from checkpoint, the shape in current model is torch.Size([512, 256, 1, 1]).
        size mismatch for 0.5.0.downsample.1.weight: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.0.downsample.1.bias: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.0.downsample.1.running_mean: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.0.downsample.1.running_var: copying a param with shape torch.Size([128]) from checkpoint, the shape in current model is torch.Size([512]).
        size mismatch for 0.5.1.conv1.weight: copying a param with shape torch.Size([128, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 512, 1, 1]).
        size mismatch for 0.5.2.conv1.weight: copying a param with shape torch.Size([128, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 512, 1, 1]).
        size mismatch for 0.5.3.conv1.weight: copying a param with shape torch.Size([128, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([128, 512, 1, 1]).
        size mismatch for 0.6.0.conv1.weight: copying a param with shape torch.Size([256, 128, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 512, 1, 1]).
        size mismatch for 0.6.0.downsample.0.weight: copying a param with shape torch.Size([256, 128, 1, 1]) from checkpoint, the shape in current model is torch.Size([1024, 512, 1, 1]).
        size mismatch for 0.6.0.downsample.1.weight: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.0.downsample.1.bias: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.0.downsample.1.running_mean: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.0.downsample.1.running_var: copying a param with shape torch.Size([256]) from checkpoint, the shape in current model is torch.Size([1024]).
        size mismatch for 0.6.1.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.2.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.3.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.4.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.6.5.conv1.weight: copying a param with shape torch.Size([256, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([256, 1024, 1, 1]).
        size mismatch for 0.7.0.conv1.weight: copying a param with shape torch.Size([512, 256, 3, 3]) from checkpoint, the shape in current model is torch.Size([512, 1024, 1, 1]).
        size mismatch for 0.7.0.downsample.0.weight: copying a param with shape torch.Size([512, 256, 1, 1]) from checkpoint, the shape in current model is torch.Size([2048, 1024, 1, 1]).
        size mismatch for 0.7.0.downsample.1.weight: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.0.downsample.1.bias: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.0.downsample.1.running_mean: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.0.downsample.1.running_var: copying a param with shape torch.Size([512]) from checkpoint, the shape in current model is torch.Size([2048]).
        size mismatch for 0.7.1.conv1.weight: copying a param with shape torch.Size([512, 512, 3, 3]) from checkpoint, the shape in current model is torch.Size([512, 2048, 1, 1]).
        size mismatch for 0.7.2.conv1.weight: copying a param with shape torch.Size([512, 512, 3, 3]) from checkpoint, the shape in current model is torch.Size([512, 2048, 1, 1]).
        size mismatch for 1.2.weight: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.2.bias: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.2.running_mean: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.2.running_var: copying a param with shape torch.Size([1024]) from checkpoint, the shape in current model is torch.Size([4096]).
        size mismatch for 1.4.weight: copying a param with shape torch.Size([512, 1024]) from checkpoint, the shape in current model is torch.Size([512, 4096]).


title: Gradient behavior in pytorch with multi-layer loss

body:
I have a loss where each layer plays into the loss. Which is the correct approach in terms of making sure the weights are updated properly?

# option 1
x2 = self.layer1(x1)
x3 = self.layer2(x2)
x4 = self.layer3(x3)

In this option, I detach when feeding into each subsequent block

    # option 2
    # x2 = self.layer1(x1.detach())
    # x3 = self.layer2(x2.detach())
    # x4 = self.layer3(x3.detach())

shared ops which calculate 4 losses and sum them.

    x4 = F.relu(self.bn1(x4))
    loss = some_loss([x1, x2, x3, x4])


title: How to reserve negative values of features in PyTorch using register_forward_hook?

body:
I am trying to extract features of a certain layer of a pretrained model. The fellowing code does work, however, the values of template_feature_map changed and I did nothing of it.

The output of 6th layer of the model should have negative values, as first print(template_feature_map) shows. But, the negative values which should maintain in second print(template_feature_map) are changed to zeros, I don’t know why. If you know the mechanism of this, please tell me how to keep the negative values.

vgg_feature = models.vgg13(pretrained=True).features
template_feature_map=[]
def save_template_feature_map(self, input, output):
    template_feature_map.append(output.detach())
    print(template_feature_map)
template_handle = vgg_feature[5].register_forward_hook(save_template_feature_map)
vgg_feature(template[0])
print(template_feature_map)

The output of two print(template_feature_map):

[tensor([[[[-5.7389e-01, -2.7154e+00, -4.0990e+00,  ...,  4.1902e+00,
            3.1757e+00,  2.2461e+00],
          [-2.2217e+00, -4.3395e+00, -6.8158e+00,  ..., -1.4454e+00,
            9.8012e-01, -2.3653e+00],
          [-4.1940e+00, -6.3235e+00, -6.8422e+00,  ..., -2.8329e+00,
            2.5570e+00, -2.7704e+00],
          ...,
          [-3.3250e+00,  1.3792e-01,  5.4926e+00,  ..., -4.1722e+00,
           -6.1008e-01, -2.6037e+00],
          [ 1.5377e+00,  6.0671e-01,  2.0974e+00,  ...,  1.2441e+00,
            1.5033e+00, -2.7246e+00],
          [ 6.8857e-01, -3.5160e-02,  6.7858e-01,  ...,  1.2052e+00,
            1.4533e+00, -1.4160e+00]],

         [[ 6.8798e-01,  1.6971e+00,  2.1629e+00,  ...,  3.1701e-01,
            8.5424e-01,  2.8768e+00],
          [ 1.4013e+00,  2.7217e+00,  2.1476e+00,  ...,  3.1156e+00,
            4.4858e+00,  3.6936e+00],
          [ 3.1807e+00,  2.2245e+00,  2.4665e+00,  ...,  1.3838e+00,
            1.0580e-02, -3.1445e-03],
          ...,
          [-4.7298e+00, -3.3037e+00, -1.2982e+00,  ...,  2.3266e-01,
            6.7711e+00,  3.8166e+00],
          [-4.7972e+00, -5.4591e+00, -2.5201e+00,  ...,  3.7584e+00,
            5.1524e+00,  2.3072e+00],
          [-2.4306e+00, -2.8033e+00, -2.0912e+00,  ...,  1.9888e+00,
            2.0582e+00,  1.9266e+00]],

         [[-4.4257e+00, -4.6331e+00, -3.3580e-03,  ..., -8.2233e+00,
           -7.4645e+00, -1.7361e+00],
          [-4.5593e+00, -8.4195e+00, -8.8428e+00,  ..., -6.7950e+00,
           -1.4665e+01, -2.5335e+00],
          [-2.3481e+00, -3.8543e+00, -3.5965e+00,  ..., -1.5105e+00,
           -1.6923e+01, -5.9852e+00],
          ...,
          [-8.0165e+00,  8.0185e+00,  6.5506e+00,  ...,  5.3241e+00,
            3.3854e+00, -1.6342e+00],
          [-1.3689e+01, -2.2930e+00,  4.7097e+00,  ...,  3.2021e+00,
            2.9208e+00, -8.0228e-01],
          [-1.3055e+01, -1.1470e+01, -8.4442e+00,  ...,  1.8155e-02,
           -6.2866e-02, -2.0333e+00]],

         ...,

         [[ 3.4622e+00, -1.2417e+00, -5.0749e+00,  ...,  5.3184e+00,
            1.4744e+01,  8.3968e+00],
          [-2.7820e+00, -9.1911e+00, -1.1069e+01,  ...,  2.5380e+00,
            9.8336e+00,  4.0623e+00],
          [-3.9794e+00, -1.0140e+01, -9.9133e+00,  ...,  3.0999e+00,
            5.5936e+00,  2.5775e+00],
          ...,
          [ 2.0299e+00,  2.1304e-01, -2.2307e+00,  ...,  1.1388e+01,
            8.8098e+00,  1.8991e+00],
          [ 8.0663e-01, -1.5073e+00,  3.3977e-01,  ...,  8.5316e+00,
            4.9923e+00, -3.6818e-01],
          [-3.5146e+00, -7.2647e+00, -5.4331e+00,  ..., -1.9781e+00,
           -3.4463e+00, -4.9034e+00]],

         [[-3.2915e+00, -7.3263e+00, -6.8458e+00,  ...,  2.3122e+00,
            9.7774e-01, -1.3498e+00],
          [-4.5396e+00, -8.6832e+00, -8.8582e+00,  ...,  7.1535e-02,
           -4.1133e+00, -4.4045e+00],
          [-4.8781e+00, -7.0239e+00, -4.7350e+00,  ..., -3.6954e+00,
           -9.6687e+00, -8.8289e+00],
          ...,
          [-4.7072e+00, -4.4823e-01,  1.7099e+00,  ...,  3.7923e+00,
            1.6887e+00, -4.3305e+00],
          [-5.5120e+00, -3.2324e+00,  2.3594e+00,  ...,  4.6031e+00,
            1.8856e+00, -4.0147e+00],
          [-5.1355e+00, -5.5335e+00, -1.7738e+00,  ...,  1.6159e+00,
           -1.3950e+00, -4.1055e+00]],

         [[-2.0252e+00, -2.3971e+00, -1.6477e+00,  ..., -3.3740e+00,
           -4.9965e+00, -2.1219e+00],
          [-7.6059e-01, -3.3901e-01, -1.8980e-01,  ..., -4.3286e+00,
           -7.1350e+00, -3.9186e+00],
          [ 8.4101e-01,  1.3403e+00,  2.5821e-01,  ..., -5.1847e+00,
           -7.1829e+00, -3.7724e+00],
          ...,
          [-6.0619e+00, -5.6475e+00, -1.6446e+00,  ..., -9.2322e+00,
           -9.1981e+00, -5.5239e+00],
          [-7.4606e+00, -7.6054e+00, -5.8401e+00,  ..., -7.6998e+00,
           -6.4111e+00, -2.9374e+00],
          [-6.4147e+00, -7.2813e+00, -6.1880e+00,  ..., -4.6726e+00,
           -3.1090e+00, -7.8383e-01]]]])]
[tensor([[[[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 4.1902e+00,
           3.1757e+00, 2.2461e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           9.8012e-01, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           2.5570e+00, 0.0000e+00],
          ...,
          [0.0000e+00, 1.3792e-01, 5.4926e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [1.5377e+00, 6.0671e-01, 2.0974e+00,  ..., 1.2441e+00,
           1.5033e+00, 0.0000e+00],
          [6.8857e-01, 0.0000e+00, 6.7858e-01,  ..., 1.2052e+00,
           1.4533e+00, 0.0000e+00]],

         [[6.8798e-01, 1.6971e+00, 2.1629e+00,  ..., 3.1701e-01,
           8.5424e-01, 2.8768e+00],
          [1.4013e+00, 2.7217e+00, 2.1476e+00,  ..., 3.1156e+00,
           4.4858e+00, 3.6936e+00],
          [3.1807e+00, 2.2245e+00, 2.4665e+00,  ..., 1.3838e+00,
           1.0580e-02, 0.0000e+00],
          ...,
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.3266e-01,
           6.7711e+00, 3.8166e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 3.7584e+00,
           5.1524e+00, 2.3072e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 1.9888e+00,
           2.0582e+00, 1.9266e+00]],

         [[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          ...,
          [0.0000e+00, 8.0185e+00, 6.5506e+00,  ..., 5.3241e+00,
           3.3854e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 4.7097e+00,  ..., 3.2021e+00,
           2.9208e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 1.8155e-02,
           0.0000e+00, 0.0000e+00]],

         ...,

         [[3.4622e+00, 0.0000e+00, 0.0000e+00,  ..., 5.3184e+00,
           1.4744e+01, 8.3968e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.5380e+00,
           9.8336e+00, 4.0623e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 3.0999e+00,
           5.5936e+00, 2.5775e+00],
          ...,
          [2.0299e+00, 2.1304e-01, 0.0000e+00,  ..., 1.1388e+01,
           8.8098e+00, 1.8991e+00],
          [8.0663e-01, 0.0000e+00, 3.3977e-01,  ..., 8.5316e+00,
           4.9923e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00]],

         [[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.3122e+00,
           9.7774e-01, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 7.1535e-02,
           0.0000e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          ...,
          [0.0000e+00, 0.0000e+00, 1.7099e+00,  ..., 3.7923e+00,
           1.6887e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 2.3594e+00,  ..., 4.6031e+00,
           1.8856e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 1.6159e+00,
           0.0000e+00, 0.0000e+00]],

         [[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [8.4101e-01, 1.3403e+00, 2.5821e-01,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          ...,
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00],
          [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 0.0000e+00,
           0.0000e+00, 0.0000e+00]]]])]


title: Pytorch - Iterating through a list of networks to evaluate forward time - cannot load next model

body:
My code does not load the next network to evaluate, it appears to just keep evaluating the same model.

The following code is run in a directory with many sub-directories of models/networks to be evaluated for forward time on a set of image sizes.

The evaluation times recorded do not change per run of the script, however since the shuffle was added the time depends on the first model loaded.

The code performs correctly when manually doing each directory separately. when run on a set of sub-directories it repeats the test on the same network model.

'''

# !/usr/bin/python

import os
import numpy as np
import torch
import time
import importlib
import sys
from random import shuffle

from PIL import Image
from argparse import ArgumentParser

from torch.autograd import Variable

import torch.backends.cudnn as cudnn
cudnn.benchmark = True

NUM_CLASSES = 20

def getEval(args):
    cwd = os.getcwd()
    modelpath = cwd + "\\" +args.loadDir + '\\' + args.loadModel + ".py"

    print ("Loading model: " + modelpath)

    MODULE_DIR = cwd + "\\" +args.loadDir + '\\'

    print("cwd", cwd)
    sys.path.append(MODULE_DIR)
    print ("sys path", sys.path)

    model_file = importlib.import_module(args.loadModel)
    model = model_file.Net(NUM_CLASSES)

    if (not args.cpu):
        model = model.cuda()#.half()    #HALF seems to be doing slower for some reason
    #model = torch.nn.DataParallel(model).cuda()

    model.eval()

    images1 = torch.randn(args.batch_size, args.num_channels, 2048, 1024)
    images2 = torch.randn(args.batch_size, args.num_channels, 1024, 512)
    images3 = torch.randn(args.batch_size, args.num_channels, 512, 256)
    images4 = torch.randn(args.batch_size, args.num_channels, 256, 256)

    images = [images1, images2, images3, images4]

    time_train = [[],[],[],[]]

    for j in range(0,4):

        inputs = Variable(images[j].cuda())

        for i in range(0,10):

            start_time = time.time()

            with torch.no_grad():
                outputs = model(inputs)

            #preds = outputs.cpu()
            if (not args.cpu):
                torch.cuda.synchronize()    #wait for cuda to finish (cuda is asynchronous!)

            if i != 0:    #first run always takes some time for setup
                fwt = time.time() - start_time
                time_train[j].append(fwt)
                print ("Forward time per img (b=%d): %.4f (Mean: %.4f)" % (args.batch_size, fwt/args.batch_size, sum(time_train[j]) / len(time_train[j]) / args.batch_size))

            time.sleep(1)   #to avoid overheating the GPU too much

    header = "2048x1024,1024x512,512x256,256x128"
    csvData = ''
    for size in range(0,4):
        csvData += '{}{:.4f}'.format("" if size == 0 else ",", sum(time_train[size]) / len(time_train[size]) / args.batch_size)
    print(header)
    print(csvData)

    # trying to clean things up from memory
    sys.path.remove(MODULE_DIR)
    del model
    del MODULE_DIR
    del model_file
    torch.cuda.empty_cache()

    return(csvData)

def main(args):
    with open('eval-win-speed.csv','w') as file:
        header = "2048x1024,1024x512,512x256,256x128"
        file.write('{},{}'.format("model_dir" ,header))
        file.write('\n')

    eval_values = []

    dirs = os.listdir(args.loadDir)
    shuffle(dirs)

    for dir in dirs:
        print(dir)

        if os.path.isdir(dir):
            print('is dir')
            class model_dir:
                loadDir = dir
                loadModel = args.loadModel
                num_channels = args.num_channels
                batch_size = args.batch_size
                cpu = args.cpu

            eval = '{},{}\n'.format(dir, getEval(model_dir))

            with open('eval-win-speed.csv','a') as file:
                file.write(eval)

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument('--loadDir',default="..\\trained_models\\")
    parser.add_argument('--loadWeights', default="model_best.pth")
    parser.add_argument('--loadModel', default="network")
    parser.add_argument('--num-channels', type=int, default=3)
    parser.add_argument('--batch-size', type=int, default=1)
    parser.add_argument('--cpu', action='store_true')

    main(parser.parse_args())

'''

expecting to record significantly different forward times for different model networks.