title: which part of the code is giving connection refused error?

body:
I am trying to understand VAE by running the code at: https://github.com/kuc2477/pytorch-vae
I changed the dataset folder and copy pasted everything to jupyter notebook (I get the same error without jupyter notebook as well). I suspect error must be coming somewhere from train.py so I modified the code there (where iterations are happening as):
tot_itrs = 0
for epoch in range(epoch_start, epochs+1):
#data_loader = utils.get_data_loader(dataset, batch_size, cuda=cuda)
data_loader = get_data_loader(dataset, batch_size, cuda=cuda)
data_stream = tqdm(enumerate(data_loader, 1))

for batch_index, (x, _) in data_stream:
    print(tot_itrs, end=' ')
    tot_itrs += 1
    # where are we?
    iteration = (epoch-1)*(len(dataset)//batch_size) + batch_index

    # prepare data on gpu if needed
    x = Variable(x).cuda() if cuda else Variable(x)

    # flush gradients and run the model forward
    optimizer.zero_grad()
    (mean, logvar), x_reconstructed = model(x)
    reconstruction_loss = model.reconstruction_loss(x_reconstructed, x)
    kl_divergence_loss = model.kl_divergence_loss(mean, logvar)
    total_loss = reconstruction_loss + kl_divergence_loss

    # backprop gradients from the loss
    total_loss.backward()
    optimizer.step()
    # update progress
    data_stream.set_description((
        'epoch: {epoch} | '
        'iteration: {iteration} | '
        'progress: [{trained}/{total}] ({progress:.0f}%) | '
        'loss => '
        'total: {total_loss:.4f} / '
        're: {reconstruction_loss:.3f} / '
        'kl: {kl_divergence_loss:.3f}'
    ).format(
        epoch=epoch,
        iteration=iteration,
        trained=batch_index * len(x),
        total=len(data_loader.dataset),
        progress=(100. * batch_index / len(data_loader)),
        total_loss=total_loss.data.item(),
        reconstruction_loss=reconstruction_loss.data.item(),
        kl_divergence_loss=kl_divergence_loss.data.item(),
    ))
    if iteration % loss_log_interval == 0:
        print("iteration % loss_log_interval == 0:")
        losses = [
            reconstruction_loss.data.item(),
            kl_divergence_loss.data.item(),
            total_loss.data.item(),
        ]
        names = ['reconstruction', 'kl divergence', 'total']
        #visual.visualize_scalars(
        visualize_scalars(
            losses, names, 'loss',
            iteration, env=model.name)

    if iteration % image_log_interval == 0:
        print("iteration % image_log_interval == 0:")
        images = model.sample(sample_size)
        #visual.visualize_images(
        visualize_images(
            images, 'generated samples',
            env=model.name
        )

# notify that we've reached to a new checkpoint.
print()
print()
print('#############')
print('# checkpoint!')
print('#############')
print()

# save the checkpoint.
#utils.save_checkpoint(model, checkpoint_dir, epoch)
save_checkpoint(model, checkpoint_dir, epoch)
print()

When I run this, I gt following error/ouput
0it [00:00, ?it/s]/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/torch/nn/_reduction.py:44: UserWarning: size_average and reduce args will be deprecated, please use reduction='sum' instead.
  warnings.warn(warning.format(ret))
epoch: 1 | iteration: 23 | progress: [1472/60000] (2%) | loss => total: 904.5577 / re: 900.121 / kl: 4.437: : 23it [00:00, 109.19it/s]
0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
epoch: 1 | iteration: 46 | progress: [2944/60000] (5%) | loss => total: 867.7250 / re: 858.849 / kl: 8.876: : 35it [00:00, 111.48it/s]
23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46
epoch: 1 | iteration: 69 | progress: [4416/60000] (7%) | loss => total: 822.9196 / re: 804.030 / kl: 18.889: : 59it [00:00, 113.42it/s]
 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
epoch: 1 | iteration: 92 | progress: [5888/60000] (10%) | loss => total: 770.0244 / re: 742.728 / kl: 27.296: : 83it [00:00, 113.52it/s]
 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
epoch: 1 | iteration: 100 | progress: [6400/60000] (11%) | loss => total: 754.4070 / re: 724.972 / kl: 29.435: : 95it [00:00, 113.77it/s]Setting up a new session...
Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 170, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/util/connection.py", line 96, in create_connection
    raise err
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/util/connection.py", line 86, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen
    chunked=chunked,
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connectionpool.py", line 394, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 234, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1229, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1275, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1224, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1016, in _send_output
    self.send(msg)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 956, in send
    self.connect()
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 200, in connect
    conn = self._new_conn()
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 182, in _new_conn
    self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f4941e8f048>: Failed to establish a new connection: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/util/retry.py", line 573, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=8097): Max retries exceeded with url: /env/VAE-128k-mnist-1x32x32 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4941e8f048>: Failed to establish a new connection: [Errno 111] Connection refused'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/visdom/__init__.py", line 711, in _send
    data=json.dumps(msg),
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/visdom/__init__.py", line 677, in _handle_post
    r = self.session.post(url, data=data)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/sessions.py", line 590, in post
    return self.request('POST', url, data=data, json=json, **kwargs)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/sessions.py", line 542, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/sessions.py", line 655, in send
    r = adapter.send(request, **kwargs)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/adapters.py", line 516, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=8097): Max retries exceeded with url: /env/VAE-128k-mnist-1x32x32 (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4941e8f048>: Failed to establish a new connection: [Errno 111] Connection refused'))
[Errno 111] Connection refused
Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 170, in _new_conn
    (self._dns_host, self.port), self.timeout, **extra_kw
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/util/connection.py", line 96, in create_connection
    raise err
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/util/connection.py", line 86, in create_connection
    sock.connect(sa)
ConnectionRefusedError: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connectionpool.py", line 706, in urlopen
    chunked=chunked,
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connectionpool.py", line 394, in _make_request
    conn.request(method, url, **httplib_request_kw)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 234, in request
    super(HTTPConnection, self).request(method, url, body=body, headers=headers)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1229, in request
    self._send_request(method, url, body, headers, encode_chunked)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1275, in _send_request
    self.endheaders(body, encode_chunked=encode_chunked)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1224, in endheaders
    self._send_output(message_body, encode_chunked=encode_chunked)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 1016, in _send_output
    self.send(msg)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/http/client.py", line 956, in send
    self.connect()
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 200, in connect
    conn = self._new_conn()
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connection.py", line 182, in _new_conn
    self, "Failed to establish a new connection: %s" % e
urllib3.exceptions.NewConnectionError: <urllib3.connection.HTTPConnection object at 0x7f4941be06a0>: Failed to establish a new connection: [Errno 111] Connection refused

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/connectionpool.py", line 756, in urlopen
    method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/urllib3/util/retry.py", line 573, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='localhost', port=8097): Max retries exceeded with url: /events (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4941be06a0>: Failed to establish a new connection: [Errno 111] Connection refused'))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/visdom/__init__.py", line 711, in _send
    data=json.dumps(msg),
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/visdom/__init__.py", line 677, in _handle_post
    r = self.session.post(url, data=data)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/sessions.py", line 590, in post
    return self.request('POST', url, data=data, json=json, **kwargs)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/sessions.py", line 542, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/sessions.py", line 655, in send
    r = adapter.send(request, **kwargs)
  File "/home/aknirala/anaconda3/envs/gan_37/lib/python3.7/site-packages/requests/adapters.py", line 516, in send
    raise ConnectionError(e, request=request)
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=8097): Max retries exceeded with url: /events (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4941be06a0>: Failed to establish a new connection: [Errno 111] Connection refused'))
epoch: 1 | iteration: 103 | progress: [6592/60000] (11%) | loss => total: 756.0789 / re: 727.468 / kl: 28.611: : 95it [00:01, 113.77it/s]
92 93 94 95 96 97 98 99 iteration % loss_log_interval == 0:
Exception in user code:
------------------------------------------------------------
Exception in user code:
------------------------------------------------------------
100 101 102 103
epoch: 1 | iteration: 127 | progress: [8128/60000] (14%) | loss => total: 711.3364 / re: 680.982 / kl: 30.354: : 119it [00:01, 94.96it/s]
104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126

From where is this exception being thrown (not the warning but the exception which ultimately says:
requests.exceptions.ConnectionError: HTTPConnectionPool(host='localhost', port=8097): Max retries exceeded with url: /events (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f4941be06a0>: Failed to establish a new connection: [Errno 111] Connection refused'))

) and how can I debug this?


title: RoBERTa Transfer learning with 1000 samples

body:
Hi I am completely a beginner. Please do correct me if I am wrong.
I have only 1000 samples of text data. I want to build a text classifier.
Building a model with the help of RoBERTa base model will it give a good accuracy or not


title: Reinterpret event_shape as batch_shape in torch.distributions

body:
Let's say I have a Gamma distribution
q = D.Independent(D.Gamma(alpha, beta), 1)
Where alpha and beta are the parameters vectors with dimensions [n_samples, batch_size, d].
The distribution will have for shapes:
print(q.batch_shape, q.event_shape)
>>> [n_samples, batch_size], [d]

the D.Independent wrapper class allows to reinterpret the batch dimensions as event dimensions (as done when initialising the Gamma distribution), but is there a way to do the opposite, that is:
Is there a way to reinterpret event dimensions as batch dimensions?
I know that I can re-define a new distribution, i.e
alpha = q.base_dist.concentration
beta = q.base_dist.rate
q = D.Gamma(alpha, beta)

that will have the correct dimensions
print(q.batch_shape, q.event_shape)
>>> [n_samples, batch_size, d], []

But that seems highly unpractical and inefficient.
Any ideas?


title: Calculating the loss in Policy Gradients

body:
I am trying to implement the loss using Policy Gradients for SeqGAN. If we do this in a function I know that parameters will be the input, target, and the rewards. These are the dimensions of the parameters:

input: (batch size x sequence length)
target: (batch size x sequence length)
rewards: (1 x batch size)

   def batchPGLoss(self, inp: torch.Tensor, target: torch.Tensor, reward: List) -> torch.Tensor:
        batch_size, seq_len = inp.size()
        # swap the dimensions
        inp = inp.permute(1, 0)  # (seq_len x batch_size)
        # swap the dimensions
        target = target.permute(1, 0)  # (seq_len x batch_size)
        # init hidden state
        h = self.init_hidden(batch_size)

        loss = torch.zeros(1)
        for i in range(seq_len):
            # pass the first tokens for each elem in batch
            out, h = self.forward(inp[i], h)
            for j in range(batch_size):
                loss += -out[j][target.data[i][j]] * reward[j] # log(P(y_t | Y_1, ..., Y_{t-1})) * Q
        return loss / batch_size

When I run this I get the following issue
RuntimeError: Can't call numpy() on Tensor that requires grad. Use tensor.detach().numpy() instead.

I check which part gives the error and I found that the -out[j][target.data[i][j]] is the issue since it requires the gradient. I could use tensor.detach().numpy() but that wouldn't solve the issue since I still need to keep out on the graph
Any suggestions?


title: Python: manylinux wheels contain unnecessary libraries using auditwheel

body:
I would like to build manylinux wheels of torchvision with FFMpeg support on aarch64.
when I executed auditwheel repair <torchvision_whl> -w output/, it added all compliers', FFMpeg's and torch's libraries, make the size of manylinux wheels too large:
[root@e6831809fd7a py39]# ls -lh whl
total 78M
-rw-r--r-- 1 root root 19M Mar 31 16:11 torchvision-0.9.1-cp39-cp39-linux_aarch64.whl
-rw-r--r-- 1 root root 60M Mar 31 16:39 torchvision-0.9.1-cp39-cp39-manylinux2014_aarch64.whl

Here is the result of auditwheel show:
[root@e6831809fd7a py39]# LD_LIBRARY_PATH=/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib /opt/python/cp39-cp39/bin/python3 /opt/_internal/tools/bin/auditwheel show /root/build/py39/output/torchvision-0.9.1-cp39-cp39-linux_aarch64.whl

torchvision-0.9.1-cp39-cp39-linux_aarch64.whl is consistent with the
following platform tag: "linux_aarch64".

The wheel references external versioned symbols in these system-
provided shared libraries: libgcc_s.so.1 with versions {'GCC_4.2.0',
'GCC_4.5.0', 'GCC_3.3', 'GCC_3.0', 'GCC_4.0.0'}, libc.so.6 with
versions {'GLIBC_2.17'}, libpthread.so.0 with versions {'GLIBC_2.17'},
libstdc++.so.6 with versions {'GLIBCXX_3.4.19', 'CXXABI_1.3',
'CXXABI_1.3.3', 'GLIBCXX_3.4.14', 'GLIBCXX_3.4.11', 'GLIBCXX_3.4.17',
'CXXABI_1.3.7', 'GLIBCXX_3.4.13', 'GLIBCXX_3.4.18', 'CXXABI_1.3.5',
'CXXABI_1.3.2', 'GLIBCXX_3.4.15', 'GLIBCXX_3.4.9', 'GLIBCXX_3.4',
'GLIBCXX_3.4.5'}, libjpeg.so.62 with versions {'LIBJPEG_6.2',
'LIBJPEGTURBO_6.2'}, libpng16.so.16 with versions {'PNG16_0'},
libswscale.so.5 with versions {'LIBSWSCALE_5'}, libswresample.so.3
with versions {'LIBSWRESAMPLE_3'}, libavutil.so.56 with versions
{'LIBAVUTIL_56'}, libavcodec.so.58 with versions {'LIBAVCODEC_58'},
libavformat.so.58 with versions {'LIBAVFORMAT_58'}, libm.so.6 with
versions {'GLIBC_2.17'}, librt.so.1 with versions {'GLIBC_2.17'},
libdl.so.2 with versions {'GLIBC_2.17'}, libgomp-d22c30c5.so.1 with
versions {'GOMP_1.0', 'OMP_1.0', 'GOMP_4.0'}, libz.so.1 with versions
{'ZLIB_1.2.3.4', 'ZLIB_1.2.0', 'ZLIB_1.2.0.2'}

The following external shared libraries are required by the wheel:
{
    "libavcodec.so.58": "/usr/local/lib/libavcodec.so.58.91.100",
    "libavformat.so.58": "/usr/local/lib/libavformat.so.58.45.100",
    "libavutil.so.56": "/usr/local/lib/libavutil.so.56.51.100",
    "libc.so.6": "/lib64/libc-2.17.so",
    "libc10.so": "/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib/libc10.so",
    "libdl.so.2": "/lib64/libdl-2.17.so",
    "libgcc_s.so.1": "/lib64/libgcc_s-4.8.5-20150702.so.1",
    "libgomp-d22c30c5.so.1": "/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib/libgomp-d22c30c5.so.1",
    "libjpeg.so.62": "/lib64/libjpeg.so.62.1.0",
    "libm.so.6": "/lib64/libm-2.17.so",
    "libpng16.so.16": "/usr/local/lib/libpng16.so.16.37.0",
    "libpthread.so.0": "/lib64/libpthread-2.17.so",
    "librt.so.1": "/lib64/librt-2.17.so",
    "libshm.so": "/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib/libshm.so",
    "libstdc++.so.6": "/lib64/libstdc++.so.6.0.19",
    "libswresample.so.3": "/usr/local/lib/libswresample.so.3.7.100",
    "libswscale.so.5": "/usr/local/lib/libswscale.so.5.7.100",
    "libtorch.so": "/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib/libtorch.so",
    "libtorch_cpu.so": "/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so",
    "libtorch_python.so": "/opt/_internal/cpython-3.9.2/lib/python3.9/site-packages/torch/lib/libtorch_python.so",
    "libz.so.1": "/lib64/libz.so.1.2.7"
}

In order to achieve the tag platform tag "manylinux_2_24_aarch64" the
following shared library dependencies will need to be eliminated:

libavcodec.so.58, libavformat.so.58, libavutil.so.56, libc10.so,
libgomp-d22c30c5.so.1, libjpeg.so.62, libpng16.so.16, libshm.so,
libswresample.so.3, libswscale.so.5, libtorch.so, libtorch_cpu.so,
libtorch_python.so, libz.so.1

In order to achieve the tag platform tag "manylinux2014_aarch64" the
following shared library dependencies will need to be eliminated:

libavcodec.so.58, libavformat.so.58, libavutil.so.56, libc10.so,
libgomp-d22c30c5.so.1, libjpeg.so.62, libpng16.so.16, libshm.so,
libswresample.so.3, libswscale.so.5, libtorch.so, libtorch_cpu.so,
libtorch_python.so, libz.so.1

In which, libav* are from FFMpeg, libtorch* are from torch, and so on. They are not expected to be include in whl files.
In addition:

the building environment is Docker, from quay.io/pypa/manylinux2014_aarch64.
torchvision-0.9.1-cp39-cp39-linux_aarch64.whl is compiled from source
FFMpeg is compiled from source
libpng is compiled from source since the latest package for CentOS 7 is 1.5.x

How can I fix this?


title: What does ToSparceTensor transform in TORCH_GEOMETRIC.TRANSFORMS exactly return?

body:
My goal is to understand ToSparseTensor transform accurately. The documentation for this transform mentions:

"ToSparseTensor: Converts the edge_index attribute of a data object
into a (transposed) torch_sparse.SparseTensor type with key adj_.t."

Looks like the adjacency list of a graph is converted to an adjacency matrix, stored as a sparse tensor.
However, I still find the description above a bit cryptic:

Is the "key" talking about a key value in a dictionary? Or key and attribute are being used interchangably in the above description? Meaning the attributes of the object are now changed?
Does "adj_.t" stand for the adjacency tensor if the data?
Why transposing is performed here? What is the significance of it?

Could you please rephrase the functionality of this method?


title: Livelossplot training graphs for Pytorch not coming up in colab

body:
Following is the line of code that i have used:
from livelossplot import PlotLosses
liveloss = PlotLosses()
liveloss.update({
'val_WER': float(val_res["valid_scores"]["wer"]),
'val_loss': float(val_res["valid_recognition_loss"])})
liveloss.draw()```

During training only *****Figure(1200x800)***** is displayed.


title: I wonder why the corresponding storage of pytorch.tensor changes every time it is resized?

body:
import torch

a = torch.tensor([3,2,3,4])
b = a.view(2,2)
c = a.resize(2,2)
d = a.resize_(2,2)
print(id(a.storage()))
print(id(b.storage()))
print(id(c.storage()))
print(id(d.storage()))

run at first time
2356950450056
2356950450056
2356950450056
2356950450056

run at second time
2206021857352
2206301638600
2206021857352
2206301638600

Why does id change sometimes but not change sometimes, I am searching for a long time on net. But no use. Please help or try to give some ideas how to achieve this. (apologies for my poor English)
Thanks in advance.


title: Issue with Gradient Accumulation

body:
Below is the code I am trying to run gradient accumulation on.
import torch
import torch.nn as nn
import numpy as np
from DataInitialization import initialize_data
from Architecture import BlackScholesMertonModel1

torch.manual_seed(123)
exercise_price = 100
sigma = 0.4
r = 0.03
dividend = 0.07
tau = 3

N_b = 100
N_init = 100
N_f = 30000

lb = [0, 0]
ub = [500, tau]

X_f, f_collocation, u_collocation, X_b, u_boundary, X_init, u_init = initialize_data(N_b, N_init, N_f, lb, ub)

# initializing the pde solver
model1 = BlackScholesMertonModel1()

# Original weight initialization didn't change the weights.
# Initialize the weights
for m in model1.modules():
    if isinstance(m, nn.Linear):
        torch.nn.init.xavier_uniform_(m.weight, gain=1.0)
        torch.nn.init.constant_(m.bias, 0)

# perform backprop
MAX_EPOCHS_1 = int(210)
LRATE = 8e-3

# use Adam for training
optimizer = torch.optim.Adam(model1.parameters(), lr=LRATE, eps=1e-7)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 140, gamma=0.5, last_epoch=-1, verbose=False)

X_f.requires_grad = True

mini_batch_size = 32

# send everything to GPU.
device = "cuda" if torch.cuda.is_available() else "cpu"
X_f = X_f.to(device)
X_b = X_b.to(device)
X_init = X_init.to(device)
model1 = model1.to(device)
u_boundary = u_boundary.to(device)
u_init = u_init.to(device)
u_collocation = u_collocation.to(device)
f_collocation = f_collocation.to(device)

accumulation_steps = 10

for epoch in range(MAX_EPOCHS_1):
    # boundary loss
    u_b_pred = model1(X_b)
    mse_u_b = torch.nn.MSELoss()(u_b_pred, u_boundary)

    # initial time loss
    u_init_pred = model1(X_init)
    mse_u_init = torch.nn.MSELoss()(u_init_pred, u_init)

    for i in range(len(X_f)):
        instance = X_f[i, :].clone()

        u_pred = model1(instance)

        u_pred_first_partials = torch.autograd.grad(u_pred.sum(), instance, allow_unused = True, retain_graph = True)[0].clone()
        u_pred_dt = u_pred_first_partials[0].clone()
        u_pred_ds = u_pred_first_partials[1].clone()

        f_pred = u_pred_dt

        f_true = f_collocation[i] / accumulation_steps

        mse_f = 100 * torch.nn.MSELoss()(f_pred, f_true)
        loss = mse_f + mse_u_init + mse_u_b
        loss.backward(retain_graph = True)
        if ((i + 1) % accumulation_steps) == 0:
            optimizer.step()
            optimizer.zero_grad()

    u_pred = model1(X_f)

    u_pred_first_partials = torch.autograd.grad(u_pred.sum(), X_f_c, create_graph = True, allow_unused = True)[0]
    u_pred_dt = u_pred_first_partials[:, 0:1]
    u_pred_ds = u_pred_first_partials[:, 1:2]

    f_pred = u_pred_dt

    mse_f = torch.nn.MSELoss()(f_pred, f_collocation)
    mse_function = torch.nn.MSELoss()(u_pred, u_collocation)

    loss_history_f_1.append(mse_f)
    loss_history_boundary_1.append(mse_u_b)
    loss_history_init_1.append(mse_u_init)
    loss_history_function_1.append(mse_function)

    if (epoch % 10) == 0:
        print("- - - - - - - - - - - - - - -")
        print("Epoch : ", epoch)
        print(f"Loss Residual:\t{loss_history_f_1[-1]:.4f}")
        print(f"Loss Boundary:\t{loss_history_boundary_1[-1]:.4f}")
        print(f"Loss Initial Time:\t{loss_history_init_1[-1]:.4f}")
        print(f"Loss Function:\t{loss_history_function_1[-1]:.4f}")
print("----------------------------------------------------------")

The error provided by the code is as follows.
> RuntimeError: one of the variables needed for gradient computation has
> been modified by an inplace operation: [torch.cuda.FloatTensor [50,
> 1]], which is output 0 of TBackward, is at version 3; expected version
> 2 instead. Hint: enable anomaly detection to find the operation that
> failed to compute its gradient, with
> torch.autograd.set_detect_anomaly(True).

I am confused as to what exactly is causing the issue. I understand that it has to do with how variables are modified (hence why I tried using .clone()) but that did not prove fruitful. However, I am having difficulty finding the inplace operations that are causing the issue because no values are being updated, they are simply used to calculate the loss.


title: how to approach leaf variables to freeze weight parameters

body:
i want to fix weights which have a value of 'zero' in model.parameters() (model is a neural network)
it means i have to freeze it in element-wise so i made a code below.
in the code, model.parameters() will be input of 'Lparam'
and when i ran the code, i got an error like 'nonetype has no attribute 'zero_()'
and when i try another trick like requires_grad=False, error says like,
"only leaf variables can use requires_grad"
then how can i approach leaf variables to fix the zero weights in the computation graph?
here is my code.
def fix_tensor_data(Lparam):
    if isinstance(Lparam,type(None)):
        return

    elif torch.is_tensor(Lparam):
        if Lparam.dim() == 0 and Lparam == 0:
            Lparam.grad.zero_()
        elif Lparam.dim() == 0 and Lparam != 0:
            return
        else:
            for tp1 in Lparam:
                fix_tensor_data(tp1)

    else: #enter this section only once when Lparam is a generator
        for tp2 in Lparam:
                fix_tensor_data(tp2)


title: Error occured while starting Spyder kernel

body:
I am using spyder4.8 with python 3.8. The kernel is working fine when I execute a single line of code. But when I executing the whole script, the kernel dies and it throws the following error.

OMP: Error #15: Initializing libiomp5md.dll, but found libiomp5md.dll
already initialized. OMP: Hint This means that multiple copies of the
OpenMP runtime have been linked into the program. That is dangerous,
since it can degrade performance or cause incorrect results. The best
thing to do is to ensure that only a single OpenMP runtime is linked
into the process, e.g. by avoiding static linking of the OpenMP
runtime in any library. As an unsafe, unsupported, undocumented
workaround you can set the environment variable
KMP_DUPLICATE_LIB_OK=TRUE to allow the program to continue to execute,
but that may cause crashes or silently produce incorrect results. For
more information, please see
http://www.intel.com/software/products/support/.

I also tried the option of setting the environment
os.environ['KMP_DUPLICATE_LIB_OK']='True'

but this is not the correct fix for this, as this produces wrong results.
I also tried some other things which are suggested here-
1- Error #15: Initializing libiomp5.dylib, but found libiomp5.dylib already initialized
2- How to properly fix the following OpenMP error?
but none of them worked.


title: Training on Sagemaker GPU is too slow

body:
I've launched a training on CelebA dataset for a binary classification with PyTorch, in Sagemaker Studio.
I've made sure all, model, tensors are sent to cuda().
My image dataset is in S3, and I'm accessing it via this import and code:
from PIL import Image
import s3fs

fs = s3fs.S3FileSystem()

# example
f = fs.open(f's3://aoha-bucket/img_celeba/dataset/000001.jpg')

And of course my PyTorch DataLoader class, which is using of s3fs to load data into DataLoaders.
class myDataset(Dataset):
    def __init__(self, csv_file, root_dir, target, length, adv = None, transform=None):
        self.annotations = pd.read_csv(csv_file).iloc[:length,:]
        self.root_dir = root_dir
        self.transform = transform
        self.target = target
        self.length = length
        self.adv = adv

    def __len__(self):
        return len(self.annotations)

    def __getitem__(self, index):
        img_path = fs.open(os.path.join(self.root_dir, self.annotations.loc[index, 'image_id']))
        image = Image.open(img_path)
        image = np.array(image)

        if self.transform:
            image = self.transform(image=image)["image"]

        image = np.transpose(image, (2, 0, 1)).astype(np.float32)
        image = torch.Tensor(image)

        y_label = torch.tensor(int(self.annotations.loc[index, str(self.target)]))

        if self.adv is None:
            return image, y_label

        if self.adv :
            z_label = torch.tensor(int(self.annotations.loc[index, 'origin']))
            return image, y_label, z_label

when I run this function, I get True:
next(model.parameters()).is_cuda

My issue is that, I don't know the training is too slow, even slower than my local CPU (not that powerful). It says for example, that one epoch is needing 1h45minutes, which is way too much.
Im' using a GPU optimized PyTorch instance of Studio.
Have you ever launched a training on GPU in Sagemaker using PyTorch ?
Could you please help ?
Thank you very much,
Habib


title: Improve pronounciation of a model

body:
I fine-tuned a dataset of Nvidia Tacotron2. While working reasonably well, there are some mispronounciations of words(I train a german dataset).
I have another set of wave files by the same speaker with according metadata.csv
How do I filter this to include mainly those sentences that teach the model the very pronounciations that are missing?


title: TypeError: pic should be Tensor or ndarray. Got . error

body:
raise TypeError('pic should be Tensor or ndarray. Got {}.'.format(type(pic)))
TypeError: pic should be Tensor or ndarray. Got <class 'NoneType'>.


title: PyTorch and GLIBC compatibility error after upgrading JetPack to 4.5

body:
A few days ago we upgraded JetPack to v4.5 on our Xavier Ubuntu 18.04 aarch64 machine.
After installing PyTorch v1.7.0 with torchvision v8.1.0 I receive the following error when running import torch in Python v3.6.9
ImportError: /lib/aarch64-linux-gnu/libc.so.6: version `GLIBC_2.28' not found (required by /home/venv/lib/python3.6/site-packages/torch/lib/libtorch_python.so)

Currently there is GLIBC v2.27 installed.
Do you know how can I solve this problem? I read online that upgrading GLIBC is not my best option as this requires a lot of effort and some risk as may other aprty of my system depend on GLIBC v2.27.


title: MNIST CNN Error at outputs = model(images)

body:
I have attempted to eliminated the raise NotImplementedError. It occurs at line outputs = model(images).
I have transformed my dataset and I have loaded them through Data Loader. The printout of the shape is immediatelty below before the main code.
The shape of the train and test data are torch.Size([300, 1, 28, 28]) and torch.Size([300, 1, 28, 28])train_data.shape
class ConvNet(nn.Module):
  def __init__(self):
    super(ConvNet, self).__init__()
    self.layer1 = nn.Sequential(
        nn.Conv2d(1, 32, kernel_size=5, stride=1, padding=2),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2))
    self.layer2 = nn.Sequential(
        nn.Conv2d(32, 64, kernel_size=5, stride=1, padding=2),
        nn.ReLU(),
        nn.MaxPool2d(kernel_size=2, stride=2))
    self.drop_out = nn.Dropout()
    self.fc1 = nn.Linear(7 * 7 * 64, 1000)
    self.fc2 = nn.Linear(1000, 10)
def forward(self, x):
  out = self.layer1(x)
  out = self.layer2(out)
  out = out.reshape(out.size(0), -1)
  out = self.drop_out(out)
  out = self.fc1(out)
  out = self.fc2(out)
  print(f"output shape: {out.shape}")
  return out

device  = torch.device("cuda" if torch.cuda.is_available() else 'cpu')
print("Device used is {}".format(device))
model = ConvNet()
model.to(device) # send model to device/GPU for training
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
num_epochs = 3

# Train the model
total_step = len(train_loader)
loss_list = []
acc_list = []
for epoch in range(num_epochs):
  for i, (images, labels) in enumerate(train_loader):
    #Run the forward pass
    # images = images.cuda()
    # label = labels.cuda()
    images = images.to(device, dtype =torch.float)
    label = label.to(device, dtype =torch.long)
    outputs = model(images)
    loss = criterion(outputs, labels)
    loss_list.append(loss.item())

    # Backprop and perform Adam optimisation
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    # Track the accuracy
    total = labels.size(0)
    _, predicted = torch.max(outputs.data, 1)
    correct = (predicted == labels).sum().item()
    acc_list.append(correct / total)

    if (i + 1) % 100 == 0:
        print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}, Accuracy: {:.2f}%'
              .format(epoch + 1, num_epochs, i + 1, total_step, loss.item(),
                      (correct / total) * 100))


title: can't transfer numpy array into a pytorch tensor

body:
I'm trying to build an ai to recognize hand-written digits.
I'm trying to expend the dataset by shifting the images to all directions by one or two pixels, and to make this process faster I'm using multiprocessing.
this is my code yet:
import pandas
import torch
import torch.nn as nn
from torch.autograd import Variable
import pandas as pd
import numpy as np
from torch.utils.data import TensorDataset
import matplotlib.pyplot as plt
from scipy.ndimage.interpolation import shift as image_shift
import math
import time
import multiprocessing as mp

def multiproc_img_sft(img, y, sfts, procNum):
    labels = []
    for i in range(784):
        labels.append(f"p{i}")
    labels.append("labels")

    #q.put(image_shift(img.reshape(28, 28), sft).reshape(1, 1, 28, 28))
    img = img.reshape(-1, 784)
    imgs = np.empty((0, img.shape[1]))
    y_expended = np.empty((0, 1))
    for i in range(img.shape[0]):
        if i % 1000 == 0:
            imgs = np.append(imgs, y_expended, axis=1)
            DF = pd.DataFrame(imgs, columns=labels)

            if i == 0:
                DF.to_csv(f"extended_{procNum}.csv", index=False)
            else:
                DF.to_csv(f"extended_{procNum}.csv", mode="a", index=False)

            imgs = np.empty((0, img.shape[1]))
            y_expended = np.empty((0, 1))
        if i % 50 == 0:
            print(f"{procNum}:   {i+1} \\ {img.shape[0]}")
        for sft in sfts:
            imgs = np.append(imgs, image_shift(img[i].reshape(28, 28), sft).reshape(-1, 784), axis=0)
            y_expended = np.append(y_expended, [y[i]], axis=0)
            #print(y[i], y_expended.shape)

    #print(procNum, y_expended, y_expended.shape, imgs.shape)
    #y_expended = np.array(y_expended)

    imgs = np.append(imgs, y_expended, axis=1)
    DF = pd.DataFrame(imgs, columns=labels)

    DF.to_csv(f"extended_{procNum}.csv", mode="a", index=False)

    print(procNum, "dataframe labels =", labels)
    print(procNum, "y_expended =", y_expended.shape, type(y_expended))

    print(f"proc {procNum} ended")

if __name__ == "__main__":
    extend_X = True
    num_of_processes = 1

    shifts = [[1, 0], [0, 1], [-1, 0], [0, -1]]#,
###              [1, 1], [1, -1], [-1, 1], [-1, -1],
###
###              [2, 0], [0, 2], [-2, 0], [0, -2],
###              [2, 2], [2, -2], [-2, 2], [-2, -2],
###              [2, 1], [1, 2],  [2, -1], [1, -2],  [-1, 2], [-2, 1],  [-1, -2], [-2, -1]]
    #shifts = [[1, 0]]

    batch_size = 100
    batch_size = 1000
    n_iters = 10000
    learning_rate = 0.003325
    learning_rate = 0.003325
    # Accuracy: 99.43386243386243, Correct: 18793, Total: 18900
    # batch_size = 100
    # n_iters = 10000
    # learning_rate = 0.003325

    train_data = pd.read_csv("train.csv")
    X, y = train_data.drop(labels=['label'], axis=1).values, train_data['label'].values

    # X = scaler.fit_transform(X)

    X_train, X_test = X[:37800, :].reshape(-1, 1, 28, 28), X[37800:, :].reshape(-1, 1, 28, 28)
    y_train, y_test = y[:37800].reshape(-1, 1), y[37800:].reshape(-1, 1)
    print(y_train.shape)
    1==1

    print("extend_X =", extend_X)

    if extend_X:
        processes = []
        for j in range(num_of_processes):
            #multiproc_img_sft(X_train, y_train, shifts, j)

            print(j)
            lower_bound = math.floor(len(X_train) / num_of_processes) * j
            upper_bound = math.floor(len(X_train) / num_of_processes) * (j+1)
            if upper_bound == len(X_train):
                upper_bound -= 1

            print(X_train.shape, lower_bound, upper_bound)
            #p = mp.Process(target=multiproc_img_sft, args=(X_train[lower_bound:upper_bound], y_train[lower_bound:upper_bound], shifts, j))
            p = mp.Process(target=multiproc_img_sft,
                           args=(X_train[0:100], y_train[0:100], shifts, j))
            processes.append(p)
            p.start()

        for j in range(num_of_processes):
            print("process", j, "has joined")
            processes[j].join()

        labels = []
        for i in range(784):
            labels.append(f"p{i}")
        DF = pandas.DataFrame(columns=labels)
        for i in range(num_of_processes):
            print(pd.read_csv(f"extended_{i}.csv").shape)
            DF = pd.concat([DF, pd.read_csv(f"extended_{i}.csv")], ignore_index=True)
            print(f"combined {i+1} dataframes \\ {num_of_processes} dataframes")
        DF.to_csv("X_Train_Extended.csv", index=False)
        print("SAVED ALL DATAFRAMES")

    extended_X = pd.read_csv("X_Train_Extended.csv")
    print(extended_X)
    print(extended_X.shape)
    extended_X = extended_X.values
    y_train = np.append(y_train, extended_X[:, -1].reshape(-1, 1), axis=0).reshape(-1)
    X_train = np.append(X_train, extended_X[:, :-1].reshape(-1, 1, 28, 28), axis=0)

    print(y_train.shape, X_train.shape)

    print("\n\n")

    X_train_size = len(X_train) - 18900
    X_val, y_val = X_train[X_train_size:], y_train[X_train_size:]
    X_train, y_train = X_train[:X_train_size], y_train[:X_train_size]

    print(X_train.shape, y_train.shape)
    print(X_val.shape, y_val.shape)
    print(X_test.shape, y_test.shape)

    X_train_tensor = torch.Tensor(X_train)
    X_val_tensor = torch.Tensor(X_val)
    X_test_tensor = torch.Tensor(X_test)

and the error that I get is the following:
TypeError: can't convert np.ndarray of type numpy.object_. The only supported types are: float64, float32, float16, complex64, complex128, int64, int32, int16, int8, uint8, and bool.

on the line:
X_train_tensor = torch.Tensor(X_train)

any help is welcom


title: Can I unpack an `nn.ModuleList` inside `nn.Sequential`?

body:
I am parametrizing the number of hidden layers of a simple ANN using nn.ModuleList. I am wondering if passing this list into a nn.Sequential module as follows would lead to any adverse effects with the execution graph.
nn.Sequential is not necessary, however it seems cleaner to me to have the entire architecture explicit in the constructor.
class ANN(nn.Module):

    def __init__(
        self,
        in_feats=3,
        in_hidden=5,
        n_hidden_layers=3,

    ):
        super(ANN, self).__init__()

        # ====== dynamically register hidden layers ======
        self.hidden_linears = nn.ModuleList()
        for i in range(n_hidden_layers):
            self.hidden_linears.append(nn.Linear(in_hidden, in_hidden))
            self.hidden_linears.append(nn.ReLU())

        # ====== sequence of layers ======
        self.layers = nn.Sequential(
            nn.Linear(in_feats, in_hidden),
            nn.ReLU(),
            *self.hidden_linears,
            nn.Linear(in_hidden, 1),
            nn.Sigmoid(),
        )

    def forward(self, X):
        return self.layers(X)

Also open to suggestions of neater ways to put this together.


title: When I was training the model, I encountered the following problem, and it haven't finished a epoch

body:
help!!!maybe someone can help me? Thank you very much!!!
I am training the model--SSN
i just use the cuda101 and torch 1.4.0


title: Apply hooks on inner layers of ResNet

body:
The pytorch official implementation of resnet results in the following model:
ResNet(
  (conv1): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  (layer1): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (shortcut): Sequential()
    )
    (1): BasicBlock(
      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (shortcut): Sequential()
    )
  )
  (layer2): Sequential(
    (0): BasicBlock(
      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (shortcut): Sequential(
        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (1): BasicBlock(
      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (shortcut): Sequential()
    )
  )
 ### Skipping layers 3 and 4
  (linear): Linear(in_features=512, out_features=10, bias=True)
)

I tried applying hook to the conv1 in the first BasicBlock of layer2 using
handle = model.layer2[0][0].register_forward_hook(batchout_pre_hook)

but got the following error :
TypeError: 'BasicBlock' object does not support indexing

I am able to apply hook to the BasicBlock using  handle = model.layer2[0].register_forward_hook(batchout_pre_hook) but cannot apply hook in the modules present inside the BasicBlock


title: Error When Importing build_model for Detectron2

body:
I'm trying to run the following:
from detectron2.modeling import build_model

Which is returning the following:
ImportError: libtorch_cuda_cu.so: cannot open shared object file: No such file or directory
I'm running this in Google Colab. When I run:
!nvidia-smi
!pip list -v | grep torch

It returns
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |
| N/A   32C    P8    29W / 149W |      0MiB / 11441MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
torch                         1.8.1+cu101        /usr/local/lib/python3.7/dist-packages pip
torchsummary                  1.5.1              /usr/local/lib/python3.7/dist-packages pip
torchtext                     0.9.1              /usr/local/lib/python3.7/dist-packages pip
torchvision                   0.9.1+cu101        /usr/local/lib/python3.7/dist-packages pip

Based on that information I installed Detectron using the command from here.
I copied and pasted that import from here
What am I missing?


title: Is there a vectorized way to create a matrix in which each element is the the row-wise dot product of a matrix?

body:
Apologize for the vagueness in the title, I spend some time rephrasing but cannot get it very well.
For example, I have a 2*3 matrix in Pytorch tensor
test = torch.tensor([[1, 10, 100],
                    [2, 20, 200]])

What I would like to have a final matrix that is
torch.tensor([10101, 20202],
             [20202, 40404])

Where we can see, the (0,0) position is the first row's dot product with itself, and (0,1) (1,0) is the dot product of first row and second row, (1,1) the the dot product of the second row with itself.
Thanks!


title: FastAI fastbook - what does it do and why do I need to setup a book?

body:
I tried running on my google colab notebook:
!pip install -Uqq fastbook
import fastbook

as it is written in the FastAI book, chapter 2.
but nor the book or anywhere on google there is an explanation on what is this liberty at all.
amazingly, the page for it does not include any explanation on what fastbook does-  only about some course for deep learning.
so,

what does it do?

also, when I run:
fastbook.setup_book()

what does that do? in which way does it setup a book, and what kind of book is it?

ty.


title: Segmentation fault from libtorch_cuda_cpp.so(Pytorch)

body:
I'm trying to run https://pytorch.org/tutorials/beginner/fgsm_tutorial.html. Everything works fine for first 651 pictures and then I'm getting segmentation fault error. I checked memory usage on GPU(GTX 1050) and seems fine. I also run the same code on my friend's GTX 1050ti and it worked fine.  I re-install Ubuntu and clean set up of driver and CUDA tools and the problem is still there. I execute the code with GNU debugger and here is what I've got after 651 pictures:
Thread 12 "python3" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffedcdb5700 (LWP 8697)]
0x00007fff32aa46ca in std::_Hashtable<at::native::ConvolutionParams, std::pair<at::native::ConvolutionParams const, cudnnConvolutionBwdDataAlgoPerf_t>, std::allocator<std::pair<at::native::ConvolutionParams const, cudnnConvolutionBwdDataAlgoPerf_t> >, std::__detail::_Select1st, at::native::ParamsEqual<at::native::ConvolutionParams>, at::native::ParamsHash<at::native::ConvolutionParams>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::_M_find_before_node(unsigned long, at::native::ConvolutionParams const&, unsigned long) const ()
   from /home/muco/.local/lib/python3.8/site-packages/torch/lib/libtorch_cuda_cpp.so

$nvcc --version

nvcc: NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2020 NVIDIA Corporation
Built on Mon_Oct_12_20:09:46_PDT_2020
Cuda compilation tools, release 11.1, V11.1.105
Build cuda_11.1.TC455_06.29190527_0

$nvidia-smi
Thu May 20 16:12:04 2021
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.80       Driver Version: 460.80       CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  GeForce GTX 1050    Off  | 00000000:01:00.0 Off |                  N/A |
| N/A   57C    P0    N/A /  N/A |   1123MiB /  4040MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A       849      G   /usr/lib/xorg/Xorg                204MiB |
|    0   N/A  N/A      1332      G   budgie-wm                          23MiB |
|    0   N/A  N/A      1673      G   ...AAAAAAAAA= --shared-files       58MiB |
|    0   N/A  N/A      6801      G   ...AAAAAAAAA= --shared-files       61MiB |
|    0   N/A  N/A      8679      C   /usr/bin/python3                  769MiB |

CuDNN Version: 8.1.0
Ubuntu Version: 20.04
GCC Version: 9.3.0
Python Version: 3.8.5
I'm wondering is this a hardware problem or some kind of bug?


title: how to create n linear layers in pytorch when n is an argument to the function

body:
I have the following model:
class model(nn.Module):
     def __init__(self,y_dim,x_dim):
         super(model,self).__init__()
         self.x_dim = x_dim
         self.y_dim = y_dim
         self.kern = 3
         self.conv1 = nn.Conv1d(y_dim,2,self.kern,padding=1)
         self.conv2 = nn.Conv1d(2,4,self.kern,padding=1)
         self.conv3 = nn.Conv1d(4,4,self.kern,padding=1)
         self.conv4 = nn.Conv1d(4,y_dim,self.kern,padding=1)
         #these are the layers I want to automate
         self.lina = nn.Linear(28,28)
         self.linb = nn.Linear(28,28)
     def forward(self,x):
         x = F.relu(self.conv1(x))
         x = F.relu(self.conv2(x))
         x = F.relu(self.conv3(x))
         x = F.relu(self.conv4(x))
         xa = self.lina(x[:,0,:]).reshape(z.shape[0],1,z.shape[2])
         xb = self.linb(x[:,1,:]).reshape(z.shape[0],1,z.shape[2])
         return torch.cat((xa,xb),1)

model = model(2,28)

So, the output channels of self.conv4 is always y_dim. I'd like to have y_dim different linear layers (which are independent) that take the output of self.conv4. As of now, I don't know a better way than explicitly writing the layers, as I did in the example above. However, is there a method to automate this, namely generating the linear layer object and then using it in the forward function appropriately?


title: Positional Embedding in Bert

body:
Can someone explain how these positional embedding code work in BERT? Also tips to make it redeable.
class PositionalEmbedding(nn.Module):

    def __init__(self, d_model, max_len=512):
        super().__init__()

        # Compute the positional encodings once in log space.
        pe = torch.zeros(max_len, d_model).float()
        pe.require_grad = False

        position = torch.arange(0, max_len).float().unsqueeze(1)
        div_term = (torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model)).exp()

        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)

        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        return self.pe[:, :x.size(1)]


title: Is it possible to load a model which is stored with model.module.state_dict() but load with model.state_dict()

body:
I want to ask a question, I have trained a model with two gpus and stored this model with
model.module.state_dict(), now I want to load this model in one gpu, can I directly load this trained model with model.state_dict()?
Thanks in advance!


title: Getting the gradients of global_model with .grad

body:
i am training a CNN model in federated learning settings. i update my local_models and then i average the weights i get from the local_model update to get the weight for my global_model. Since i use loss.backward() during my local_model training, i can get the gradients by using :
updates_list = []
for item in model.parameters():
    updates_list.append(copy.deepcopy(item.grad))
print(updates_list)

But i can not use .grad for my global_model as there is no backward() function usage during the averaging of local_model weights. My question is, how do i get the updated gradients of my global_model?


title: Is torch.empty_like() dependent on the input value as well as input size?

body:
The description in the torch docs for torch.empty_like says:

torch.empty_like(input, *, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) → Tensor
Returns an uninitialized tensor with the same size as input. torch.empty_like(input) is equivalent to torch.empty(input.size(), dtype=input.dtype, layout=input.layout, device=input.device).
Parameters
input (Tensor) – the size of input will determine size of the output tensor.

What I do is :
>>> torch.empty(3,4)
tensor([[-1.8597e+15,  4.5657e-41, -1.8597e+15,  4.5657e-41],
        [ 4.4842e-44,  0.0000e+00,  8.9683e-44,  0.0000e+00],
        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00]])

>>> c1
tensor([[ 0,  1,  2,  3],
        [ 4,  5,  6,  7],
        [ 8,  9, 10, 11]])

>>> torch.empty_like(c1)
tensor([[139942262173040,  93851872482144,               1,               0],
        [              0,               0,  93851872492496,               0],
        [              0,               0,               0,               0]])

>>> d
tensor([[0., 0., 0., 0.],
        [0., 0., 0., 0.],
        [0., 0., 0., 0.]])

>>> torch.empty_like(d)
tensor([[-8.6092e-25,  3.0620e-41,  0.0000e+00,  0.0000e+00],
        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00],
        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00]])

It seems that the tensor returned by torch.empty_like depends on input value, contrary to the description in the docs. Can someone explain this?


title: tqdm notebook - 2 inner bars

body:
I'm trying to print training progress bar using tqdm.
I'd like to track the progress of the epochs, and for each epoch i have 2 progress bars: the train_loader minibatches, and validation_loader minibatches.
The code is something like that:
for epoch in tqdm(range(epochs_num)):
    for inputs, labels in tqdm(train_loader, "Train progress", leave=False):
        # train...
    with torch.no_grad():
        for inputs, labels in tqdm(validation_loader, "Validation progress", leave=False):
            # calc validation loss

Using the leave argument, the progress bars deleted every epoch, but I'd like to delete them togather, right after the validation process ends.
There's any way to do it?
Thanks


title: What are some of the best tools to augment 3D medical images for pytorch model

body:
I have medical images in dicom format. I have read the images and extracted the pixel values and stored as a numpy array. The final shape of array of a single patient is in the form [channels, frames/slices, height, width]. The number of channels is fixed and that is 1. The images are always grayscale.
I want to use these images for a simple task of binary classification. I want to augment the images for each patient(random rotation, zooming, translation etc.) and I was wondering what will be the best tool that can do the augmentation on this array.
I am not sure how we can really augment images randomly when there are more than one slice per image.


title: apply a function over all combination of tensor rows in pytorch

body:
I want to make a function f1(arg_tensor) which gets a pytorch tensor as an argument.
In this function I use another function:
f2(tensor_row_1, tensor_row_2) which gets two pytorch's tensor rows as an arguments and outputs a scalar.
f2(..) should be applied over all combinations of tensor's rows [1..n] (i.e. apply function f2(..) on tensor rows' indices: [0,1], [0,2], [0,3]...[0,n-1]...[n-1,0]..[n-1,n-1]).
The output of f1(..) should be a tensor such that at element [0,0] there will the output value of f2(tensor_rows[0], tensor_rows[0]) and so on...
Is there a way to perform it efficiently (and not with double for loop)?


title: Pytorch tensor **2computing

body:
for i in range(180000):
    p[indices[0][i], :, indices[1][i], indices[2][i]] = \
        p[indices[0][i], :, indices[1][i], indices[2][i]] ** 2

It needs to comsume 60+ seconds, and p = p ** 2 can attain the result right now. I just want part of P to change. So, what should I do to achive this efficiency? Coding by Pytorch.


title: Suggestion for the hybrid model

body:
I am currently using a hybrid model using PyTorch by hybrid I mean Autoencoder and CNN and here is my model :

convnet( (fc_encoder): Sequential( (0): Linear(in_features=9950,
out_features=200, bias=True) ) (decoder): Sequential( (0):
Linear(in_features=200, out_features=9950, bias=True) ) (con1):
Conv1d(1, 2, kernel_size=(5,), stride=(1,)) (max1):
MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1,
ceil_mode=False) (con2): Conv1d(2, 4, kernel_size=(5,), stride=(1,))
(max2): MaxPool1d(kernel_size=4, stride=4, padding=0, dilation=1,
ceil_mode=False) (l1): Linear(in_features=2480, out_features=1024,
bias=True) (drop): Dropout(p=0.5, inplace=False) (l2):
Linear(in_features=1024, out_features=512, bias=True) (l3):
Linear(in_features=512, out_features=128, bias=True) (l4):
Linear(in_features=128, out_features=32, bias=True) (l5):
Linear(in_features=32, out_features=1, bias=True) )

I have some doubts regarding:

I am using one loss function to evaluate my model which is BCELoss. I want to know if the hybrid model is implemented do I need to use 2 loss functions? What will happen if I evaluate my model just on one loss function?
The second question is if I am changing my learning rate I have a big difference in the ROC curve why this is happening?
Any clarification needed for the shape of the input I can provide.

This is my class to define the model:
class convnet(nn.Module):
def init(self,num_inputs=9950):
super(convnet,self).init()
self.num_inputs=num_inputs
self.fc_encoder= nn.Sequential(
nn.Linear(num_inputs,200)
)
self.decoder = nn.Sequential(
nn.Linear(200,num_inputs)
)
self.con1=nn.Conv1d(1, 2, 5)
self.max1=nn.MaxPool1d(4)
self.con2=nn.Conv1d(2, 4, 5)
self.max2=nn.MaxPool1d(4)
self.l1=nn.Linear(2480,1024)
self.drop=nn.Dropout(p= 0.5)
self.l2=nn.Linear(1024,512)
self.l3=nn.Linear(512,128)
self.l4=nn.Linear(128,32)
self.l5=nn.Linear(32,1)
def forward(self, x):
x = self.fc_encoder(x)
x = self.decoder(x)
x=x.unsqueeze(1)
x=self.con1(x)
x=self.max1(x)
x=self.con2(x)
x=self.max2(x)
x = x.view(x.size(0), -1)
x=self.l1(x)
x=self.drop(x)
x=self.l2(x)
x=self.l3(x)
x=self.l4(x)
x=self.l5(x)
return torch.sigmoid(x)
m = convnet()
m.to(device)

Thanks in advance


title: Custom defined a layer in Pytorch but cannot back propogate properly

body:
I have a custom loss function
class distance_loss(torch.nn.Module):
  def __init__(self, in_features, out_features = 10):
    super(distance_loss, self).__init__()
    self.in_features = in_features
    self.out_features = out_features
    self.center = torch.nn.Parameter(torch.randn(in_features), requires_grad= True)

  def forward(self, input):
    output =  torch.sum(torch.norm(input - self.center.cuda(), 2))
    print(self.center.requires_grad)
    output.requires_grad = True
    return output

I want to concatenate it with some model using
dtype = torch.float32
print_every = 100
device = torch.device('cuda')
def train(model, optimizer, loss_func, epochs=1):
    model = model.to(device=device)  # move the model parameters to CPU/GPU
    for e in range(epochs):
        for t, (x,y) in enumerate(trainsetLoader):
            model.train()
            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU
            loss = loss_func(scores)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

This code does not really work and when I print self.center.grad, it just returns none. All the parameters are not updating at all during training. Initially I do not have output.requires_grad = True, it returns "element 0 of tensors does not require grad and does not have a grad_fn". Then I changed it to the current setting but gradients are not calculated properly.


title: Pytorch model.load call ambiguity

body:
Simple one:
what does this do?
model.load_state_dict({name :
        weights_before[name] + (weights_after[name] - weights_before[name]) * outerstepsize
        for name in weights_before})

Thank you very much!


title: Autograd on Triplet Loss with Different Input Shapes

body: I’m using triplet loss on a siamese network where each half receives a different sized input (128 and 512), how would you recommend that I compute the loss? Would expanding the 128 by copying it 4 times mess with autograd? For now I am just padding the input but I was wondering if there was a more efficient way.


title: Using find-links with flit

body:
I'm trying to make a module using PyTorch as a dependency. To my understanding, this requires torch to be listed in the TOML file. My hope is to use flit or pip to install it with the module
[build-system]
requires = [
    "flit_core >=2,<4",
    "torch==1.6.0+cpu",
    "torchvision==0.7.0+cpu",
]
build-backend = "flit_core.buildapi"
...

My issue is that torch would ordinarily be installed using a find-link
pip3 install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html

Ideally it would be nice to include it in the meta-data of the toml, but I can't find anyway of providing flit with find-links.
Fortunately I can still use pip, but it's far from entuitive
pip3 install . -f https://download.pytorch.org/whl/torch_stable.html


title: How to make weights into leaf node after using an assignment operation on them

body:
I'm adapting a 10-class classification model to an 11-class classification model where I add one extra class. As well as keeping the prior layer weights, I want to make use of as much of the final classifier weights as possible, so I do something like this:
def make_backbone(self, load=''):
    self.backbone = SFNetV1()
    if len(load):
        self.backbone.load(load)

    # replace fully connected layer
    if len(load):
        prior_weight = self.backbone.fc1.weight
        prior_bias = self.backbone.fc1.bias
    prior_in_features = self.backbone.fc1.in_features
    prior_out_features = self.backbone.fc1.out_features

    # add "empty" token
    self.backbone.fc1 = nn.Linear(prior_in_features, prior_out_features+1)

    print(self.backbone.fc1.weight.is_leaf)
    print(prior_weight.is_leaf)

    # reuse whatever weights and biases we still can
    if len(load):
        self.backbone.fc1.weight[:prior_out_features, :] = prior_weight
        self.backbone.fc1.bias[:prior_out_features] = prior_bias

    print(self.backbone.fc1.weight.is_leaf)

This outputs
True
True
False

So my problem is that last False. So how do I "reset" self.backbone.fc1.weight to be a leaf node (also bias)?
Bonus side question: Is there a better way to do what I'm trying to do?


title: I am trying to modify the code to train my transformer image caption model with multiple gpus for paralelism.I don't know what 'list' refer to

body:
The error comes when I am tring to modify the code to train my image caption model with multiple gpus.I don't know what 'list' refer to. Maybe it is problem of inputs, but I don't know why is wrong.
Meshed-Memory Transformer Training
Let's use 3 GPUs!
Training starts
Epoch 0 - train:   0%|                                                                                | 0/9440 [00:08<?, ?it/s]
Traceback (most recent call last):
  File "train.py", line 257, in <module>
    train_loss = train_xe(model, dataloader_train, optim, text_field)
  File "train.py", line 82, in train_xe
    out = model(detections, captions)
  File "/data/zzw/anaconda3/envs/m2release/lib/python3.6/site-packages/torch/nn/modules/module.py", line 493, in __call__
    result = self.forward(*input, **kwargs)
  File "/data/zzw/anaconda3/envs/m2release/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 151, in forward
    replicas = self.replicate(self.module, self.device_ids[:len(inputs)])
  File "/data/zzw/anaconda3/envs/m2release/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 156, in replicate
    return replicate(module, device_ids)
  File "/data/zzw/anaconda3/envs/m2release/lib/python3.6/site-packages/torch/nn/parallel/replicate.py", line 174, in replicate
    replica._buffers[key] = buffer_copies[j][buffer_idx]
IndexError: list index out of range


title: Name of Modules to compute sparsity

body:
I'm writing a function that computes the sparsity of the weight matrices of the following fully connected network:
class FCN(nn.Module):
    def __init__(self):
        super(FCN, self).__init__()

        self.fc1 = nn.Linear(input_dim, hidden_dim)
        self.relu1 = nn.ReLU()
        self.fc2 = nn.Linear(hidden_dim, hidden_dim)
        self.relu2 = nn.ReLU()
        self.fc3 = nn.Linear(hidden_dim, hidden_dim)
        self.relu3 = nn.ReLU()
        self.fc4 = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):

        out = self.fc1(x)
        out = self.relu1(out)
        out = self.fc2(out)
        out = self.relu2(out)
        out = self.fc3(out)
        out = self.relu3(out)
        out = self.fc4(out)

        return out

The function I have written is the following:
def print_layer_sparsity(model):
    for name,module in model.named_modules():
        if 'fc' in name:
            zeros = 100. * float(torch.sum(model.name.weight == 0))
            tot = float(model.name.weight.nelement())
            print("Sparsity in {}.weight: {:.2f}%".format(name, zeros/tot))

But it gives me the following error:

torch.nn.modules.module.ModuleAttributeError: 'FCN' object has no attribute 'name'

It works fine when I manually enter the name of the layers (e.g.,

(model.fc1.weight == 0)
(model.fc2.weight == 0)
(model.fc3.weight == 0) ....

but I'd like to make it independent from the network. In other words, I'd like to adapt my function in a way that, given any sparse network, it prints the sparsity of every layer. Any suggestions?
Thanks!!


title: Audio signal processing using LSTM

body:
I am a beginner for LSTM and now I am stuck by setting the input for the LSTM. I am trying to use LSTM in pytorch to predict the pitch of the frames for signals. In the prepossing step, the signal is divided into frames and get the autocorrelations from each frame. For example, the signal's length is 1000 the frame length is 10, then there are 10 frames.
the data looks like this:
time stamps:    0s, 1s, 2s, ...
autocorrelation of each frames (length: len_auto):
autocorrelation of the frame at 0s
autocorrection of the frame at  1s
autocorrection of the frame at  2s...
but the input of LSTM in pytorch consists of (seq_len, batch_size, input_size). Now let s just set batch_size as b, which means each batch has b frames. The input_size should be the length of the autocorrelation (len_auto). If I want to consider the time stamps also into feature, then the input_size should be Lauto + 1, right? but I am not sure what the seq_len should be.


title: What should be output size of image classifier model?

body: I'm performing an image classification task . Images are labeled as 0 1 2. Should be the size of the last linear layer in the model output be 3 or 1 ? In general, for a 3-class operation, the output is set to 3, and as a result of these three, the maximum probability is returned. But I saw that the last layer is set as 1 in some codes. I think it is actually logical. What do you think about ? ( Also I dont use softmax or sigmoid function in last layer.)


title: does tensor label for loss calculation in crossentropy must has dtype? why my obejct has no dtype?

body:
I created a custom dataset, and I'm wondering why the answer object has no data type?
the dataloader singel output:
{'answer': tensor([1]),
 'image': tensor([[[0.0000, 0.6180, 0.0000,  ..., 0.0000, 0.0000, 0.0000]]],
        device='cuda:0'),
 'question': tensor([[   3,    6,    8, 1274,  593,    0,    0,    0,    0,    0,    0,    0,
             0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
             0,    0,    0,    0,    0,    0]], dtype=torch.int32)}

the custom dataset:
class VQAdataset(Dataset):

    def __init__(self,img_data,  que_data, ans_data, top_1000_answers):

        self.que_data=que_data
        self.ans_data=ans_data
        self.img_data=img_data
        self.top_answers=top_1000_answers

    def __getitem__(self,index):
        if torch.is_tensor(index):
            index = index.tolist()
        question_item=self.que_data[index]
        answer_item=self.ans_data[index]
        answer_item_index=torch.tensor(self.top_answers.index(answer_item),dtype=torch.int64)
        image_item=self.img_data[index]
        example={'image': image_item, 'question': question_item, 'answer': answer_item_index}

        return example

    def __len__(self):
      return len(self.que_data)

Does the data type attribute important to the loss calculation?


title: Is there a way to replace the 'allreduce_hook' used for DDP(DistributedDataParallel) in Pytorch?

body:
I know that Pytorch DDP uses 'allreduce_hook' as the default communication hook. Is there a way to replace this default hook with 'quantization_pertensor_hook' or 'powerSGD_hook'. There is an official Pytorch documentation introducing the DDP communication hooks, but I still got confused about how to do this in practice.
This is how I initiate the process group and create the DDP model.
import torch.distributed as dist
import torch.nn as nn

dist.init_process_group(backend='nccl', init_method='env://', world_size=args.world_size, rank=rank)
model = nn.parallel.DistributedDataParallel(model, device_ids=[0])

Is there any way to declare the hook that I want based on this code?


title: Perturbing a tensor values in all components by epsilon value

body:
I have a task where I need to perturb a 1d tensor of 3 values by a small value. The 3 values essential are the three coordinate values of a position in a grid.
eg.
A = tensor([[ 2.0851e+00, -1.2280e+00,  9.6665e-01],
        [ 2.6299e+00, -8.0376e-01,  2.2364e+00],
        [ 2.8123e+00, -2.0852e+00,  5.9921e-02],
        [ 6.1601e-01, -1.9922e+00,  1.2721e+00]])

For this i need to perturb the positions in all directions (individually and also cross-combination) by a small value. That includes operations like (assuming the second tensor is broadcasted)
A1 = A + [0.5, 0, 0]
A2 = A + [0, -0.5, 0]
A3 = A + [0, 0, 0.5]
A4 = A + [0.5, 0.5, 0]
...

Whats the best way to vectorize this ? Appreciate any ideas


title: obtain matrix from a tensor in pytorch with arbitrary number of trailing dimensions

body:
I have a pytorch tensor with an arbitrary number of dimensions : ...X,Y,Z
I would like to have a function such that I give a number C, and I get the ...,C,Y,Z
my_matrix = [:,:,C,:,:]

But I dont know how many trailing dimensions are before C, I saw an answer with using tuples of slices  but can seem to get it to work.
Partial slices in pytorch / numpy with arbitrary and variable number of dimensions


title: StratifiedKFold overfitting

body:
I'm working on a multimodal classifier (text + image) using pytorch (only 2 classes).
Since I don't have a lot of data, i've decided to use StratifiedKFold to avoid overfitting.
I noticed a strange behavior on training/testing curves.

My training accuracy quickly converges forward a unique value for few epochs before evolving again.
With these results I directly thought of overfitting, .67 being the maximum accuracy of the model.
With the rest of the data separated by the KFold, I tested my model in evaluation mode.

I've been quite surprised since test accuracy follows (quite exactly) the training accuracy while the loss (CrossEntropyLoss) still evolves.
Note : changing the batch size only make growing of accuracy delays or brings closer the moment the loss evolves.
Any ideas about this behaviour ?


title: Varying-length sequence，if there has be nn.Embedding padding_idx=···，still need pack_padded_sequence pad_packed_sequence？

body:
Varying-length sequence，if there has be nn.Embedding padding_idx，still need pack_padded_sequence pad_packed_sequence？
self.embedding = nn.Embedding(input_dim, embedding_dim, padding_idx=pad_idx)

def forward(self, text):
    embedded = self.dropout(self.embedding(text))

    outputs, (hidden, cell) = self.lstm(embedded)

    predictions = self.fc(self.dropout(outputs))

    return predictions


title: convert matrix vector product computation from single example to mini-batch

body:
I have a network, and part of the forward function includes a multiplication of an input metrix by an input vector.
v = torch.tensor([[0.0,1.0]])
m =  torch.tensor([[0.0,2.0],[4,1]])
torch.matmul(v,m)

which works great.
now I want to get my network to run mini-batchs of size n, to better utilize the GPU.
So I guess I want, in some way, to compute a concatenation of the vectors by a concatenation of the matrices in a way that will result in the matrix, which is composed from the vector result I would get by computing the vector-matrix product on at the time.
Or maybe there is a different way to convert to mini-batchs, I don't know.
I tried playing with dimensions and concatenations, but to no success:
v1 = torch.tensor([[0.0,1.0]])
v2 = torch.tensor([[1.0,0.0]])
v = torch.cat([v1,v2], dim=0).view((2,2))
m1 =  torch.tensor([[0.0,2.0],[4,1]]).unsqueeze(2)
m2 =  torch.tensor([[0.0,2.0],[4,1]]).unsqueeze(2)
m = torch.cat([m1,m2], dim=2)
torch.matmul(v,m)

this does not produce what I want...
any ideas?


title: Is there any code available for temporal graph network where graph topology is fixed?

body: I have been trying to implement the temporal graph neural network for graph classification problem with only two classes where only node features evolve over time. The graph I have is a complete graph with only a few number of nodes. I am from non-programming background and I have been on the same page for long time. I am taking the reference of the paper "Temporal graph network for deep learning on dynamic graphs". Please someone help me to find code or some  readings related to this problem.


title: Is batch normalisation a nonlinear operation?

body:
Can batch normalisation be considered a nonlinear operation like relu or sigmoid?
I have come across a resnet block like:
: IBasicBlock(
      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (prelu): PReLU(num_parameters=512)
      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )

so I wonder if batchnormalisation can act as non linear operation


title: Issue about Training MobileDet

body:
I am trying to train mobiledet[1] in PyTorch. The experiment is following the paper setting[2], but the initial learning rate(0.8) is too large to coverage.
Can I get the training log of mobiledet to find problems? Thanks!
Links:
[1]https://arxiv.org/abs/2004.14525
[2]https://github.com/tensorflow/models/blob/master/research/object_detection/samples/configs/ssdlite_mobiledet_dsp_320x320_coco_sync_4x4.config


title: Pytorch Load Balance and Scalability

body:
I need some ideas to execute a test on a PyTorch image generative model for load balancing and scalability?
Are there any specific tools for that? Do I need to execute a parallel process?


title: How to move data_parallel model to a specific cuda device?

body:
I currently need to use a pretrained model by setting it on a specific cuda device. The pretrained model is defined as below:
DataParallel(
  (module): MobileFaceNet(
    (conv1): Conv_block(
      (conv): Conv2d(3, 64, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (prelu): PReLU(num_parameters=64)
    )
    (conv2_dw): Conv_block(
      (conv): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=64, bias=False)
      (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (prelu): PReLU(num_parameters=64)
    )
    (conv_23): Depth_Wise(
      (conv): Conv_block(
        (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (prelu): PReLU(num_parameters=128)
      )
      (conv_dw): Conv_block(
        (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=128, bias=False)
        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (prelu): PReLU(num_parameters=128)
      )
      (project): Linear_block(
        (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (conv_3): Residual(
      (model): Sequential(
        (0): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128, bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (project): Linear_block(
            (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (1): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128, bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (project): Linear_block(
            (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (2): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128, bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (project): Linear_block(
            (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (3): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(64, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=128, bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=128)
          )
          (project): Linear_block(
            (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
      )
    )
    (conv_34): Depth_Wise(
      (conv): Conv_block(
        (conv): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (prelu): PReLU(num_parameters=256)
      )
      (conv_dw): Conv_block(
        (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=256, bias=False)
        (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (prelu): PReLU(num_parameters=256)
      )
      (project): Linear_block(
        (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (conv_4): Residual(
      (model): Sequential(
        (0): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (1): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (2): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (3): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (4): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (5): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
      )
    )
    (conv_45): Depth_Wise(
      (conv): Conv_block(
        (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (prelu): PReLU(num_parameters=512)
      )
      (conv_dw): Conv_block(
        (conv): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=512, bias=False)
        (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
        (prelu): PReLU(num_parameters=512)
      )
      (project): Linear_block(
        (conv): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
        (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      )
    )
    (conv_5): Residual(
      (model): Sequential(
        (0): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
        (1): Depth_Wise(
          (conv): Conv_block(
            (conv): Conv2d(128, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (conv_dw): Conv_block(
            (conv): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=256, bias=False)
            (bn): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
            (prelu): PReLU(num_parameters=256)
          )
          (project): Linear_block(
            (conv): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)
            (bn): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          )
        )
      )
    )
    (conv_6_sep): Conv_block(
      (conv): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)
      (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
      (prelu): PReLU(num_parameters=512)
    )
    (conv_6_dw): Linear_block(
      (conv): Conv2d(512, 512, kernel_size=(7, 7), stride=(1, 1), groups=512, bias=False)
      (bn): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    )
    (conv_6_flatten): Flatten()
    (linear): Linear(in_features=512, out_features=512, bias=False)
    (bn): BatchNorm1d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
  )
)

If I conventionally declare
model.to(device)

with device on cuda:1, then it makes error when forwarding:
model(imgs)

RuntimeError: module must have its parameters and buffers on device cuda:0 (device_ids[0]) but found one of them on device: cuda:1

I think this is because the model was previously trained with data parallel utils in pytorch.
How can I properly set the model to the device that I specifically want?


title: how to convert the output of a nural network to long type while maintaining the trainability

body:
The output of my pytorch neural network is a float64 type of data. This variable has to be used as a pixel offset and as such I need to convert it to long type.
However I have just discovered that a conversion out=out.long() switches the variable attribute ".requires_grad" to False.
How can I convert it to long maintaining ".requires_grad" to true?


title: The training loss fluctuates violently, and validation loss has dropped normally. What are the reasons and solutions?

body: As shown in the figure, orange line is training loss and blue line is validation loss. The training process uses pytorch DDP.


title: classifier object has no attribute train

body:
I am getting this error in the datatset.py module, which is used to create positive and negative pairs for the siamese network.  Any idea why it is giving me this error??
I am calling this Siamese MNIST my main module 'itr1.py' on train dataset for my own custom dataset.

siamese_train_dataset = SiameseMNIST(train_dataset)
siamese_test_dataset = SiameseMNIST(test_dataset)

class SiameseMNIST(Dataset):
    def __init__(self, dataset, train=True):
        self.dataset = dataset
        self.train = self.dataset.train
        self.transform = self.dataset.transform

        if self.train:
            self.train_labels = self.dataset.train_labels
            self.train_data = self.dataset.train_data
            self.labels_set = set(self.train_labels.numpy())
            self.label_to_indices = {label: np.where(self.train_labels.numpy() == label)[0]
                                     for label in self.labels_set}
        else:
            # generate fixed pairs for testing
            self.test_labels = self.dataset.test_labels
            self.test_data = self.dataset.test_data
            self.labels_set = set(self.test_labels.numpy())
            self.label_to_indices = {label: np.where(self.test_labels.numpy() == label)[0]
                                     for label in self.labels_set}

            random_state = np.random.RandomState(29)

            positive_pairs = [[i,
                               random_state.choice(self.label_to_indices[self.test_labels[i].item()]),
                               1]
                              for i in range(0, len(self.test_data), 2)]

            negative_pairs = [[i,
                               random_state.choice(self.label_to_indices[
                                                       np.random.choice(
                                                           list(self.labels_set - set([self.test_labels[i].item()]))
                                                       )
                                                   ]),
                               0]
                              for i in range(1, len(self.test_data), 2)]
            self.test_pairs = positive_pairs + negative_pairs

    def __getitem__(self, index):
        if self.train:
            target = np.random.randint(0, 2)
            img1, label1 = self.train_data[index], self.train_labels[index].item()
            if target == 1:
                siamese_index = index
                while siamese_index == index:
                    siamese_index = np.random.choice(self.label_to_indices[label1])
            else:
                siamese_label = np.random.choice(list(self.labels_set - set([label1])))
                siamese_index = np.random.choice(self.label_to_indices[siamese_label])
            img2 = self.train_data[siamese_index]
        else:
            img1 = self.test_data[self.test_pairs[index][0]]
            img2 = self.test_data[self.test_pairs[index][1]]
            target = self.test_pairs[index][2]

        img1 = Image.fromarray(img1.numpy(), mode='L')
        img2 = Image.fromarray(img2.numpy(), mode='L')
        if self.transform is not None:
            img1 = self.transform(img1)
            img2 = self.transform(img2)
        return (img1, img2), target

    def __len__(self):
        return len(self.dataset)