title: Tensorflow 2 : Todo Error with custom layers

body:
I'm converting Face Alignment Network(FAN) in PyTorch to Tensorflow 2.
I got a error, but I don't know why this error occurs.
Original code is here.

The code below is my code.

import tensorflow as tf
import numpy as np
from tensorflow.keras import datasets, layers, models

class ConvBlock(layers.Layer):
    def __init__(self, in_planes, out_planes):
        super(ConvBlock, self).__init__()
        self.bn1 = layers.BatchNormalization()
        self.conv1 = layers.Conv2D(int(out_planes / 2), (3, 3), padding='same', strides=(1, 1))
        self.bn2 = layers.BatchNormalization()
        self.conv2 = layers.Conv2D(int(out_planes / 4), (3, 3), padding='same', strides=(1, 1))
        self.bn3 = layers.BatchNormalization()
        self.conv3 = layers.Conv2D(int(out_planes / 4), (3, 3), padding='same', strides=(1, 1))

        if in_planes != out_planes:
            self.downsample = tf.keras.Sequential()
            self.downsample.add(layers.BatchNormalization())
            self.downsample.add(layers.ReLU())
            self.downsample.add(layers.Conv2D(out_planes, (1, 1), padding='same', strides=(1, 1)))

        else:
            self.downsample = None

    def call(self, input_tensor, training=False):
        residual = input_tensor

        out1 = self.bn1(input_tensor)
        out1 = tf.nn.relu(out1)
        out1 = self.conv1(out1)

        out2 = self.bn2(out1)
        out2 = tf.nn.relu(out2)
        out2 = self.conv2(out2)

        out3 = self.bn3(out2)
        out3 = tf.nn.relu(out3)
        out3 = self.conv3(out3)

        out3 = tf.concat([out1, out2, out3], 3)

        if self.downsample is not None:
            residual = self.downsample(residual)

        out3 += residual

        return out3

class Bottleneck(layers.Layer):

    expansion = 4

    def __init__(self, in_planes, planes, stride=1, downsample=None):
        super(Bottleneck, self).__init__()
        self.conv1 = layers.Conv2D(planes, (1, 1), padding='valid')
        self.bn1 = layers.BatchNormalization()
        self.conv2 = layers.Conv2D(planes, (3, 3), padding='same', strides=1)
        self.bn2 = layers.BatchNormalization()
        self.conv3 = layers.Conv2D(planes * 4 , (1, 1), padding='valid')
        self.bn3 = layers.BatchNormalization()
        self.downsample = downsample
        self.stride = stride

    def call(self, input_tensor, training=False):
        residual = input_tensor

        out = self.conv1(input_tensor)
        out = self.bn1(out)
        out = tf.nn.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = tf.nn.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)
        out = tf.nn.relu(out)

        if self.downsample is not None:
            residual = self.downsample(residual)

        out += residual
        out = tf.nn.relu(out)

        return out

class HourGlass(layers.Layer):
    def __init__(self, num_modules, depth, num_features):
        super(HourGlass, self).__init__()
        self.num_modules = num_modules
        self.depth = depth
        self.features = num_features

        self._layers = dict()

        self._generate_network(self.depth)

    def _generate_network(self, level):
        self._layers['b1_' + str(level)] = ConvBlock(self.features, self.features)
        self._layers['b2_' + str(level)] = ConvBlock(self.features, self.features)

        if level > 1:
            self._generate_network(level - 1)
        else:
            self._layers['b2_plus_' + str(level)] = ConvBlock(self.features, self.features)

        self._layers['b3_' + str(level)] = ConvBlock(self.features, self.features)

    def _call(self, level, inp):
        # Upper branch
        up1 = inp
        up1 = self._layers['b1_' + str(level)](up1)

        # Lower branch
        low1 = layers.AveragePooling2D((2, 2), stride=2)(inp)
        low1 = self._layers['b2_' + str(level)](low1)

        if level > 1:
            low2 = self._call(level - 1, low1)
        else:
            low2 = low1
            low2 = self._layers['b2_plus_' + str(level)](low2)

        low3 = low2
        low3 = self._layers['b3_' + str(level)](low3)

        up2 = layers.UpSampling2D(low3, interpolation='nearest') #########

        return up1 + up2

    def call(self, x):
        return self._call(self.depth, x)

class FAN(layers.Layer):
    def __init__(self, num_modules=1):
        super(FAN, self).__init__()
        self.num_modules = num_modules

        #base part
        self.conv1 = layers.Conv2D(64, (7, 7), padding='same', strides=2)
        self.bn1 = layers.BatchNormalization()
        self.conv2 = ConvBlock(64, 128)
        self.conv3 = ConvBlock(128, 128)
        self.conv4 = ConvBlock(128, 128)
        self._layers = dict()

        # Stacking part
        for hg_module in range(self.num_modules):
            self._layers['m' + str(hg_module)] = HourGlass(1, 4, 256)
            self._layers['top_m_' + str(hg_module)] = ConvBlock(256, 256)
            self._layers['conv_last' + str(hg_module)] = layers.Conv2D(256, (1, 1), padding='same', strides=1)
            self._layers['bn_end' + str(hg_module)] = layers.BatchNormalization()
            self._layers['l' + str(hg_module)] = layers.Conv2D(68, (1, 1), padding='same', strides=1)

            if hg_module < self.num_modules -1:
                self._layers['bl' + str(hg_module)] = layers.Conv2D(256, (1, 1), padding='same', strides=1)
                self._layers['al' + str(hg_module)] = layers.Conv2D(256, (1, 1), padding='same', strides=1)

    def call(self, x):
        x = tf.nn.relu(self.bn1(self.conv1(x)))
        x = layers.AveragePooling2D((7, 7))(x)
        x = self.conv3(x)
        x = self.conv4(x)

        previous = x
        outputs = []

        for i in range(self.num_modules):
            hg = self._layers['m' + str(i)](previous)

            ll = hg
            ll = self._module['top_m_' + str(i)](ll)

            ll = tf.nn.relu(self._layers['bn_end' + str(i)]
                            (self._layers['conv_last' + str(i)](ll)))

            # Predict heatmaps
            tmp_out = self._layers['l' + str(i)](ll)
            outputs.append(tmp_out)

            if i < self.num_modules - 1:
                ll = self._layers['bl' + str(i)](ll)
                tmp_out_ = self._layers['al' + str(i)](tmp_out)
                previous = previous + ll + tmp_out_

        return outputs

class ResNetDepth(layers.Layer):
    def __init__(self, block=Bottleneck, layers=[3, 8, 36, 3], num_classes=208):
        self.inplanes = 208
        super(ResNetDepth, self).__init__()
        self.conv1 = layers.Conv2D(208, (7, 7), padding='same', stride=2)
        self.bn1 = layers.BatchNormalization()
        self.relu = layers.ReLu()
        self.maxpool = layers.MaxPool2D(pool_size=(3, 3), stride=2, padding='same')
        self.layer1 = self._make_layer(block, 64, layers[0])
        self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
        self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
        self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
        self.avgpool = layers.AveragePooling2D((7, 7))
        self.fc = layers.Dense(num_classes)

    def _make_layer(sellf, block, planes, blocks, stride=1):
        dowmsample = None
        if stride != 1 or self.inplanes != planes * block.expansion:
            downsample == tf.keras.Sequential(
                layers.Conv2D(planes * block.expansion, (1, 1), strides=1, padding='same'), # 확인요망
                layers.BatchNormalization()
            )

        layers = []
        layers.append(block(self.inplanes, planes, stride, downsample))
        self.inplanes = planes * block.expansion
        for i in range(1, blocks):
            layers.append(block(self.inplanes, planes))

        return tf.keras.Sequential(*layers)

    def call(self, x):
        x = self.conv1(x)
        x = self.bn1(x)
        x = self.relu(x)
        x = self.maxpool(x)

        x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)

        x = self.avgpool(x)
        x = x.view(x.size(0), -1)
        x = self.fc(x)

        return x

model = tf.keras.Sequential()
model.add(FAN())

This is an error.

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-19-721cf81b7676> in <module>
----> 1 model.add(FAN())

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\training\tracking\base.py in _method_wrapper(self, *args, **kwargs)
    455     self._self_setattr_tracking = False  # pylint: disable=protected-access
    456     try:
--> 457       result = method(self, *args, **kwargs)
    458     finally:
    459       self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\keras\engine\sequential.py in add(self, layer)
    219       self._layers.append(layer)
    220     if self._layers:
--> 221       self._track_layers(self._layers)
    222
    223     self._layer_call_argspecs[layer] = tf_inspect.getfullargspec(layer.call)

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\keras\engine\network.py in _track_layers(self, layers)
    403     for layer_index, layer in enumerate(layers):
    404       try:
--> 405         if layer.weights:
    406           # Keep a separate index for layers which have weights. This allows
    407           # users to insert Layers without weights anywhere in the network

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\keras\engine\base_layer.py in weights(self)
    913       A list of variables.
    914     """
--> 915     return self.trainable_weights + self.non_trainable_weights
    916
    917   @property

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\keras\engine\base_layer.py in trainable_weights(self)
    890   def trainable_weights(self):
    891     if self.trainable:
--> 892       nested = self._gather_children_attribute('trainable_weights')
    893       return self._dedup_weights(self._trainable_weights + nested)
    894     else:

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\keras\engine\base_layer.py in _gather_children_attribute(self, attribute)
   2319       return list(
   2320           itertools.chain.from_iterable(
-> 2321               getattr(layer, attribute) for layer in nested_layers))
   2322     return []
   2323

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\keras\engine\base_layer.py in <genexpr>(.0)
   2319       return list(
   2320           itertools.chain.from_iterable(
-> 2321               getattr(layer, attribute) for layer in nested_layers))
   2322     return []
   2323

~\anaconda3\envs\tf2\lib\site-packages\tensorflow_core\python\training\tracking\layer_utils.py in filter_empty_layer_containers(layer_list)
    236   # TODO(b/130381733): Make this an attribute in base_layer.Layer.
    237   existing = object_identity.ObjectIdentitySet()
--> 238   to_visit = layer_list[::-1]
    239   while to_visit:
    240     obj = to_visit.pop()

TypeError: unhashable type: 'slice'

I noticed the message [# TODO(b/130381733): Make this an attribute in base_layer.Layer.], but I can't understand where I fix the code.


title: Pytorch with Modified Derivatives

body:
I'm in the process of rewriting the TRACX2 model, a variation of a recurrent neural network used for training encodings in the context of word segmentation from continuous speech or text. The writer of the original code manually wrote the network in Numpy, while I want to optimize it with Pytorch. However, they implement something they call "temperature" and a "fahlman offset":

This clearly isn't the actual derivative of tanh(x), one of their activation functions, but they used this derivative instead. How would I go about implementing this modification in Pytorch?


title: Pytorch MNIST code is returning IndexError

body:
I've followed the Pytorch documentation and have made an extremely simple classifier for the MNIST dataset. Below is my code:
import numpy as np

import torch
import torchvision
from torchvision import transforms, datasets

import torch.nn as nn
import torch.nn.functional as F

transform = transforms.Compose([
    transforms.ToTensor(),
    transforms.Normalize([0.5], [0.5])
    ])

train = datasets.MNIST('', train=True, download=True, transform=transform)
test = datasets.MNIST('', train=False, download=True, transform=transform)
trainset = torch.utils.data.DataLoader(train, batch_size=1, shuffle=True)
testset = torch.utils.data.DataLoader(test, batch_size=1, shuffle=False)

class Classifier(nn.Module):
    def __init__(self, D_in, H, D_out):
        super(Classifier, self).__init__()
        self.linear_1 = torch.nn.Linear(D_in, H)
        self.linear_2 = torch.nn.Linear(H, D_out)

    def forward(self, x):
        x = self.linear_1(x).clamp(min=0)
        x = self.linear_2(x)
        return F.log_softmax(x, dim=1)

net = Classifier(28*28, 128, 10)
optimizer = torch.optim.Adam(net.parameters(), lr=1e-3)

for epoch in range(3):
    running_loss = 0.0
    for X, label in iter(trainset):
        X = X.view(28*28, -1)

        optimizer.zero_grad()

        output = net(torch.flatten(X))
        loss = nn.CrossEntropyLoss(output, label)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print(f'[{epoch + 1}, {i + 1}] loss: {running_loss / 2000}')
            running_loss = 0.0
print("Finished training.")

torch.save(net.state_dict(), './classifier.pth')

For some reason, I'm getting the output
IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)

at the line: output = net(torch.flatten(X)
Thanks in advance for your help!


title: Why does code A runs well whereas B got an error?

body:
I encountered a question when I try to understand a line of code:
scores = [s[tuple(k.t())] for s, k in zip(scores, keypoints)].
Here I am going to give the code that can reproduce it:
import torch # v1.5.1

scores = [torch.rand(480, 640)]
keypoints = [torch.randint(0, 480, [978, 2])]

scores = [s[tuple(k.t())] for s, k in zip(scores, keypoints)] # label A, OK

for s, k in zip(scores, keypoints):
    print(s[tuple(k.t())]) # label B, Error

>>> IndexError: too many indices for tensor of dimension 1

I think the above two kinds of code (A and B respectively) are almost the same, but the latter threw out an error. I discussed it with my roommate but we have no idea what's going on.
Could someone help me? Thanks in advance!


title: Skipping first line of a file when using linecache

body:
I've created a lazy dataloader in Pytorch using linecache. It pulls from a tsv file that I also use to building the vocabulary with Pytorch's build_vocab, so I need to have a header line of titles for each of the columns.
For the dataset I'm using getitem:
def __getitem__(self, index):
    "Generates one sample of data"
    line = linecache.getline(self._filepath, index + 1)

However, since linecache doesn't load the whole file in memory, there's no obvious way to skip the first line/header of the tsv file. I tried "if index == 0: pass" but this obviously returned None that threw a different error.
My current solution is just to have two tsvs, one with a header and one without.


title: Converting Tensorflow code to Pytorch - performance metrics very different

body:
I have converted a tensorflow code for timeseries analysis to pytorch and performance difference is very high, in fact pytorch layers cannot account for seasonality at all. It feels like I must be missing something important.
Please help find where the pytorch code is lacking  that the learning is not up to the par. I noticed that loss values has high jumps when it encounters the season change and is not learning that. With the same layers, nodes and every other thing, I imagined the performance to be close.
# tensorflow code
window_size = 20
batch_size = 32
shuffle_buffer_size = 1000

def windowed_dataset(series, window_size, batch_size, shuffle_buffer):
  dataset = tf.data.Dataset.from_tensor_slices(series)
  dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
  dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))
  dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))
  dataset = dataset.batch(batch_size).prefetch(1)
  return dataset

dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)

model = tf.keras.models.Sequential([
    tf.keras.layers.Dense(100, input_shape=[window_size], activation="relu"),
    tf.keras.layers.Dense(10, activation="relu"),
    tf.keras.layers.Dense(1)
])

model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(lr=1e-6, momentum=0.9))
model.fit(dataset,epochs=100,verbose=0)
forecast = []
for time in range(len(series) - window_size):
  forecast.append(model.predict(series[time:time + window_size][np.newaxis]))

forecast = forecast[split_time-window_size:]
results = np.array(forecast)[:, 0, 0]

plt.figure(figsize=(10, 6))

plot_series(time_valid, x_valid)
plot_series(time_valid, results)

tf.keras.metrics.mean_absolute_error(x_valid, results).numpy()

# pytorch code
window_size = 20
batch_size = 32
shuffle_buffer_size = 1000

class tsdataset(Dataset):
  def __init__(self, series, window_size):
    self.series = series
    self.window_size = window_size
    self.dataset, self.labels = self.preprocess()

  def preprocess(self):
    series = self.series
    final, labels = [], []
    for i in range(len(series)-self.window_size):
      final.append(np.array(series[i:i+window_size]))
      labels.append(np.array(series[i+window_size]))
    return torch.from_numpy(np.array(final)), torch.from_numpy(np.array(labels))

  def __getitem__(self,index):
    # print(self.dataset[index], self.labels[index], index)
    return self.dataset[index], self.labels[index]

  def __len__(self):
    return len(self.dataset)

train_dataset = tsdataset(x_train, window_size)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)

class tspredictor(nn.Module):
  def __init__(self, window_size, out1, out2, out3):
    super(tspredictor, self).__init__()
    self.l1 = nn.Linear(window_size, out1)
    self.l2 = nn.Linear(out1, out2)
    self.l3 = nn.Linear(out2, out3)

  def forward(self,seq):
    l1 = F.relu(self.l1(seq))
    l2 = F.relu(self.l2(l1))
    l3 = self.l3(l2)
    return l3

model = tspredictor(20, 100,10,1)
loss_function = nn.MSELoss()
optimizer = optim.SGD(model.parameters(), lr=1e-6, momentum=0.9)

for epoch in range(100):
    for t,l in train_dataloader:
      model.zero_grad()
      tag_scores = model(t)
      loss = loss_function(tag_scores, l)
      loss.backward()
      optimizer.step()
    # print("Epoch is {}, loss is {}".format(epoch, loss.data))

forecast = []
for time in range(len(series) - window_size):
    prediction = model(torch.from_numpy(series[time:time + window_size][np.newaxis]))
    forecast.append(prediction)

forecast = forecast[split_time-window_size:]

results = np.array(forecast)

plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, results)

To generate data, you can use:
def plot_series(time, series, format="-", start=0, end=None):
    plt.plot(time[start:end], series[start:end], format)
    plt.xlabel("Time")
    plt.ylabel("Value")
    plt.grid(False)

def trend(time, slope=0):
    return slope * time

def seasonal_pattern(season_time):
    """Just an arbitrary pattern, you can change it if you wish"""
    return np.where(season_time < 0.1,
                    np.cos(season_time * 6 * np.pi),
                    2 / np.exp(9 * season_time))

def seasonality(time, period, amplitude=1, phase=0):
    """Repeats the same pattern at each period"""
    season_time = ((time + phase) % period) / period
    return amplitude * seasonal_pattern(season_time)

def noise(time, noise_level=1, seed=None):
    rnd = np.random.RandomState(seed)
    return rnd.randn(len(time)) * noise_level

time = np.arange(10 * 365 + 1, dtype="float32")
baseline = 10
series = trend(time, 0.1)
baseline = 10
amplitude = 40
slope = 0.005
noise_level = 3

# Create the series
series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)
# Update with noise
series += noise(time, noise_level, seed=51)

split_time = 3000
time_train = time[:split_time]
x_train = series[:split_time]
time_valid = time[split_time:]
x_valid = series[split_time:]


title: pytorch - package importing error...i have no idea

body:
Image of code and pytorch and my error
I have no idea why this perfect baseline code is not running on my pc.
I fail to install "processes" package.
I fail to use modules of "tools".
Please help me...
The Perfect baseline is below.
https://github.com/audio-captioning/dcase-2020-baseline


title: Pytorch Distributed Data Parallel over the Internet

body:
I am trying to train a network with "Distributed Data Parallel" on multiple nodes, each having a different public IP address by sshing simultaneously into these nodes using "pdsh" coordination tool as suggested in this tutorial.
Specifically, given a local machine with public IP address "Ip0" and 2 remote nodes with Public IP address "Ip1" and  "Ip2" respectively on which training is to be performed remotely from the local machine, how to go about making such a set-up?
Also, how to ensure before running the training script on each remote node that the 2 remote nodes have access to each other?
Thanks in advance.


title: Directly update the optimizer learning rate

body:
I have a specific learning rate schedule in mind. It is based on the epoch but differs from the generally available ones I am aware of including StepLR.
Is there something that would perform the equivalent to:
optimizer.set_lr(lr)
or
optimizer.set_param('lr,',lr)
I would then simply invoke that method at the end of each epoch (or possibly even more frequently)
Context: I am using the adam optimizer as so:
    optimizer = torch.optim.Adam(model.parameters(), lr=LrMax, weight_decay=decay) # , betas=(args.beta1, args.beta2)

Update  I found this information https://discuss.pytorch.org/t/change-learning-rate-in-pytorch/14653:
for param_group in optimizer.param_groups:
        param_group['lr'] = lr

Is there a way to ascertain that the adam optimizer being used is employing the new learning rate?


title: Multiprocesssing for deep learning cross-validation

body:
I am working on a research project which involves cross-validation for deep learning models. The data is pretty small (under 1000 matrices of size 200 x 200), as are models (e.g. 2-3 layers perceptron). I am implementing the learning process in PyTorch with PyTorch Lightning, Weights and Biases logging, and Hydra configuration framework.
So, the question is: what is the best practice to parallelize the learning on the folds? Currently, I use concurrent.futures.ProcessPoolExecutor to run several jobs corresponding to the folds. Is it a good practice or better tools exist?
Besides, I have 2 GPUs, each 8 Gb memory, and GPU memory usage while learning one model is not big (1-2 Gb depending on model type). The problem is CPU usage, with 32 cores even 2 processes take 100% of it. What practices exist which can reduce it? Also, how can I choose automatically the GPU for the process (say, the one with the most memory out of the visible GPUs)?


title: How to learn two functions simultaneously in using python (either pytorch or tensorflow)?

body:
I have three series of observations, namely Y, T, and X. I would like to study the differences between the predicted values of the two models. The first model is to learn g such that Y=g(T, X). The second model is to learn L and f such that Y=L(T)f(X). I have no problem in learning the first model using the PyTorch package or the Tensorflow package. However, I am not sure how to learn L and f. In using the PyTorch package, I can set up two feedforward MLPs with different hidden layers and inputs. For simplicity, I define a Feedforward MLP class as follows:
class Feedforward(t.nn.Module): # the definition of a feedforward neural network
    # Basic definition
    def __init__(self, input_size, hidden_size):
        super(Feedforward, self).__init__()
        self.input_size = input_size
        self.hidden_size  = hidden_size
        self.fc1 = t.nn.Linear(self.input_size, self.hidden_size)
        self.relu = t.nn.ReLU()
        self.fc2 = t.nn.Linear(self.hidden_size, 1)
        self.sigmoid = t.nn.Sigmoid()
    # Advance definition
    def forward(self, x):
        hidden = self.fc1(x)
        relu = self.relu(hidden)
        output = self.fc2(relu)
        output = self.sigmoid(output)
        return output

Suppose L=Feedforward(2,10) and L=Feedforward(3,9). From my understanding, I can only learn either L or f, but not both simultaneously. Is it possible to learn L and f simultaneously using Y, T, and X?


title: How to load a pre-trained tensorflow model and then inference by torch?

body: Is there any trick to parallel all the parameter in tensorflow model and torch model?


title: MLP always returns same predicted result

body:
I am using pytorch to implement a simple multilayer perceptron. The input data I have is of 450 dimensions and output is 120. I have normalized my input data. I use MSE as my loss function and the training converged.
When I testing the model, I found that no matter what input is, the output almost always remains same (for all 120 dimensions). I have tried make the model simpler (2 hidden layers) or more complex (up to 7 hidden layers), but this still happened.
Here is how I build my model:
class MLP(torch.nn.Module):
def __init__(self, D_in, D_out):
    super(MLP, self).__init__()
    self.linear_1 = torch.nn.Linear(D_in, 1000)
    self.linear_2 = torch.nn.Linear(1000, 1500)
    self.linear_3 = torch.nn.Linear(1500, 1000)
    self.linear_4 = torch.nn.Linear(1000, 750)
    self.linear_5 = torch.nn.Linear(750, 500)
    self.linear_6 = torch.nn.Linear(500, 250)
    self.linear_7 = torch.nn.Linear(250, D_out)
    self.sigmoid = torch.nn.Sigmoid()

def forward(self, x):

    x = self.sigmoid(self.linear_1(x))
    x = self.sigmoid(self.linear_2(x))
    x = self.sigmoid(self.linear_3(x))
    x = self.sigmoid(self.linear_4(x))
    x = self.sigmoid(self.linear_5(x))
    x = self.sigmoid(self.linear_6(x))
    y_pred = self.linear_7(x)

    return y_pred

Can anyone give any insights into this? Thanks in advance!


title: Sequential hyperparameter tuning ray tune

body:
I am a newcomer to ray tune and I until now I was using the ASHAScheduler for hyperparameter tuning.
It was working well as long as I used small networks but now that I am trying to tune bigger networks the program runs out of memory as I only have two GPUs available.
More precisely, one big network takes both GPUs memory so when the program runs two of them in parallel the second one runs out of memory.
My question is: does a sequential scheduler exist in ray tune ? So that I can run each trial one at a time, even if I am aware that it will take more time.


title: Looking for the implementation of the math operations in the pytorch library. (such as torch.add, troch.mm etc.)

body:
I want to have the source code for the math operations of pytorch. I know they are not all in the same file but hopefully someone can help me. I saw that there is an Aten folder on the github of pytorch but for me its quite confusing to go through.
Its my first question here. Sorry for anything annoying.


title: how can I clear graphic card memory after training in pytorch?

body:
I am dealing with pytorch in colab
While training, pytorch consumes enormous memory
after training, I saved model, and loaded model to another notebook(note 2).
in note 2, after loading state_dict and everything, pytorch consumes way less memory than in training state.
So, I wonder 'useless' data is stored in graphic card memory while training(in my case, about 13gb)...
If so, how do I delete useless data after training?
plus. I tried to delete variables used while training, but wasn't big enough(about 2gb)


title: Why is the shape of the weight different?(Pytorch)

body:
Im trying to to do SoftMaxRegression in 2ways
First Way:
x_train = torch.FloatTensor([[1, 2, 1, 1],
           [2, 1, 3, 2],
           [3, 1, 3, 4],
           [4, 1, 5, 5],
           [1, 7, 5, 5],
           [1, 2, 5, 6],
           [1, 6, 6, 6],
           [1, 7, 7, 7]])
y_train=torch.LongTensor([2, 2, 2, 1, 1, 1, 0, 0])

y_one_hot=torch.zeros(8,3)
y_one_hot.scatter_(1,y_train.unsqueeze(1),1)

W=torch.zeros((4,3),requires_grad=True)
b=torch.zeros((1,3),requires_grad=True)
optimizer=optim.SGD([W,b],lr=0.1)

nb_epoch=1000
for epoch in range(nb_epoch+1):
    hypothesis=F.softmax(x_train.matmul(W)+b,dim=1)
    cost=(y_one_hot*-torch.log(hypothesis)).sum(dim=1).mean()
    optimizer.zero_grad()
    cost.backward()
    optimizer.step()
    if epoch % 100==0:
        print('{0}th epoch Cost:{1} '.format(epoch,cost.item()))
print('W:',W.shape)
print('B:',b.shape)

Then, W.shape is torch.Size([4, 3])
However, If I USE THE second Way:
class SoftmaxClassifierModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(4, 3)

    def forward(self, x):
        return self.linear(x)
model = SoftmaxClassifierModel()
optimizer = optim.SGD(model.parameters(), lr=0.1)

nb_epochs = 1000
for epoch in range(nb_epochs + 1):

    prediction = model(x_train)

    cost = F.cross_entropy(prediction, y_train)

    optimizer.zero_grad()
    cost.backward()
    optimizer.step()

    if epoch % 100 == 0:
        print('Epoch {:4d}/{} Cost: {:.6f}'.format(
            epoch, nb_epochs, cost.item()
        ))
for name, param in model.state_dict().items():
    print(name, param.size())

Then, the output is linear.weight torch.Size([3, 4])
linear.bias torch.Size([3])
Shouldn't the size of the linear.weight torch.Size be (4,3)?


I am working on a cpp extension for pytorch, and followed the official tutorial, using libtorch and cmake to compile the program. But I met the problem of creating tensor.
This code can work.
#include <torch/torch.h>
#include <iostream>
int main(){
    std::vector<int64_t> test_data = {1, 2, 3, 4, 5, 6, 7, 8, 9};
    torch::Tensor s = torch::from_blob(test_data.data(), {3, 3}, torch::kInt64);
    std::cout << "test case pass" << std::endl;
}

But this code can't work.
int main(){
    auto option = torch::TensorOptions().dtype(torch::KInt16);
    auto b = torch::zeros({2,3}, option);
    std::cout << "test case pass" << std::endl;
}

and the compile error log is here.
error: no member named 'KInt16' in namespace 'torch'; did you mean 'kInt16'?


title: Simple case with regression not working (PyTorch)

body:
This simple PyTorch code from https://medium.com/@benjamin.phillips22/simple-regression-with-neural-networks-in-pytorch-313f06910379 doesn't find the regression expected on figure 2, from data on figure 1.
I tried several layers (nb & size), optimizer, loss, learning rate, epoch.. The result is only a flat horizontal line. As a newbie, I think I missed something.
Data:

Approx. expected result:

####
#code from https://medium.com/@benjamin.phillips22/simple-regression-with-neural-networks-in-pytorch-313f06910379
####

import torch
from torch.autograd import Variable
import torch.nn.functional as F
import torch.utils.data as Data
import matplotlib.pyplot as plt
# %matplotlib inline
import numpy as np
import imageio
import pandas as pd

torch.manual_seed(1)    # reproducible

x=torch.from_numpy(pd.read_csv('./x.csv').to_numpy())
y=torch.from_numpy(pd.read_csv('./y.csv').to_numpy())
x=x.float()
y=y.float()

# torch can only train on Variable, so convert them to Variable
x, y = Variable(x), Variable(y)

# view data
plt.figure(figsize=(10,4))
plt.scatter(x.data.numpy(), y.data.numpy(), color = "orange")
plt.title('regression')
plt.xlabel('x input')
plt.ylabel('y output')
plt.show()

net = torch.nn.Sequential(
 torch.nn.Linear(1,20),
 torch.nn.Sigmoid(),
 torch.nn.Linear(20,20),
 torch.nn.Sigmoid(),
 torch.nn.Linear(20,1),
)
optimizer = torch.optim.SGD(net.parameters(), lr=0.1)
loss_func = torch.nn.MSELoss()  # this is for regression mean squared loss

my_images = []
fig, ax = plt.subplots(figsize=(12,7))

# train the network
for t in range(500):

    prediction = net(x)     # input x and predict based on x

    loss = loss_func(prediction, y)     # must be (1. nn output, 2. target)

    optimizer.zero_grad()   # clear gradients for next train
    loss.backward()         # backpropagation, compute gradients
    optimizer.step()        # apply gradients

    # plot and show learning process
    plt.cla()
    ax.set_xlabel('x input',fontsize=12)
    ax.set_ylabel('y output',fontsize=12)
    ax.set_xlim(min(x),max(x))
    ax.set_ylim(min(y),max(y))
    ax.scatter(x.data.numpy(), y.data.numpy(), color = "orange")
    ax.plot(x.data.numpy(), prediction.data.numpy(), 'g-', lw=3)
    ax.set_title('regression step='+str(t)+" loss="+str(loss.data.numpy()),fontsize=16)

    # Used to return the plot as an image array
    # (https://ndres.me/post/matplotlib-animated-gifs-easily/)
    fig.canvas.draw()       # draw the canvas, cache the renderer
    image = np.frombuffer(fig.canvas.tostring_rgb(), dtype='uint8')
    image  = image.reshape(fig.canvas.get_width_height()[::-1] + (3,))

    my_images.append(image)
    print(str(t)+" "+str(loss.data.numpy()))

# save images as a gif
imageio.mimsave('./result.gif', my_images, fps=10)

x.csv data: from 0 to 191, step 1
y.csv data:
0.316905
0.322015
0.332582
0.310465
0.250653
0.292722
0.297352
0.276525
0.308451
0.283465
0.276011
0.29934
0.307176
0.28573
0.248667
0.288614
0.249795
0.255556
0.258393
0.235972
0.225121
0.207828
0.190252
0.192881
0.204084
0.167646
0.155202
0.146516
0.162462
0.182906
0.160287
0.213769
0.186362
0.201151
0.186125
0.190625
0.146851
0.169204
0.207855
0.196557
0.208835
0.2244
0.206303
0.193485
0.185266
0.205616
0.229315
0.196254
0.219849
0.209988
0.197361
0.195402
0.210149
0.240754
0.210418
0.191776
0.189532
0.206153
0.165696
0.187938
0.157561
0.163148
0.19473
0.18966
0.162334
0.189277
0.166506
0.198193
0.157867
0.135192
0.152216
0.137521
0.142007
0.121252
0.136517
0.118812
0.126124
0.141713
0.13222
0.2032
0.156077
0.166526
0.167117
0.130817
0.167058
0.188566
0.178803
0.224779
0.217089
0.194542
0.199796
0.246194
0.249908
0.23034
0.204611
0.222958
0.24259
0.234767
0.278205
0.267297
0.275127
0.264059
0.25439
0.287421
0.267725
0.252964
0.256326
0.229031
0.276914
0.244985
0.273892
0.298103
0.256733
0.27219
0.301747
0.278291
0.274979
0.300091
0.310184
0.333836
0.297877
0.279405
0.278263
0.291442
0.278518
0.28268
0.321826
0.355584
0.315503
0.338342
0.39687
0.388692
0.353228
0.368169
0.328025
0.407137
0.38092
0.357814
0.362786
0.405149
0.354694
0.348222
0.295455
0.307671
0.290612
0.24626
0.229377
0.26535
0.217139
0.206268
0.230013
0.255796
0.27014
0.246626
0.224845
0.272181
0.201281
0.252555
0.270198
0.289443
0.243552
0.238465
0.207842
0.197373
0.238857
0.224703
0.259659
0.288809
0.24757
0.264744
0.250775
0.245659
0.193861
0.296178
0.234242
0.219704
0.264879
0.290614
0.296195
0.237291
0.208546
0.295197
0.272628
0.288054
0.293539
0.305374
0.328142
0.328574
0.307407
0.298107
0.315636
0.301924


title: How to load multi-task model but only predict one of the tasks?

body:
Here is the Model structure. I combined almost everything together to for the convenience of hyper-parameters tuning.
'''
class MultiTaskDNN(nn.Module):
def __init__(self, n_tasks,
             input_dim=1024,
             output_dim=1,
             hidden_dim=[1024, 100],
             inits=['xavier_normal', 'kaiming_uniform'],
             act_function=['relu', 'leaky_relu'],
             dropouts=[0.10, 0.25],
             batch_norm=True):

    super(MultiTaskDNN, self).__init__()
    self.n_tasks = n_tasks
    self.input_dim = input_dim
    self.output_dim = output_dim
    self.hidden_dim = hidden_dim
    self.act_function = act_function
    self.batch_norm = batch_norm
    current_dim = input_dim
    self.layers = nn.ModuleList()
    self.dropouts = nn.ModuleList()
    self.bns = nn.ModuleList()

    for k, hdim in enumerate(hidden_dim):
        self.layers.append(nn.Linear(current_dim, hdim))
        self.bns.append(nn.BatchNorm1d(hdim, eps=2e-1))

        current_dim = hdim

        if inits[k] == 'xavier_normal':
            nn.init.xavier_normal_(self.layers[k].weight)
        elif inits[k] == 'kaiming_normal':
            nn.init.kaiming_normal_(self.layers[k].weight)
        elif inits[k] == 'xavier_uniform':
            nn.init.xavier_uniform_(self.layers[k].weight)
        elif inits[k] == 'kaiming_uniform':
            nn.init.kaiming_uniform_(self.layers[k].weight)

        self.dropouts.append(nn.Dropout(dropouts[k]))
    # n_targets
    self.heads = nn.ModuleList()
    for _ in range(self.n_tasks):
        self.heads.append(nn.Linear(current_dim, output_dim))

def forward(self, x):

    for k, layer in enumerate(self.layers):
        x = layer(x)
        if self.act_function[k] == 'sigmoid':
            x = torch.sigmoid(x)
        elif self.act_function[k] == 'relu':
            x = F.relu(x)
        elif self.act_function[k] == 'leaky_relu':
            x = F.leaky_relu(x)

        if self.batch_norm == True:
            x = self.bns[k](x)

        x = self.dropouts[k](x)

    outputs = []
    for head in self.heads:
        outputs.append(head(x))

    return outputs

'''
Please also let me know if the structure looks right. After training this multi-task model which has, say, 10 tasks (heads). I only want to predict task 7 which is head No.7. How should I load the model and do the prediction? Thank you.
model.state_dict()

MultiTaskDNN(
  (layers): ModuleList(
    (0): Linear(in_features=1024, out_features=128, bias=True)
    (1): Linear(in_features=128, out_features=128, bias=True)
  )
  (dropouts): ModuleList(
    (0): Dropout(p=0.25, inplace=False)
    (1): Dropout(p=0.25, inplace=False)
  )
  (bns): ModuleList(
    (0): BatchNorm1d(128, eps=0.2, momentum=0.1, affine=True, track_running_stats=True)
    (1): BatchNorm1d(128, eps=0.2, momentum=0.1, affine=True, track_running_stats=True)
  )
  (heads): ModuleList(
    (0): Linear(in_features=128, out_features=1, bias=True)
    (1): Linear(in_features=128, out_features=1, bias=True)
    (2): Linear(in_features=128, out_features=1, bias=True)
    (3): Linear(in_features=128, out_features=1, bias=True)
    (4): Linear(in_features=128, out_features=1, bias=True)
    (5): Linear(in_features=128, out_features=1, bias=True)
    (6): Linear(in_features=128, out_features=1, bias=True)
    (7): Linear(in_features=128, out_features=1, bias=True)
    (8): Linear(in_features=128, out_features=1, bias=True)
    (9): Linear(in_features=128, out_features=1, bias=True)
    (10): Linear(in_features=128, out_features=1, bias=True)
    )
  )


title: Multiprocessing PicklingError

body:
I am sorry that I can not reproduce my error because my code is too complex.
When I try to train a deep learning model build in PyTorch, I got the following error.
Traceback (most recent call last):
  File "/home/xxx/.conda/envs/env_name/lib/python3.8//queues.py", line 239, in _feed
    obj = _ForkingPickler.dumps(obj)
  File "/home/xxx/.conda/envs/env_name/lib/python3.8/multiprocessing/reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
_pickle.PicklingError: Can't pickle <class 'numpy.core._exceptions.UFuncTypeError'>: it's not the same object as numpy.core._exceptions.UFuncTypeError

I cannot locate the location of the error. Can anyone give some advice about what's the type of this error?


title: Add normalization layer at the begining of a pre-trained model

body:
I have a pretrained UNet model with the following architecture
UNet(
  (encoder1): Sequential(
    (enc1conv1): Conv2d(3, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc1norm1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc1relu1): ReLU(inplace=True)
    (enc1conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc1norm2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc1relu2): ReLU(inplace=True)
  )
  (pool1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  (encoder2): Sequential(
    (enc2conv1): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc2norm1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc2relu1): ReLU(inplace=True)
    (enc2conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc2norm2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc2relu2): ReLU(inplace=True)
  )
  (pool2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  (encoder3): Sequential(
    (enc3conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc3norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc3relu1): ReLU(inplace=True)
    (enc3conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc3norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc3relu2): ReLU(inplace=True)
  )
  (pool3): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  (encoder4): Sequential(
    (enc4conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc4norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc4relu1): ReLU(inplace=True)
    (enc4conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (enc4norm2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (enc4relu2): ReLU(inplace=True)
  )
  (pool4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
  (bottleneck): Sequential(
    (bottleneckconv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (bottlenecknorm1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (bottleneckrelu1): ReLU(inplace=True)
    (bottleneckconv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (bottlenecknorm2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (bottleneckrelu2): ReLU(inplace=True)
  )
  (upconv4): ConvTranspose2d(512, 256, kernel_size=(2, 2), stride=(2, 2))
  (decoder4): Sequential(
    (dec4conv1): Conv2d(512, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec4norm1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec4relu1): ReLU(inplace=True)
    (dec4conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec4norm2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec4relu2): ReLU(inplace=True)
  )
  (upconv3): ConvTranspose2d(256, 128, kernel_size=(2, 2), stride=(2, 2))
  (decoder3): Sequential(
    (dec3conv1): Conv2d(256, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec3norm1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec3relu1): ReLU(inplace=True)
    (dec3conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec3norm2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec3relu2): ReLU(inplace=True)
  )
  (upconv2): ConvTranspose2d(128, 64, kernel_size=(2, 2), stride=(2, 2))
  (decoder2): Sequential(
    (dec2conv1): Conv2d(128, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec2norm1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec2relu1): ReLU(inplace=True)
    (dec2conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec2norm2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec2relu2): ReLU(inplace=True)
  )
  (upconv1): ConvTranspose2d(64, 32, kernel_size=(2, 2), stride=(2, 2))
  (decoder1): Sequential(
    (dec1conv1): Conv2d(64, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec1norm1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec1relu1): ReLU(inplace=True)
    (dec1conv2): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
    (dec1norm2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
    (dec1relu2): ReLU(inplace=True)
  )
  (conv): Conv2d(32, 1, kernel_size=(1, 1), stride=(1, 1))
)

The model takes an input which has been normalized using min-max normalization. Instead, I want to add a batch/layer norm layer at the beginning so that I can feed the image as it is without normalization.
I don't want to use torchvision.transforms to normalize the image, instead I want to add a layer at the beginning that does the same work for me.


title: Error while turning a torch model into a pytorch model

body:
I want to turn this torch model into a pytorch model:
https://github.com/WangYueFt/dcp/blob/master/pretrained/dcp_v1.t7

But when I'm running the convert_torch.py that one can get from here:
https://github.com/clcarwin/convert_torch_to_pytorch

I get this error:
torch.utils.serialization.read_lua_file.T7ReaderException: unknown type id. The file may be corrupted.

Any idea how can I solve this issue?
Thank you in advance.


title: Getting an error while training Resnet50 on Imagenet at 14th Epoch

body:
I am training Resnet50 on imagenet using the script provided from PyTorch (with a slight trivial tweak for my purpose). However, I am getting the following error after 14 epochs of training. I have allocated 4 gpus in the server I'm using to run this. Any pointers as to what this error is about would be appreciated. Thanks a lot!
Epoch: [14][5000/5005]  Time 1.910 (2.018)  Data 0.000 (0.191)  Loss 2.6954 (2.7783)    Total 2.6954 (2.7783)   Reg 0.0000  Prec@1 42.969 (40.556)  Prec@5 64.844 (65.368)
Test: [0/196]   Time 86.722 (86.722)    Loss 1.9551 (1.9551)    Prec@1 51.562 (51.562)  Prec@5 81.641 (81.641)
Traceback (most recent call last):
  File "main_group.py", line 549, in <module>
  File "main_group.py", line 256, in main

  File "main_group.py", line 466, in validate
    if args.gpu is not None:
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 801, in __next__
    return self._process_data(data)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 846, in _process_data
    data.reraise()
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/_utils.py", line 385, in reraise
    raise self.exc_type(msg)
OSError: Caught OSError in DataLoader worker process 11.
Original Traceback (most recent call last):
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/_utils/worker.py", line 178, in _worker_loop
    data = fetcher.fetch(index)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 44, in fetch
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torch/utils/data/_utils/fetch.py", line 44, in <listcomp>
    data = [self.dataset[idx] for idx in possibly_batched_index]
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torchvision/datasets/folder.py", line 138, in __getitem__
    sample = self.loader(path)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torchvision/datasets/folder.py", line 174, in default_loader
    return pil_loader(path)
  File "/home/users/oiler/anaconda3/envs/ml/lib/python3.7/site-packages/torchvision/datasets/folder.py", line 155, in pil_loader
    with open(path, 'rb') as f:
OSError: [Errno 5] Input/output error: '/data/users2/oiler/github/imagenet-data/val/n02102973/ILSVRC2012_val_00009130.JPEG'


title: Is it possible to save Tensorboad session?

body:
I'm using Tensorboard and would like to save and send my report (email outside my organization), without losing interactive abilities. I've tried to save it as a complete html but that didn't work.
Anyone encountered the same issue and found a solution?


title: Running into IndexError when following PyTorch CrossEntropyLoss example for classification

body:
I am trying to follow exemplary usage of PyTorch's CrossEntropyLoss function for a classification problem using the fashionMNIST dataset.
I think the issue has to do with the dimension of the data output from my model. I'm not sure though, which makes figuring this out more difficult. Only running on Google Colab right now so I don't have easy access to a debugger (as far as I know).
Here's my code, please let me know what I am doing wrong!
import torch.nn as nn
import torch.optim as optim

model = nn.Sequential(
    nn.Linear(784, 1024),
    nn.ReLU(),
    nn.Linear(1024, 10),
    nn.ReLU(),
)
optimizer = optim.SGD(model.parameters(), lr=0.001, momentum=0.0)
loss_fn = nn.CrossEntropyLoss()

# Model Training
for epoch in range(1):
  for data, label in trainset:
    optimizer.zero_grad()
    output = model(torch.flatten(data))
    loss = loss_fn(output, label)
    loss.backward()
    optimizer.step()

And then the error:
IndexError                                Traceback (most recent call last)
<ipython-input-3-6d9a9ea2e921> in <module>()
     16     optimizer.zero_grad()
     17     output = model(torch.flatten(data))
---> 18     loss = loss_fn(output, label)
     19     loss.backward()
     20     optimizer.step()

3 frames
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in log_softmax(input, dim, _stacklevel, dtype)
   1603         dim = _get_softmax_dim('log_softmax', input.dim(), _stacklevel)
   1604     if dtype is None:
-> 1605         ret = input.log_softmax(dim)
   1606     else:
   1607         ret = input.log_softmax(dim, dtype=dtype)

IndexError: Dimension out of range (expected to be in range of [-1, 0], but got 1)


title: LSTM classifier from h_n layer (pytorch)

body:
I'm working on a classifier for a sample of long wikipedia articles (5000+ tokens in each article) using a simple LSTM model with 2 layers as part of a school project. The goal is to classify every article into 2 classes.
Due to the length of each article, I had splitted up every article into subsequences of 100 tokens long (and padding on the last subsequence), and trained the model using these subsequences while evaluation is based on the perplexity score (inverse probability of sentence) of similar subsequence from the test set.
The questions are:

The experience is that the perplexity raises quickly after every epoch. I suspect that it is due to the fact that the unique sentences/words in the test set never appears during training hence "unlikely" to appear. Is it true? The only solution I have is shuffling the subsequence of train set and test set together and make new sets of train and test sets (thereby introducing the unknown words to the training loop).

The main goal is to classify. The last hidden layer (h_n) after every loop is saved. The last layer should contain information about the text hence possible to be used as feature list in a classifier (I planned for a linear SVM), but I get a h_n for every article subsequence. Is it possible to combine the h_n values from every subsequence back into their respective "original" article?

Instead of having the subsequences as individual sample in the model, should I maybe introduce an additional nested loop in the model training and use every output from an iteration of subsequence as input for the next subsequence (if they are from the same article)?

Thanks for the help :)


title: NLP: Using other features alongside glove embeddings for sentiment analysis

body:
I want to classify tweets as negative, neutral or positive. I have build a NN using pytorch that does uses pre-trained glove embeddings and an LSTM layer and does a decent job:
class LSTMClassifier(nn.Module):
    def __init__(self, input_dim, embedding_dim, hidden_dim, output_dim):
        super(LSTMClassifier, self).__init__()
        self.glove_embedding = nn.Embedding(input_dim, embedding_dim)
        self.glove_embedding.weight.data.copy_(TEXT.vocab.vectors)
        self.lstm = nn.LSTM(embedding_dim, hidden_dim, dropout = 0.3, num_layers = 2, bidirectional = True)
        self.linear = nn.Linear(hidden_dim , output_dim)

    def forward(self, text):
      embeds = self.glove_embedding(text)
      lstm_out, (hm, cm) = self.lstm(embeds )
      last_hm = hm[-1]
      linear_out = self.linear(last_hm )
      log_probs = F.log_softmax(linear_out, dim = 0)
      return log_probs

Now I would like to play around with this NN a bit and see if I can improve things. Particullarly, I want to try if also feeding POS-tags alongside the glove embeddings into the LSTM layer improves anything. I one hot encode the POS-tags and concatenate them to the embeddings before feeding it to the LSTM:
POS.build_vocab(train_set) #building a dictionary for POS, which is a pytorch field containing my POS-tags
POS_NUM = len(POS.vocab)
.
.
.

class LSTMClassifier(nn.Module):
    def __init__(self, input_dim, pos_num, embedding_dim, hidden_dim, output_dim):
        super(LSTMClassifier, self).__init__()
        self.glove_embedding = nn.Embedding(input_dim, embedding_dim)
        self.pos_len = pos_num
        self.glove_embedding.weight.data.copy_(TEXT.vocab.vectors)
        self.lstm = nn.LSTM(embedding_dim + self.pos_len, hidden_dim, dropout = 0.3, num_layers = 2, bidirectional = True)
        self.linear = nn.Linear(hidden_dim, output_dim)

    def forward(self, fields):
      text, pos = fields
      pos_one_hot = torch.nn.functional.one_hot(pos, self.pos_len)
      embeds = self.glove_embedding(text)
      super_vector = torch.cat([pos_one_hot, embeds], dim=2)
      lstm_out, (hm, cm) = self.lstm(super_vector)
      last_hm = hm[-1]
      linear_out = self.linear(last_hm )
      log_probs = F.log_softmax(linear_out, dim = 0)
      return log_probs

This also works, however, it does not affect performence in any way (so it does not make it better but also does not make it worse). I am uncertain if I did something wrong or if POS-tags just do not matter for classifying my data.
Does the second network I posted above actually make use of the POS-tags or did I do something wrong?


title: Validation loss is only increasing. What could be wrong?

body:
I have stumbled upon EfficientNet and I have tried to use this architecture to train a model on a X-Ray Chest Pneumonia dataset. I have used this dataset before on a ResNet-9 and achieved a 75 % accuracy score, which is fine but it could be better.
This is a binary classification problem with pictures labeled either 0 (Normal) and 1 (Pneumonia). The dataset has around 6000 pictures in the training set, 600 in the test set and 16 in the validation set. Since the validation set has such a small amount of pictures I am using the test set as the validation set.
I looked at different implementations of EfficientNet in PyTorch and implemented a barebone version of it. I set up the dataloaders, loaded the model (in this case EfficientNetB0, not pretrained), unfroze the weights, setup the input features and output size and wrote a basic training loop.
After training several times with different configurations, the training loss is decreasing as usual but the validation loss is just increasing since the first epoch. Surprisingly, the validation accuracy is also increasing but not by a lot. In general the accuracy stays between 48-52% all the time. Basically it seems like the network is not training at all.
There are several things that come to mind:

First of all of course is overfitting and I had problems with overfitting on the ResNet9 model, but it only occured after around 6-7 epochs. Here, the loss is increasing immediately.
The EfficientNet model may be just to complicated for such a task. It may have a low amount of parameters but the architecture is still a bit more complicated than the resnet9, but I don't know.

Here are the results after 5 epochs
{'val_loss': 0.7583010792732239, 'val_acc': 0.5066964030265808, 'train_loss': 0.28630712628364563}
{'val_loss': 1.0902423858642578, 'val_acc': 0.45625001192092896, 'train_loss': 0.15664264559745789}
{'val_loss': 2.254948139190674, 'val_acc': 0.5127232074737549, 'train_loss': 0.11335344612598419}
{'val_loss': 3.511770009994507, 'val_acc': 0.5158482193946838, 'train_loss': 0.10181811451911926}
{'val_loss': 6.068212509155273, 'val_acc': 0.5207589268684387, 'train_loss': 0.09102503210306168}

I have tried different things. I augmented my data a bit more. I have been using a OneCycle learning rate scheduler and weight decay. I have gone back and forth looking at the different implemenations that I have been looking at to find where I could find a mistake but I cannot find anything. If I am not mistaken the training was normal the first time I implemented the core of the model but after I've tried to implement some of the optimizations, the results were getting worse and I could not go back to where I have been before.
Model
model = EfficientNet.from_name("efficientnet-b0")

for param in model.parameters():
  param.requires_grad = True

num_features = model._fc.in_features
model._fc = nn.Linear(num_features, 1)

model = model.to('cuda')

optimizer = optim.Adam(model.parameters())
loss_func = nn.BCELoss()

epochs = 5

Training and validation loop
torch.cuda.empty_cache()
history = []

# Training
for epoch in range(epochs):
  model.train()
  train_losses = []
  lrs = []

  for batch in train_dl:

    optimizer.zero_grad()

    loss = training_step(batch, loss_func)

    train_losses.append(loss)
    loss.backward()

    optimizer.step()

    #lrs.append(get_lr(optimizer))
    #sched.step()

  # Validation
  result = evaluate(valid_dl)
  result['train_loss'] = torch.stack(train_losses).mean().item()
  #result['lrs'] = lrs
  print(result)
  history.append(result)

I have put the training steps in separate methods
@torch.no_grad()
def evaluate(val_loader):
  model.eval()
  outputs = [validation_step(batch, loss_func) for batch in val_loader]
  return validation_epoch_end(outputs)

def accuracy(outputs, labels):
  preds = torch.sigmoid(outputs)
  preds = preds >= 0.5
  return torch.tensor(torch.sum(preds == labels).item() / len(preds))

def training_step(batch, loss_func):
  images, targets = batch
  images, targets = images.cuda(), targets.cuda()
  targets = targets.float()
  output = torch.squeeze(model(images))
  m = nn.Sigmoid()
  return loss_func(m(output), targets)

def validation_step(batch,  loss_func):                 # Generate predictions
  images, labels = batch
  images, labels = images.cuda(), labels.cuda()
  labels = labels.float()

  m = nn.Sigmoid()
  images = torch.squeeze(model(images))
  loss = loss_func(m(images), labels)  # Calculate loss
  acc = accuracy(images, labels)           # Calculate accuracy
  return {'val_loss': loss.detach(), 'val_acc': acc}

Some more information that may be useful:

Running it on colab
Batch size 64
Image size 224*224 (as used for EfficientNetb0)
Transformations for augmentation: Normalization, RandomRotation, RandomHorizontalFlip, Resize
Using a WeightedRandomSampler as the validation set as well as the training set are highly unbalanced.


title: Why does precision start dropping with MaskRCNN?

body:
I am experimenting with different optimizers and learning rate schedulers to improve the performance of a MaskRCNN model (40K images input, instance segmentation, 1 class). The baseline I am comparing with is the:
PURPLE LINE which uses
SGD
StepLR(step_size=3, gamma=0.1)

My two attempts at improving is are the
YELLOW LINE
AdamW(lr=5e-5)
ReduceLROnPlateau(patience=3, factor=0.75

BLUE LINE
AdamW(lr=5e-5)
StepLR(step_size=3, gamma=0.1)
2x number of input images

While I'm excited to see the metrics go above the purple line, I'm very interested to show what is happening with the AP@ IoU 0.5 which start to wiggle and drop, where as AP@ IoU 0.75 seems more stable.

The losses for each Mask-RCNN metric are:

Could someone help me understand why this is happening and what it means?


title: Pytorch model 2D regression given an scalar input

body:
I want to create a model to perform this regression:

My dataset looks like:
t,x,y
0.0,-,0.5759052335487023
0.01,-,-
0.02,1.1159124144549086,-
0.03,-,-
0.04,1.0054825084650338,0.4775267298487888
0.05,-,-

I'm having some troubles with loss, dataset load, batch_size, and Net structure (I add one single layer to simplify the problem)
Thats my code:
Net:
class Net(nn.Module):
    '''Model to regress 2d time series values given scalar input.'''
    def __init__(self):
        super(Net, self).__init__()
        #Layers
        self.predict = nn.Linear(1, 2)

    def forward(self, x):
        x = self.predict(x)
        return x

Dataset load
class TimeSeriesDataset(torch.utils.data.Dataset):

    def __init__(self, csv_file):
        #Load the dataset
        #Load the csv file as a dataframe
        df = pd.read_csv(csv_file, header=0, na_values='-')
        #Store the inputs and outputs
        self.x = df.values[:,:-2].astype('float32')
        self.y = df.values[:,1:].astype('float32')
        #Ensure target has the right shape
        self.y = self.y.reshape((len(self.y),2))

    def __len__(self):
        #Return the number of rows in the dataset
        return len(self.x)

    def __getitem__(self, idx):
        #Return a row at an index
        return [self.x[idx], self.y[idx]]

Trainloader, loss, optimizer
dataset = TimeSeriesDataset('data.csv')
trainloader = torch.utils.data.DataLoader(
    dataset, batch_size=32, shuffle=True, num_workers=2)

def lossFunc(outputs, labels):
  # nn.MSELoss() #Mean Squared Error, works fine with regression problems and with small numbers (x-y)^2
  return torch.mean((outputs-labels)**2)

net = Net()
optimizer = torch.optim.SGD(net.parameters(), lr=0.01)
print(net)

Trainning:
for epoch in range(300):

    running_loss = 0.0
    for i, data in enumerate(trainloader, 0):
        # TODO get the data
        # inputs, labels
        inputs, labels = data
        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        #print("Inputs", inputs)
        #print("labels", labels)
        #print("outputs", outputs)
        loss = lossFunc(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 20 == 19:    # print every 20 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 20))
            running_loss = 0.0

print('Finished Training')

Outputs looks this way:
tensor([[nan, nan],
        [nan, nan],
        [nan, nan],
...

And when I execute the 300 epochs error value doesn't change and prints nan


title: Found 0 Defected examples, But In the directory there are 10 images | Found 0 Perfect examples |

body:
I am trying to read images from a directory. There are some images, but the code is saying 0 images.
I checked with os.listdir(r'C:\Users\hafizurr\Documents\classifier\dataset\Perfect'), which is showing all of the images. But my code is now showing. I am putting my codes below. Please take a look and help me to figure out the issue. Hoping for your help:
class_names = ['Perfect', 'Defected']
root_dir = r'C:\Users\hafizurr\Documents\classifier\dataset'
source_dirs = ['Perfect', 'Defected']`

`if os.path.isdir(os.path.join(root_dir, source_dirs[1])):
    os.mkdir(os.path.join(root_dir, 'test'))`

    for i, d in enumerate(source_dirs):
        os.rename(os.path.join(root_dir, d), os.path.join(root_dir, class_names[i]))

    for c in class_names:
        os.mkdir(os.path.join(root_dir, 'test', c))

    for c in class_names:
        images = [x for x in os.listdir(os.path.join(root_dir, c)) if x.lower().endswith('.JPG')]
        selected_images = random.sample(images, 0)
        for image in selected_images:
            source_path = os.path.join(root_dir, c, image)
            target_path = os.path.join(root_dir, 'test', c, image)
            shutil.move(source_path, target_path)

`class ImgDataset(torch.utils.data.Dataset):
    def __init__(self, image_dirs, transform):
        def get_images(class_name):
            images = [x for x in os.listdir(image_dirs[class_name]) if x.lower().endswith('.JPG')]
            print(f'Found {len(images)} {class_name} examples')
            return images
        self.images = {}
        self.class_names = ['Perfect', 'Defected']
        for class_name in self.class_names:
            self.images[class_name] = get_images(class_name)
        self.image_dirs = image_dirs
        self.transform = transform`

    def __len__(self):
        return sum([len(self.images[class_name]) for class_name in self.class_names])

    def __getitem__(self, index):
        class_name = random.choice(self.class_names)
        index = index % len(self.images[class_name])
        image_name = self.images[class_name][index]
        image_path = os.path.join(self.image_dirs[class_name], image_name)
        image = Image.open(image_path).convert('RGB')
        return self.transform(image), self.class_names.index(class_name)

train_transform = torchvision.transforms.Compose([
    torchvision.transforms.Resize(size=(224, 224)),
    torchvision.transforms.RandomHorizontalFlip(),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])

test_transform = torchvision.transforms.Compose([
    torchvision.transforms.Resize(size=(224, 224)),
    torchvision.transforms.ToTensor(),
    torchvision.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])

train_dirs = {
    'Perfect': f'{root_dir}/Perfect',
    'Defected': f'{root_dir}/Defected'
}

train_dataset = ImgDataset(train_dirs, train_transform)


title: FastAi Learner changes the dataloader Why do they do it? How is this the right thing to do?

body:
The code below shows that passing a dataloader to a learner changes it. This seems like a very odd behavior. Why is it done this way, what is the logic for the change and how can I turn it off?
More importantly, the dataloader also has the val and test data in it. If the learner goes around changing it then it should be very well documented on what its doing. Nothing is mentioned about changing the data loader in cnn_learner and Learner.
dls = ImageDataLoaders.from_name_func(path, get_image_files(path), valid_pct=0.2,
label_func=is_tb,item_tfms=Resize(224))

x,y=next(iter(dls[0]))
print(x.min(),x.max())

This gives 0 and 1 respectively. However, initiating a learner

learn = cnn_learner(dls, resnet34, metrics=[accuracy],n_out=2,loss_func=CrossEntropyLossFlat())

x,y=next(iter(dls[0]))
print(x.min(),x.max())

I get -2.11 and 2.64 respectively.


title: Torch multiprocessing with multiple GPUs, extra processes appearing

body:
I want to run 16 processes in parallel, spread evenly over 4 GPUs. When I monitor using nvidia-smi I see there are 4 processes on each GPU for a while, then just as the processes are completing, about 8 additional processes show up on one GPU. They are usually on GPU0 but not always. The exact number of surplus processes varies. This is causing a memory error even though there should be more than enough memory in total.
Here is a minimal example.
import torch
import torch.multiprocessing as mp

def some_computation(some_square_tensor):
    for _ in range(1000):
        some_square_tensor * some_square_tensor

if __name__ == "__main__":
    x0 = torch.ones((2000,2000), device=torch.device(0))
    x1 = torch.ones((2000,2000), device=torch.device(1))
    x2 = torch.ones((2000,2000), device=torch.device(2))
    x3 = torch.ones((2000,2000), device=torch.device(3))
    tensors = [x0,x1,x2,x3]*4
    ctx = mp.get_context("spawn")
    with ctx.Pool(processes=16) as pool:
        new_list = pool.map(some_computation, tensors)

Expect: 4 processes on each GPU
Actual: 4 on 3 GPUs, >4 on the other GPU, memory error


In vision we usually process multiple images at once. I believe this is possible because most images are the same size or we can easily pad them with zeros if they are not (and thus process many at once).
However, I don't see a simple way to do this for structure data like data (e.g. programs, code, not NLP) in the form of Trees (e.g. while using a TreeLSTM). It seems that the processing of the data and the forward pass of the TreeLSTM are tightly coupled. e.g. if our TreeLSTM is processing the Abstract Syntax Tree (AST) top-down or bottom-up my guess is that the batching has to be different and custom for each of these. What I am imagining very vaguely and abstracting is that we get each node from the AST, generate the tensor for that node and then do the 1 step forward pass. If we were to do this for many ASTs I'd assume to do this style of batching we'd need to collect ASTs of the same topology (not only size, since I assume even if two trees are the same size their branches everywhere have to match somehow...unless we padd them with zeros?). That way I'd imagine the code to be something like this:
def forward(asts):
    """ asts have already been collated according to their matching topology. """
    # order trees according to how the TreeNN will process it e.g. post order for bottom up
    order_asts_according_to_traversal = oder_asts(asts)  ## e.g. order in a post order
    # do the forward pass
    current_node_embeddings =  torch.zeros(batch_size, D_embedding)
    for batch_node in order_asts_according_to_traversal:
        current_node_embeddings = node_step_forward(batch_node, current_node_embeddings)
    return current_node_embeddings

in addition, we'd have to collate the asts (pseudo-code that I didn't write since I'm not sure how to do this). For me, my data is usually already parsed (e.g. in json or s-expressions, but I prefer and plan to use json for each data point).
The problem that I have above does not feel new. TreeNNs (e.g. TreeLSTMs) have been implemented before so I'd rather use a tested method - the standard way to process and do forward passes on tree data for pytorch. Is there such a tutorial or library for pytorch one could easily use?

Research I've done
I've found the following options:

using CoqGym's example (https://github.com/princeton-vl/CoqGym/discussions/48 and https://github.com/princeton-vl/CoqGym/issues/16) though that general repo feels very hacky and their way of doing things seem very very custom, which worries me.
this option: https://medium.com/near-ai/pytorch-dynamic-batching-f4df3dbe09ef
a pytorch implementation of tree lstms: https://github.com/unbounce/pytorch-tree-lstm
yet another link: https://developer.nvidia.com/blog/recursive-neural-networks-pytorch/

what worries me is that none of these seemed to be part of the standard pytorch ecosystem (https://pytorch.org/ecosystem/), which raises doubt to me which one to use (if any of them). Should I be using any of these above? My guess is that perhaps the 4th one is the one I should be using? Ideally, I'd like to be able to implement a custom data set object that returns ASTs (from my Json file or s-expressions) and create a batch that takes advantage of GPU acceleration.

Another Solution: using Graph Neural Network libraries
I also realized that tree are really just a special case of graphs (perhaps not in the way we do the forward pass since GNNs seem to have a more complex set of "cycles" to produce embeddings). So with that I found these links:

question from the pytorch forum: https://discuss.pytorch.org/t/what-is-an-efficient-way-to-do-a-forward-pass-of-a-batch-of-data-for-an-graph-neural-network-gnn/107086/2
deep graph library: https://www.dgl.ai/
geometric library: https://github.com/rusty1s/pytorch_geometric

with the last two being part of the pytorch ecosystem. Though these are built for GNNs, so I am unsure if they are really worth the effort to check out to decide what to use.
Overall: What is the standard/recommended way to generate batches and process batches of tree data in pytorch to leverage the power of GPUs?

cross posted:

What is the standard way to batch and do a forward pass through tree structured data (ASTs) in pytorch so to leverage the power of GPUs?
https://discuss.pytorch.org/t/what-is-the-standard-way-to-batch-and-do-a-forward-pass-through-tree-structured-data-asts-in-pytorch-so-to-leverage-the-power-of-gpus/110939
quora: https://qr.ae/pN6NsY
reddit: https://www.reddit.com/r/pytorch/comments/lcn8xz/what_is_the_standard_way_to_batch_and_do_a/


title: Storing a dictionary with random indices as keys and simulated values as values in hdf5 possibly using pytorch?

body:
UPDATED QUESTION:
Each entry in an nd-array (say Sim_nDArray) correspond to a combination of parameters chosen from 8D search space. I have used Sim_nDArray.ravel() to convert it to 1D equivalent. Since I can not search from ~100 million entries, I decided to choose ~1 million random entries. I have corresponding ~1 million simulated values.
I have been able to simulate and save it. However, it seems that I have not been able to load data properly. I am getting error while overloading "len" during the declaration of the object: "dataset".
I am planning to use hdf5 to store and read data. Can someone please guide me how to achieve this?
def add_trace(arrInd, arr):
    """ Add one trace to the dataset, keeping count of the # of traces written """
    global ntraces
    dset1[ntraces, :] = arrInd
    dset2[ntraces, :] = arr
    ntraces += 1

def done():
    """ After all calls to add_trace_2, trim the dataset to size """
    dset1.resize((ntraces, 1000))
    dset2.resize((ntraces, 1000))

import torch
from torch.utils.data import Dataset, DataLoader

class Dataset(torch.utils.data.Dataset):
    # Characterizes a dataset for PyTorch
    def __init__(self, dset1, dset2):
        'Initialization'
        self.dset1 = dset1
        self.dset2 = dset2
        self._data_len = len(dset1)

def __len__(self):
    # Denotes the total number of samples
    return len(self._data_len)

def __getitem__(self, index):
    # Generates one sample of data
    # Select sample
    ID = self.dset1[index]
    SimData = self.dset2[index]
    return ID, SimData

# Running the main.
if __name__ == '__main__':
    import h5py
    import numpy as np
    import timeit

    """ Re-initialize both datasets for the tests """
    global data, N, dset1, dset2, ntraces
    N = 1000
    ################ WRITE #############################################################################################
    ## Creating two datasets
    f = h5py.File("randomDataset2.hdf5", 'w')
    dset1 = f.create_dataset('dataset1', (5000, 1000), maxshape=(None, 1000), dtype="float32", chunks=(1, 1000))
    dset2 = f.create_dataset('dataset2', (5000, 1000), maxshape=(None, 1000),
                             dtype="float32")  # DK: why faster if I do not define chunk

    dset1.resize((10001, 1000))  # Allocating extra space
    dset2.resize((10001, 1000))  # Allocating extra space

    ## TEST 1: Less efficient way of writing to hdf5
    ntraces = 0
    start1 = timeit.default_timer()
    for idx in range(N):
        IndxVec1 = np.random.randint(low=0, high=1000, size=1000);
        DataVec1 = np.random.random(1000)
        add_trace(IndxVec1, DataVec1)
    done()

    # All the program statements
    stop1 = timeit.default_timer()
    execution_time = stop1 - start1
    print("Program Executed in " + str(execution_time))  # It returns time in seconds
    f.close()
    ##################################
    ## READING HDF files
    fr = h5py.File("randomDataset2.hdf5", 'r')
    dset10 = fr['dataset1']
    dset20 = fr['dataset2']
    fr.close()

    # Parameters
    params = {'batch_size': 64, 'shuffle': True, 'num_workers': 6}
    max_epochs = 100

    # Generators
    training_set = Dataset(dset10, dset20)
    training_generator = torch.utils.data.DataLoader(training_set, batch_size= 64, shuffle=True, num_workers= 6)

Error:
(PipInConda_DKU) dushyant20@DESKTOP-U96RKFC:/mnt/c/PyImageSearch/Sim_Write_n_Read$ python3 main.
py
Program Executed in 2.265893899995717
Traceback (most recent call last):
  File "main.py", line 89, in <module>
    training_set = Dataset(dset10, dset20)
  File "main.py", line 30, in __init__
    self._data_len = len(dset1)
  File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
  File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
  File "/home/dushyant20/miniconda3/envs/PipInConda_DKU/lib/python3.8/site-packages/h5py/_hl/dat
aset.py", line 447, in __len__
    size = self.len()
  File "/home/dushyant20/miniconda3/envs/PipInConda_DKU/lib/python3.8/site-packages/h5py/_hl/dat
aset.py", line 459, in len
    shape = self.shape
  File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
  File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
  File "/home/dushyant20/miniconda3/envs/PipInConda_DKU/lib/python3.8/site-packages/h5py/_hl/dataset.py", line 286, in shape
    return self.id.shape
  File "h5py/h5d.pyx", line 132, in h5py.h5d.DatasetID.shape.__get__
  File "h5py/h5d.pyx", line 133, in h5py.h5d.DatasetID.shape.__get__
  File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
  File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
  File "h5py/h5d.pyx", line 289, in h5py.h5d.DatasetID.get_space
ValueError: Not a dataset (not a dataset)


title: How does one install ocaml-pytorch in mac OS X without errors?

body:
I recently tried to install opam install torch and I get a linker issue. What was the commands you run to install on mac os x?
My error:
(automl-meta-learning) brando~/ML4Coq ❯ opam install torch
[NOTE] It seems you have not updated your repositories for a while. Consider updating them with:
       opam update

The following actions will be performed:
  - install torch 0.10

<><> Gathering sources ><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
[torch.0.10] found in cache

<><> Processing actions <><><><><><><><><><><><><><><><><><><><><><><><><><><><>
[ERROR] The compilation of torch failed at "/Users/brando/.opam/opam-init/hooks/sandbox.sh build dune build -p torch -j 15".

#=== ERROR while compiling torch.0.10 =========================================#
# context     2.0.7 | macos/x86_64 | ocaml-variants.4.07.1+flambda | https://opam.ocaml.org#bcb3c300
# path        ~/.opam/4.07.1+flambda/.opam-switch/build/torch.0.10
# command     ~/.opam/opam-init/hooks/sandbox.sh build dune build -p torch -j 15
# exit-code   1
# env-file    ~/.opam/log/torch-46975-6d07ae.env
# output-file ~/.opam/log/torch-46975-6d07ae.out
### output ###
#   ocamlmklib src/wrapper/dlltorch_core_stubs.so,src/wrapper/libtorch_core_stubs.a (exit 2)
# (cd _build/default && /Users/brando/.opam/4.07.1+flambda/bin/ocamlmklib.opt -g -o src/wrapper/torch_core_stubs src/wrapper/torch_api.o src/wrapper/torch_stubs.o -lstdc++ -Wl,-rpath,/Users/brando/anaconda3/envs/automl-meta-learning/lib/python3.8/site-packages/torch/lib -L/Users/brando/anaconda3/envs/automl-meta-learning/lib/python3.8/site-packages/torch/lib -lc10 -ltorch_cpu -ltorch -Wl,-rpath[...]
# ld: file not found: @rpath/libiomp5.dylib for architecture x86_64
# clang: error: linker command failed with exit code 1 (use -v to see invocation)

<><> Error report <><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>
+- The following actions failed
| - build torch 0.10
+-
- No changes have been performed
# Run eval $(opam env) to update the current shell environment

I've ran the following .sh script so far:

opam update

#apt-get install pkg-config
brew install pkg-config

# apt-get install libffi
brew install libffi

opam install torch

cross posted: https://github.com/LaurentMazare/ocaml-torch/issues/2


title: Pytorch logistic regression weird accuracy scores

body:
I am using PyTorch logistic regression for a binary image classification problem. I've tried the same code for training the model for another task and the accuracy formula was working properly. But in this case the accuracy exceeds thousands!
correct.item() / total
is supposed to get multiplied with 100 but if I don't multiply with 100 then it gives proper scores.
I have manually initiated the labels. I really can't identify what's wrong.
Here is my code:
# Library
import pandas as pd
import numpy as np

# Load Data
train1 = pd.read_csv("/content/gdrive/MyDrive/Ekushe/Male/maleDigits.csv")

# Separating Data and Label

X_train1 = train1.drop(labels = ["label"],axis = 1)
Y_train1 = np.full(X_train1.shape[0], 0)
# Y_train1 = train1["label"]

# Normalize the data
X_train1 = X_train1 / 255.0

# Reshape the array into 28 x 28 pixel
X_train1 = X_train1.values.reshape(-1,28,28,1)

# Load Data
train2 = pd.read_csv("/content/gdrive/MyDrive/Ekushe/Female/femaleDigits.csv")

# Separating Data and Label
X_train2 = train2.drop(labels = ["label"],axis = 1)
Y_train2 = np.full(X_train2.shape[0], 1)
# Y_train2 = train2["label"]

# Normalize the data
X_train2 = X_train2 / 255.0

# Reshape the array into 28 x 28 pixel
X_train2 = X_train2.values.reshape(-1,28,28,1)

from sklearn.model_selection import train_test_split
X_train1, X_test1, y_train1, y_test1 = train_test_split(X_train1, Y_train1, test_size=0.05, random_state=42)
from sklearn.model_selection import train_test_split
X_train2, X_test2, y_train2, y_test2 = train_test_split(X_train2, Y_train2, test_size=0.05, random_state=42)
X_train=np.concatenate((X_train1, X_train2), axis=0)
Y_train=np.concatenate((y_train1, y_train2), axis=0)
X_test=np.concatenate((X_test1, X_test2), axis=0)
Y_test=np.concatenate((y_test1, y_test2), axis=0)
Y_train=Y_train.reshape(29287,1)
Y_test=Y_test.reshape(1543,1)

# Hyperparameters

batch_size = 128
num_iters = 6000
input_dim = 28*28 # num_features = 784
output_dim = 2

learning_rate = 0.0001

# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

num_epochs = num_iters / (len(X_train) / batch_size)
num_epochs = int(num_epochs)

class LogisticRegressionModel(nn.Module):
    def __init__(self, input_size, num_classes):
        super().__init__()
        self.linear = nn.Linear(input_size, num_classes)

    def forward(self, x):
        logits  = self.linear(x)
        probas = F.softmax(logits, dim=1)
        return logits, probas

'''
INSTANTIATE MODEL CLASS
'''
model = LogisticRegressionModel(input_size=input_dim,
                                num_classes=output_dim)
# To enable GPU
model.to(device)

# INSTANTIATE OPTIMIZER CLASS
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

train_loader = torch.utils.data.DataLoader(dataset= list(zip(X_train,Y_train)),
                                           batch_size=batch_size,
                                           shuffle=True)   # It's better to shuffle the whole training dataset!

test_loader = torch.utils.data.DataLoader(dataset= list(zip(X_test,Y_test)),
                                          batch_size=batch_size,
                                          shuffle=False)

'''
TRAIN THE MODEL
'''
#X_train=torch.from_numpy(X_train)
#y_train=torch.from_numpy(y_train)
iteration_loss = []
iter = 0
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):

        images = images.view(-1, 28*28).to(device)
        labels = labels.flatten().to(device)

        # Clear gradients w.r.t. parameters
        optimizer.zero_grad()

        # Forward pass to get output/logits
        logits, probas = model(images.float())

        # Calculate Loss: PyTorch implementation of CrossEntropyLoss works with logits, not probabilities
        loss = F.cross_entropy(logits, labels)

        # Getting gradients w.r.t. parameters
        loss.backward()

        # Updating parameters
        optimizer.step()

        iter += 1

        if iter % 500 == 0:
            # Calculate Accuracy
            correct = 0
            total = 0
            # Iterate through test dataset
            for images, labels in test_loader:

                images = images.view(-1, 28*28).to(device)

                # Forward pass only to get logits/output
                logits, probas = model(images.float())

                # Get predictions from the maximum value
                _, predicted = torch.max(probas, 1)

                # Total number of labels
                total += labels.size(0)

                # Total correct predictions
                if torch.cuda.is_available():
                    correct += (predicted.cpu() == labels.cpu()).sum()
                else:
                    correct += (predicted == labels).sum()

            accuracy =  100*correct.item() / total

            # Print Loss
            iteration_loss.append(loss.item())
            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.item(), accuracy))

Here is the output:
enter image description here
Dataset link:
enter link description here


title: spyder kernel dies during training

body: I am trying to train an fairly complex GCN network on my 10GB GPU. It runs smoothly until epoch 87 but then the spyder kernel restarts. Is it because of memory issue, if so how can I handle it.


title: How can I prevent PyTorch from making little changes to my assigned values

body:
PyTorch makes little changes to my assigned values, which causes really different results in my neural network. E.g.:
a = [234678.5462495405945]
b = torch.tensor(a)
print(b.item())

The output is:
234678.546875

The little change PyTorch made to my variable a caused an entirely different result in my neural network. My neural network is a very sensitive one. How can I prevent PyTorch from making little changes to assigned values?


title: How can I Replicate PyTorch BatchNorm2d with TensorFlow BatchNormalization?

body:
How can i replicate PyTorch BatchNorm2d() behaviour
with TensorFlows BatchNormalization() Layer?
In the code below you will find me creating
a random datasample of shape (batch_size, height, width, channels)=(1,2,4,3). Then creating a TensorFlow and a PyTorch tensor
out of this datasample. To get correct input_shape for PyTorch
I transposed the dimensions of the PyTorch Tensor. To compare PyTorchs results with TensorFlow ones I transposed the PyTorch Tensor back to (batch_size, height, width,channels).
This is my code:
import torch
import numpy as np
import tensorflow as tf
from copy import deepcopy

input_shape = (1, 2, 4, 3)
image = np.random.random_sample(input_shape) + 50
image2 = deepcopy(image)  # for pyt batchnorm
print("image: \n", image)

# tensorflow tensor
tft = tf.convert_to_tensor(image, dtype=tf.float64)

# pytorch tensor
ptt = torch.from_numpy(image2)

# batchnormalization pytorch
ptt.transpose_(1, 3).transpose_(2, 3)
ptBN = torch.nn.BatchNorm2d(3)
# print("\n Torch tensor: \n", ptt)
ptOutBN = ptBN(ptt.float())
ptOutBN.transpose_(2, 3).transpose_(1, 3)
print("\n Normalized Torch tensor: \n", ptOutBN)

# batchnormalization tensorflow
tfBN = tf.keras.layers.BatchNormalization(axis=-1, momentum=0.1,epsilon=1e-05)
tfOutBN = tfBN(tft)
print("\n Normalized TensorFlow tensor: \n", tfOutBN)

Google Colab
This is my output:
image:
 [[[[50.39217663 50.87261735 50.03217405]
   [50.60820831 50.74885536 50.34138617]
   [50.89196386 50.32315424 50.13149054]
   [50.76395015 50.03932698 50.12598469]]

  [[50.93456584 50.65474443 50.86897314]
   [50.7955281  50.63487641 50.90706063]
   [50.01449488 50.359608   50.94558701]
   [50.35211077 50.33072223 50.95769909]]]]

 Normalized Torch tensor:
 tensor([[[[-0.6781,  1.4588, -1.2986],
          [ 0.0473,  0.9800, -0.5060],
          [ 1.0000, -0.6666, -1.0440],
          [ 0.5702, -1.7645, -1.0581]],

         [[ 1.1431,  0.6160,  0.8463],
          [ 0.6762,  0.5392,  0.9440],
          [-1.9462, -0.5256,  1.0427],
          [-0.8126, -0.6373,  1.0737]]]], grad_fn=<TransposeBackward1>)

 Normalized TensorFlow tensor:
 tf.Tensor(
[[[[50.391926 50.87236  50.03192 ]
   [50.607956 50.7486   50.341133]
   [50.89171  50.322903 50.131237]
   [50.763695 50.039074 50.125732]]

  [[50.93431  50.65449  50.868717]
   [50.795273 50.634624 50.906807]
   [50.014244 50.359356 50.94533 ]
   [50.35186  50.33047  50.957443]]]], shape=(1, 2, 4, 3), dtype=float32)

As we can see the tf.keras.layers.BatchNormalization() layer does not apply any normalization like the torch.nn.BatchNorm2d() function.
Is there any way to use TensorFlow to reproduce my
PyTorch BatchNorm2d output?


title: Convert my model predicted array to categorical using a condition

body:
I've built a MLP neural network -> Trained -> Saved -> loaded -> now I'm testing my loaded model.
I've used y_pred=loaded_model(Variable(featuresTest)) to predict using pytorch then converted my tensor to array y_pred_arr and here's my result:
array([[-1.1663326 ,  0.369073  ],
       [-1.4255922 ,  0.23584184],
       [-2.045612  ,  1.4165274 ],
       ...,
       [ 4.327711  , -4.1331964 ],
       [-1.255816  ,  0.65834284],
       [ 6.642277  , -7.4430957 ]], dtype=float32)

My true labels array y_test which I'm trying to compare to are categorical
array([[0., 1.],
       [0., 1.],
       [0., 1.],
       ...,
       [1., 0.],
       [0., 1.],
       [1., 0.]], dtype=float32)

now I'm trying to use sklearn.metrics ->classification report however as my predicted array is obviously continuous I'm trying to convert it to a categorical which uses a condition converting the highest positive number in each of the array to a 1 for instance:
my first set in the predicted array is  [-1.4255922 ,  0.23584184], <- this would be converted to [ 0 , 1] as 0.2358 is the highest positive class in the prediction.
I've tried to use:
from keras.utils.np_utils import to_categorical
labels123 = to_categorical(y_pred_arr, dtype = "int64")

however, got this error:
IndexError: index -75 is out of bounds for axis 1 with size 74

Could I please get some assistance with this. Any help would be appreciated


title: how to run a pytorch model server inside docker

body:
i am trying to create a docker image to make it run as a server for serving a model in pytorch
i converted the .pt model file to .MAR file in my local machine
and i copied the .MAR file inside the docker image
i created a dockerfile
FROM ubuntu:18.04
ENV TZ=Asia/Shanghai
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update \
    && apt-get install --yes --no-install-recommends \
    tzdata

RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime\
    && echo ${TZ} > /etc/timezone \
    && dpkg-reconfigure -f noninteractive tzdata

RUN apt-get install python3 python3-dev python3-pip openjdk-11-jre-headless git wget curl -y
RUN python3 -m pip install torch torchvision torch-model-archiver torchserve==0.2.0
COPY densenet161.mar /model_store/
CMD torchserve --start --model-store model_store --models densenet161=densenet161.mar
EXPOSE 8080

i was able to create the image but i was not able to access that container
when i treid to open the image and run the code it works
docker exec -it 4b126bd87f21 sh

# torchserve --start --ncs --model-store model_store --models densenet161.mar

the server is running
when i run the docker image it is not working
docker container is running but i was not able to access the server
i don't know  what is the problem


title: Performance/Memory Problems when porting PyTorch Code to TensorFlow (GPU)

body:
I am trying to port an operation that uses convolutional kernels to transform time series for further tasks (e.g. classification, etc).
The following code is written in PyTorch and is intended for use on GPU:
class ROCKET(nn.Module):
    def __init__(self, c_in, seq_len, n_kernels=10_000, kss=[7, 9, 11], device=None, verbose=False):
        super().__init__()
        device = ifnone(device, default_device())
        kss = [ks for ks in kss if ks < seq_len]
        convs = nn.ModuleList()
        for i in range(n_kernels):
            ks = np.random.choice(kss)
            dilation = 2**np.random.uniform(0, np.log2((seq_len - 1) // (ks - 1)))
            padding = int((ks - 1) * dilation // 2) if np.random.randint(2) == 1 else 0
            weight = torch.randn(1, c_in, ks)
            weight -= weight.mean()
            bias = 2 * (torch.rand(1) - .5)
            layer = nn.Conv1d(c_in, 1, ks, padding=2 * padding, dilation=int(dilation), bias=True)
            layer.weight = torch.nn.Parameter(weight, requires_grad=False)
            layer.bias = torch.nn.Parameter(bias, requires_grad=False)
            convs.append(layer)
        self.convs = convs
        self.n_kernels = n_kernels
        self.kss = kss
        self.to(device=device)
        self.verbose=verbose

    def forward(self, x):
        _output = []
        for i in progress_bar(range(self.n_kernels), display=self.verbose, leave=False, comment='kernel/kernels'):
            out = self.convs[i](x).cpu()
            _max = out.max(dim=-1)[0]
            _ppv = torch.gt(out, 0).sum(dim=-1).float() / out.shape[-1]
            _output.append(_max)
            _output.append(_ppv)
        return torch.cat(_output, dim=1)

My approach in tensorflow (GPU) so far looks like this and performs the same computations:
from tqdm import trange
class ROCKET():
    def __init__(self, c_in, seq_len, n_kernels = 10_000, kss=[7,9,11]):
        kss = [ks for ks in kss if ks < seq_len]
        weights = []
        biases = []
        dilations = []
        for i in (t := trange(n_kernels)):
            ks = np.random.choice(kss)
            dilation = 2**np.random.uniform(0, np.log2((seq_len - 1) // (ks - 1)))

            weight = tf.random.normal([ks,c_in,1], dtype=tf.double)
            weight -= tf.math.reduce_mean(weight)
            weight = tf.Variable(weight)

            bias = tf.Variable(2 * (tf.random.normal([1], dtype=tf.double) - .5), dtype=tf.double)

            weights.append(weight)
            biases.append(bias)
            dilations.append(dilation)

            t.set_description("set kernels")

        self.weights = weights
        self.biases = biases
        self.dilations = dilations
        self.n_kernels = n_kernels

    def forward(self, x):
        _output = []
        for i in (t := trange(self.n_kernels)):
            weight = self.weights[i]
            bias = self.biases[i]
            dilation = self.dilations[i]

            tensor = tf.nn.conv1d(x, filters=weight, stride=1, padding='VALID', dilations=dilation)
            tensor = tf.nn.bias_add(tensor, bias)

            _max = tf.Variable(tf.math.reduce_max(tensor, axis=-1))

            temp = tf.cast(tf.math.greater(tensor, 0), tf.double)
            _ppv = tf.math.reduce_sum(temp, axis=-1) / tensor.shape[-1]

            _output.append(_max)
            _output.append(_ppv)

            t.set_description("apply kernels")

        return tf.concat(_output, axis=1)

However, this approach performs much slower (30secs for just the constructor).
Further, since my GPU quickly runs out of memory (6GB VRAM), I can only use it on small datasets.
Are there some immediate performance benefits I am missing out on?
I also noticed that the computations are performed by my CPU and just stored in GPU memory.
Would it be beneficial to use tf.placeholders?


title: Why do I get RuntimeError when reshaping tensor with torch.view?

body:
I am trying to analyze FaceForensics detector XceptionNet with this LRP implementation in Pytorch, but I end up stuck when trying to reshape the relevance_in tensor between the network layers.
def conv_nd_inverse(self, m, relevance_in):

# In case the output had been reshaped for a linear layer,
# make sure the relevance is put into the same shape as before.
relevance_in = relevance_in.view(m.out_shape)

m is a conv layer in the network and relevance_in is the relevance tensor received from the future layer. At relevance_in = relevance_in.view(m.out_shape) I am getting this error:
RuntimeError: shape '[1, 2048, 10, 10]' is invalid for input of size 2048
relevance_in has size torch.Size([1, 2048]) and m.out_shape is a list [1, 2048, 10, 10]. Does anyone know why I am getting this error and how I can solve it?


title: Numpy/Torch : Selecting elements using indices over a dimension

body:
Given an array like below:
np.arange(12).reshape(4,3)
Out[119]:
array([[ 0,  1,  2],
       [ 3,  4,  5],
       [ 6,  7,  8],
       [ 9, 10, 11]])

I want to select a single element from each of the rows using a list of indices [0, 2, 1, 2] to create a 4x1 array of [0, 5, 7, 11].
Is there any easy way to do this indexing. The closest I could see was the gather method in pytorch.


title: How to return a k-dim pytorch tensor truncated based on 1D mask

body:
So suppose I have a k-dim tensor and a 1-dim mask, that is used a lot in pytorch for variable length sequences, and I want to return a tensor that represents the elements up to the first false value in mask.  Here is an example:
import torch

a = torch.tensor([[1,2],[3,4],[5,6],[0,0],[0,0],[0,0]])
b = torch.tensor([True,True,True,False,False,False])

# magic goes here, result of c should be:

print(c)
>>> [[1,2],[3,4],[5,6]]

In this example, the input tensor is 2D, but it could be k-d, with any number of values across those dimensions.  Only the first dimension needs to match the mask dimension.  So doing torch.masked_select doesn't work since the tensor to be truncated isn't 1D like the mask, and since you don't know the dimensionality, squeeze and unsqueeze is not a solution either.
The mask is always going to be true for the first k elements and false for the remaining elements, though if your solution does not "depend" on this, that is fine.
This seems like people would be doing it all the time, yet I cannot find anywhere where this question has been answered.