title: Why my model training on one epoch takes almost 2 hours?

body:
I'm working on Neural Machine Translation (Fr to En). I developed (with Quadro P5000 GPU 16278MiB) a my first model which is a simple sequence-to-sequence with LSTM.
# Model definition

class EncoderLSTM(nn.Module):

    def __init__(
        self,
        embedding_size,
        vocab_size,
        hidden_size,
        n_layers,
        dropout,
        recurrent_dropout
    ):
        super(EncoderLSTM, self).__init__()
        self.embedding_size = embedding_size
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.recurrent_dropout = recurrent_dropout
        self.embedding = nn.Embedding(vocab_size, embedding_size)
        self.lstm = nn.LSTM(embedding_size, hidden_size,
                            num_layers=n_layers,
                            bidirectional=True,
                            dropout=(recurrent_dropout if n_layers > 1 else 0))

    def load_pretrained_embeddings(self, embeddings):
        self.embedding.weight = nn.Parameter(embeddings)

    def fine_tuning_embeddings(self, fine_tune=True):
        for p in self.embedding.parameters():
            p.requires_grad = fine_tune

    def forward(self, input_sequences, sequence_lengths):
        """
        :params
            input_sequences: Tensor[seq_len, batch_size]
            sequence_lengths: Tensor[batch_size,]

        :return
            outputs: Tensor[seq_len, batch_size, 2 * hidden_size]
            hn: Tensor[n_layers * 2, batch_size, hidden_size]
            cn: Tensor[n_layers * 2, batch_size, hidden_size]
        """
        embedded = self.embedding(input_sequences)
        embedded = F.dropout(embedded, p=self.dropout)
        packed = nn.utils.rnn.pack_padded_sequence(embedded, sequence_lengths)
        outputs, (hn, cn) = self.lstm(packed)
        outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs)
        return outputs, hn, cn

class DecoderLSTM(nn.Module):

    def __init__(
        self,
        embedding_size,
        vocab_size,
        hidden_size,
        n_layers,
        dropout,
        recurrent_dropout
    ):
        super(DecoderLSTM, self).__init__()
        self.embedding_size = embedding_size
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.recurrent_dropout = recurrent_dropout
        self.embedding = nn.Embedding(vocab_size, embedding_size)
        self.lstm = nn.LSTM(embedding_size, hidden_size,
                            num_layers=n_layers,
                            dropout=(recurrent_dropout if n_layers > 1 else 0))
        self.fc = nn.Linear(hidden_size, vocab_size)

    def load_pretrained_embeddings(self, embeddings):
        self.embedding.weight = nn.Parameter(embeddings)

    def fine_tuning_embeddings(self, fine_tune=True):
        for p in self.embedding.parameters():
            p.requires_grad = fine_tune

    def forward(self, input_word_index, h_state, c_state):
        """
        :params
            input_word_index: Tensor[batch_size,]
            h_state: Tensor[num_layers, batch_size, hidden_size]
            c_state: Tensor[num_layers, batch_size, hidden_size]

        :return
            logit: Tensor[batch_size, vocab_size]
            h_state: Tensor[num_layers, batch_size, hidden_size]
            c_state: Tensor[num_layers, batch_size, hidden_size]
        """
        embedded = self.embedding(input_word_index.unsqueeze(0))
        outputs, (h_state, c_state) = self.lstm(embedded, (h_state, c_state))
        logit = self.fc(F.dropout(outputs, p=self.dropout))
        logit = logit.squeeze(0)
        return logit, h_state, c_state

class SeqToSeqLSTM(nn.Module):

    def __init__(self, encoder, decoder, device):
        assert encoder.n_layers == decoder.n_layers, \
            'Encoder and Decoder must have the same number of reccurent layers'
        assert encoder.hidden_size == decoder.hidden_size, \
            'Encoder and Decoder must have the same number of reccurrent hidden units'

        super(SeqToSeqLSTM, self).__init__()
        self.encoder = encoder
        self.decoder = decoder
        self.init_h0 = nn.Linear(decoder.n_layers * 2, decoder.n_layers)
        self.init_c0 = nn.Linear(decoder.n_layers * 2, decoder.n_layers)
        self.device = device

    def forward(self, src_sequences, src_lengths, dest_sequences, dest_lengths, tf_ratio):
        """
        :params
            src_sequences: Tensor[seq_len, batch_size]
            src_lengths: Tensor[batch_size,]
            dest_sequences: Tensor[seq_len, batch_size]
            dest_lengths: Tensor[batch_size,]
            tf_ratio: float

        :return
            logits: Tensor[max(decode_lengths), batch_size, vocab_size]
            sorted_dest_sequences: Tensor[seq_len, batch_size]
            sorted_decode_lengths: Tensor[batch_size,]
            sorted_indices: Tensor[batch_size,]
        """
        # Encoding
        _, h_state, c_state = self.encoder(
            input_sequences=src_sequences,
            sequence_lengths=src_lengths
        )
        # h_state: [n_layers * 2, batch_size, hidden_size]
        # c_state: [n_layers * 2, batch_size, hidden_size]

        # Sort the batch (dest) by decreasing lengths
        sorted_dest_lengths, sorted_indices = torch.sort(dest_lengths, dim=0, descending=True)
        sorted_dest_sequences = dest_sequences[:, sorted_indices]
        h_state = h_state[:, sorted_indices, :]
        c_state = c_state[:, sorted_indices, :]

        # Init hidden and memory states
        h_state = self.init_h0(h_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]
        c_state = self.init_c0(c_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]
        h_state = h_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]
        c_state = c_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]

        # We won't decode at the <eos> position, since we've finished generating as soon as we generate <eos>
        # So, decoding lengths are actual lengths - 1
        sorted_decode_lengths = (sorted_dest_lengths - 1).tolist()

        # Decoding
        batch_size, last = dest_sequences.size(1), None
        logits = torch.zeros(max(sorted_decode_lengths), batch_size, self.decoder.vocab_size).to(self.device)
        for t in range(max(sorted_decode_lengths)):
            batch_size_t = sum([l > t for l in sorted_decode_lengths])
            if last is not None:
                if random.random() < tf_ratio:
                    in_ = last[:batch_size_t]
                else:
                    in_ = sorted_dest_sequences[t, :batch_size_t]
            else:
                in_ = sorted_dest_sequences[t, :batch_size_t]
            # in_ [batch_size,]
            logit, h_state, c_state = self.decoder(
                in_,
                h_state[:, :batch_size_t, :].contiguous(),
                c_state[:, :batch_size_t, :].contiguous()
            )
            # logit: [batch_size, vocab_size]
            # h_state: [num_layers, batch_size, hidden_size]
            # c_state: [num_layers, batch_size, hidden_size]
            logits[t, :batch_size_t, :] = logit
            last = torch.argmax(F.softmax(logit, dim=1), dim=1) # [batch_size,]

        return logits, sorted_dest_sequences, sorted_decode_lengths, sorted_indices

# Model initialization
MODEL_NAME = 'seq2seq-lstm'
N_LAYERS = 4
HIDDEN_SIZE = 512
EMBEDDING_SIZE = 300
ENC_DROPOUT = 0.3
ENC_RECURRENT_DROPOUT = 0.25
DEC_DROPOUT = 0.15
DEC_RECURRENT_DROPOUT = 0.2
N_EPOCHS = 15
BATCH_SIZE = 64
LEARNING_RATE = 1e-3
GRAD_CLIP = 1.0
TF_RATIO = 1.0

encoder = EncoderLSTM(
    embedding_size=EMBEDDING_SIZE,
    vocab_size=len(FR.vocab),
    hidden_size=HIDDEN_SIZE,
    n_layers=N_LAYERS,
    dropout=ENC_DROPOUT,
    recurrent_dropout=ENC_RECURRENT_DROPOUT
)
encoder.load_pretrained_embeddings(fr_embeddings)
encoder.fine_tuning_embeddings(fine_tune=True)
decoder = DecoderLSTM(
    embedding_size=EMBEDDING_SIZE,
    vocab_size=len(EN.vocab),
    hidden_size=HIDDEN_SIZE,
    n_layers=N_LAYERS,
    dropout=DEC_DROPOUT,
    recurrent_dropout=DEC_RECURRENT_DROPOUT
)
decoder.load_pretrained_embeddings(en_embeddings)
decoder.fine_tuning_embeddings(fine_tune=True)
seq2seq = SeqToSeqLSTM(encoder=encoder, decoder=decoder, device=DEVICE)
seq2seq.apply(torch_utils.xavier_init_weights)
seq2seq.to(DEVICE)
optimizer = optim.RMSprop(params=seq2seq.parameters(), lr=LEARNING_RATE)
criterion = nn.CrossEntropyLoss()
print(f'Number of parameters of the model: {torch_utils.count_parameters(seq2seq):,}')
# Number of parameters of the model: 41,471,097

With the model above, the training time for one epoch is 04:31 minutes.
When I add attention mechanism (Luong style), the training time for one epoch is 1:55:47 hours. I'm wondering why?
# Model definition
class EncoderLSTM(nn.Module):

    def __init__(
        self,
        embedding_size,
        vocab_size,
        hidden_size,
        n_layers,
        dropout,
        recurrent_dropout
    ):
        super(EncoderLSTM, self).__init__()
        self.embedding_size = embedding_size
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.recurrent_dropout = recurrent_dropout
        self.embedding = nn.Embedding(vocab_size, embedding_size)
        self.lstm = nn.LSTM(embedding_size, hidden_size,
                            num_layers=n_layers,
                            bidirectional=True,
                            dropout=(recurrent_dropout if n_layers > 1 else 0))

    def load_pretrained_embeddings(self, embeddings):
        self.embedding.weight = nn.Parameter(embeddings)

    def fine_tuning_embeddings(self, fine_tune=True):
        for p in self.embedding.parameters():
            p.requires_grad = fine_tune

    def forward(self, input_sequences, sequence_lengths):
        """
        :params
            input_sequences: Tensor[seq_len, batch_size]
            sequence_lengths: Tensor[batch_size,]

        :return
            outputs: Tensor[seq_len, batch_size, 2 * hidden_size]
            hn: Tensor[n_layers * 2, batch_size, hidden_size]
            cn: Tensor[n_layers * 2, batch_size, hidden_size]
        """
        embedded = self.embedding(input_sequences)
        embedded = F.dropout(embedded, p=self.dropout)
        packed = nn.utils.rnn.pack_padded_sequence(embedded, sequence_lengths)
        outputs, (hn, cn) = self.lstm(packed)
        outputs, _ = nn.utils.rnn.pad_packed_sequence(outputs)
        return outputs, hn, cn

class LuongAttention(nn.Module):

    def __init__(self, hidden_size, method):
        if method not in ['dot', 'concat']:
            raise NotImplemented(f'The {method} attention is not defined!')

        super(LuongAttention, self).__init__()
        self.hidden_size = hidden_size
        self.method = method
        if method == 'dot':
            pass
        elif method == 'concat':
            self.W = nn.Linear(hidden_size, hidden_size)
            self.V = nn.Linear(hidden_size, 1)
        else:
            raise NotImplementedError(f'{method} not implemented!')

    def forward(self, h_state, enc_outputs, mask):
        """
        :args
        h_state: Tensor[n_layers, batch_size, hidden_size]
        enc_outputs: Tensor[seq_len, batch_size, hidden_size]
        mask: Tensor[seq_len, batch_size]

        :return
            attn_weights: Tensor[seq_len, batch_size, 1]
        """
        if h_state.shape[0] > 1:
            h_state = h_state.sum(dim=0) # [batch_size, hidden_size]
            h_state = h_state.unsqueeze(0) # [1, batch_size, hidden_size]

        # Calculating the alignment scores
        if self.method == 'dot':
            scores = torch.sum(h_state * enc_outputs, dim=2)
            scores = scores.unsqueeze(dim=2) / np.sqrt(self.hidden_size) # [seq_len, batch_size, 1]
        elif self.method == 'concat':
            scores = self.V(
                torch.tanh(self.W(
                    enc_outputs + h_state # [seq_len, batch_size, hidden_size]
                ))
            ) # [seq_len, batch_size, 1]
        else:
            raise NotImplementedError(f'{method} not implemented!')

        # Apply mask to ignore <pad> tokens
        mask = mask.unsqueeze(2) # [seq_len, batch_size, 1]
        scores = scores.masked_fill(mask == 0, -1e10)

        # Calculating the attention weights by softmaxing the alignment scores
        attn_weights = F.softmax(scores, dim=1) # [seq_len, batch_size, 1]

        return attn_weights

class DecoderLSTM(nn.Module):

    def __init__(
        self,
        embedding_size,
        vocab_size,
        hidden_size,
        n_layers,
        dropout,
        recurrent_dropout,
        attention
    ):
        super(DecoderLSTM, self).__init__()
        self.embedding_size = embedding_size
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.dropout = dropout
        self.recurrent_dropout = recurrent_dropout
        self.attention = attention
        self.embedding = nn.Embedding(vocab_size, embedding_size)
        self.lstm = nn.LSTM(embedding_size, hidden_size,
                            num_layers=n_layers,
                            dropout=(recurrent_dropout if n_layers > 1 else 0))
        self.fc1 = nn.Linear(hidden_size * 2, hidden_size)
        self.fc2 = nn.Linear(hidden_size, vocab_size)

    def load_pretrained_embeddings(self, embeddings):
        self.embedding.weight = nn.Parameter(embeddings)

    def fine_tuning_embeddings(self, fine_tune=True):
        for p in self.embedding.parameters():
            p.requires_grad = fine_tune

    def forward(self, input_word_index, h_state, c_state, enc_outputs, mask):
        """
        :params
            input_word_index: Tensor[batch_size,]
            h_state: Tensor[num_layers, batch_size, hidden_size]
            c_state: Tensor[num_layers, batch_size, hidden_size]
            enc_outputs: Tensor[seq_len, batch_size, hidden_size]
            mask: Tensor[seq_len, batch_size]

        :return
            logit: Tensor[batch_size, vocab_size]
            h_state: Tensor[num_layers, batch_size, hidden_size]
            c_state: Tensor[num_layers, batch_size, hidden_size]
            attn_weights: Tensor[batch_size, seq_len]
        """
        embedded = self.embedding(input_word_index.unsqueeze(0)) # [seq_len=1, batch_size, embedding_size]
        outputs, (h_state, c_state) = self.lstm(embedded, (h_state, c_state))
        # outputs: [seq_len=1, batch_size, hidden_size]
        # h_state: [n_layers, batch_size, hidden_size]
        # c_state: [n_layers, batch_size, hidden_size]

        # Compute Attention Weights
        attn_weights = self.attention(h_state=outputs,
                                      enc_outputs=enc_outputs,
                                      mask=mask) # [seq_len, batch_size, 1]

        # Compute Context Vector
        context_vector = torch.bmm(
            enc_outputs.permute(1, 2, 0), # [batch_size, hidden_size, seq_len]
            attn_weights.permute(1, 0, 2), # [batch_size, seq_len, 1]
        ).permute(2, 0, 1) # [1, batch_size, hidden_size]

        # New input: concatenate context_vector with hidden_states
        new_input = torch.cat((context_vector, outputs), dim=2) # [1, batch_size, hidden_size * 2]

        # Get logit
        out = torch.tanh(self.fc1(new_input.squeeze(0)))
        logit = self.fc2(out) # [batch_size, vocab_size]

        return logit, h_state, c_state, attn_weights.squeeze(2)

class SeqToSeqLSTM(nn.Module):

    def __init__(self, encoder, decoder, pad_index, device):
        assert encoder.n_layers == decoder.n_layers, \
            'Encoder and Decoder must have the same number of reccurent layers'
        assert encoder.hidden_size == decoder.hidden_size, \
            'Encoder and Decoder must have the same number of reccurrent hidden units'

        super(SeqToSeqLSTM, self).__init__()
        self.encoder = encoder
        self.decoder = decoder
        self.pad_index = pad_index
        self.init_h0 = nn.Linear(decoder.n_layers * 2, decoder.n_layers)
        self.init_c0 = nn.Linear(decoder.n_layers * 2, decoder.n_layers)
        self.fc = nn.Linear(2 * encoder.hidden_size, encoder.hidden_size)
        self.device = device

    def create_mask(self, src_sequences):
        """
        :params
            src_sequences: Tensor[seq_len, batch_size]

        :return
            mask: Tensor[seq_len, batch_size]
        """
        mask = (src_sequences != self.pad_index)
        return mask

    def forward(self, src_sequences, src_lengths, dest_sequences, dest_lengths, tf_ratio):
        """
        :params
            src_sequences: Tensor[seq_len, batch_size]
            src_lengths: Tensor[batch_size,]
            dest_sequences: Tensor[seq_len, batch_size]
            dest_lengths: Tensor[batch_size,]
            tf_ratio: float

        :return
            logits: Tensor[max(decode_lengths), batch_size, vocab_size]
            sorted_dest_sequences: Tensor[seq_len, batch_size]
            sorted_decode_lengths: Tensor[batch_size,]
            sorted_indices: Tensor[batch_size,]
        """
        mask = self.create_mask(src_sequences) # [seq_len, batch_size]

        # Encoding
        enc_outputs, h_state, c_state = self.encoder(
            input_sequences=src_sequences,
            sequence_lengths=src_lengths
        )
        # enc_outputs: [seq_len, batch_size, 2 * hidden_size]
        # h_state: [n_layers * 2, batch_size, hidden_size]
        # c_state: [n_layers * 2, batch_size, hidden_size]

        enc_outputs = self.fc(enc_outputs)
        # enc_outputs: [seq_len, batch_size, hidden_size]

        # Sort the batch (dest) by decreasing lengths
        sorted_dest_lengths, sorted_indices = torch.sort(dest_lengths, dim=0, descending=True)
        sorted_dest_sequences = dest_sequences[:, sorted_indices]
        enc_outputs = enc_outputs[:, sorted_indices, :]
        h_state = h_state[:, sorted_indices, :]
        c_state = c_state[:, sorted_indices, :]

        # Init hidden and memory states
        h_state = self.init_h0(h_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]
        c_state = self.init_c0(c_state.permute(1, 2, 0)) # [batch_size, hidden_size, n_layers]
        h_state = h_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]
        c_state = c_state.permute(2, 0, 1) # [n_layers, batch_size, hidden_size]

        # We won't decode at the <eos> position, since we've finished generating as soon as we generate <eos>
        # So, decoding lengths are actual lengths - 1
        sorted_decode_lengths = (sorted_dest_lengths - 1).tolist()

        # Decoding
        batch_size, last = dest_sequences.size(1), None
        logits = torch.zeros(max(sorted_decode_lengths), batch_size, self.decoder.vocab_size).to(self.device)
        for t in range(max(sorted_decode_lengths)):
            batch_size_t = sum([l > t for l in sorted_decode_lengths])
            if last is not None:
                if random.random() < tf_ratio:
                    in_ = last[:batch_size_t]
                else:
                    in_ = sorted_dest_sequences[t, :batch_size_t]
            else:
                in_ = sorted_dest_sequences[t, :batch_size_t]
            # in_ [batch_size,]
            logit, h_state, c_state, _ = self.decoder(
                in_,
                h_state[:, :batch_size_t, :].contiguous(),
                c_state[:, :batch_size_t, :].contiguous(),
                enc_outputs[:, :batch_size_t, :],
                mask[:, :batch_size_t]
            )
            # logit: [batch_size, vocab_size]
            # h_state: [num_layers, batch_size, hidden_size]
            # c_state: [num_layers, batch_size, hidden_size]
            logits[t, :batch_size_t, :] = logit
            last = torch.argmax(F.softmax(logit, dim=1), dim=1) # [batch_size,]

        return logits, sorted_dest_sequences, sorted_decode_lengths, sorted_indices

# Model initialization
encoder = EncoderLSTM(embedding_size=EMBEDDING_SIZE,
                      vocab_size=len(FR.vocab),
                      hidden_size=HIDDEN_SIZE,
                      n_layers=N_LAYERS,
                      dropout=ENC_DROPOUT,
                      recurrent_dropout=ENC_RECURRENT_DROPOUT)
encoder.load_pretrained_embeddings(fr_embeddings)
encoder.fine_tuning_embeddings(fine_tune=True)
attention = LuongAttention(hidden_size=HIDDEN_SIZE, method='dot')
decoder = DecoderLSTM(embedding_size=EMBEDDING_SIZE,
                      vocab_size=len(EN.vocab),
                      hidden_size=HIDDEN_SIZE,
                      n_layers=N_LAYERS,
                      dropout=DEC_DROPOUT,
                      recurrent_dropout=DEC_RECURRENT_DROPOUT,
                      attention=attention)
decoder.load_pretrained_embeddings(en_embeddings)
decoder.fine_tuning_embeddings(fine_tune=True)
seq2seq = SeqToSeqLSTM(encoder=encoder,
                       decoder=decoder,
                       pad_index=EN.vocab.stoi[EN.pad_token],
                       device=DEVICE)
seq2seq.apply(torch_utils.xavier_init_weights)
seq2seq.to(DEVICE)
optimizer = optim.RMSprop(params=seq2seq.parameters(), lr=LEARNING_RATE)
criterion = nn.CrossEntropyLoss()
print(f'Number of parameters of the model: {torch_utils.count_parameters(seq2seq):,}')
# Number of parameters of the model: 42,520,697

The two models have nearly the same number of parameters.
The data come from the europarl parallel corpora with 89,752 sampled examples where each example have a length between 15 and 25.
Do someone know why the model with attention mechanism takes so long for one epoch?


title: Minimizing a function with two layers

body:
I'm working on optimal transport and i have to solve this problem:

where

where  have been simulated.
I can approximate T the transfer plan and the function u as  and try to find the theta and w with neural networks.
Using Arrow_Huwicz algorithm I have the following simple program : we draw some k in [1,N] and we minimize in w and  this function iteratively , such that at step n,  and
until itermax is attained
def Arrow_Hurwicz_algorithm(dim,NMC,S12,itermax):
w = torch.autograd.Variable(torch.rand(dim, 1), requires_grad=True)
theta = torch.autograd.Variable(torch.rand(dim, 1), requires_grad=True)
step_size = 1e-6
for i in range(itermax):
    k = randrange(NMC)
    L = JN(S12,k,theta,w)
    L.backward()
    theta.data -= step_size * theta.grad.data # step
    F = JN(S12,k,theta,w)
    F.backward()
    w.data += step_size * w.grad.data
    w.grad.data.zero_()
    theta.grad.data.zero_()
optw = w.detach().numpy()[0][0]
optth = theta.detach().numpy()[0][0]
return JN2(S12,optth,optw)

How can i perfrom the same thing using Adam stochastic gradient in pytorch ?


title: Replacing a decoder linear layer with an LSTM

body:
I am using this NET:
https://www.kaggle.com/ceshine/pytorch-temporal-convolutional-networks
I would like to replace the decoder with an LSTM but I am getting an error.

Old:

class TCNModel(nn.Module):
    def __init__(self, num_channels, kernel_size=2, dropout,history):
        super(TCNModel, self).__init__()
        self.tcn = TemporalConvNet(
            history, num_channels, kernel_size=kernel_size, dropout=dropout)
        self.dropout = nn.Dropout(dropout)
        self.decoder = nn.Linear(num_channels[-1], 1)

     def forward(self, x):
        #print(np.shape( self.dropout(self.tcn(x)[:, :, -1])) )
        return self.decoder(self.dropout(self.tcn(x)[:, :, -1]))

New (dont work)

class TCNModel(nn.Module):
    def __init__(self, num_channels, kernel_size=2, dropout=0.02,history_len = 3):
        super(TCNModel, self).__init__()
        self.tcn = TemporalConvNet(
            history_len, num_channels, kernel_size=kernel_size, dropout=dropout)
        self.dropout = nn.Dropout(dropout)
        #self.decoder = nn.Linear(num_channels[-1], 1)
        self.hidden_dim = 2

        self.decoder = nn.LSTM(input_size=num_channels[-1],hidden_size=self.hidden_dim ,num_layers=2,bidirectional=False)

        #self.decoder = LLinear(num_channels[-1], 1)

    def forward(self, x):
        print(np.shape( self.dropout(self.tcn(x)[:, :, -1])) )
        #return self.decoder(self.dropout(self.tcn(x)[:, :, -1]))
        y = self.tcn(x)[:, :, -1]
        h0 = torch.zeros(2, y.size(0), self.hidden_dim).requires_grad_()
        # Initialize cell state
        c0 = torch.zeros(2, y.size(0), self.hidden_dim).requires_grad_()

        out, (hn, cn) = self.decoder(y, (h0.detach(), c0.detach()))

        return self.decoder(self.dropout(out) )


title: Is Imagenet_1k dataset enough to train Resnet50 model?

body: I used nervana distiller to train resnet50 baseline model with imagenet_1k dataset.I've observed after 100 epochs, Top5 accuracy is about 10%.The validation accuracy remains zero for long step.


Trying to access subset of mnist dataset in pytorch [equal samples from each class] but getting this error
prng = RandomState(42)
random_permute = prng.permutation(np.arange(0, 6000))[0:3000]
indx = np.concatenate([np.where(np.array(mnist_data.targets) == classe)[0][random_permute] for classe in range(0,10)])

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-178-038015f76b77> in <module>
----> 1 indx = np.concatenate([np.where(np.array(mnist_data.targets) == classe)[0][random_permute] for classe in range(0,10)])

<ipython-input-178-038015f76b77> in <listcomp>(.0)
----> 1 indx = np.concatenate([np.where(np.array(mnist_data.targets) == classe)[0][random_permute] for classe in range(0,10)])

IndexError: index 5992 is out of bounds for axis 0 with size 5923


title: Can output of final classification FC layer can be negative?

body: I am getting final output Fc layer values all negative? Can it be negative? my final layer looks like(bnorm->dropout->linear)


title: Generic Computation of Distance Matrices in Pytorch

body:
I have two tensors a & b of shape (m,n), and I would like to compute a distance matrix m using some distance metric d. That is, I want m[i][j] = d(a[i], b[j]). This is somewhat like cdist(a,b) but assuming a generic distance function d which is not necessarily a p-norm distance. Is there a generic way to implement this in PyTorch?
And a more specific side question: Is there an efficient way to perform this with the following metric
d(x,y) = 1 - cos(x,y)

edit
I've solved the specific case above using this answer:
def metric(a, b, eps=1e-8):
    a_norm, b_norm = a.norm(dim=1)[:, None], b.norm(dim=1)[:, None]
    a_norm = a / torch.max(a_norm, eps * torch.ones_like(a_norm))
    b_norm = b / torch.max(b_norm, eps * torch.ones_like(b_norm))
    similarity_matrix = torch.mm(a_norm, b_norm.transpose(0, 1))
    return 1 - similarity_matrix


title: Pytorch error when computing loss between two tensors. TypeError: __init__() takes 1 positional argument but 3 were given

body:
When trying to compute the loss between two tensors rPPG = (shape(torch.Size([4, 128])) and BVP_label = (shape(torch.Size([4, 128]))) using the following function:
class Neg_Pearson(nn.Module):    # Pearson range [-1, 1] so if < 0, abs|loss| ; if >0, 1- loss
    def __init__(self):
        super(Neg_Pearson,self).__init__()
        return
    def forward(self, preds, labels):       # tensor [Batch, Temporal]
        loss = 0
        for i in range(preds.shape[0]):
            sum_x = torch.sum(preds[i])                # x
            sum_y = torch.sum(labels[i])               # y
            sum_xy = torch.sum(preds[i]*labels[i])        # xy
            sum_x2 = torch.sum(torch.pow(preds[i],2))  # x^2
            sum_y2 = torch.sum(torch.pow(labels[i],2)) # y^2
            N = preds.shape[1]
            pearson = (N*sum_xy - sum_x*sum_y)/(torch.sqrt((N*sum_x2 - torch.pow(sum_x,2))*(N*sum_y2 - torch.pow(sum_y,2))))

            print(N)
            #if (pearson>=0).data.cpu().numpy():    # torch.cuda.ByteTensor -->  numpy
            #    loss += 1 - pearson
            #else:
            #    loss += 1 - torch.abs(pearson)

            loss += 1 - pearson

        loss = loss/preds.shape[0]
        return loss

#3. Calculate the loss
loss_ecg = Neg_Pearson(rPPG, BVP_label)

I keep getting the following error:
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-15-f14cbf0fc84b> in <module>
      1 #3. Calculate the loss
----> 2 loss_ecg = Neg_Pearson(rPPG, BVP_label)

TypeError: __init__() takes 1 positional argument but 3 were given

I am new to Pytorch, I am not sure what is going on here. Any suggestions?


title: Training/Predicting with CNN / ResNet on all classes each iteration - concatenation of input data + Hungarian algorithm

body:
So I've got a simple pytorch example of how to train a ResNet CNN to learn MNIST labeling from this link:
https://zablo.net/blog/post/using-resnet-for-mnist-in-pytorch-tutorial/index.html
It's working great, but I want to hack it a bit so that it does 2 things. First, instead of predicting digits, it predicts animal shapes/colors for a project I'm working on. That's already working quite well already and am happy with it.
Second, I'd like to hack the training (and possibly layers) so that predictions is done in parallel on multiple images at a time. In the MNIST example, basically prediction (or output) would be done for an image that has 10 digits at a time concatenated by me. For clarity, each 10-image input will have the digits 0-9 appearing only once each. The key here is that each of the 10 digit gets a unique class/label from the CNN/ResNet and each class gets assigned exactly once. And that digits that have high confidence will prevent other digits with lower confidence from using that label (a Hungarian algorithm type of approach).
So in my use case I want to train on concatenated images (not single images) as in Fig A below and force the classifier to learn to predict the best unique label for each of the concatenated images and do this all at once. Such an approach should outperform single image classification - and it's particularly useful for my animal classification because otherwise the CNN can sometimes return the same ID for multiple animals which is impossible in my application.
I can already predict in series as in Fig B below. And indeed looking at the confidence of each prediction I am able to implement a Hungarian-algorithm like approach post-prediction to assign the best (most confident) unique IDs in each batch of 4 animals. But this doesn't always work and I'm wondering if ResNet can try and learn the greedy Hungarian assignment as well.

In particular, it's not clear that implementing A simply requires augmenting the data input and labels in the training set will do it automatically - because I don't know how to penalize or dissalow returning the same label twice for each group of images.  So for now I can generate these training datasets like this:
print (train_loader.dataset.data.shape)
print (train_loader.dataset.targets.shape)
torch.Size([60000, 28, 28])
torch.Size([60000])

And I guess I would want the targets to be [60000, 10]. And each input image would be [1, 28, 28, 10]?  But I'm not sure what the correct approach would be.
Any advice or available links?
I think this is a specific type of training, but I forgot the name.


title: Efficiently filling torch.Tensor at equal index positions

body:
I have a 6 dimensional all-zero pytorch tensor lrel_w that I want to fill with 1s at positions where the indices of the first three dimensions and the indices of the last three dimensions match. I'm currently solving this trivially using 3 nested for loops:
lrel_w = torch.zeros(
  input_size[0], input_size[1], input_size[2],
  input_size[0], input_size[1], input_size[2]
)
for c in range(input_size[0]):
  for x in range(input_size[1]):
    for y in range(input_size[2]):
      lrel_w[c,x,y,c,x,y] = 1

I'm sure there must be a more efficient way of doing this, however I have not been able to figure it out.


title: How to access conv2d layers in resnet50?

body:
I am using a Resnet50 model and want to visualize the features of certain conv2d layer after forwarding an image through the network, but somehow I am not able to access the inner conv2d layers in the Bottleneck blocks.
My code so far:
def forward_res(model, conv_layer, x):
    count = 0
    for child in model.children(): #outer layers
        x = child(x)
        if isinstance(child, nn.Conv2d):
            if count == conv_layer:
                return x
            count = count+1

        if isinstance(child, nn.Sequential):
            for bottleneck in child:
                for lay in bottleneck.children(): #inner layers
                    x = lay(x)
                    if isinstance(lay, nn.Conv2d):
                        if count == conv_layer:
                            return x
                        count = count + 1

When I try to use my code like this, I get the following error:

RuntimeError: Given groups=1, weight of size 1 3 1 1, expected
input[1, 1, 512, 512] to have 3 channels, but got 1 channels instead

Can somebody help me how to access the inner conv2d layers of a Renset model? Or tell me what is wrong with my code?


title: How to give proper path to Pytorch model in Github?

body:
My repo is like this
repo:
   model:
         fasterrcnn.pth
   streamlit.py

I am using
WEIGHTS_FILE = './model/fasterrcnn.pth'

and this to upload the model
model.load_state_dict(torch.load(WEIGHTS_FILE))

but error gives
UnpicklingError: invalid load key, 'v'.
Traceback:
File "/usr/local/lib/python3.7/site-packages/streamlit/script_runner.py", line 332, in _run_script
    exec(code, module.__dict__)
File "/app/streamlit/streamlit.py", line 71, in <module>
    model.load_state_dict(torch.load(WEIGHTS_FILE, map_location=device))
File "/home/appuser/.local/lib/python3.7/site-packages/torch/serialization.py", line 529, in load
    return _legacy_load(opened_file, map_location, pickle_module, **pickle_load_args)
File "/home/appuser/.local/lib/python3.7/site-packages/torch/serialization.py", line 692, in _legacy_load
    magic_number = pickle_module.load(f, **pickle_load_args)

The app works on my desktop,
Thanks
Edit: I understood the question better, I have put the .pth that is 168 file in GitHub through git lfs, but it seems it can't run that big file in github


title: How do I make Input type and weight type same?

body:
I am getting a runtime error that says inputs and weights must be on same. However I made sure that my model and input are on the same device yet I cannot get rid of the error. As far as I read, I know that my input data is not on GPU . Since, In this case image is an input so I tried img = torch.from_numpy(img).to(device) and pred = model(img)[0].to(device but no luck. Please let me know what can be done.
Here's the code:
source = '0'
webcam = source == '0'
image_size = 640
imgsz = check_img_size(image_size)
# Load the model
filepath = 'weights/mask.pt'
# device = torch.device('cpu')
device = select_device()
# half = device.type != 'cpu'
model = attempt_load(filepath, map_location = device)
model.to(device).eval()
# if half:
#     model.half()

# Second stage classifier
classify = False
if classify:
    modelc = torch_utils.load_classifier(name = 'resnet101', n = 2)
    modelc.load_state_dict(torch.load('weights/resnet101.pt', map_location = device)['modelc']) ###########
    modelc.to(device).eval()

vid_path, vid_writer = None, None
if webcam:
    view_img = True
    cudnn.benchmark = True
    dataset = LoadStreams(source, img_size = imgsz)

names = model.module.names if hasattr(model, 'module') else model.names
print(names)

def process_image(image):
    h, w = image.shape[:2]
    desired_size = 416
    ratio = desired_size/w
    print("Ratio",ratio)
    img = cv2.resize(image, (0, 0), fx = ratio, fy = ratio)
    h, w = img.shape[:2]
    img = cv2.copyMakeBorder(img, int((416-h)/2), int((416-h)/2), 0, 0, cv2.BORDER_CONSTANT)
    img = img[:, :, ::-1].transpose(2, 0, 1)
    img = np.ascontiguousarray(img)
    img = torch.from_numpy(img).to(device)
    img = img.float()
    img /=255.0
    if img.ndimension() == 3:
        img = img.unsqueeze(0)

    return img

def classify(image):
    # device = torch.device("cpu")

    #image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    #im = Image.fromarray(image)
    img = process_image(image)
    print('Image processed')
    # img = image.unsqueeze_(0)
    # img = image.float()

    pred = model(img)[0]

    # Apply NMS
    pred = non_max_suppression(pred, 0.4, 0.5, classes = [0, 1, 2], agnostic = None )
    if classify:
        pred = apply_classifier(pred, modelc, img, im0s)
    print("1 ", pred)

    model.eval()
    model.cpu()

    classification = torch.cat(pred)[:, -1]
    if len(classification) == 0:
        return None

    index = int(classification[0])
    print(names[index])
    return names[index]

def detect(frame):
    source = '0'
    webcam = source == '0'

    image_size = 640
    imgsz = check_img_size(image_size)

    # Load model
    file_path = 'weights/yolov5s.pt'
    #device = torch.device('cpu')
    device = select_device()
    # half = device.type != 'cpu'
    model = attempt_load(file_path, map_location = device)
    model.to(device).eval()
    # if half:
    #     model.half()
    names = model.module.names if hasattr(model, 'module') else model.names
    colors = [[75, 125, 2]]
    img = process_image(frame)
    pred = model(img)[0]
    pred = non_max_suppression(pred, 0.4, 0.5, classes = [0], agnostic = None)
    if classify:
        pred = apply_classifier(pred, modelc, img, im0s)
    gn = torch.tensor(frame.shape)[[1,0,1,0]]
    for i, det in enumerate(pred):
        det[:,:4] = scale_coords(img.shape[2:], det[:,:4], frame.shape).round()

        for *xyxy, conf, cls in reversed(det):
            xywh = (xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist()  # normalized xywh
            label = '%s %.2f' % (names[int(cls)], conf)
            if label is not None:
                if (label.split())[0] == 'person':
                    plot_one_box(xyxy, frame, label = label, color = colors[0], line_thickness = 1)  # utils.general

Here's the main code:
with tf.Graph().as_default():
    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.6)
    sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
    with sess.as_default():
        pnet, rnet, onet = detect_face.create_mtcnn(sess, './models/')

        minsize = 20  # minimum size of face
        threshold = [0.6, 0.7, 0.7]  # three steps's threshold
        factor = 0.709  # scale factor
        margin = 44
        frame_interval = 3
        batch_size = 1000
        image_size = 182
        input_image_size = 160

        print('Loading feature extraction model')
        modeldir = './models/'
        facenet.load_model(modeldir)

        images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
        embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
        phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
        embedding_size = embeddings.get_shape()[1]

        classifier_filename = './myclassifier/my_classifier.pkl'
        classifier_filename_exp = os.path.expanduser(classifier_filename)
        with open(classifier_filename_exp, 'rb') as infile:
            (model, class_names) = pickle.load(infile)
            print('load classifier file-> %s' % type(class_names))
        HumanNames = class_names
        video_capture = cv2.VideoCapture(0)
        c = 0

        print('Start!')
        prevTime = 0
        while True:
            ret, frame = video_capture.read()

            # frame = cv2.resize(frame, (0,0), fx=0.5, fy=0.5)    #resize frame (optional)

            curTime = time.time()    # calc fps
            timeF = frame_interval

            if (c % timeF == 0):
                find_results = []

                if frame.ndim == 2:
                    frame = facenet.to_rgb(frame)
                frame = frame[:, :, 0:3]
                bounding_boxes, _ = detect_face.detect_face(frame, minsize, pnet, rnet, onet, threshold, factor)
                nrof_faces = bounding_boxes.shape[0]
                # print('Bounding Boxes: ', bounding_boxes, 'Shape: ', bounding_boxes.shape, 'nrof_faces:: ', nrof_faces)
                # print('Detected_FaceNum: %d' % nrof_faces)

                if nrof_faces > 0:
                    detect(frame)
                    label = classify(frame)
                    if label == "a":
                        det = bounding_boxes[:, 0:4]
                        img_size = np.asarray(frame.shape)[0:2]

                        cropped = []
                        scaled = []
                        scaled_reshape = []
                        bb = np.zeros((nrof_faces,4), dtype=np.int32)

                        for i in range(nrof_faces):
                            emb_array = np.zeros((1, embedding_size))
                            # print("Embeddinigs:::::")
                            # print(emb_array)
                            # print("Embeddinigs:::::")
                            bb[i][0] = det[i][0]
                            bb[i][1] = det[i][1]
                            bb[i][2] = det[i][2]
                            bb[i][3] = det[i][3]

                            if bb[i][0] <= 0 or bb[i][1] <= 0 or bb[i][2] >= len(frame[0]) or bb[i][3] >= len(frame):
                                print('face is inner of range!')
                                continue

                            cropped.append(frame[bb[i][1]:bb[i][3], bb[i][0]:bb[i][2], :])
                            cropped[0] = facenet.flip(cropped[0], False)
                            scaled.append(misc.imresize(cropped[0], (image_size, image_size), interp='bilinear'))
                            scaled[0] = cv2.resize(scaled[0], (input_image_size,input_image_size),
                                                   interpolation=cv2.INTER_CUBIC)
                            scaled[0] = facenet.prewhiten(scaled[0])
                            scaled_reshape.append(scaled[0].reshape(-1,input_image_size,input_image_size,3))
                            feed_dict = {images_placeholder: scaled_reshape[0], phase_train_placeholder: False}
                            emb_array[0, :] = sess.run(embeddings, feed_dict=feed_dict)

                            predictions = model.predict_proba(emb_array)
                            best_class_indices = np.argmax(predictions, axis=1)
                            best_class_probabilities = predictions[np.arange(len(best_class_indices)), best_class_indices]
                            cv2.rectangle(frame, (bb[i][0], bb[i][1]), (bb[i][2], bb[i][3]), (0, 255, 0), 2)
                            text_x = bb[i][0]
                            text_y = bb[i][3] + 20

(Edit) Error:
Traceback (most recent call last):
  File "realtime.py", line 105, in <module>
    label = classify(frame)
  File "yolov5-master\myutils.py", line 117, in classify
    pred = model(img)[0]
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "yolov5-master\models\yolo.py", line 122, in forward
    return self.forward_once(x, profile)  # single-scale inference, train
  File "yolov5-master\models\yolo.py", line 138, in forward_once
    x = m(x)  # run
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "yolov5-master\models\common.py", line 94, in forward
    return self.conv(torch.cat([x[..., ::2, ::2], x[..., 1::2, ::2], x[..., ::2, 1::2], x[..., 1::2, 1::2]], 1))
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "yolov5-master\models\common.py", line 38, in fuseforward
    return self.act(self.conv(x))
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\conv.py", line 419, in forward
    return self._conv_forward(input, self.weight)
  File "C:\Users\Anuj\anaconda3\envs\py36\lib\site-packages\torch\nn\modules\conv.py", line 416, in _conv_forward
    self.padding, self.dilation, self.groups)
RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same


Happy New year and good day.
I am trying to load the JSON using json.loads() method, but I am getting the error as "JSONDecodeError: Expecting property name enclosed in double quotes: line 2 column 1 (char 2)".annotations = [] with open("annotations.json") as f: for line in f: annotations.append(json.loads(line))
The Json file in context is https://www.dropbox.com/s/ydr3asu406hl4ho/dataset.tar.gz?dl=0&file_subpath=%2Fdataset%2Fannotations.json
I have verified the JSON and it is a perfect JSON document, It did not got any error.
Help me to identify the bottleneck here.


title: torch row wise cosinus similarity

body:
Assuming I am having 2 tensors with dimension:

tensor A: [512, 100]
tensor B: [512, 200, 100]

and I would like to use torch.cosine_similarity to produce a tensor C of dimensions [512, 1, 200] or [512, 200].
Using torch.cosine_similarity(A, B) I get an error:
'{RuntimeError}The size of tensor a (512) must match the size of tensor b (200) at non-singleton dimension 1'
I guess it can be done by the following:
desired_result = torch.stack([torch.cosine_similarity(A_row, B_row, axis=-1) for B_row, A_row in zip(B, A)])

but there should be a more optimized way. Any help/hint?


title: Torch: Why is this collate function so much faster than this other one?

body:
I have developed two collate functions to read in data from h5py files (I tried to create some synthetic data for a MWE here but it is not going to plan).
The difference between the two in processing my data is about 10x -- a very large increase and I am unsure as to why and I am curious for insights for my future collate functions.
def slow(batch):
    '''
    This function retrieves the data emitted from the H5 torch data set.
    It alters the emitted dimensions from the dataloader
    from: [batch_sz, layers, tokens, features], to:
    [layers, batch_sz, tokens, features]
    '''
    embeddings = []
    start_ids = []
    end_ids = []
    idxs = []
    for i in range(len(batch)):
        embeddings.append(batch[i]['embeddings'])
        start_ids.append(batch[i]['start_ids'])
        end_ids.append(batch[i]['end_ids'])
        idxs.append(batch[i]['idx'])
    # package data;    # swap to expected [layers, batch_sz, tokens, features]
    sample = {'embeddings': torch.as_tensor(embeddings).permute(1, 0, 2, 3),
              'start_ids': torch.as_tensor(start_ids),
              'end_ids': torch.as_tensor(end_ids),
              'idx': torch.as_tensor(idxs)}
    return sample

I thought the one below, with more loops, would be slower, but it is far from the case.
def fast(batch):
    ''' This function alters the emitted dimensions from the dataloader
    from: [batch_sz, layers, tokens, features]
    to: [layers, batch_sz, tokens, features] for the embeddings
    '''
    # turn data to tensors
    embeddings = torch.stack([torch.as_tensor(item['embeddings']) for item in batch])
    # swap to expected [layers, batch_sz, tokens, features]
    embeddings = embeddings.permute(1, 0, 2, 3)
    # get start ids
    start_ids = torch.stack([torch.as_tensor(item['start_ids']) for item in batch])
    # get end ids
    end_ids = torch.stack([torch.as_tensor(item['end_ids']) for item in batch])
    # get idxs
    idxs = torch.stack([torch.as_tensor(item['idx']) for item in batch])
    # repackage
    sample = {'embeddings': embeddings,
              'start_ids': start_ids,
              'end_ids': end_ids}
    return sample

Edit: I tried swapping to this:
It still is about 10x slower compared to 'fast'.
def slow(batch):
    '''
    This function retrieves the data emitted from the H5 torch data set.
    It alters the emitted dimensions from the dataloader
    from: [batch_sz, layers, tokens, features], to:
    [layers, batch_sz, tokens, features]
    '''
    embeddings = []
    start_ids = []
    end_ids = []
    idxs = []
    for item in batch:
        embeddings.append(item['embeddings'])
        start_ids.append(item['start_ids'])
        end_ids.append(item['end_ids'])
        idxs.append(item['idx'])
    # package data;    # swap to expected [layers, batch_sz, tokens, features]
    sample = {'embeddings': torch.as_tensor(embeddings).permute(1, 0, 2, 3),
              'start_ids': torch.as_tensor(start_ids),
              'end_ids': torch.as_tensor(end_ids),
              'idx': torch.as_tensor(idxs)}
    return sample


title: while importing torch- shows - [WinError 126] The specified module could not be found

body:
I have tried to install python torch by using
  !pip install torch

But I got the error OSError: [WinError 126] The specified module could not be found
Then I tried with
pip install torch -f https://download.pytorch.org/whl/torch_stable.html

The running log is as follows,

Then I have installed the torch by the below command to install the CPU version of the torch.
pip install torch==1.7.1+cpu torchvision==0.8.2+cpu torchaudio===0.7.2 -f https://download.pytorch.org/whl/torch_stable.html

Then, when I try to test by the command,
import torch
print(torch.__version__)

In the Spyder, it still could not recognize,

However, in python.exe it could recognize and provides the correct output.


title: diffrence between a[:,:,0] and a[:][:][0]

body:
Hi I was studying slicing in python and I found something strange and I don't understand
import torch

a = torch.tensor([
    [
        [1, 2, 3],
        [4, 5, 6]
    ],
    [
        [7, 2, 3],
        [8, 5, 6]
    ]
])

>>> a[:][:][0]
tensor([[1, 2, 3],
        [4, 5, 6]])

>>> a[:,:,0]
tensor([[1, 4],
        [7, 8]])

I tried to pull out [[1,4,7,8]] from the corresponding torch list, so I entered a[:][:][0] and the result of it is [[1,2,3], [4,5,6]]. Then, when I input a[:,:,0], then [[1,4,7,8]] appeared.
I thought they have no different but different result appeared.
the torch and numpy operators, there was an operation like a[:,0]. How exactly can it be different from a[:][0]?


title: Linux Pytorch.cuda.is_available() returns false

body:
I tried to use pytorch 1.4.0 with GPU but cuda.is_available returned false.
I installed pytorch using conda:
conda install pytorch==1.4.0 torchvision==0.5.0 cudatoolkit=10.1

this is the output of cat /proc/version:
Linux version 5.8.0-40-generic (buildd@lcy01-amd64-014) (gcc (Ubuntu 9.3.0-17ubuntu1~20.04) 9.3.0, GNU ld (GNU Binutils for Ubuntu) 2.34) #45~20.04.1-Ubuntu SMP

and this is the output of nvidia-smi:
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  GeForce RTX 3080    Off  | 00000000:65:00.0 Off |                  N/A |
| 33%   27C    P8     4W / 320W |     47MiB / 10010MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      1134      G   /usr/lib/xorg/Xorg                 36MiB |
|    0   N/A  N/A      1297      G   /usr/bin/gnome-shell                9MiB |
+-----------------------------------------------------------------------------+

and the output of conda list(part of it):
Name                    Version                   Build  Channel
cudatoolkit               10.1.243             h6bb024c_0
ninja                     1.10.2           py38hff7bd54_0
python                    3.8.2               hcff3b4d_14
pytorch                   1.4.0           py3.8_cuda10.1.243_cudnn7.6.3_0    https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch
torchvision               0.5.0                py38_cu101    https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/pytorch
yacs                      0.1.6                      py_0    conda-forge
yaml                      0.2.5                h7b6447c_0
zlib                      1.2.11               h7b6447c_3
zstd                      1.4.5                h9ceee32_0

As far as I know my Linux server could support pytorch 1.4.0 but it didn't. what should I do?


title: why torch.Tensor subtract works well when tensor size is different?

body:
This example will make it easier to understand. The following fails:
A = tensor.torch([[1, 2, 3], [4, 5, 6]])   # shape : (2, 3)
B = tensor.torch([[1, 2], [3, 4], [5, 6]]) # shape : (3, 2)
print((A - B).shape)

# RuntimeError: The size of tensor A (3) must match the size of tensor B (2) at non-singleton dimension 1
# ==================================================================
A = tensor.torch([[1, 2], [3, 4], [5, 6]])   # shape : (3, 2)
B = tensor.torch([[1, 2], [3, 4],]) # shape : (2, 2)
print((A - B).shape)

# RuntimeError: The size of tensor A (3) must match the size of tensor B (2) at non-singleton dimension 0

But the following works well:
a = torch.ones(8).unsqueeze(0).unsqueeze(-1).expand(4, 8, 7)
a_temp = a.unsqueeze(2)                            # shape : ( 4, 8, 1, 7 )
b_temp = torch.transpose(a_temp, 1, 2)             # shape : ( 4, 1, 8, 7 )
print(a_temp-b_temp)                               # shape : ( 4, 8, 8, 7 )

Why does the latter work, but not the former?
How/why has the result shape been expanded?


title: How to plot multi-epoch x-y line graph in Tensorboard, which is not a histogram?

body:
I want to plot a standard line graph in tensorboard - per epoch.
In Matplotlib I would just use plt.plot(np.arange(100), np.arange(100)) [and not plt.hist()], but would be left with multiple graphs, instead of the convenient tensorboard UI.
I want such a graph to be generated every epoch, and be plotted near each other, to have the output look similar to that of histogram, like this, but display data as it- just plot it in the relevant epoch.
I would use the histogram mechanism, but I don't know how to use it to just plot values rather than histograms (which distort the X-axis, and count on the Y-axis, instead of just displaying the data as-is).

I am using Pytorch [and Pytorch lightning], so a compatible solution will be preferred.


title: Training Pytorch Model

body:
I've got a problem about training the PyTorch model. Following the PyTorch model found from  GitHub, I run that source code perfectly that can assume the original idea of training the SEC model works well. Then, I'm trying to modify a little bit:

Implement algorithms

I applied the DSRG algorithm for this
weakly-supervised semantic segmentation task into my PyTorch model. It technically takes the
final feature maps out, and perform some mechanism to come up with new
"semantic map (G)" (size: 21x41x41).

I suppose it works well, since I run the code with this modification, and the training and inferencing works fine.

Implement loss function

Also from the DSRG, I calculate the pixel-wise seeding-loss which is nothing but the cross-entropy loss.

def dsrg_seed_loss_layer(probs, labels):

    count_bg = torch.sum(labels[:,0:1,:,:], dim=(2, 3, 1), keepdim=True)
    loss_bg = -torch.mean(torch.sum(labels[:,0:1,:,:] * torch.log(probs[:,0:1,:,:]), dim=(2, 3, 1), keepdim=True) / (count_bg+MIN_PROB))

    count_fg = torch.sum(labels[:,1:,:,:], dim=(2, 3, 1), keepdim=True)
    loss_fg = -torch.mean(torch.sum(labels[:,1:,:,:] * torch.log(probs[:,1:,:,:]), dim=(2, 3, 1), keepdim=True) / (count_fg+MIN_PROB))
    loss_balanced = loss_bg+loss_fg

    count_bg_avg = torch.mean(count_bg.squeeze())
    count_fg_avg = torch.mean(count_fg.squeeze())
    return loss_balanced, loss_bg, count_bg_avg, loss_fg, count_fg_avg

As same as above, it should works!!

Modify model network structure

I tried to attach more layers to the original model network. (Original
model is vgg16-deeplabv1 while I implemented vgg16-deeplabv2)

class DeepLabV2LargeFOV(nn.Module):

    def __init__(self, num_classes, original_model, small = False):
        super(DeepLabV2LargeFOV, self).__init__()
        self.num_classes = num_classes
        self.features =  nn.Sequential(*list(original_model.features.children()))

        if small:
            rates = [2, 4, 8, 12]
        else:
            rates = [6, 12, 18, 24]

        self.aspp1 = self.atrous_layers(rates[0], self.num_classes, "1")
        self.aspp2 = self.atrous_layers(rates[1], self.num_classes, "2")
        self.aspp3 = self.atrous_layers(rates[2], self.num_classes, "3")
        self.aspp4 = self.atrous_layers(rates[3], self.num_classes, "4")

    def forward(self, x):
        x = self.features(x)

        # Atrous Layers
        aspp1 = self.aspp1(x)
        aspp2 = self.aspp2(x)
        aspp3 = self.aspp3(x)
        aspp4 = self.aspp4(x)

        x = aspp1 + aspp2 + aspp3 + aspp4

        return x

    def atrous_layers(self, rate, num_classes, num):
        layers = nn.Sequential()
        layers.add_module("fc6_{0:s}".format(num), nn.Conv2d(512, 1024, kernel_size=3, padding=rate, dilation=rate))
        layers.add_module("relu6_{0:s}".format(num), nn.ReLU())
        layers.add_module("drop6_{0:s}".format(num), nn.Dropout2d(0.5))
        layers.add_module("fc7_{0:s}".format(num), nn.Conv2d(1024, 1024, kernel_size=1, stride=1, padding=0))
        layers.add_module("relu7_{0:s}".format(num), nn.ReLU())
        layers.add_module("drop7_{0:s}".format(num), nn.Dropout2d(0.5))
        layers.add_module("fc8-SEC_{0:s}".format(num), nn.Conv2d(1024, num_classes, kernel_size=1, stride=1, padding=0))
        weight_init(layers)
        return layers

def weight_init(m):
    for name, layer in m.named_modules():
        if isinstance(layer, nn.Conv2d):
            nn.init.normal_(layer.weight, mean=0, std=0.01)
            nn.init.constant_(layer.bias, 0)

def deeplabv2_large_fov(num_classes, original_model):
    return DeepLabV2LargeFOV(num_classes=num_classes, original_model=original_model)

After I implemented this part, I tried to train the model by loading only given original PyTorch weights (for former layer). However, it seems that the model is not trained normally.

Load pretrained weight

I tried to load the pretrained weight into the new model.
However, the pretrained weight was nothing but python numpy
dictionary. I develop another file to convert such numpy file
(.npy) into the PyTorch mode (.pth.tar)

I also spend time to load weights and bias to model layer-by-layer and save as .pth.tar. I pretty sure the size of each of every layers is matched while the parameters are all saved. But the result is also as similar as before, so it proves that either partial-pretrained-weight, or complete-pretrained-weight is not workable in my program.

Does anyone have an idea about my technical training problem. I actually have few ideas

Since the loss goes down generally, can I assume my model is built properly with complete computational graph and all connected gradient flow (like requires_grad=True? If no, how can I check that?

Is is possible that the problem happens on optimizer? What I mean is it seems that the source code set dynamic learning rate on parameters from different layers (actually 2, the final and other). Does anyone know how it works
 optimizer = torch.optim.SGD([
 {'params': get_parameters(model, bias=False, final=False), 'lr':args.lr, 'weight_decay': args.wd},
 {'params': get_parameters(model, bias=True, final=False), 'lr':args.lr * 2, 'weight_decay': 0},
 {'params': get_parameters(model, bias=False, final=True), 'lr':args.lr * 10, 'weight_decay': args.wd},
 {'params': get_parameters(model, bias=True, final=True), 'lr':args.lr * 20, 'weight_decay': 0}], momentum=args.momentum)

It combines with regular learning-rate decay techniques. How can I check if it works correctly?
     if global_iter % args.lr_decay == 0:
     args.lr = args.lr * 0.1
     optimizer = adjust_learning_rate(optimizer, args.lr)

Hyper-parameters tuning, this is what I am trying do since a lot of hyper-parameters are involved. It would be really happy if anyone could bring me experiences on this issues. (The whole training takes more than 8 hours to finish)


title: Looking for a pytorch function to repeat a vector

body:
I am looking for a pytorch function that is similar to tf's tile function.
I saw that PyTorch used to have a tile function, but apparently it was removed.
An example for the functionality I am looking for:
Let's say I have a tensor of dimensions (1,1,1,1000), I want to repeat it several times so I get a (1,40,40,1000) tensor.


title: How to install CMake to install PyTorch in root without root authorization

body:
I'm installing pytorch in Linux on a slurm cluster. Since I don't have sudo authorization, I have to install everything from source code.
I'm using the following codes for the PyTorch installation:
$ git clone https://github.com/torch/distro.git ~/torch --recursive
$ sed -i 's/-pPR/-PR/' ~/torch/exe/luajit-rocks/luarocks/src/luarocks/fs/unix/tools.lua
$ cd ~/torch; ./install.sh

While I'm running the cd ~/torch; ./install.sh command, I got the following error:
Installing Lua version: LUAJIT21
./install.sh: line 73: cmake: command not found

So my job becomes installing cmake. I followed this post:
$ wget http://www.cmake.org/files/v2.8/cmake-2.8.3.tar.gz
$ tar xzf cmake-2.8.3.tar.gz
$ cd cmake-2.8.3
$ ./configure --help
$ ./configure --prefix=/opt/cmake

While when I run the last line, I got :
 Could NOT find OpenSSL, try to set the path to OpenSSL root folder in the system variable OPENSSL_ROOT_DIR (missing: OPENSSL_CRYPTO_LIBRARY OPENSSL_INCLUDE_DIR)
CMake Error at Utilities/cmcurl/CMakeLists.txt:511 (message):
  Could not find OpenSSL.  Install an OpenSSL development package or
  configure CMake with -DCMAKE_USE_OPENSSL=OFF to build without OpenSSL.

And though I installed OpenSSL, and have OpenSSL has been successfully configured, the same Could NOT find OpenSSL happens. Everything would be easy if I have the sudo authorization, but I would like to know how to install without it.


title: Does the PyTorch Lightning average metrics over the whole epoch?

body:
I am looking at the example provided on PyTorch-Lightning official documentation https://pytorch-lightning.readthedocs.io/en/0.9.0/lightning-module.html.
Here the loss and metric is calculated on the concrete batch. But when logging one is not interested in the accuracy for a particular batch, which can be rather small and not representative, but the averaged over all epoch. Do I understand correctly, that there is some code performing the averaging over all batches, passed through the epoch?
 import pytorch_lightning as pl
 from pytorch_lightning.metrics import functional as FM

 class ClassificationTask(pl.LightningModule):

 def __init__(self, model):
     super().__init__()
     self.model = model

 def training_step(self, batch, batch_idx):
     x, y = batch
     y_hat = self.model(x)
     loss = F.cross_entropy(y_hat, y)
     return pl.TrainResult(loss)

 def validation_step(self, batch, batch_idx):
    x, y = batch
    y_hat = self.model(x)
    loss = F.cross_entropy(y_hat, y)
    acc = FM.accuracy(y_hat, y)
    result = pl.EvalResult(checkpoint_on=loss)
    result.log_dict({'val_acc': acc, 'val_loss': loss})
    return result

 def test_step(self, batch, batch_idx):
    result = self.validation_step(batch, batch_idx)
    result.rename_keys({'val_acc': 'test_acc', 'val_loss': 'test_loss'})
    return result

 def configure_optimizers(self):
     return torch.optim.Adam(self.model.parameters(), lr=0.02)


title: How does the output of the Discriminator of a Convolutional Generative Adversarial Network work, can it have a Fully Connected Layer?

body:
I'm building a DCGAN, and I am having a problem with the shape of the output, it is not matching the shape of the labels when I try calculating the BCELoss.
To generate the discriminator output, do I have to use convolutions all the way down or can I add a Linear layer at some point to match the shape I want?
I mean, do I have to reduce the shape by adding more convolutional layers or can I add a fully connected one? I thought it should have a fully connected layer, but on every tutorial I checked the discriminator had no fully connected layer.
import random
import torch.nn as nn
import torch.optim as optim
import torch.utils.data
import torchvision.datasets as torch_dataset
import torchvision.transforms as transforms
import torchvision.utils as vutils
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from IPython.display import HTML

seed = 1
print("Random Seed: ", seed)
random.seed(seed)
torch.manual_seed(seed)
images_folder_path = "./spectrograms/"

batch_size = 1
image_size = 256
n_channels = 1
z_vector = 100
n_features_generator = 32
n_features_discriminator = 32
num_epochs = 5
lr = 0.0002
beta1 = 0.5

dataset = torch_dataset.ImageFolder(
    root=images_folder_path, transform=transforms.Compose(
        [
            transforms.Grayscale(num_output_channels=1),
            transforms.Resize(image_size),
            transforms.CenterCrop(image_size),
            transforms.ToTensor(),
            transforms.Normalize(0.5, 0.5)
         ]
    )
)

dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True, num_workers=0)

device = torch.device("cuda:0" if (torch.cuda.is_available()) else "cpu")

def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        nn.init.normal_(m.weight.data, 0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        nn.init.normal_(m.weight.data, 1.0, 0.02)
        nn.init.constant_(m.bias.data, 0)

class Generator(nn.Module):
    def __init__(self):
        super(Generator, self).__init__()
        self.main = nn.Sequential(
            nn.ConvTranspose2d(z_vector, n_features_generator * 8, 4, 1, bias=False),
            nn.BatchNorm2d(n_features_generator * 8),
            nn.ReLU(True),
            nn.ConvTranspose2d(n_features_generator * 8, n_features_generator * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(n_features_generator * 4),
            nn.ReLU(True),
            nn.ConvTranspose2d(n_features_generator * 4, n_features_generator * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(n_features_generator * 2),
            nn.ReLU(True),
            nn.ConvTranspose2d(n_features_generator * 2, n_features_generator, 4, 2, 1, bias=False),
            nn.BatchNorm2d(n_features_generator),
            nn.ReLU(True),
            nn.ConvTranspose2d(n_features_generator, n_channels, 4, 2, 1, bias=False),
            nn.Tanh()
        )

    def forward(self, inputs):
        return self.main(inputs)

# Convolutional Layer Output Shape = [(W−K+2P)/S]+1
# W is the input volume
# K is the Kernel size
# P is the padding
# S is the stride
class Discriminator(nn.Module):
    def __init__(self):
        super(Discriminator, self).__init__()
        self.main = nn.Sequential(
            nn.Conv2d(n_channels, n_features_discriminator, 4, 2, 1, bias=False),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(n_features_discriminator, n_features_discriminator * 2, 4, 2, 1, bias=False),
            nn.BatchNorm2d(n_features_discriminator * 2),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(n_features_discriminator * 2, n_features_discriminator * 4, 4, 2, 1, bias=False),
            nn.BatchNorm2d(n_features_discriminator * 4),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(n_features_discriminator * 4, n_features_discriminator * 8, 4, 2, 1, bias=False),
            nn.BatchNorm2d(n_features_discriminator * 8),
            nn.LeakyReLU(0.2, inplace=True),
            nn.Conv2d(n_features_discriminator * 8, 1, 4, 1, bias=False),
        )

    def forward(self, inputs):
        return self.main(inputs)

netG = Generator().to(device)
if device.type == 'cuda':
    netG = nn.DataParallel(netG)
netG.apply(weights_init)
print(netG)

netD = Discriminator().to(device)
if device.type == 'cuda':
    netD = nn.DataParallel(netD)
netD.apply(weights_init)
print(netD)

criterion = nn.BCEWithLogitsLoss()

fixed_noise = torch.randn(64, z_vector, 1, 1, device=device)

real_label = 1.
fake_label = 0.

optimizerD = optim.Adam(netD.parameters(), lr=lr, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters(), lr=lr, betas=(beta1, 0.999))

img_list = []
G_losses = []
D_losses = []
iters = 0

print("Starting Training Loop...")
for epoch in range(num_epochs):
    for i, data in enumerate(dataloader, 0):
        netD.zero_grad()
        real_cpu = data[0].to(device)
        b_size = real_cpu.size(0)
        label = torch.full((b_size,), real_label, dtype=torch.float, device=device)
        output = netD(real_cpu)
        print(output.shape)
        print(label.shape)
        output = output.view(-1)
        errD_real = criterion(output, label)
        errD_real.backward()
        D_x = output.mean().item()

        noise = torch.randn(b_size, z_vector, 1, 1, device=device)
        fake = netG(noise)
        label.fill_(fake_label)
        output = netD(fake.detach()).view(-1)
        errD_fake = criterion(output, label)
        errD_fake.backward()
        D_G_z1 = output.mean().item()
        errD = errD_real + errD_fake
        optimizerD.step()

        netG.zero_grad()
        label.fill_(real_label)
        output = netD(fake).view(-1)
        errG = criterion(output, label)
        errG.backward()
        D_G_z2 = output.mean().item()
        optimizerG.step()

        if i % 50 == 0:
            print('[%d/%d][%d/%d]\tLoss_D: %.4f\tLoss_G: %.4f\tD(x): %.4f\tD(G(z)): %.4f / %.4f'
                  % (epoch, num_epochs, i, len(dataloader),
                     errD.item(), errG.item(), D_x, D_G_z1, D_G_z2))

        G_losses.append(errG.item())
        D_losses.append(errD.item())

        if (iters % 500 == 0) or ((epoch == num_epochs-1) and (i == len(dataloader)-1)):
            with torch.no_grad():
                fake = netG(fixed_noise).detach().cpu()
            img_list.append(vutils.make_grid(fake, padding=2, normalize=True))

        iters += 1

The error I'm getting:
Traceback (most recent call last):
  File "G:/Pastas Estruturadas/Conhecimento/CEFET/IA/SpectroGAN/dcgan.py", line 140, in <module>
    errD_real = criterion(output, label)
  File "C:\Users\Ramon\anaconda3\envs\vision\lib\site-packages\torch\nn\modules\module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "C:\Users\Ramon\anaconda3\envs\vision\lib\site-packages\torch\nn\modules\loss.py", line 631, in forward
    reduction=self.reduction)
  File "C:\Users\Ramon\anaconda3\envs\vision\lib\site-packages\torch\nn\functional.py", line 2538, in binary_cross_entropy_with_logits
    raise ValueError("Target size ({}) must be the same as input size ({})".format(target.size(), input.size()))
ValueError: Target size (torch.Size([1])) must be the same as input size (torch.Size([169]))

The shape of output: torch.Size([1, 1, 13, 13]), and shape of label: torch.Size([1]).


title: How to shift columns (or rows) in a tensor with different offsets in PyTorch?

body:
In PyTorch, the build-in torch.roll function is only able to shift columns (or rows) with same offsets. But I want to shift columns with different offsets. Suppose the input tensor is
[[1,2,3],
 [4,5,6],
 [7,8,9]]

Let's say, I want to shift with offset i for the i-th column. Thus, the expected output is
[[1,8,6],
 [4,2,9],
 [7,5,3]]

An option to do so is to separately shift every column using torch.roll and concat each of them. But for the consideration of effectiveness and code compactness, I don't want to introduce the loop structure. Is there a better way？


title: ValueError: optimizer got an empty parameter list. Is this a valid way to create a neural network in PyTorch?

body:
I have been trying to implelemnt SRGAN for a project, however when i try to create a optimizer for my generator I run into error:
ValueError: optimizer got an empty parameter list.

I have checked my parameters and they are indeed an empty list, the issue is that I don't know how to access those parameters, when I was implementing other network architectures I have met no such problem.
I think that due to my lack of experience with the framework i might have messed up when creating generator class so I have included code snippet below.
class Generator(nn.Module):
  def __init__(self, in_channels, out_channels):
    super(Generator, self).__init__()
    self.in_channels = in_channels
    self.out_channels = out_channels

  @staticmethod
  def block(in_channels, out_channels, kernel_size=3, padding=1):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding, bias=False),
        nn.BatchNorm2d(out_channels),
        nn.PReLU()
    )

  @staticmethod
  def input_block(in_channels, out_channels, kernel_size=9, padding=4):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding, bias=False),
        nn.PReLU()
    )

  @staticmethod
  def output_block(in_channels, out_channels, kernel_size=3, padding=1):
    return nn.Sequential(
        nn.Conv2d(in_channels, out_channels, kernel_size, padding=padding, bias=False),
        nn.PixelShuffle(2),
        nn.PReLU()
    )

  def forward(self, x):
    block_in = self.input_block(self.in_channels, self.out_channels)(x)
    block1 = T.add(self.block(self.out_channels, self.out_channels)(block_in), block_in)
    block2 = T.add(self.block(self.out_channels, self.out_channels)(block1), block1)
    block3 = T.add(self.block(self.out_channels, self.out_channels)(block2), block2)
    block4 = T.add(self.block(self.out_channels, self.out_channels)(block3), block3)
    block_bridge = T.add(block4, block_in)
    block5 = self.output_block(self.out_channels, self.out_channels * 4)(block_bridge)
    block6 = self.output_block(self.out_channels, self.out_channels * 4)(block5)
    _output = nn.Conv2d(self.out_channels, self.in_channels, 9, padding=4, bias=False)(block6)
    return _output


title: How can torch.cat only have one tensor?

body:
I think there is an error in line 53 of the following code:
CycleGan, Buffer for Images
It says:
return_images = torch.cat(return_images, 0)  # collect all the images and return

What do you think, what should it read correctly? After having gone throught the code, I am unfortunately not sure what this specific line is supposed to do, but I think I understand the rest until line 52.


title: Output evaluation loss after every n-batches instead of epochs with pytorch

body:
Instead of printing the evaluation loss every epoch I would like to output it after every n-batches.
I have around 150'000 batches per epoch. I would like to output the evaluation loss every 50'000 batches.
Is this even possible? I am using pytorch and a pretrained bert model from huggingface.
My train loop:
best_valid_loss = float('inf')
train_losses=[]
valid_losses=[]

for epoch in range(params['epochs']):

      print('\n Epoch {:} / {:}'.format(epoch + 1, params['epochs']))

      #train model
      train_loss = train(scheduler, optimizer)

      #evaluate model
      valid_loss = evaluate()

      #save the best model
      if valid_loss < best_valid_loss:
          best_valid_loss = valid_loss
          torch.save(model.state_dict(), model_file)

      # append training and validation loss
      train_losses.append(train_loss)
      valid_losses.append(valid_loss)

      print(f'\nTraining Loss: {train_loss:.3f}')
      print(f'Validation Loss: {valid_loss:.3f}')


title: Is there any alternate for tfagents in pytorch

body:
We are using tfagents in tensorflow for reinforcement learning, because of limitations with static computation graphs we are planning to migrate our code to pytorch.
tfagents is great and have very good documentation and reduce a lot of time doing the same task again
We are wondering if the pytorch community have a similar kind of stuff?


title: Modifying a Numpy float32 bitwise and returning the modified np.float32 value

body:
I'm currently working on a bit chopping algorithm for simulation of memory footprint reduction when training neural networks. I'm using PyTorch to achieve this.
However, what I'm basically trying to do is set to 0 the less significant bits of the mantissa of the float32 value to see for now if the neural network will train and how much precision it will lose depending on the number of bits that are set to 0.
My problem is that every value on the tensors is of type Numpy float32, and I would like to get the literal bit representation of the value (how the actual float32 value is represented in memory) or as an integer, apply the bitwise modification and convert it back to np.float32.
However, I've tried the following (Note, x is of type numpy.ndarray):
    print(x)
    value_as_int = self.x.view(np.int32)
    print(value_as_int)
    value_as_int = value_as_int&0xFFFFFFFE
    print(value_as_int)
    new_float = value_as_int.view(np.float32)
    print(new_float)

Here's an example output of the part that works:
0.13498048
1040857171
1040857170

This does convert the value to its literal bit integer representation and allows me to set to 0 the last bit, although when trying to convert it back to np.float32 I get the following error:

ValueError: Changing the dtype of a 0d array is only supported if the itemsize is unchanged

Is there a proper way to do this? Am I missing something in my code or is it the wrong approach?
Thank you in advance


title: pytorch change input image size

body:
I am new to pytorch and I am following a tutorial but when i try to modify the code to use 64x64x3 images instead of 32x32x3 images, i get a buch of errors. Here is the code from the tutorial:
import torch
from torch.utils.data import DataLoader
import torchvision
import torchvision.transforms as transforms
from torchvision.datasets import ImageFolder

transform = transforms.Compose(
    [transforms.ToTensor(),
     transforms.Resize(32),
     transforms.RandomCrop(32),
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

batch_size = 4

trainset = ImageFolder("Train", transform=transform)
trainloader = DataLoader(trainset, shuffle=True, batch_size=batch_size, num_workers=0)

classes = ('Dog', 'Cat')

import matplotlib.pyplot as plt
import numpy as np

# functions to show an image

def imshow(img):
    img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

# get some random training images
dataiter = iter(trainloader)
images, labels = dataiter.next()

# show images
imshow(torchvision.utils.make_grid(images))
# print labels
print(' '.join('%5s' % classes[labels[j]] for j in range(batch_size)))

import torch.nn as nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 5 * 5)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

net = Net()

import torch.optim as optim

criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

print("training started")

from tqdm import tqdm

for epoch in range(5):  # loop over the dataset multiple times

    running_loss = 0.0
    for i, data in tqdm(enumerate(trainloader, 0), desc=f"epoch: {epoch + 1}"):
        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        running_loss += loss.item()
        if i % 2000 == 1999:    # print every 2000 mini-batches
            print('[%d, %5d] loss: %.3f' %
                  (epoch + 1, i + 1, running_loss / 2000))
            running_loss = 0.0

print('Finished Training')

PATH = './net.pth'
torch.save(net.state_dict(), PATH)

If I change the ´transforms.Resize(32)´ and ´transforms.RandomCrop(32)´ to 64 (to get 64x64x3 images), I get this error
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
~\Documents\pyth\classifier\train_classifier.py in <module>
     86
     87         # forward + backward + optimize
---> 88         outputs = net(inputs)
     89         loss = criterion(outputs, labels)
     90         loss.backward()

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~\Documents\pyth\classifier\train_classifier.py in forward(self, x)
     57         x = self.pool(F.relu(self.conv1(x)))
     58         x = self.pool(F.relu(self.conv2(x)))
---> 59         x = x.view(-1, 10816+1)
     60         x = F.relu(self.fc1(x))
     61         x = F.relu(self.fc2(x))

RuntimeError: shape '[-1, 10817]' is invalid for input of size 10816
´´´

and if i try to change the parameters of ´x.view(...)´ i get this error

´´´
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
~\Documents\pyth\classifier\train_classifier.py in <module>
     86
     87         # forward + backward + optimize
---> 88         outputs = net(inputs)
     89         loss = criterion(outputs, labels)
     90         loss.backward()

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~\Documents\pyth\classifier\train_classifier.py in forward(self, x)
     58         x = self.pool(F.relu(self.conv2(x)))
     59         x = x.view(-1, 16 * 2 * 5 * 5)
---> 60         x = F.relu(self.fc1(x))
     61         x = F.relu(self.fc2(x))
     62         x = self.fc3(x)

~\Anaconda3\lib\site-packages\torch\nn\modules\module.py in _call_impl(self, *input, **kwargs)
    887             result = self._slow_forward(*input, **kwargs)
    888         else:
--> 889             result = self.forward(*input, **kwargs)
    890         for hook in itertools.chain(
    891                 _global_forward_hooks.values(),

~\Anaconda3\lib\site-packages\torch\nn\modules\linear.py in forward(self, input)
     92
     93     def forward(self, input: Tensor) -> Tensor:
---> 94         return F.linear(input, self.weight, self.bias)
     95
     96     def extra_repr(self) -> str:

~\Anaconda3\lib\site-packages\torch\nn\functional.py in linear(input, weight, bias)
   1751     if has_torch_function_variadic(input, weight):
   1752         return handle_torch_function(linear, (input, weight), input, weight, bias=bias)
-> 1753     return torch._C._nn.linear(input, weight, bias)
   1754
   1755

RuntimeError: mat1 and mat2 shapes cannot be multiplied (2x800 and 400x120)
´´´


title: Differentiable loss from the difference of 2 lists

body:
I have a model
import torch
from torch.autograd import grad
import torch.nn as nn
import torch.optim as optim

class net_x(nn.Module):
        def __init__(self):
            super(net_x, self).__init__()
            self.fc1=nn.Linear(1, 1)
            self.out=nn.Linear(1, 1)

        def forward(self, x):
            x=self.fc1(x)
            x=self.out(x)
            return x

nx = net_x()
#inputs
t = torch.tensor([1.0, 2.0, 3.2], requires_grad = True) #input vector
t = torch.reshape(t, (3,1)) #reshape for batch

and 2 lists:
pred_lst = []
goal_lst = list(range(10))

I was trying to get the loss between these two lists as follows:
for epoch in range(10):
    optimizer.zero_grad()
    y = nx(t)
    if torch.sum(y) > 5:
        pred_lst.append(epoch)
    else:
        pass
    loss = len(set(pred_lst).symmetric_difference(set(goal_lst)))
    loss = torch.tensor(float(loss), requires_grad = True)
    print('loss: ', loss)
    loss.backward()

But the parameters were not updating because symmetric_difference is a non differentiable operation.
How can I modify/use something else that will take these 2 lists and give me a differentiable loss that I can backpropagate?


I want to train a gpt2 model in my laptop and I have a GPU in it and my os is windows , but I always got this error in python:
torch._C._cuda_init()
RuntimeError: No CUDA GPUs are available

when I tried to check the availability of GPU in the python console, I got true:
import torch
torch.cuda.is_available()
Out[4]: True

but I can't get the version by
nvcc version
#or  nvcc --version
NameError: name 'nvcc' is not defined

I use this command to install CUDA
conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch

What can I do to make the GPU available for python?


title: pytorch multiprocessing - TypeError: 'str' object is not callable

body:
I want to configure the Multiple gpu environment using ‘torch.multiprocessing’ and ‘torch.distributed’. However, I received the following error message.
Traceback (most recent call last):
  File "train_custom.py", line 398, in <module>
    mp.spawn(init_process, args=(world_size, backend), nprocs=world_size, join=True)
  File "/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py", line 230, in spawn
    return start_processes(fn, args, nprocs, join, daemon, start_method='spawn')
  File "/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py", line 188, in start_processes
while not context.join():
  File "/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py", line 150, in join
raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException:

-- Process 0 terminated with the following error:
Traceback (most recent call last):
  File "/usr/local/lib/python3.8/dist-packages/torch/multiprocessing/spawn.py", line 59, in _wrap
fn(i, *args)
  File "/root/USRGAN_step2/train_custom.py", line 390, in init_process
fn(rank, size2)
TypeError: 'str' object is not callable

My code is as follows.
def run(rank, size2):
...

def init_process(rank, size2, fn, backend='gloo'):
    """ Initialize the distributed environment. """
    os.environ['MASTER_ADDR'] = '127.0.0.1'
    os.environ['MASTER_PORT'] = '29500'
    dist.init_process_group(backend, rank=rank, world_size=torch.cuda.device_count())
    fn(rank, size2)

###################################

if __name__ == "__main__":
    world_size = torch.cuda.device_count()
    backend = 'gloo'
    mp.spawn(init_process, args=(world_size, backend), nprocs=world_size, join=True)
    #mp.set_start_method('spawn', force=True)
    processes = []
    size2 = 4
    for rank in range(size2):
        p = Process(target=init_process, args=(rank, size2, run))
        p.start()
        processes.append(p)

    for p in processes:
        p.join()

I followed the tutorial(https://pytorch.org/tutorials/intermediate/dist_tuto.html) as it was, but this error occurred. And I don't find the cause of error.


title: Pytorch - batches doesn't update

body:
I was training my model but some issues occurred.
The dataloader was supposed to update batches but instead I got this:

Output

[0,     0] loss: 0.009
[1,     0] loss: 0.009
[2,     0] loss: 0.009
[3,     0] loss: 0.009
[4,     0] loss: 0.009
[5,     0] loss: 0.009
[6,     0] loss: 0.009
[7,     0] loss: 0.009
[8,     0] loss: 0.009
[9,     0] loss: 0.009

Anyone know what's wrong? My codes are as below

DataLoader

class MSourceDataSet(Dataset):

    def __init__(self, clean_dir, mix_dir, clean_label_dir, mix_label_dir):

        with open(clean_dir + 'clean1.json') as f:
            clean0 = torch.Tensor(json.load(f))

        with open(mix_dir + 'mix1.json') as f:
            mix0 = torch.Tensor(json.load(f))

        with open(clean_label_dir + 'clean_label1.json') as f:
            clean_label0 = torch.Tensor(json.load(f))

        with open(mix_label_dir + 'mix_label1.json') as f:
            mix_label0 = torch.Tensor(json.load(f))

        self.spec = torch.cat([clean0, mix0], 0)
        self.label = torch.cat([clean_label0, mix_label0], 0)

    def __len__(self):
        return self.spec.shape[0]

    def __getitem__(self, index):

        spec = self.spec[index]
        label = self.label[index]
        return spec, label

trainset = MSourceDataSet(clean_dir, mix_dir, clean_label_dir, mix_label_dir)

trainloader = torch.utils.data.DataLoader(dataset = trainset,
                                                batch_size = 4,
                                                shuffle = True)

# testloader = torch.utils.data.DataLoader(dataset = testset,
#                                                batch_size = 4,
#                                                shuffle = True)

Training

model.train()

for epoch in range(10):
    running_loss = 0

    for i, data in enumerate(trainloader, 0):

        inputs, labels = data

        optimizer.zero_grad()

        outputs = model(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
        if i % 1000 == 0:
            print ('[%d, %5d] loss: %.3f' % (epoch, i, running_loss/ 1000))
            running_loss = 0

torch.save(model, 'FeatureNet.pkl')


title: Is there any other reason why we make sequence length the same using padding?

body: Is there any other reason why we make sequence length the same length using padding? Other than in order to do matrix multiplication (therefore doing parallel computation).


title: How to convert ndarray to autograd variable in GPU format?

body:
I am trying to do something like this,

data = torch.autograd.Variable(torch.from_numpy(nd_array))

It comes under the type as Variable[torch.FloatTensor], But I need Variable[torch.cuda.FloatTensor] also I want to do this in pytorch version 0.3.0 which lacks few methods like to(device) or set_default_device


title: Memory utilization much higher than it should be

body:
I'm using a simple method to extract descriptors from images and save them to disk into a .csv file. I have around 1M images and my network returns 512 features per image (float32).

Therefore, I estimate that at the end of the loop I would have 1e6 * 512 * 32/4 / 1e9 = 4.1GB. However, I observed that it is using more than twice the memory.

index is a string and class_id is a int64, so I don't think they are the culprit here.

I have already tried using gc.collect() without any success. Do you think my code is leaving references behind?

Here is the method:

def prepare_gallery(self, data_loader, TTA, pbar=False, dump_path=None):
    '''Compute embeddings for a data_loader and store it in model.
    This is required before predicting to a test set.
    New entries should be removed from data before calling this function
    to avoid inferring on useless images.
    data_loader: A linear loader containing the database that test is
    compared against.'''
    self.set_mode('valid')
    self.net.cuda()
    n_iter = len(data_loader.dataset) / data_loader.batch_size
    if pbar:
        loader = tqdm(enumerate(data_loader), total=n_iter)
    else:
        loader = enumerate(data_loader)
    # Run inference and get embeddings
    feat_list = []
    index_list = []
    class_list = []
    for i, (index, im, class_id) in loader:
        with torch.no_grad():
            feat = tta(self.net, im)
            # Returns something like np.random.random((32, 512))

        feat_list.extend(feat)
        index_list.extend(index)
        class_list.extend(class_id.item())

    if dump_path is not None:
        np.save(dump_path + '_ids', index_list)
        np.save(dump_path + '_cls', class_list)
        np.save(dump_path + '_feat', feat_list)

    return np.asarray(index_list), np.asarray(feat_list), np.asarray(class_list)


title: Using GPU takes even longer time for broadcasting in PyTorch

body:
print("CUDA available: ", torch.cuda.is_available())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

CUDA available:  True

x = torch.tensor([1, 2, 3]).to(device)
y = torch.tensor([1, 2, 3])

%%timeit -n 10000
z = x + 1

10000 loops, best of 3: 15.6 µs per loop

%%timeit -n 10000
z = y + 1

10000 loops, best of 3: 5.19 µs per loop


title: Resolving pytorch distributed execution printing multiple log statements for each process spawned?

body:
I am running pytorch distributed environment to train some models and in the same script I am also using logging to print status of the program. The problem is that with pytorch distributed since its spawning multiple processes I see my log statements being printed n times where n is the number of processes being spawned. Here's an example of it:

1.0, 0.05, 2.1823, 0.1703, 1.9799, 0.2352
1.0, 0.05, 2.1804, 0.1674, 1.9767, 0.2406
1.0, 0.05, 2.1814, 0.1697, 2.0053, 0.2154
2.0, 0.05, 2.1593, 0.1741, 2.0935, 0.192
2.0, 0.05, 2.1526, 0.1779, 2.1166, 0.1908
2.0, 0.05, 2.1562, 0.1812, 2.0868, 0.2076
3.0, 0.05, 1.9319, 0.2473, 1.8041, 0.2903
3.0, 0.05, 1.9386, 0.2413, 1.8037, 0.3017
3.0, 0.05, 1.9286, 0.2443, 1.815, 0.2939
4.0, 0.05, 1.7522, 0.3153, 1.828, 0.3131
4.0, 0.05, 1.7504, 0.3207, 1.7613, 0.3245
4.0, 0.05, 1.7522, 0.3223, 1.7841, 0.3209
5.0, 0.05, 1.5815, 0.3951, 1.5559, 0.4307
5.0, 0.05, 1.5767, 0.3939, 1.5326, 0.4205
5.0, 0.05, 1.588, 0.3909, 1.5882, 0.3995

Any ideas on how to avoid or resolve this issue? Thanks!


title: Seq to Seq model training

body:
I have couple of questions:

In a seq to seq model with varying input length, if you don't use the attention mask the RNN may end up computing the hidden state value for padded element? So thus it mean attention mask is mandatory else my output will be wrong?
How to deal with varying length labels then, let's say I have padded for passing it in batch. Now I don't want my padded elements to have an impact on my loss, so how do I ignore that?