title: Finetuning BERT with LSTM via PyTorch and transformers library. Metrics remain the same with hyperparameter changes

body:
I know for a fact that changing hyperparameters of an LSTM model or selecting different BERT layers causes changes in the classification result. I have tested this out using TensorFlow and Keras. I recently switched to Pytorch to do the same design, but no matter what I change, the result remains the same. Below is the code. Am I doing anything wrong?

def pad_sents(sents, pad_token):  #Pad list of sentences according to the longest sentence in the batch.
    sents_padded = []
    max_len = max(len(s) for s in sents)
    batch_size = len(sents)

    for s in sents:
        padded = [pad_token] * max_len
        padded[:len(s)] = s
        sents_padded.append(padded)

    return sents_padded

def sents_to_tensor(tokenizer, sents, device):
    tokens_list = [tokenizer.tokenize(str(sent)) for sent in sents]
    sents_lengths = [len(tokens) for tokens in tokens_list]
    tokens_list_padded = pad_sents(tokens_list, '[PAD]')
    sents_lengths = torch.tensor(sents_lengths, device=device)

    masks = []
    for tokens in tokens_list_padded:
        mask = [0 if token=='[PAD]' else 1 for token in tokens]
        masks.append(mask)
    masks_tensor = torch.tensor(masks, dtype=torch.long, device=device)
    tokens_id_list = [tokenizer.convert_tokens_to_ids(tokens) for tokens in tokens_list_padded]
    sents_tensor = torch.tensor(tokens_id_list, dtype=torch.long, device=device)

    return sents_tensor, masks_tensor, sents_lengths

class BERT_LSTM_Model(nn.Module):

    def __init__(self, device, dropout_rate, n_class, lstm_hidden_size=None):
        super(BERT_LSTM_Model, self).__init__()

        self.bert_config = BertConfig.from_pretrained('bert-base-uncased', output_hidden_states=True)
        self.bert = BertModel.from_pretrained('bert-base-uncased',config =self.bert_config)
        self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',config =self.bert_config)

        if not lstm_hidden_size:
            self.lstm_hidden_size = self.bert.config.hidden_size
        else:
            self.lstm_hidden_size = lstm_hidden_size
        self.n_class = n_class
        self.dropout_rate = dropout_rate
        self.lstm = nn.LSTM(self.bert.config.hidden_size, self.lstm_hidden_size, bidirectional=True)
        self.hidden_to_softmax = nn.Linear(self.lstm_hidden_size * 2, n_class, bias=True)
        self.dropout = nn.Dropout(p=self.dropout_rate)
        self.device = device

    def forward(self, sents):
        sents_tensor, masks_tensor, sents_lengths = sents_to_tensor(self.tokenizer, sents, self.device)
        encoded_layers = self.bert(input_ids=sents_tensor, attention_mask=masks_tensor)[2] #,output_all_encoded_layers=False)   #output_hidden_states output_hidden_states=True
        bert_hidden_layer = encoded_layers[12]
        bert_hidden_layer = bert_hidden_layer.permute(1, 0, 2)   #permute rotates the tensor. if tensor.shape = 3,4,5  tensor.permute(1,0,2), then tensor,shape= 4,3,5  (batch_size, sequence_length, hidden_size)

        enc_hiddens, (last_hidden, last_cell) = self.lstm(pack_padded_sequence(bert_hidden_layer, sents_lengths, enforce_sorted=False)) #enforce_sorted=False  #pack_padded_sequence(data and batch_sizes
        output_hidden = torch.cat((last_hidden[0], last_hidden[1]), dim=1)  # (batch_size, 2*hidden_size)
        output_hidden = self.dropout(output_hidden)
        pre_softmax = self.hidden_to_softmax(output_hidden)

        return pre_softmax

def batch_iter(data, batch_size, shuffle=False, bert=None):
    batch_num = math.ceil(data.shape[0] / batch_size)
    index_array = list(range(data.shape[0]))

    if shuffle:
        data = data.sample(frac=1)

    for i in range(batch_num):
        indices = index_array[i * batch_size: (i + 1) * batch_size]
        examples = data.iloc[indices]
        targets = list(examples.train_label.values)
        yield sents, targets  # list[list[str]] if not bert else list[str], list[int]

def validation(model, df_val, loss_func, device):
    was_training = model.training
    model.eval()
    train_BERT_tweet = list(df_val.train_BERT_tweet)
    train_label = list(df_val.train_label)
    val_batch_size = 16

    n_batch = int(np.ceil(df_val.shape[0]/val_batch_size))

    total_loss = 0.

    with torch.no_grad():
        for i in range(n_batch):
            sents =  train_BERT_tweet[i*val_batch_size: (i+1)*val_batch_size]
            targets = torch.tensor(train_label[i*val_batch_size: (i+1)*val_batch_size],
                                   dtype=torch.long, device=device)
            batch_size = len(sents)
            pre_softmax = model(sents)
            batch_loss = loss_func(pre_softmax, targets)
            total_loss += batch_loss.item()*batch_size

    if was_training:
        model.train()

    return total_loss/df_val.shape[0]

def train():
    label_name = ['Yes', 'Maybe', 'No']
    if torch.cuda.is_available():
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")

    start_time = time.time()
    print('Importing data...', file=sys.stderr)
    df_train = pd.read_csv('trainn.csv') #, index_col=0)
    df_val = pd.read_csv('valn.csv')   #, index_col=0)
    train_label = dict(df_train.train_label.value_counts())

    label_max = float(max(train_label.values()))

    train_label_weight = torch.tensor([label_max/train_label[i] for i in range(len(train_label))], device=device)

    print('Done! time elapsed %.2f sec' % (time.time() - start_time), file=sys.stderr)
    print('-' * 80, file=sys.stderr)

    start_time = time.time()
    print('Set up model...', file=sys.stderr)

    model = BERT_LSTM_Model(device=device, dropout_rate=0.2, n_class=len(label_name),lstm_hidden_size=768)
    optimizer = AdamW(model.parameters(), lr=1e-3, correct_bias=False)
    scheduler = get_linear_schedule_with_warmup(optimizer, warmup_steps=100, t_total=1000)  #changed the last 2 arguments to old ones

    model = model.to(device)
    print('Use device: %s' % device, file=sys.stderr)
    print('Done! time elapsed %.2f sec' % (time.time() - start_time), file=sys.stderr)
    print('-' * 80, file=sys.stderr)

    model.train()

    cn_loss = torch.nn.CrossEntropyLoss(weight=train_label_weight, reduction='mean')
    torch.save(cn_loss, 'loss_func3')  # for later testing

    train_batch_size =16
    valid_niter = 500
    log_every = 10
    model_save_path = 'NonLinear_bert_uncased_model.bin'

    num_trial = 0
    train_iter = patience = cum_loss = report_loss = 0
    cum_examples = report_examples = epoch = 0
    hist_valid_scores = []
    train_time = begin_time = time.time()
    print('Begin Maximum Likelihood training...')

    for epoch in range(20):

        for sents, targets in batch_iter(df_train, batch_size=train_batch_size, shuffle=True):  # for each epoch
            train_iter += 1
            optimizer.zero_grad()
            batch_size = len(sents)
            pre_softmax = model(sents)
            loss = cn_loss(pre_softmax, torch.tensor(targets, dtype=torch.long, device=device))
            loss.backward()
            optimizer.step()
            scheduler.step()
            batch_losses_val = loss.item() * batch_size
            report_loss += batch_losses_val
            cum_loss += batch_losses_val
            report_examples += batch_size
            cum_examples += batch_size

            if train_iter % log_every == 0:
                print('epoch %d, iter %d, avg. loss %.2f, '
                      'cum. examples %d, speed %.2f examples/sec, '
                      'time elapsed %.2f sec' % (epoch, train_iter,
                                                                                report_loss / report_examples,
                                                                                cum_examples,
                                                                                report_examples / (time.time() - train_time),
                                                                                time.time() - begin_time), file=sys.stderr)

                train_time = time.time()
                report_loss = report_examples = 0.

    #torch.save(model.state_dict(), 'LSTM_bert_uncased_model.bin')

            # perform validation
            if train_iter % valid_niter == 0:
                print('epoch %d, iter %d, cum. loss %.2f, cum. examples %d' % (epoch, train_iter,
                                                                                cum_loss / cum_examples,
                                                                                cum_examples), file=sys.stderr)
                cum_loss = cum_examples = 0.

                print('begin validation ...', file=sys.stderr)

                validation_loss = validation(model, df_val, cn_loss, device=device)   # dev batch size can be a bit larger

                print('validation: iter %d, loss %f' % (train_iter, validation_loss), file=sys.stderr)

                is_better = len(hist_valid_scores) == 0 or validation_loss < min(hist_valid_scores)
                hist_valid_scores.append(validation_loss)

                if is_better:
                    patience = 0
                    print('save currently the best model to [%s]' % model_save_path, file=sys.stderr)

                    torch.save(model.state_dict(), 'LSTM_bert_uncased_model.bin')
                    # also save the optimizers' state
                    torch.save(optimizer.state_dict(), model_save_path + '.optim')

                elif patience < 5:
                    patience += 1
                    print('hit patience %d' % patience, file=sys.stderr)

                    if patience == 20:
                        num_trial += 1
                        print('hit #%d trial' % num_trial, file=sys.stderr)
                        if num_trial == 3:
                            print('early stop!', file=sys.stderr)
                            exit(0)

                        # decay lr, and restore from previously best checkpoint
                        print('load previously best model and decay learning rate to %f%%' %
                              (0.1*100), file=sys.stderr)

                        # load model                                       model.load_state_dict(torch.load('LSTM_bert_uncased_model.bin'))
                        model = model.to(device)

                        print('restore parameters of the optimizers', file=sys.stderr)
                        optimizer.load_state_dict(torch.load(model_save_path + '.optim'))

                        # set new lr
                        for param_group in optimizer.param_groups:
                            param_group['lr'] *= 0.5

                        # reset patience
                        patience = 0

                if epoch == 100:
                    print('reached maximum number of epochs!', file=sys.stderr)
                    exit(0)

def test():
    label_name = ['Yes', 'Maybe', 'No']
    if torch.cuda.is_available():
        device = torch.device("cuda")
    else:
        device = torch.device("cpu")
    model = BERT_LSTM_Model(device=device, dropout_rate=0.3, n_class=len(label_name), lstm_hidden_size=768)

    model.load_state_dict(torch.load('LSTM_bert_uncased_model.bin'))
    model.to(device)
    model.eval()
    df_test = pd.read_csv('testn.csv')
    test_batch_size = 16
    n_batch = int(np.ceil(df_test.shape[0]/test_batch_size))
    cn_loss = torch.load('loss_func3', map_location=lambda storage, loc: storage).to(device)
    train_BERT_tweet = list(df_test.train_BERT_tweet)
    train_label = list(df_test.train_label)
    test_loss = 0.
    prediction = []
    prob = []
    softmax = torch.nn.Softmax(dim=1)

    with torch.no_grad():
        for i in range(n_batch):
            sents = train_BERT_tweet[i*test_batch_size: (i+1)*test_batch_size]
            targets = torch.tensor(train_label[i * test_batch_size: (i + 1) * test_batch_size],
                                   dtype=torch.long, device=device)
            batch_size = len(sents)

            pre_softmax = model(sents)
            batch_loss = cn_loss(pre_softmax, targets)
            test_loss += batch_loss.item()*batch_size
            prob_batch = softmax(pre_softmax)
            prob.append(prob_batch)

            prediction.extend([t.item() for t in list(torch.argmax(prob_batch, dim=1))])

    accuracy = accuracy_score(df_test.train_label.values, prediction)
    matthews = matthews_corrcoef(df_test.train_label.values, prediction)
    f1_macro = f1_score(df_test.train_label.values, prediction, average='macro')
    print('accuracy: %.2f' % accuracy)
    print('matthews coef: %.2f' % matthews)
    print('f1_macro: %.2f' % f1_macro)

TrainingModel = train()
TestingModel = test()

The data can be accessed from https://github.com/Kosisochi/DataSnippet
I didnt know how else to create a synthetic data.

Also, the training and validation loss remains quite high with the lowest being around 0.93.
I also tried a CNN and the same issue remained. Is there something I'm over looking? thanks for your help.


title: Loading huge text files for neural machine translation with Pytorch

body:
In PyTorch, I have written a dataset loading class for loading 2 text files as source and targets, for neural machine translation purpose. Each file has 93577946 lines, and each of them allocates 8GB memory on Hard Disc.

The class is as the following:

class LoadUniModal(Dataset):
    sources = []
    targets = []
    maxlen = 0
    lengths = []

    def __init__(self, src, trg, src_vocab, trg_vocab):
        self.src_vocab = src_vocab
        self.trg_vocab = trg_vocab

        with codecs.open(src, encoding="utf-8") as f:
            for line in f:
                tokens = line.replace("\n", "").split()
                self.maxlen = max(self.maxlen, len(tokens))
                self.sources.append(tokens)
        with codecs.open(trg, encoding="utf-8") as f:
            for line in f:
                tokens = line.replace("\n", "").split()
                self.maxlen = max(self.maxlen, len(tokens))
                self.targets.append(tokens)
                self.lengths.append(len(tokens)+2)

    # Overrride to give PyTorch access to any image on the dataset
    def __getitem__(self, index):

        # Source sentence processing
        tokens = self.sources[index]
        ntokens = [self.src_vocab['<START>']]
        for a in range(self.maxlen):
            if a <= (len(tokens) - 1):
                if tokens[a] in self.src_vocab.keys():
                    ntokens.append(self.src_vocab[tokens[a]])
                else:
                    ntokens.append(self.src_vocab['<UNK>'])
            elif a == len(tokens):
                ntokens.append(self.src_vocab['<END>'])
            elif a > len(tokens):
                ntokens.append(self.src_vocab['<PAD>'])

        source = torch.from_numpy(np.asarray(ntokens)).long()

        # Target sentence processing
        tokens = self.targets[index]
                ntokens = [self.trg_vocab['<START>']]
                for a in range(self.maxlen):
                        if a <= (len(tokens) - 1):
                                if tokens[a] in self.trg_vocab.keys():
                                        ntokens.append(self.trg_vocab[tokens[a]])
                                else:
                                        ntokens.append(self.trg_vocab['<UNK>'])
                        elif a == len(tokens):
                                ntokens.append(self.trg_vocab['<END>'])
                        elif a > len(tokens):
                                ntokens.append(self.trg_vocab['<PAD>'])

                target = torch.from_numpy(np.asarray(ntokens)).long()

        length = self.lengths[index]

        return [0], source, target, length

    def __len__(self):
        return len(self.sources)

I use the class in order to load dataset as follows:

def load_text_train_data(train_dir, src_vocab, trg_vocab, lang_pair, batch_size):

        tpl = ast.literal_eval(lang_pair)
        slang = tpl[1]
        tlang = tpl[2]

        strain_file = os.path.join(train_dir, "train"+slang)
        ttrain_file = os.path.join(train_dir, "train"+tlang)

        data_iter = LoadUniModal(strain_file, ttrain_file, src_vocab, trg_vocab)
        data_iter = DataLoader(data_iter, batch_size=batch_size)

        return data_iter

When I am trying to load the data, I get memory error.

How would it be possible to load the data without memory problem?

Thanks,


title: pytorch model -> onnx -> tensorflow

body:
I made pytorch model for machine learning and I want to convert this to tensorflow model. I think I converted pytorch model to onnx.
so now I want to convert onnx to tensorflow(pb) model.
Here is my code.
import onnx
from onnx_tf.backend import prepare
onnx_model = onnx.load("./sales_predict_model.onnx")  # load onnx model
tf_rep = prepare(onnx_model)  # prepare tf representation
tf_rep.export_graph("sales_predict_model.pb")  # export the model

And I got error here.
AssertionError: Tried to export a function which references untracked object Tensor("1076:0", shape=(), dtype=resource)`.
TensorFlow objects (e.g. `tf.Variable`) captured by functions must be tracked by assigning them to an attribute of a tracked object or assigned to an attribute of the main object directly.

I am using TensorFlow version 1.14.0.
Maybe Onnx version 1.7.0 (I checked this pip show onnx)
onnx-tf version 1.6.0 (pip show onnx-tf)
Here is the code below when I converted pytorch model to ONNX.
class LSTM(nn.Module):
    def __init__(self, input_dim, hidden_dim, num_layers, output_dim):
        super(LSTM, self).__init__()
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers
        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True)

        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):

        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).requires_grad_()
        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).requires_grad_()

        x = torch.tensor(x, dtype = torch.float32)
        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))
        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))

        out = out[:, -1, :]
        out = self.fc(out)
        return out

PATH = './model/file300input5lb1ep100drop0W_V7'

model = torch.load(PATH + 'model.pt')
model.eval()

x = torch.randn(1, 1, 5, requires_grad=True)
torch_out = model(x)

torch.onnx.export(model,
                 x,
                 "sales_predict_model.onnx",
                 export_params=True,
                 opset_version=10,
                 do_constant_folding=True,
                 input_names = ['input'],
                 output_names =  ['x'],
                 dynamic_axes={'input' : {0 : 'batch_size'},
                              'output' : {0 : 'batch_size'}})


title: Convert nngraph model to nn model

body:
I want to convert the pix2pix Image to Image translation model (https://github.com/phillipi/pix2pix) which is built using nngraph.

When I try to convert it to caffe model using torch to caffe tool (https://github.com/facebook/fb-caffe-exts#torch2caffe) I get the error "unknown class nn.gModule".

I also try to load torch model in pytorch by load_lua from torch.utils.serialization where I get similar error.

Since, it seems nngraph is not supported for conversion. Is there a way to convert nngraph model to nn model without the need of rewriting or retraining? Down the line, I want to convert the pix2pix model into various formats like caffe, caffe2, coreML and others to test the performance in mobile environment.

Thanks


title: Pytorch LSTM not using GPU

body:
I'm trying to train a pytorch LSTM model connected with couple of MLP layers. The model is coded as follows:

class RNNBlock(nn.Module):

def __init__(self, in_dim, hidden_dim, num_layer=1, dropout=0):
    super(RNNBlock, self).__init__()

    self.hidden_dim = hidden_dim
    self.num_layer = num_layer

    self.lstm = nn.LSTM(in_dim, hidden_dim, num_layer, dropout)

def forward(self, onehot, length):
    batch_size = onehot.shape[0]

    h_in = nn.Parameter(torch.randn(self.num_layer, batch_size, self.hidden_dim))
    c_in = nn.Parameter(torch.randn(self.num_layer, batch_size, self.hidden_dim))
    packed = nn.utils.rnn.pack_padded_sequence(onehot, length, batch_first=True)

    output, (h_out, c_out) = self.lstm(packed, (h_in, c_in))
    unpacked, unpacked_length = nn.utils.rnn.pad_packed_sequence(output, batch_first=True)
    vectors = list()
    for i, vector in enumerate(unpacked):
        vectors.append(unpacked[i, unpacked_length[i]-1, :].view(1, -1))
    out = torch.cat(vectors, 0)
    return out

class Predictor(nn.Module):

def __init__(self, in_dim, out_dim, act=None):
    super(Predictor, self).__init__()

    self.linear = nn.Linear(in_dim, out_dim)
    nn.init.xavier_normal_(self.linear.weight)
    self.activation = act

def forward(self, x):
    out = self.linear(x)
    if self.activation != None:
        out = self.activation(out)
    return out

class RNNNet(nn.Module):

def __init__(self, args):
    super(RNNNet, self).__init__()

    self.rnnBlock = RNNBlock(args.in_dim, args.hidden_dim, args.num_layer, args.dropout)
    self.pred1 = Predictor(args.hidden_dim, args.pred_dim1, act=nn.ReLU())
    self.pred2 = Predictor(args.pred_dim1, args.pred_dim2, act=nn.ReLU())
    self.pred3 = Predictor(args.pred_dim2, args.out_dim)

def forward(self, onehot, length):
    out = self.rnnBlock(onehot, length)
    out = self.pred1(out)
    out = self.pred2(out)
    out = self.pred3(out)
    return out

and this is my train and experiment functions

def train(model, device, optimizer, criterion, data_train, bar, args):
epoch_train_loss = 0
epoch_train_mae = 0

for i, batch in enumerate(data_train):
    list_onehot = torch.tensor(batch[0]).cuda().float()
    list_length = torch.tensor(batch[1]).cuda()
    list_logP = torch.tensor(batch[2]).cuda().float()
    # Sort onehot tensor with respect to the sequence length.
    list_length, list_index = torch.sort(list_length, descending=True)
    list_length.cuda()
    list_index.cuda()
    list_onehot = torch.Tensor([list_onehot.tolist()[i] for i in list_index]).cuda().float()

    model.train()
    optimizer.zero_grad()

    list_pred_logP = model(list_onehot, list_length).squeeze().cuda()
    list_pred_logP.require_grad = False

    train_loss = criterion(list_pred_logP, list_logP)
    train_mae = mean_absolute_error(list_pred_logP.tolist(), list_logP.tolist())
    epoch_train_loss += train_loss.item()
    epoch_train_mae += train_mae

    train_loss.backward()
    optimizer.step()

    bar.update(len(list_onehot))

epoch_train_loss /= len(data_train)
epoch_train_mae /= len(data_train)

return model, epoch_train_loss, epoch_train_mae

def experiment(dict_partition, device, bar, args):
time_start = time.time()

model = RNNNet(args)
model.cuda()

if args.optim == 'Adam':
    optimizer = optim.Adam(model.parameters(),
                           lr=args.lr,
                           weight_decay=args.l2_coef)
elif args.optim == 'RMSprop':
    optimizer = optim.RMSprop(model.parameters(),
                           lr=args.lr,
                           weight_decay=args.l2_coef)
elif args.optim == 'SGD':
    optimizer = optim.SGD(model.parameters(),
                           lr=args.lr,
                           weight_decay=args.l2_coef)
else:
    assert False, 'Undefined Optimizer Type'

criterion = nn.MSELoss()
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma)

list_train_loss = list()
list_val_loss = list()
list_train_mae = list()
list_val_mae = list()

data_train = DataLoader(dict_partition['train'], batch_size=args.batch_size, shuffle=args.shuffle)
data_val = DataLoader(dict_partition['val'], batch_size=args.batch_size, shuffle=args.shuffle)

for epoch in range(args.epoch):
    scheduler.step()

    model, train_loss, train_mae = train(model, device, optimizer, criterion, data_train, bar, args)
    list_train_loss.append(train_loss)
    list_train_mae.append(train_mae)

    mode, val_loss, val_mae = validate(model, device, criterion, data_val, bar, args)
    list_val_loss.append(val_loss)
    list_val_mae.append(val_mae)

data_test = DataLoader(dict_partition['test'], batch_size=args.batch_size, shuffle=args.shuffle)

mae, std, logP_total, pred_logP_total = test(model, device, data_test, args)

time_end = time.time()
time_required = time_end - time_start

args.list_train_loss = list_train_loss
args.list_val_loss = list_val_loss
args.list_train_mae = list_train_mae
args.list_val_mae = list_val_mae
args.logP_total = logP_total
args.pred_logP_total = pred_logP_total
args.mae = mae
args.std = std
args.time_required = time_required

return args

The list_onehot and list_length tensors are loaded from the DataLoader and uploaded to GPU. Then, to use packed sequence as input, I’ve sorted the both list_onehot and list_length and uploaded to GPU. The model was uploaded to GPU and h_in, c_in tensors and packed sequence object were also uploaded to the GPU. However, when I try to run this code, it does not use GPU but only use CPU. What should I do to use GPU to train this model?


title: ImageDataBunch not loading in my files/labels

body:
I have a dataset of images stored on my local disk that I name as such:

path = '../input/resized/Resized Dataset.zip/Resized Data'

And I am calling an instance of an ImageDataBunch from the fastai.vision library using the following code:

data = ImageDataBunch.from_folder(path, train=".", ds_tfms=get_transforms(),size=224, valid_pct=0.2)

but for some reason I am getting an error

File "../src/script.py", line 17, in <module>
    data = ImageDataBunch.from_folder(path, train=".", ds_tfms=get_transforms(),size=20, valid_pct=0.2)
  File "/opt/conda/lib/python3.6/site-packages/fastai/vision/data.py", line 110, in from_folder
    src = src.label_from_folder(classes=classes)
  File "/opt/conda/lib/python3.6/site-packages/fastai/data_block.py", line 463, in _inner
    self.train = ft(*args, from_item_lists=True, **kwargs)
  File "/opt/conda/lib/python3.6/site-packages/fastai/data_block.py", line 292, in label_from_folder
    label_cls=label_cls, **kwargs)
  File "/opt/conda/lib/python3.6/site-packages/fastai/data_block.py", line 287, in label_from_func
    return self._label_from_list([func(o) for o in self.items], label_cls=label_cls, **kwargs)
  File "/opt/conda/lib/python3.6/site-packages/fastai/data_block.py", line 262, in _label_from_list
    label_cls = self.get_label_cls(labels, label_cls=label_cls, **kwargs)
  File "/opt/conda/lib/python3.6/site-packages/fastai/data_block.py", line 251, in get_label_cls
    it = index_row(labels,0)
  File "/opt/conda/lib/python3.6/site-packages/fastai/core.py", line 250, in index_row
3.5s
3
    return a[idxs]
IndexError: index 0 is out of bounds for axis 0 with size 0

I have used multiple different datasets and have the same problem


title: GAN measurement - FID score

body:
I have two GANs and I want to compare their results using FID (Fréchet Inception Distance).
I have trianed the networks with the same dataset of frogs images, and by looking at the results (the generated images) one network yields better results but it's FID score is higher.
I computed the FID score between the original dataset and the generated images of each network.

I have read that lower FID values mean better image quality and diversity,
which is not consistent with the results I have seen.

Is there an explanation for that?


title: PyTorch AutoEncoder - Decoded output dimension not the same as input

body:
I am building a Custom Autoencoder to train on a dataset. My model is as follows

class AutoEncoder(nn.Module):
    def __init__(self):
        super(AutoEncoder,self).__init__()

        self.encoder = nn.Sequential(
        nn.Conv2d(in_channels = 3, out_channels = 32, kernel_size=3,stride=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(in_channels = 32, out_channels = 64, kernel_size=3,stride=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(in_channels = 64, out_channels = 128, kernel_size=3,stride=1),
        nn.ReLU(inplace=True),
        nn.Conv2d(in_channels=128,out_channels=256,kernel_size=5,stride=2),
        nn.ReLU(inplace=True),
        nn.Conv2d(in_channels=256,out_channels=512,kernel_size=5,stride=2),
        nn.ReLU(inplace=True),
        nn.Conv2d(in_channels=512,out_channels=1024,kernel_size=5,stride=2),
        nn.ReLU(inplace=True)
        )

        self.decoder = nn.Sequential(
        nn.ConvTranspose2d(in_channels=1024,out_channels=512,kernel_size=5,stride=2),
        nn.ReLU(inplace=True),
        nn.ConvTranspose2d(in_channels=512,out_channels=256,kernel_size=5,stride=2),
        nn.ReLU(inplace=True),
        nn.ConvTranspose2d(in_channels=256,out_channels=128,kernel_size=5,stride=2),
        nn.ReLU(inplace=True),
        nn.ConvTranspose2d(in_channels=128,out_channels=64,kernel_size=3,stride=1),
        nn.ReLU(inplace=True),
        nn.ConvTranspose2d(in_channels=64,out_channels=32,kernel_size=3,stride=1),
        nn.ReLU(inplace=True),
        nn.ConvTranspose2d(in_channels=32,out_channels=3,kernel_size=3,stride=1),
        nn.ReLU(inplace=True)
        )

    def forward(self,x):
        x = self.encoder(x)
        print(x.shape)
        x = self.decoder(x)
        return x

def unit_test():
    num_minibatch = 16
    img = torch.randn(num_minibatch, 3, 512, 640).cuda(0)
    model = AutoEncoder().cuda()
    model = nn.DataParallel(model)
    output = model(img)
    print(output.shape)

if __name__ == '__main__':
    unit_test()

As you can see, my input dimension is (3, 512, 640) but my output after passing it through the decoder is (3, 507, 635). Am I missing something while adding the Conv2D Transpose layers ?

Any help would be appreciated. Thanks


title: PyTorch - convert ProGAN agent from pth to onnx

body:
I trained a ProGAN agent using this PyTorch reimplementation, and I saved the agent as a .pth. Now I need to convert the agent into the .onnx format, which I am doing using this scipt:

from torch.autograd import Variable

import torch.onnx
import torchvision
import torch

device = torch.device("cuda")

dummy_input = torch.randn(1, 3, 64, 64)
state_dict = torch.load("GAN_agent.pth", map_location = device)

torch.onnx.export(state_dict, dummy_input, "GAN_agent.onnx")

Once I run it, I get the error AttributeError: 'collections.OrderedDict' object has no attribute 'state_dict' (full prompt below). As far as I understood, the problem is that converting the agent into .onnx requires more information. Am I missing something?

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-2-c64481d4eddd> in <module>
     10 state_dict = torch.load("GAN_agent.pth", map_location = device)
     11
---> 12 torch.onnx.export(state_dict, dummy_input, "GAN_agent.onnx")

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\__init__.py in export(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes, keep_initializers_as_inputs)
    146                         operator_export_type, opset_version, _retain_param_name,
    147                         do_constant_folding, example_outputs,
--> 148                         strip_doc_string, dynamic_axes, keep_initializers_as_inputs)
    149
    150

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in export(model, args, f, export_params, verbose, training, input_names, output_names, aten, export_raw_ir, operator_export_type, opset_version, _retain_param_name, do_constant_folding, example_outputs, strip_doc_string, dynamic_axes, keep_initializers_as_inputs)
     64             _retain_param_name=_retain_param_name, do_constant_folding=do_constant_folding,
     65             example_outputs=example_outputs, strip_doc_string=strip_doc_string,
---> 66             dynamic_axes=dynamic_axes, keep_initializers_as_inputs=keep_initializers_as_inputs)
     67
     68

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in _export(model, args, f, export_params, verbose, training, input_names, output_names, operator_export_type, export_type, example_outputs, propagate, opset_version, _retain_param_name, do_constant_folding, strip_doc_string, dynamic_axes, keep_initializers_as_inputs, fixed_batch_size)
    414                                                         example_outputs, propagate,
    415                                                         _retain_param_name, do_constant_folding,
--> 416                                                         fixed_batch_size=fixed_batch_size)
    417
    418         # TODO: Don't allocate a in-memory string for the protobuf

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in _model_to_graph(model, args, verbose, training, input_names, output_names, operator_export_type, example_outputs, propagate, _retain_param_name, do_constant_folding, _disable_torch_constant_prop, fixed_batch_size)
    277             model.graph, tuple(in_vars), False, propagate)
    278     else:
--> 279         graph, torch_out = _trace_and_get_graph_from_model(model, args, training)
    280         state_dict = _unique_state_dict(model)
    281         params = list(state_dict.values())

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\onnx\utils.py in _trace_and_get_graph_from_model(model, args, training)
    226     # A basic sanity check: make sure the state_dict keys are the same
    227     # before and after running the model.  Fail fast!
--> 228     orig_state_dict_keys = _unique_state_dict(model).keys()
    229
    230     # By default, training=False, which is good because running a model in

~\anaconda3\envs\Basemap_upres\lib\site-packages\torch\jit\__init__.py in _unique_state_dict(module, keep_vars)
    283     # id(v) doesn't work with it. So we always get the Parameter or Buffer
    284     # as values, and deduplicate the params using Parameters and Buffers
--> 285     state_dict = module.state_dict(keep_vars=True)
    286     filtered_dict = type(state_dict)()
    287     seen_ids = set()

AttributeError: 'collections.OrderedDict' object has no attribute 'state_dict'


title: Torchviz doesn't show image

body:
I am trying to use torchviz but it's not giving any output. No error or warning either..

import numpy as np
import torch
from torchviz import make_dot
yhat= a+b*x_train_tensor
make_dot(yhat).render("attached", format="png")

I also tried using make_dot(yhat) but that doesn't work either. In the above only a small file with the name "attached" is being created but is not an image. Obviously x_train_tensor and yhat are all tensors and compute just fine. It's the visualization that I'm having trouble with.

I have already seen these posts.

How do I visualize a net in Pytorch?

https://github.com/szagoruyko/pytorchviz/issues/5

But couldn't solve the problem.
I'm using python 3.7.4 64 bit in vscode


title: How conv1d pytorch operates on a sequence of characters or frames?

body:
I understand convolution filters when applied to an image (e.g. an 224x224 image with 3 in-channels transformed by 56 total filters of 5x5 conv to a 224x224 image with 56 out-channels). The key is that there are 56 different filters each with 5x5x3 weights that end up producing output image 224x224, 56 (term after comma is output channels).

But I can't seem to understand how conv1d filter works in seq2seq models on a sequence of characters. One of the models i was looking at https://arxiv.org/pdf/1712.05884.pdf has a "post-net layer is comprised of 512 filters with shape 5×1" that operates on a spectrogram frame 80-d (means 80 different float values in the frame), and the result of filter is a 512-d frame.

I don't understand what in_channels, out_channels mean in pytorch conv1d definition as in images I can easily understand what in-channels/out-channels mean, but for sequence of 80-float values frames I'm at loss. What do they mean in the context of seq2seq model like this above?
How do 512, 5x1 filters on 80 float values produce 512 float values?**
Wouldn't a 5x1 filter when operating on 80 float values just produce 80 float values (by just taking 5 consecutive values at a time of those 80)? How many weights total these 512 filters have?**

The layer when printed in pytorch shows up as:

(conv): Conv1d(80, 512, kernel_size=(5,), stride=(1,), padding=(2,))

and the parametes in this layer show up as:

postnet.convolutions.0.0.conv.weight : 512x80x5 = 204800

Shouldn't the weights in this layer instead be 512*5*1 as it only has 512 filters each of which is 5x1?


title: Intermittent "RuntimeError: CUDA out of memory" error in Google Colab Fine Tuning BERT Base Cased with Transformers and PyTorch

body:
I'm running the following code to fine-tune a BERT Base Cased model in Google Colab. Sometimes the code runs fine first time without error. Other times, the same code, using the same data, results in a "CUDA out of memory" error. Previously, restarting the runtime or exiting the notebook, going back into the notebook, doing a factory runtime restart, and re-running the code runs successfully without error. Just now though, I've tried a restart and re-try 5 times and got the error every time.

The issue doesn't appear to be the combination of data and code that I'm using because sometimes it works without error. So it appears to be something to do with the Google Colab runtime.

Does anyone know why this is happening, why it is intermittent, and/or what I can do about it?

I'm using Huggingface's transformers library and PyTorch.

The code cell that results in an error:

# train the model
%%time

history = defaultdict(list)

for epoch in range(EPOCHS):

  print(f'Epoch {epoch + 1}/{EPOCHS}')
  print('-' * 10)

  train_acc, train_loss = train_epoch(
    model,
    train_data_loader,
    loss_fn,
    optimizer,
    device,
    scheduler,
    train_set_length
  )

  print(f'Train loss {train_loss} accuracy {train_acc}')

  dev_acc, dev_loss = eval_model(
    model,
    dev_data_loader,
    loss_fn,
    device,
    evaluation_set_length
  )

  print(f'Dev   loss {dev_loss} accuracy {dev_acc}')

  history['train_acc'].append(train_acc)
  history['train_loss'].append(train_loss)
  history['dev_acc'].append(dev_acc)
  history['dev_loss'].append(dev_loss)

  model_filename = f'model_{epoch}_state.bin'
  torch.save(model.state_dict(), model_filename)

The full error:

RuntimeError                              Traceback (most recent call last)
<ipython-input-29-a13774d7aa75> in <module>()
----> 1 get_ipython().run_cell_magic('time', '', "\nhistory = defaultdict(list)\n\nfor epoch in range(EPOCHS):\n\n  print(f'Epoch {epoch + 1}/{EPOCHS}')\n  print('-' * 10)\n\n  train_acc, train_loss = train_epoch(\n    model,\n    train_data_loader,    \n    loss_fn, \n    optimizer, \n    device, \n    scheduler, \n    train_set_length\n  )\n\n  print(f'Train loss {train_loss} accuracy {train_acc}')\n\n  dev_acc, dev_loss = eval_model(\n    model,\n    dev_data_loader,\n    loss_fn, \n    device, \n    evaluation_set_length\n  )\n\n  print(f'Dev   loss {dev_loss} accuracy {dev_acc}')\n\n  history['train_acc'].append(train_acc)\n  history['train_loss'].append(train_loss)\n  history['dev_acc'].append(dev_acc)\n  history['dev_loss'].append(dev_loss)\n  \n  model_filename = f'model_{epoch}_state.bin'\n  torch.save(model.state_dict(), model_filename)")

15 frames
<decorator-gen-60> in time(self, line, cell, local_ns)

<timed exec> in <module>()

/usr/local/lib/python3.6/dist-packages/transformers/modeling_bert.py in forward(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask)
    234         # Take the dot product between "query" and "key" to get the raw attention scores.
    235         attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
--> 236         attention_scores = attention_scores / math.sqrt(self.attention_head_size)
    237         if attention_mask is not None:
    238             # Apply the attention mask is (precomputed for all layers in BertModel forward() function)

RuntimeError: CUDA out of memory. Tried to allocate 24.00 MiB (GPU 0; 7.43 GiB total capacity; 5.42 GiB already allocated; 8.94 MiB free; 5.79 GiB reserved in total by PyTorch)


title: Error Compiling C++/Cuda extension with Pytorch Cuda c++ in MSVC using CMake

body:
I am trying to build a c++/cuda extension with Pytorch following the tutorial here, (with instructions how to use pytorch with c++ here). My environment details are:

Using Microsoft Visual Studio 2019 version 16.6.5
Windows 10
libtorch c++ debug 1.70 with cuda 11.0 installed from the pytorch website

I am using this cmake code where I set the include directory for python 3.6 and the library for python36.lib
cmake_minimum_required (VERSION 3.8)

project ("DAConvolution")

find_package(Torch REQUIRED)

# Add source to this project's executable.
add_executable (DAConvolution "DAConvolution.cpp" "DAConvolution.h")

include_directories("C:/Users/James/Anaconda3/envs/masters/include")

target_link_libraries(DAConvolution "${TORCH_LIBRARIES}" "C:/Users/James/Anaconda3/envs/masters/libs/python36.lib")

if (MSVC)
  file(GLOB TORCH_DLLS "${TORCH_INSTALL_PREFIX}/lib/*.dll")
  add_custom_command(TARGET DAConvolution
                     POST_BUILD
                     COMMAND ${CMAKE_COMMAND} -E copy_if_different
                     ${TORCH_DLLS}
                     $<TARGET_FILE_DIR:DAConvolution>)
endif (MSVC)

I set the CMake command arguments to be -DCMAKE_PREFIX_PATH=C:\libtorch  (my path to libtorch debug mentioned above). I am building with the x64-debug option in MSVC version (as building with the x-64 Release option gives me a torch-NOTFOUND error).
The example DAConvolution.cpp file is:

#ifdef _DEBUG
#undef _DEBUG
#include <python.h>
#define _DEBUG
#else
#include <python.h>
#endif

#include <torch/extension.h>

Where I have undefined the _DEBUG flag so that the linker does not look for the python36_d.lib file (which I do not have).
I am getting a linking error:

Simply including torch.h works fine, but when I want to include the extension header thats when I get these problems, as it uses Pybind 11 I believe. Any insights much appreciated. I have tried to include all the info I can, but would be happy to give more information.


title: TypeError: expected CPU (got CUDA)

body:
import torch
torch.cuda.is_available()
torch.cuda.current_device()
torch.cuda.get_device_name(0)
torch.cuda.memory_reserved()
torch.cuda.memory_allocated()
torch.cuda.memory_allocated()
var1=torch.FloatTensor([1.0,2.0,3.0]).cuda()
var1
var1.device
import pandas as pd
df=pd.read_csv('diabetes.csv')
df.head()
df.isnull().sum()

import seaborn as sns
import numpy as np
df['Outcome']=np.where(df['Outcome']==1,"Diabetic","No Diabetic")
df.head()
sns.pairplot(df,hue="Outcome")
X=df.drop('Outcome',axis=1).values### independent features
y=df['Outcome'].values###dependent features
from sklearn.model_selection import train_test_split

X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0)
y_train
import torch
import torch.nn as nn
import torch.nn.functional as F
X_train=torch.FloatTensor(X_train).cuda()
X_test=torch.FloatTensor(X_test).cuda()
y_train=torch.LongTensor(y_train).cuda()
y_test=torch.LongTensor(y_test).cuda()

when I Run this code I got this error:
Traceback (most recent call last):
  File "<stdin>", line 24, in <module>
TypeError: expected CPU (got CUDA)

How to can I solve this error?


title: How to implement LSTM layer with multiple cells per memory block in Pytorch?

body:
I intend to implement an LSTM in Pytorch with multiple memory cell blocks - or multiple LSTM units, an LSTM unit being the set of a memory block and its gates - per layer, but it seems that the base class torch.nn.LSTM enables only to implement a multi-layer LSTM with one LSTM unit per layer:

lstm = torch.nn.LSTM(input_size, hidden_size, num_layers)

where (from the Pytorch's documentation):

input_size is the input dimension of the network,
hidden_size is the hidden state dimension for every layer (i.e. the dimension of every layer),
num_layer is the number of layers of the network

Thereupon, from above, each LSTM unit has exactly one cell (the cell state for each LSTM unit is thus a scalar) because for each layer the dimension of the cell state corresponds to the dimension of the hidden state (i.e. hidden_size).

However in the original LSTM model proposed by Hochreiter and Schmidhuber
[1997], each LSTM block/unit can contains several cells:
LSTM Network [Hochreiter, 1997]

Is there a way to do so?


title: Changing the np array does not change the Torch Tensor automatically?

body:
I was going through the basic tutorials of PyTorch and came across conversion between NumPy arrays and Torch tensors. The documentation says:

  The Torch Tensor and NumPy array will share their underlying memory locations, and changing one will change the other.

But, this does not seem to be the case in the below code:

import numpy as np

a = np.ones((3,3))
b = torch.from_numpy(a)

np.add(a,1,out=a)
print(a)
print(b)

In the above case I see the changes automatically reflected in the output:

[[2. 2. 2.]
 [2. 2. 2.]
 [2. 2. 2.]]
tensor([[2., 2., 2.],
        [2., 2., 2.],
        [2., 2., 2.]], dtype=torch.float64)

But same doesn't happen when I write something like this:

a = np.ones((3,3))
b = torch.from_numpy(a)

a = a + 1
print(a)
print(b)

I get the following output:

[[2. 2. 2.]
 [2. 2. 2.]
 [2. 2. 2.]]
tensor([[1., 1., 1.],
        [1., 1., 1.],
        [1., 1., 1.]], dtype=torch.float64)

What am I missing here?


title: Output from LSTM not changing for different inputs

body:
I have the an LSTM implemented in PyTorch as below.

import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

class LSTM(nn.Module):
    """
    Defines an LSTM.
    """

    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):

        super(LSTM, self).__init__()

        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True)

    def forward(self, input_data):

        lstm_out_pre, _ = self.lstm(input_data)
        return lstm_out_pre

model = LSTM(input_dim=2, hidden_dim=2, output_dim=1, num_layers=8)

random_data1 = torch.Tensor(np.random.standard_normal(size=(1, 5, 2)))
random_data2 = torch.Tensor(np.random.standard_normal(size=(1, 5, 2)))

out1 = model(random_data1).detach().numpy()
out2 = model(random_data2).detach().numpy()

print(out1)
print(out2)

I am simply creating an LSTM network and passing two random inputs into it. The outputs does not make sense because no matter what random_data1 and random_data2 is, out1 and out2 are always the same. This does not make any sense to me as random inputs multiplied with random weights should give different outputs.

This does not seem to be the case if I use less number of hidden layers. With num_layers=2, this effect seems to be nil. And as you increase it, out1 and out2 keeps on getting closer. This does not make sense to me because with more layers of the LSTM stacked on top of each other, we are multiplying the input with more number of random weights which should magnify the differences in the input and give a very different output.

Can someone please explain this behavior? Is there something wrong with my implementation?

In one particular run, random_data1 is

tensor([[[-2.1247, -0.1857],
         [ 0.0633, -0.1089],
         [-0.6460, -0.1079],
         [-0.2451,  0.9908],
         [ 0.4027,  0.3619]]])

random_data2 is

tensor([[[-0.9725,  1.2400],
         [-0.4309, -0.7264],
         [ 0.5053, -0.9404],
         [-0.6050,  0.9021],
         [ 1.4355,  0.5596]]])

out1 is

[[[0.12221643 0.11449362]
  [0.18342148 0.1620608 ]
  [0.2154751  0.18075559]
  [0.23373817 0.18768947]
  [0.24482158 0.18987371]]]

out2 is

[[[0.12221643 0.11449362]
  [0.18342148 0.1620608 ]
  [0.2154751  0.18075559]
  [0.23373817 0.18768945]
  [0.24482158 0.18987371]]]

EDIT:
I am running on the following configurations -

PyTorch - 1.0.1.post2
Python - 3.6.8 with GCC 7.3.0
OS - Pop!_OS 18.04 (Ubuntu 18.04, more-or-less)
CUDA - 9.1.85
Nvidia driver - 410.78


title: How to load/covert pandas dataframe with string values into pytorch dataset

body:
I am working on windows machine.
I am trying to load pandas dataframe which has one columns as text(string) and another column as label (int). I want to load this dataframe into pytorch dataset.
Let's say above data is in dataframe df.

I am using below

train_target = torch.DoubleTensor(df['Text'].values)

I also used below to load directly from csv file.

TEXT1 = data.Field(tokenize = 'spacy', include_lengths = True)
LABEL1 = data.LabelField(dtype = torch.float)
fields = [('text',TEXT1),(label,LABEL1)]
train_data, valid_data, test_data = data.TabularDataset.splits(
                                        path = '.',
                                        train = '1.csv',
                                        validation = '1.csv',
                                        test = '1.csv',
                                        format = 'csv',
                                        fields = fields,
                                        skip_header = True
)

But it is giving error as

OverflowError: Python int too large to convert to C long

I am searching for such text conversion into pytorch,tensor object .


title: Torch use of Spacy -- Strange Errors w/ Standard Dataset

body:
In the following code:

from torchtext.datasets import Multi30k
from torchtext.data import Field, BucketIterator
import spacy
from spacy.tokenizer import Tokenizer

nlp = spacy.load("en_core_web_sm")
tokenizer = Tokenizer(nlp.vocab)

SRC = Field(tokenize = tokenizer,
            init_token = '<sos>',
            eos_token = '<eos>',
            lower = True)

TRG = SRC

train_data, valid_data, test_data = Multi30k.splits(exts = ('.en', '.en'), fields = (SRC, TRG), root='.')

Running this code produces the following error:

descriptor 'lower' requires a 'str' object but received a 'spacy.tokens.doc.Doc

Using a debugger, I find that the error occurs at the following line (in classmethod preprocess(x) of class Field(RawField) of the Torch file 'pipeline.py'

if self.lower:
   x = Pipeline(six.text_type.lower)(x)

where indeed the arg 'x' is of type spacy.tokens.doc.Doc. But, to my surprise, when I move up the stack one step I find that the line which calls preprocess(x) is this --

setattr(ex, name, field.preprocess(val))

but in this case the type of 'val' is 'str', as required for use with 'lower' (!!??)

My code is a completely standard use case, so what am I missing??

If instead I use 'lower'=False, the code runs without error, and the Train/Test/Val data produced are correct (the train_data produces 29,000 pairs). However, if I then run --

SRC.build_vocab(train_data, min_freq = 2)

I get a vocab with only 4 items, namely:

Vocab stoi:
 [('<unk>', 0), ('<pad>', 1), ('<sos>', 2), ('<eos>', 3)]

So something is wrong here as well.

Finally, if I use tokenize='spacy' in the Field constructor, instead of passing a reference to a tokenizer function -- as I do in the code above, all of these problems disappear, except that I can only do this in a collab notebook. If I try to run with tokenize='spacy' on my machine, I get this error (vscode in Windows 10):

"OSError: [E050] Can't find model 'en'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory."

But clearly the spacy model has been downloaded since I was able to pass the corresponding tokenizer function to the same keyword arg (I checked this also with the debugger -- the tokenizer gets passed in).

I'd really like to understand what's going on . . .

(I've set SRC=TRG to simplify, but I get identical behavior if I use a language pair, e.g. SRC=en, TRG=de)


PyTorch

I'm trying to build CNN on my own Dataset which has 39 different classes.

I face one Runtime Error

Error

RuntimeError                              Traceback (most recent call last)
<ipython-input-42-67f1329ebfd8> in <module>()
     16         images, labels = images.to(device), labels.to(device)
     17
---> 18         train = Variable(images.view(100, 1, 28, 28))
     19         labels = Variable(labels)
     20

RuntimeError: shape '[100, 1, 28, 28]' is invalid for input of size 62720

RuntimeError: shape '[100, 1, 28, 28]' is invalid for input of size 62720.


title: How to apply linear layer to 2D layer only in one dimension (by row or by column) - partially connected layers

body:
I'm trying to apply a linear layer to a 2D matrix of tensors connecting it only by column as in the picture below.

The input shape is (batch_size, 3, 50). I first tried with 2D convolution, adding a 1 channel dimension, so input shape is (batch_size, 1, 3, 50)

import torch.nn as nn
import torch

class ColumnConv(nn.Module):
    def __init__(self):
        self.layers = nn.Sequential(
            nn.Conv2d(
                in_channels=1,
                out_channels=1,
                kernel_size=(3, 1),
                stride=1,
            ),  # shape is B, 1, 1, 50
            nn.ReLU(),
            nn.Flatten() #shape is B, 50
        )

    def forward(self, x):
        return self.layers(x)

But it doesn't seem to work.
I'm planning to use a list of 50 nn.Linear layers and apply them to column slices of input, but it seems much more like a workaround not optimized for performance.

Is there a more "pytorchic" way of doing this?


title: AttributeError: 'Example' object has no attribute 'Insult' when build vocab using build_vocab() from torchtext

body:
I tried to build vocabulary using .build_vocab() from torchtext in colab. It returns the error message: AttributeError: 'Example' object has no attribute 'Insult'

My question is similar to @szymix12 s question. His answer is to make sure the order of the fields passed is the same as the csv headers. I confirm that the order I assign is correct. The csv data got two columns:"Insult" (LABEL) and "Comment"(TEXT). "Insult" is a binary label indicator (0 or 1).

The code is as below, I have also made a colab notebook. Feel free to run.

import os
import torch
from torchtext import data, datasets
from torchtext.utils import download_from_url

CSV_FILENAME = 'data.csv'
CSV_GDRIVE_URL = 'https://drive.google.com/open?id=1ctPKO-_sJbmc8RodpBZ5-3EyYBWlUy5Pbtjio5pyq00'
download_from_url(CSV_GDRIVE_URL, CSV_FILENAME)

TEXT = data.Field(tokenize = 'spacy', batch_first = True, lower=True)  #from torchtext import data
LABEL = data.LabelField(sequential=False, dtype = torch.float)

train = data.TabularDataset(path=os.path.join('/content', CSV_FILENAME),
                            format='csv',
                            fields = [('Insult', LABEL), ('Comment', TEXT)],
                            skip_header=True)

print(vars(train[0]),vars(train[1]),vars(train[2]))

TEXT.build_vocab(train)


title: Mask certain indices for every entry in a batch, when using torch.max()

body:
I am incremently sampling a batch of size torch.Size([n, 8]).
I also have a list valid_indices of length n which contains tuples of indices that are valid for each entry in the batch.
For instance valid_indices[0] may look like this: (0,1,3,4,5,7) , which suggests that indices 2 and 6 should be excluded from the first entry in batch along dim 1.
Particularly I need to exclude these values for when I use torch.max(batch, dim=1, keepdim=True).
Indices to be excluded (if any) may differ from entry to entry within the batch.
Any ideas? Thanks in advance.


title: How to convert a PyTorch sparse_coo_tensor into a PyTorch dense tensor?

body:
I create a sparse_coo tensor in PyTorch:
import torch

# create indices
i = torch.tensor([[0, 1, 1],
                  [2, 0, 2]])
# create values
v = torch.tensor([3, 4, 5], dtype=torch.float32)

# create sparse_coo_tensor
sparse_tensor = torch.sparse_coo_tensor(i, v, [2, 4])

Now I want to convert a PyTorch sparse tensor into a PyTorch dense tensor. Which function can be used?


title: when I run p.start(), I got TypeError: 'generator' object is not callable

body:
if __name__ == '__main__':
        # num_processes = 4
        # NOTE: this is required for the ``fork`` method to work
        torch.manual_seed(args.seed)
        cnn.share_memory()
        processes = []
        for rank in range(args.num_processes):
            p = mu.Process(target=train.train, args=(rank, train_iter, dev_iter, cnn, args,))
            p.start()
            processes.append(p)
        for p in processes:
            p.join()

when I run p.start(),have a error in here,The following is the error message,anyone can help me,thanks:

Traceback (most recent call last):
  File "E:/workspace/pycharmworkspace/cnn-text-classification-pytorch-update/main.py", line 144, in <module>
    p.start()
  File "C:\Users\bamtercelboo\Anaconda3\lib\multiprocessing\process.py", line 105, in start
    self._popen = self._Popen(self)
  File "C:\Users\bamtercelboo\Anaconda3\lib\multiprocessing\context.py", line 223, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "C:\Users\bamtercelboo\Anaconda3\lib\multiprocessing\context.py", line 322, in _Popen
    return Popen(process_obj)
  File "C:\Users\bamtercelboo\Anaconda3\lib\multiprocessing\popen_spawn_win32.py", line 65, in __init__
    reduction.dump(process_obj, to_child)
  File "C:\Users\bamtercelboo\Anaconda3\lib\multiprocessing\reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
TypeError: 'generator' object is not callable


title: DataLoader num_workers vs torch.set_num_threads

body: Is there a difference between the parallelization that takes place between these two options? I’m assuming num_workers is solely concerned with the parallelizing the data loading. But is setting torch.set_num_threads for training in general? Trying to understand the difference between these options. Thanks!


title: Why am I able to change the value of a tensor without the computation graph knowing about it in Pytorch with detach?

body:
I can change the value of a tensor that requires grad without autograd knowing about it:

def error_unexpected_way_to_by_pass_safety():
    import torch
    a = torch.tensor([1,2,3.], requires_grad=True)
    # are detached tensor's leafs? yes they are
    a_detached = a.detach()
    #a.fill_(2) # illegal, warns you that a tensor which requires grads is used in an inplace op (so it won't be recorded in computation graph so it wont take the right derivative of the forward path as this op won't be in it)
    a_detached.fill_(2) # weird that this one is allowed, seems to allow me to bypass the error check from the previous comment...?!
    print(f'a = {a}')
    print(f'a_detached = {a_detached}')
    a.sum().backward()

this throws no errors. Though, I am able to change the contents of a which is a tensor that requires grad without autograd knowing about. Which means the computation graph does not know about this op (filling with 2). This seems wrong. Can anyone shed light what is going on?


title: How to utilize all GPUs when dealing with pytorch code?

body:
I have 2 GPUs and when I am working with pytorch code, only one GPU is used. I tried CUDA_VISIBLE_DEVICES=0,1 python xxx.py, but occurs

  'CUDA_VISIBLE_DEVICES: command not found'

problems. I have also tried to add the following lines in object py file:

 import os
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"

but still only one GPU is utilized.


title: How to use DistributedDataParallel() and init_process_group() in a HPC?

body:
I will use HPC for my research and I don't know a lot about parallel or distributed computing.
I really don't understand the DistributedDataParallel() in pytorch. Especially init_process_group().
What is the meaning of initializing processes group? and what is

  init_method : URL specifying how to initialize the package.

for example (I found those in the documentation):
'tcp://10.1.1.20:23456' or 'file:///mnt/nfs/sharedfile'
What are those URLs?

What is the Rank of the current process?
Is world_size the number of GPUs?

It would be really appreciated if someone explained to me what is and How to use DistributedDataParallel() and init_process_group() because I don't know parallel or distributed computing.

I will use things like Slurm(sbatch) in the HPC.


title: How to fit custom data into Pytorch DataLoader?

body:
I have pre-processed and normalized my data, and split into training set and testing set. I have the following dimensions for my x_train and y_train:
Shape of X_Train:  (708, 256, 3)
Shape of Y_Train:  (708, 4)

As you can see, x_train is 3-D. How can I go about inputting it into the pytorch dataloader? What do I put for the class block?

class training_set(Dataset):
    def __init__(self,X,Y):

    def __len__(self):
        return

    def __getitem__(self, idx):
        return

training_set = torch.utils.data.TensorDataset(x_train, y_train)
train_loader = torch.utils.data.DataLoader(training_set, batch_size=50, shuffle=True)


title: CUDA error: device-side assert triggered error on AWS EC2 instance

body:
I try to train my model using PyTorch and a AWS EC2 Instance (g3s.xlarge), but there is a common error, the famous RuntimeError: CUDA error: device-side assert triggered

So have a more clear stacktrace, I run my sscript using CUDA_LAUNCH_BLOCKING=1, and there is all the stacktrace I have:

  /pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [0,0,0], thread: [44,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
/pytorch/aten/src/ATen/native/cuda/IndexKernel.cu:53: lambda [](int)->auto::operator()(int)->auto: block: [0,0,0], thread: [26,0,0] Assertion `index >= -sizes[i] && index < sizes[i] && "index out of bounds"` failed.
THCudaCheck FAIL file=/pytorch/aten/src/THC/THCCachingHostAllocator.cpp line=265 error=59 : device-side assert triggered
Traceback (most recent call last):
  File "train.py", line 367, in <module>
    backup_every=args.backup_every)
  File "train.py", line 267, in train
    if torch.isnan(loss):
RuntimeError: CUDA error: device-side assert triggered

This error came during the first epoch, or during the test of the first epoch (it depends)

I check my dataset and:
- There is no negative number
- All the output files are in Yolo format
- There is exactly the same number of files and labels

The number of classes I have is 17 (so I use 17 to generate my cfg, and I enter 17 in my .data file)

I can't identify where is the mistake, do you have any idea ?

EDIT:

I try to run the same command, with same dataset on CPU, and the following error happend:

Using CPU

Traceback (most recent call last):
  File "train.py", line 367, in <module>
    backup_every=args.backup_every)
  File "train.py", line 198, in train
    load_darknet_weights(model, f'weights/{cutoff_name}')
  File "/home/florian/EYESR/eyesr-psycle/yolo_utils/models.py", line 278, in load_darknet_weights
    conv_w = torch.from_numpy(weights[ptr:ptr + num_w]).view_as(conv_layer.weight)
RuntimeError: shape '[512, 256, 3, 3]' is invalid for input of size 776331

Maybe when I resolve this, can affect the error on GPU ?


title: PyTorch - Effect of normal() initialization on gradients

body:
Suppose I have a neural network where I use a normal distribution initialization and I want to use the mean value which is used for initialization as a parameter of the network.

I have a small example:

import torch
parameter_vector = torch.tensor(range(10), dtype=torch.float, requires_grad=True)
sigma = torch.ones(parameter_vector.size(0), dtype=torch.float)*0.1
init_result = torch.normal(parameter_vector, sigma)
print('requires_grad:', init_result.requires_grad)
print('result:       ', init_result)

This results in:

requires_grad: True
result:        tensor([ 0.1026,  0.9183,  1.9586,  3.1778,  4.0538,  4.8056,  5.9561,
         6.9501,  7.7653,  8.9583])

So the requires_grad flag was obviously taken over from the mean value tensor resp. parameter_vector.

But does this automatically mean that the parameter_vector will be updated through backward() in a larger network where init_result does affect the end result?

Especially as normal() does not really seem like normal operation because it involves randomness.


title: Unable to load data Google Colab

body:
I am trying to load data after downloading it through kaggle cli. This exercise is from this course on Udacity.

!kaggle competitions download -c dogs-vs-cats
!unzip {content}/competitions/dogs-vs-cats/train.zip -d {content}/competitions/dogs-vs-cats/
!unzip {content}/competitions/dogs-vs-cats/test1.zip -d {content}/competitions/dogs-vs-cats/

Counting the number of images

!ls '{content}/competitions/dogs-vs-cats/train/' | wc -l
# 25000

Then I try to load the data

data_dir = '{content}/competitions/dogs-vs-cats/train/'

transform = transforms.Compose([transforms.Resize(255),
                               transforms.CenterCrop(224),
                               transforms.ToTensor()]) # TODO: compose transforms here
dataset = datasets.ImageFolder(data_dir, transform=transform) # TODO: create the ImageFolder
dataloader = torch.utils.data.DataLoader(dataset,batch_size=32,shuffle=True) # TODO: use the ImageFolder dataset to create the DataLoader

Error

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-70-9c49d0bdcdb0> in <module>()
      4                                transforms.CenterCrop(224),
      5                                transforms.ToTensor()]) # TODO: compose transforms here
----> 6 dataset = datasets.ImageFolder(data_dir, transform=transform) # TODO: create the ImageFolder
      7 dataloader = torch.utils.data.DataLoader(dataset,batch_size=32,shuffle=True) # TODO: use the ImageFolder dataset to create the DataLoader

1 frames
/usr/local/lib/python3.6/dist-packages/torchvision/datasets/folder.py in __init__(self, root, transform, target_transform, loader, is_valid_file)
    207                                           transform=transform,
    208                                           target_transform=target_transform,
--> 209                                           is_valid_file=is_valid_file)
    210         self.imgs = self.samples

/usr/local/lib/python3.6/dist-packages/torchvision/datasets/folder.py in __init__(self, root, loader, extensions, transform, target_transform, is_valid_file)
     95         if len(samples) == 0:
     96             raise (RuntimeError("Found 0 files in subfolders of: " + self.root + "\n"
---> 97                                 "Supported extensions are: " + ",".join(extensions)))
     98
     99         self.loader = loader

RuntimeError: Found 0 files in subfolders of: {content}/competitions/dogs-vs-cats/train/
Supported extensions are: .jpg,.jpeg,.png,.ppm,.bmp,.pgm,.tif,.tiff,.webp


title: How do I fix RuntimeError: An attempt has been made to start a new process before the current process has finished its bootstrapping phase?

body:
I'm in the midst of learning fastai and I'm constantly running into the '[Errno 32] Broken pipe' error, I have the necessary packages in pip and conda,
and also running an Intel i5-2520M 2.5GHz, Windows 10 64-bit,
Python

3.7.4

, and the CPU version of Pytorch since I don't have a dedicated GPU, and the latest Pytorch

torch           1.3.0+cpu
torchvision     0.4.1+cpu

And fastai

fastai          1.0.58

After launching the script in conda in my environment I get the following in my terminal

------->> biwi_head_pose\09\frame_00667_rgb.jpg
---- Done Gathering Data
epoch     train_loss  valid_loss  time
------->> biwi_head_pose\09\frame_00667_rgb.jpg--------------------------| 0.00% [0/237 00:00<00:00]
---- Done Gathering Data
epoch     train_loss  valid_loss  time
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\spawn.py", line 105, in spawn_main
    exitcode = _main(fd)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\spawn.py", line 114, in _main
    prepare(preparation_data)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\spawn.py", line 225, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\spawn.py", line 277, in _fixup_main_from_path
    run_name="__mp_main__")
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\runpy.py", line 263, in run_path
    pkg_name=pkg_name, script_name=fname)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\runpy.py", line 96, in _run_module_code
    mod_name, mod_spec, pkg_name, script_name)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "C:\Users\Eric\Desktop\fastai Practice\L3\HeadPose.py", line 39, in <module>
    learn.lr_find()
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\train.py", line 41, in lr_find
    learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\basic_train.py", line 200, in fit
    fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\basic_train.py", line 99, in fit
    for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastprogress\fastprogress.py", line 72, in __iter__
    for i,o in enumerate(self._gen):
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\basic_data.py", line 75, in __iter__
    for b in self.dl: yield self.proc_batch(b)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py", line 193, in __iter__
    return _DataLoaderIter(self)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py", line 469, in __init__
    w.start()
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\process.py", line 112, in start
    self._popen = self._Popen(self)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\context.py", line 223, in _Popen
LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.
Traceback (most recent call last):
  File "HeadPose.py", line 39, in <module>
    learn.lr_find()
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\train.py", line 41, in lr_find
    learn.fit(epochs, start_lr, callbacks=[cb], wd=wd)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\basic_train.py", line 200, in fit
    fit(epochs, self, metrics=self.metrics, callbacks=self.callbacks+callbacks)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\basic_train.py", line 99, in fit
    for xb,yb in progress_bar(learn.data.train_dl, parent=pbar):
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastprogress\fastprogress.py", line 72, in __iter__
    for i,o in enumerate(self._gen):
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\fastai\basic_data.py", line 75, in __iter__
return _default_context.get_context().Process._Popen(process_obj)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\context.py", line 322, in _Popen
    for b in self.dl: yield self.proc_batch(b)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py", line 193, in __iter__
    return Popen(process_obj)
      File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\popen_spawn_win32.py", line 46, in __init__
return _DataLoaderIter(self)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\site-packages\torch\utils\data\dataloader.py", line 469, in __init__
    w.start()
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\process.py", line 112, in start
    self._popen = self._Popen(self)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\context.py", line 223, in _Popen
    return _default_context.get_context().Process._Popen(process_obj)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\context.py", line 322, in _Popen
    return Popen(process_obj)
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\popen_spawn_win32.py", line 89, in __init__
        reduction.dump(process_obj, to_child)prep_data = spawn.get_preparation_data(process_obj._name)

  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\reduction.py", line 60, in dump
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\spawn.py", line 143, in get_preparation_data
    _check_not_importing_main()
  File "C:\Users\Eric\Anaconda3\envs\fastai\lib\multiprocessing\spawn.py", line 136, in _check_not_importing_main
    is not going to be frozen to produce an executable.''')
RuntimeError:
        An attempt has been made to start a new process before the
        current process has finished its bootstrapping phase.

        This probably means that you are not using fork to start your
        child processes and you have forgotten to use the proper idiom
        in the main module:

            if __name__ == '__main__':
                freeze_support()
                ...

        The "freeze_support()" line can be omitted if the program
        is not going to be frozen to produce an executable.
    ForkingPickler(file, protocol).dump(obj)
BrokenPipeError: [Errno 32] Broken pipe

As you can see Errno 32 opens up, I have looked around and people say put if __name__ == '__main__': before the training code, however when I do this the error goes away but it does not train.

So this is what I have done(HeadPose.py), first we get the data from the path and convert the images to coords to train

from fastai.vision import *
import matplotlib.pyplot as plt

# Get Data & Convert to Matrices
path = 'C:\\Users\\Eric\\Desktop\\fastai Practice\\L3\\'
cal = np.genfromtxt(path+'biwi_head_pose\\01\\rgb.cal', skip_footer=6); cal
fname = 'biwi_head_pose\\09\\frame_00667_rgb.jpg'

def img2txt_name(f): return path+f'{str(f)[:-7]}pose.txt'
img=open_image(path+fname)
#img.show()

ctr = np.genfromtxt(img2txt_name(fname), skip_header=3); ctr
def convert_biwi(coords):
    c1 = coords[0] * cal[0][0]/coords[2] + cal[0][2]
    c2 = coords[1] * cal[1][1]/coords[2] + cal[1][2]
    return tensor([c2,c1])

def get_ctr(f):
    ctr = np.genfromtxt(img2txt_name(f), skip_header=3)
    return convert_biwi(ctr)

def get_ip(img,pts): return ImagePoints(FlowField(img.size, pts), scale=True)
get_ctr(fname)
ctr = get_ctr(fname)
#img.show(y=get_ip(img,ctr), figsize=(6,6))

then this Code creates a dataset from the files in the biwi_head_pose folder

# Creating a dataset
data = (PointsItemList.from_folder('biwi_head_pose')
        .split_by_valid_func(lambda o: o.parent.name=='13')
        .label_from_func(get_ctr)
        .transform(get_transforms(), tfm_y=True, size=(120,160))
        .databunch().normalize(imagenet_stats))

data.show_batch(3, figsize=(9,6))
print('---- Done Gathering Data')

and this is the code that trains the data with a Convolutional Neural Network.

#Train model
learn = cnn_learner(data, models.resnet34)
learn.lr_find()
learn.recorder.plot()
lr = 2e-2
learn.fit_one_cycle(5, slice(lr))
print('---- Saving as 3-stage-1')
learn.save(p+'3-stage-1')
print('---- Saved!')
print('---- Loading 3-stage-1')
learn.load(p+'3-stage-1');
learn.show_results()
print('---- Done!')
plt.show()

I'm following the tutorial from this fastai github page:
https://github.com/fastai/course-v3/blob/master/nbs/dl1/lesson3-head-pose.ipynb


title: Get feature vectors from BertForSequenceClassification

body:
I have successfully build a sentiment analysis tool with BertForSequenceClassification from huggingface/transformers to classify $tsla tweets as positive or negative.

However, I can't find out how I can obtain the feature vectors per tweet (more specifically the embedding of [CLS]) from my finetuned model.

more info of used model:

model = BertForSequenceClassification.from_pretrained(OUTPUT_DIR, num_labels=num_labels)
model.config.output_hidden_states = True
tokenizer = BertTokenizer(OUTPUT_DIR+'vocab.txt')

However, when I run the code below the output variable only consists of the logits.

model.eval()
eval_loss = 0
nb_eval_steps = 0
preds = []

for input_ids, input_mask, segment_ids, label_ids in tqdm_notebook(eval_dataloader, desc="Evaluating"):
    input_ids = input_ids.to(device)
    input_mask = input_mask.to(device)
    segment_ids = segment_ids.to(device)
    label_ids = label_ids.to(device)

    with torch.no_grad():
        output = model(input_ids,token_type_ids= segment_ids,attention_mask= input_mask)


title: Certain members of a torch module aren't moved to GPU even if model.to(device) is called

body:
An mwe is as follows:

import torch
import torch.nn as nn

class model(nn.Module):
    def __init__(self):
        super(model,self).__init__()
        self.mat = torch.randn(2,2)

    def forward(self,x):
        print('self.mat.device is',self.mat.device)
        x = torch.mv(self.mat,x)
        return x

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
m = model()
m.to(device)

x = torch.tensor([2.,1.])
x = x.to(device)

m(x)

The output is

self.mat.device is cpu

and right after that comes

Traceback (most recent call last):
  File "Z:\cudatest.py", line 21, in <module>
    print(m(x))
  File "E:\Python37\lib\site-packages\torch\nn\modules\module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "Z:\cudatest.py", line 11, in forward
    x = torch.mv(self.mat,x)
RuntimeError: Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_mv

The code works fine if I set device = torch.device('cpu'). It seems that the problem is model.mat is not moved to GPU even after m.to(device) is called. Why doesn't whis work? How can I fix this? Please note the following:

Even though this particular example can be fixed by using self.mat = nn.Linear(2,2) and x = self.mat(x) instead, in my original program, I need a temporary tensor to store some data in forward() that is also used in some arithmetics. How can I construct such a tensor and send it to GPU when calling m.to(device)
It is not known in advance whether the computer has GPU or not. Therefore, writing self.mat = self.mat.cuda() is not a good solution for my case.


title: ValueError: Target and input must have the same number of elements. target nelement (50) != input nelement (100)

body:
I'm new to Pytorch so I tried to learn it by creating simple dogs vs cats classification.
The code:

class DogCatClassifier(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(1, 32, 5)
        self.conv2 = nn.Conv2d(32, 64, 5)
        self.conv3 = nn.Conv2d(64, 128, 5)

        self.fc1 = nn.Linear(512, 256)
        self.fc2 = nn.Linear(256, 2)

    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        print("1-st: ", x.shape)

        x = F.max_pool2d(F.relu(self.conv2(x)), (2, 2))
        print("2-nd: ", x.shape)

        x = F.max_pool2d(F.relu(self.conv3(x)), (2, 2))
        print("3-rd: ", x.shape)

        x = torch.flatten(x, start_dim=1)

        x = F.relu(self.fc1(x))
        print("6-th: ", x.shape)

        x = self.fc2(x)  # bc this is our output layer. No activation here.
        print("7-th: ", x.shape)

        x = F.sigmoid(x)
        print("8-th: ", x.shape)

        return x

I pass a single batch of data (data shape is (50, 1, 50, 50)

model = DogCatClassifier()

images, labels = next(iter(train_loader))

preds = model(images)
print(pred)

loss = F.binary_cross_entropy(preds, labels)

My prediction shape is (50, 2), so as I understand F.binary_cross_entropy(preds, labels) checks both predictions from a single image and that's why I get 100 predictions against 50 labels. Coming from tensorflow I thought that I could just implement the same logic like using sigmoid as last activation and binary_cross_entropy as loss function. What I don't understand is how to make this piece of code work.


title: Pytorch MSE loss function nan during training

body:
I am trying linear regression from boston dataset. MSE loss function is nan since the first iteration. I tried altering learning rate and batch_size but of no use.
from torch.utils.data import TensorDataset , DataLoader
inputs  = torch.from_numpy(Features).to(torch.float32)
targets = torch.from_numpy(target).to(torch.float32)
train_ds = TensorDataset(inputs , targets)
train_dl = DataLoader(train_ds , batch_size = 5 , shuffle = True)

model = nn.Linear(13,1)
opt = optim.SGD(model.parameters(), lr=1e-5)
loss_fn = F.mse_loss

def fit(num_epochs, model, loss_fn, opt, train_dl):

    # Repeat for given number of epochs
    for epoch in range(num_epochs):

        # Train with batches of data
        for xb,yb in train_dl:

            # 1. Generate predictions
            pred = model(xb)

            # 2. Calculate loss
            loss = loss_fn(pred, yb)

            # 3. Compute gradients
            loss.backward()

            # 4. Update parameters using gradients
            opt.step()

            # 5. Reset the gradients to zero
            opt.zero_grad()

        # Print the progress
        if (epoch+1) % 10 == 0:
            print('Epoch [{}/{}], Loss: {}'.format(epoch+1, num_epochs, loss.item()))

fit(100, model, loss_fn , opt , train_dl)

output