title: BERT always predicts same class (Fine-Tuning)

body:
I am fine-tuning BERT on a financial news dataset.
Unfortunately BERT seems to be trapped in a local minimum. It is content with learning to always predict the same class.

balancing the dataset didnt work
tuning parameters didnt work as well

I am honestly not sure what is causing this problem. With the simpletransformers library I am getting very good results. I would really appreciate if somebody could help me. thanks a lot!
Full code on github:
https://github.com/Bene939/BERT_News_Sentiment_Classifier
Code:
from transformers import BertForSequenceClassification, AdamW, BertTokenizer, get_linear_schedule_with_warmup, Trainer, TrainingArguments
import torch
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
import pandas as pd
from pathlib import Path
import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, precision_recall_fscore_support
import numpy as np
from torch.nn import functional as F
from collections import defaultdict
import random

#defining tokenizer, model and optimizer
tokenizer = BertTokenizer.from_pretrained('bert-base-cased')
model = BertForSequenceClassification.from_pretrained('bert-base-cased', num_labels=3)

if torch.cuda.is_available():
  print("\nUsing: ", torch.cuda.get_device_name(0))
  device = torch.device('cuda')
else:
  print("\nUsing: CPU")
  device = torch.device('cpu')
model = model.to(device)

#loading dataset
labeled_dataset = "news_headlines_sentiment.csv"
labeled_dataset_file = Path(labeled_dataset)
file_loaded = False
while not file_loaded:
  if labeled_dataset_file.exists():
    labeled_dataset = pd.read_csv(labeled_dataset_file)
    file_loaded = True
    print("Dataset Loaded")
  else:
    print("File not Found")
print(labeled_dataset)

#counting sentiments
negative = 0
neutral = 0
positive = 0
for idx, row in labeled_dataset.iterrows():
  if row["sentiment"] == 0:
    negative += 1
  elif row["sentiment"] == 1:
    neutral += 1
  else:
    positive += 1
print("Unbalanced Dataset")
print("negative: ", negative)
print("neutral: ", neutral)
print("positive: ", positive)

#balancing dataset to 1/3 per sentiment
for idx, row in labeled_dataset.iterrows():
  if row["sentiment"] == 0:
    if negative - neutral != 0:
      index_name = labeled_dataset[labeled_dataset["news"] == row["news"]].index
      labeled_dataset.drop(index_name, inplace=True)
      negative -= 1
  elif row["sentiment"] == 2:
    if positive - neutral != 0:
      index_name = labeled_dataset[labeled_dataset["news"] == row["news"]].index
      labeled_dataset.drop(index_name, inplace=True)
      positive -= 1

#custom dataset class
class NewsSentimentDataset(torch.utils.data.Dataset):
  def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

  def __getitem__(self, idx):
      item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
      item['labels'] = torch.tensor(self.labels[idx])
      return item

  def __len__(self):
      return len(self.labels)

#method for tokenizing dataset list
def tokenize_headlines(headlines, labels, tokenizer):

  encodings = tokenizer.batch_encode_plus(
      headlines,
      add_special_tokens = True,
      truncation = True,
      padding = 'max_length',
      return_attention_mask = True,
      return_token_type_ids = True
  )

  dataset = NewsSentimentDataset(encodings, labels)
  return dataset

#splitting dataset into training and validation set
#load news sentiment dataset
all_headlines = labeled_dataset['news'].tolist()
all_labels = labeled_dataset['sentiment'].tolist()

train_headlines, val_headlines, train_labels, val_labels = train_test_split(all_headlines, all_labels, test_size=.2)

val_dataset = tokenize_headlines(val_headlines, val_labels, tokenizer)
train_dataset = tokenize_headlines(train_headlines, val_labels, tokenizer)

#data loader
train_batch_size = 8
val_batch_size = 8

train_data_loader = DataLoader(train_dataset, batch_size = train_batch_size, shuffle=True)
val_data_loader = DataLoader(val_dataset, batch_size = val_batch_size, sampler=SequentialSampler(val_dataset))

#optimizer and scheduler
num_epochs = 1
num_steps = len(train_data_loader) * num_epochs
optimizer = AdamW(model.parameters(), lr=5e-5, eps=1e-8)
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_steps*0.06, num_training_steps=num_steps)

#training and evaluation
seed_val = 64

random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)

for epoch in range(num_epochs):

  print("\n###################################################")
  print("Epoch: {}/{}".format(epoch+1, num_epochs))
  print("###################################################\n")

  #training phase

  average_train_loss = 0
  average_train_acc = 0
  model.train()
  for step, batch in enumerate(train_data_loader):

      input_ids = batch['input_ids'].to(device)
      attention_mask = batch['attention_mask'].to(device)
      labels = batch['labels'].to(device)
      token_type_ids = batch['token_type_ids'].to(device)

      outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids = token_type_ids)

      loss = F.cross_entropy(outputs[0], labels)
      average_train_loss += loss

      if step % 40 == 0:
        print("Training Loss: ", loss)

      logits = outputs[0].detach().cpu().numpy()
      label_ids = labels.to('cpu').numpy()

      average_train_acc += sklearn.metrics.accuracy_score(label_ids, np.argmax(logits, axis=1))
      print("predictions: ",np.argmax(logits, axis=1))
      print("labels:      ",label_ids)
      print("#############")
      optimizer.zero_grad()
      loss.backward()
      #maximum gradient clipping
      torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

      optimizer.step()
      scheduler.step()
      model.zero_grad()

  average_train_loss = average_train_loss / len(train_data_loader)
  average_train_acc = average_train_acc / len(train_data_loader)
  print("======Average Training Loss: {:.5f}======".format(average_train_loss))
  print("======Average Training Accuracy: {:.2f}%======".format(average_train_acc*100))

  #validation phase
  average_val_loss = 0
  average_val_acc = 0
  model.eval()
  for step,batch in enumerate(val_data_loader):
    input_ids = batch['input_ids'].to(device)
    attention_mask = batch['attention_mask'].to(device)
    labels = batch['labels'].to(device)
    token_type_ids = batch['token_type_ids'].to(device)

    pred = []
    with torch.no_grad():

      outputs = model(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)

      loss = F.cross_entropy(outputs[0], labels)
      average_val_loss += loss

      logits = outputs[0].detach().cpu().numpy()
      label_ids = labels.to('cpu').numpy()
      print("predictions: ",np.argmax(logits, axis=1))
      print("labels:      ",label_ids)
      print("#############")

      average_val_acc += sklearn.metrics.accuracy_score(label_ids, np.argmax(logits, axis=1))

  average_val_loss = average_val_loss / len(val_data_loader)
  average_val_acc = average_val_acc / len(val_data_loader)

  print("======Average Validation Loss: {:.5f}======".format(average_val_loss))
  print("======Average Validation Accuracy: {:.2f}%======".format(average_val_acc*100))
###################################################
Epoch: 1/1
###################################################

Training Loss:  tensor(1.1006, device='cuda:0', grad_fn=<NllLossBackward>)
predictions:  [1 0 2 0 0 0 2 0]
labels:       [2 0 1 1 0 1 0 1]
#############
predictions:  [2 2 0 0 0 2 0 0]
labels:       [1 2 1 0 2 0 1 2]
#############
predictions:  [0 0 0 0 1 0 0 1]
labels:       [0 1 1 0 1 1 2 0]
#############
predictions:  [0 0 0 2 0 1 0 0]
labels:       [0 0 0 2 0 0 2 1]
#############
predictions:  [1 0 0 0 0 0 2 0]
labels:       [0 2 2 1 0 0 0 0]
#############
predictions:  [0 0 0 0 0 1 0 0]
labels:       [1 0 2 2 2 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 2 2 2 0 2 0]
#############
predictions:  [0 1 0 0 0 0 0 0]
labels:       [2 2 0 2 0 0 0 1]
#############
predictions:  [0 0 0 0 0 2 0 1]
labels:       [0 1 0 2 2 0 1 2]
#############
predictions:  [0 0 2 0 0 0 1 0]
labels:       [0 0 0 1 2 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 1 0 1 0 1 1]
#############
predictions:  [0 2 0 0 0 0 0 0]
labels:       [2 2 0 1 0 1 2 1]
#############
predictions:  [0 1 0 0 0 0 1 2]
labels:       [2 2 1 0 2 0 0 2]
#############
predictions:  [0 0 1 1 1 1 0 1]
labels:       [1 2 1 1 1 1 2 2]
#############
predictions:  [1 0 0 0 0 1 2 1]
labels:       [1 0 1 1 0 0 0 2]
#############
predictions:  [0 1 1 1 1 0 2 1]
labels:       [2 2 1 2 2 1 1 2]
#############
predictions:  [0 0 1 0 1 1 0 0]
labels:       [1 0 0 1 0 1 0 2]
#############
predictions:  [1 2 0 0 1 2 0 0]
labels:       [0 2 2 1 2 0 1 0]
#############
predictions:  [0 2 1 1 0 1 1 0]
labels:       [2 2 0 1 1 0 1 2]
#############
predictions:  [1 0 1 1 1 1 1 0]
labels:       [0 2 0 1 0 1 2 2]
#############
predictions:  [0 2 1 2 0 0 1 1]
labels:       [2 1 1 1 1 2 2 0]
#############
predictions:  [0 1 2 2 2 1 1 2]
labels:       [2 2 1 1 2 1 0 1]
#############
predictions:  [2 2 2 1 2 1 1 1]
labels:       [0 1 1 0 0 2 2 1]
#############
predictions:  [1 2 2 2 1 2 1 2]
labels:       [0 0 0 0 2 0 1 2]
#############
predictions:  [2 1 1 1 2 2 2 2]
labels:       [1 0 2 2 1 0 0 0]
#############
predictions:  [2 1 2 2 2 1 2 2]
labels:       [2 1 1 1 1 1 2 2]
#############
predictions:  [1 1 0 2 1 2 1 2]
labels:       [2 2 0 2 0 1 2 0]
#############
predictions:  [0 1 1 2 0 1 2 1]
labels:       [2 2 2 1 2 2 0 1]
#############
predictions:  [2 1 1 1 1 2 1 1]
labels:       [0 1 1 2 1 0 0 2]
#############
predictions:  [1 2 2 0 1 1 1 2]
labels:       [0 1 2 1 2 1 0 1]
#############
predictions:  [0 1 1 1 1 1 1 0]
labels:       [0 2 0 1 1 2 2 2]
#############
predictions:  [1 2 1 1 2 1 1 0]
labels:       [0 2 2 2 0 0 1 0]
#############
predictions:  [2 2 2 1 2 1 1 2]
labels:       [2 2 1 2 1 0 0 0]
#############
predictions:  [2 2 1 2 2 2 1 2]
labels:       [1 1 2 2 2 0 2 1]
#############
predictions:  [2 2 2 2 2 0 2 2]
labels:       [2 2 1 2 0 1 1 2]
#############
predictions:  [1 1 2 1 2 2 0 1]
labels:       [2 1 1 1 0 0 2 2]
#############
predictions:  [2 1 2 2 2 2 1 0]
labels:       [0 2 0 2 0 0 0 0]
#############
predictions:  [2 2 2 2 2 2 2 2]
labels:       [1 1 0 2 0 1 2 1]
#############
predictions:  [2 2 2 2 1 2 2 2]
labels:       [1 0 0 1 1 0 0 0]
#############
predictions:  [2 2 2 1 2 2 2 2]
labels:       [1 0 1 1 0 2 2 0]
#############
Training Loss:  tensor(1.1104, device='cuda:0', grad_fn=<NllLossBackward>)
predictions:  [2 0 1 2 1 2 2 0]
labels:       [2 2 0 0 1 0 0 2]
#############
predictions:  [0 2 2 0 2 1 1 1]
labels:       [0 0 0 1 0 0 1 0]
#############
predictions:  [0 2 2 0 1 1 1 2]
labels:       [2 1 1 1 2 2 1 0]
#############
predictions:  [2 1 1 2 2 0 2 0]
labels:       [1 2 1 2 1 0 2 1]
#############
predictions:  [0 2 2 0 0 2 1 2]
labels:       [0 0 2 2 0 0 2 0]
#############
predictions:  [0 0 1 2 2 0 2 2]
labels:       [0 0 0 0 0 0 0 0]
#############
predictions:  [1 1 2 1 2 0 1 2]
labels:       [0 0 2 0 0 0 1 1]
#############
predictions:  [0 0 2 1 0 2 0 1]
labels:       [1 1 2 1 1 0 2 0]
#############
predictions:  [0 0 0 0 1 0 0 0]
labels:       [2 2 1 1 2 1 1 1]
#############
predictions:  [0 0 0 0 1 0 0 0]
labels:       [1 1 2 2 1 1 2 0]
#############
predictions:  [0 0 0 0 0 1 1 1]
labels:       [2 0 1 1 0 1 2 2]
#############
predictions:  [0 0 1 0 0 1 2 1]
labels:       [1 2 0 2 2 0 2 1]
#############
predictions:  [1 1 1 1 0 1 0 1]
labels:       [2 0 1 0 1 0 1 2]
#############
predictions:  [1 2 2 0 0 0 1 1]
labels:       [2 0 0 2 1 2 2 2]
#############
predictions:  [1 0 2 1 0 2 2 0]
labels:       [0 0 2 1 2 1 1 1]
#############
predictions:  [0 0 0 1 1 1 1 1]
labels:       [1 2 1 0 0 0 1 0]
#############
predictions:  [1 1 1 0 1 1 0 1]
labels:       [0 2 1 2 1 2 2 0]
#############
predictions:  [2 1 0 1 1 2 0 0]
labels:       [0 1 0 0 1 2 0 2]
#############
predictions:  [0 1 1 0 0 1 0 1]
labels:       [1 0 0 2 2 1 1 2]
#############
predictions:  [1 1 1 1 1 1 1 1]
labels:       [2 0 1 0 2 0 0 2]
#############
predictions:  [1 0 0 1 0 1 0 2]
labels:       [1 0 0 1 1 2 2 1]
#############
predictions:  [1 1 1 1 1 1 0 0]
labels:       [1 1 0 2 1 0 2 0]
#############
predictions:  [1 1 2 1 0 1 0 0]
labels:       [0 2 1 2 1 1 0 2]
#############
predictions:  [1 1 0 0 1 2 1 1]
labels:       [0 2 1 0 2 2 0 1]
#############
predictions:  [0 1 1 0 0 1 0 1]
labels:       [0 0 1 2 2 0 1 2]
#############
predictions:  [1 0 2 2 2 1 1 0]
labels:       [2 2 1 0 0 1 1 2]
#############
predictions:  [1 2 2 1 1 2 1 1]
labels:       [1 0 0 1 0 0 0 0]
#############
predictions:  [0 2 0 2 2 0 2 2]
labels:       [2 0 0 0 2 1 1 2]
#############
predictions:  [0 0 1 0 1 0 2 2]
labels:       [0 0 1 0 1 0 2 0]
#############
predictions:  [0 2 0 1 1 2 2 0]
labels:       [0 2 0 2 0 2 0 0]
#############
predictions:  [2 2 2 2 2 2 2 1]
labels:       [2 2 1 1 0 0 2 2]
#############
predictions:  [2 0 0 2 2 1 1 0]
labels:       [1 0 0 1 0 2 1 2]
#############
predictions:  [2 0 0 2 0 2 2 0]
labels:       [2 2 2 2 0 1 1 1]
#############
predictions:  [0 2 2 0 2 2 0 0]
labels:       [1 0 1 2 0 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 2]
labels:       [2 1 1 0 0 0 1 2]
#############
predictions:  [2 0 2 0 2 1 0 2]
labels:       [2 1 1 2 1 1 0 0]
#############
predictions:  [1 1 2 0 2 0 2 2]
labels:       [0 2 1 2 1 2 1 0]
#############
predictions:  [2 0 1 1 0 2 0 0]
labels:       [2 1 0 1 1 0 2 0]
#############
predictions:  [2 0 0 2 0 2 1 0]
labels:       [0 0 0 0 2 1 0 1]
#############
predictions:  [1 2 1 0 0 2 0 2]
labels:       [2 0 2 1 0 0 1 1]
#############
Training Loss:  tensor(1.1162, device='cuda:0', grad_fn=<NllLossBackward>)
predictions:  [2 0 0 1 1 1 0 1]
labels:       [0 1 1 1 1 2 2 1]
#############
predictions:  [0 2 0 1 2 0 0 1]
labels:       [2 2 1 0 1 0 0 0]
#############
predictions:  [0 0 1 0 0 0 0 1]
labels:       [1 0 2 0 0 2 2 0]
#############
predictions:  [2 1 2 2 0 1 2 0]
labels:       [2 0 1 0 2 1 0 1]
#############
predictions:  [1 0 0 2 0 0 1 1]
labels:       [2 2 0 2 0 2 0 0]
#############
predictions:  [0 0 1 0 0 0 0 0]
labels:       [2 2 2 1 2 2 2 2]
#############
predictions:  [0 0 1 1 0 1 1 0]
labels:       [2 1 1 1 0 2 1 0]
#############
predictions:  [0 0 0 1 0 0 1 0]
labels:       [2 0 2 2 0 0 1 2]
#############
predictions:  [1 0 1 0 0 2 0 0]
labels:       [1 1 2 0 0 1 0 0]
#############
predictions:  [2 1 0 0 0 1 0 0]
labels:       [1 2 0 0 0 0 0 0]
#############
predictions:  [0 2 0 0 0 0 0 0]
labels:       [2 0 1 1 2 2 1 1]
#############
predictions:  [0 1 0 0 0 1 0 2]
labels:       [0 2 1 1 0 0 1 2]
#############
predictions:  [0 2 1 0 0 1 1 1]
labels:       [1 1 0 2 0 1 1 0]
#############
predictions:  [0 1 1 0 0 0 1 0]
labels:       [0 0 1 0 1 2 1 1]
#############
predictions:  [0 1 1 0 1 0 0 0]
labels:       [0 1 1 1 2 2 2 0]
#############
predictions:  [0 0 0 0 1 1 0 0]
labels:       [2 0 2 2 1 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 0 2 2 0 1 1]
#############
predictions:  [0 1 0 0 0 0 0 0]
labels:       [0 2 0 1 1 2 0 2]
#############
predictions:  [1 1 0 1 0 1 0 2]
labels:       [1 2 0 0 2 2 2 1]
#############
predictions:  [1 1 0 0 0 1 2 1]
labels:       [0 0 1 2 2 1 2 2]
#############
predictions:  [1 1 1 0 1 1 2 0]
labels:       [0 0 0 2 0 1 0 2]
#############
predictions:  [0 1 0 0 1 1 2 1]
labels:       [2 0 0 1 2 2 1 2]
#############
predictions:  [1 0 0 0 1 0 0 1]
labels:       [1 2 2 2 2 1 0 1]
#############
predictions:  [2 0 0 0 0 0 0 0]
labels:       [1 2 0 2 2 1 1 1]
#############
predictions:  [2 0 1 1 0 0 1 0]
labels:       [0 0 0 0 2 2 1 1]
#############
predictions:  [2 0 0 1 0 0 1 1]
labels:       [2 2 1 1 0 0 1 0]
#############
predictions:  [1 1 1 1 1 2 0 0]
labels:       [0 0 2 1 0 0 0 0]
#############
predictions:  [1 1 2 0 1 2 0 1]
labels:       [0 2 1 0 2 0 0 1]
#############
predictions:  [0 0 2 1 0 2 0 1]
labels:       [1 2 0 2 2 1 0 0]
#############
predictions:  [0 0 2 0 2 1 1 2]
labels:       [2 2 1 2 2 2 0 0]
#############
predictions:  [0 1 0 0 0 0 2 1]
labels:       [1 1 0 1 1 1 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 0 0 2 0 0 2]
#############
predictions:  [2 2 2 0 1 1 1 0]
labels:       [1 0 2 1 1 2 0 0]
#############
predictions:  [0 0 1 0 0 0 2 0]
labels:       [0 1 2 1 1 0 0 0]
#############
predictions:  [0 2 0 1 0 2 0 0]
labels:       [0 0 2 1 1 0 2 2]
#############
predictions:  [0 0 1 2 0 2 0 1]
labels:       [2 2 0 0 0 2 2 2]
#############
predictions:  [1 0 0 0 2 0 0 1]
labels:       [2 0 1 1 1 0 0 1]
#############
predictions:  [0 1 0 0 0 0 0 2]
labels:       [1 1 1 0 0 0 2 2]
#############
predictions:  [0 2 0 1 0 2 0 0]
labels:       [1 1 1 1 2 2 1 0]
#############
predictions:  [1 2 0 0 0 0 0 0]
labels:       [2 0 2 1 0 1 1 1]
#############
Training Loss:  tensor(1.2082, device='cuda:0', grad_fn=<NllLossBackward>)
predictions:  [0 2 0 0 0 0 2 0]
labels:       [1 0 2 1 2 2 1 1]
#############
predictions:  [2 0 0 0 0 0 1 0]
labels:       [1 0 0 0 0 2 1 0]
#############
predictions:  [0 0 0 0 2 1 1 1]
labels:       [0 2 2 0 1 2 1 1]
#############
predictions:  [2 1 0 1 0 0 2 0]
labels:       [1 0 2 1 0 2 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 0 0 0 0 1 0]
#############
predictions:  [0 2 1 0 0 0 1 1]
labels:       [0 2 2 2 2 1 1 0]
#############
predictions:  [0 0 0 1 1 0 0 1]
labels:       [0 1 0 1 2 2 2 2]
#############
predictions:  [0 0 0 1 1 1 1 2]
labels:       [2 2 1 2 0 1 1 1]
#############
predictions:  [0 1 2 0 0 1 0 0]
labels:       [0 2 1 0 0 1 0 0]
#############
predictions:  [1 1 1 1 0 0 0 0]
labels:       [2 1 2 1 0 2 2 1]
#############
predictions:  [0 1 2 0 0 1 1 0]
labels:       [2 0 2 1 1 1 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 0 1 1 0 0]
#############
predictions:  [0 0 0 0 0 1 2 2]
labels:       [2 2 1 1 0 2 1 2]
#############
predictions:  [0 1 0 0 1 1 0 1]
labels:       [0 1 0 2 1 0 0 1]
#############
predictions:  [0 2 2 0 0 0 0 2]
labels:       [0 0 2 1 2 2 0 1]
#############
predictions:  [2 0 0 2 2 0 2 0]
labels:       [2 1 0 2 2 0 1 0]
#############
predictions:  [0 2 2 0 2 1 1 2]
labels:       [1 1 0 0 2 1 0 0]
#############
predictions:  [1 1 2 2 0 0 1 2]
labels:       [2 0 2 0 1 1 1 1]
#############
predictions:  [0 1 1 0 0 1 1 0]
labels:       [0 2 1 0 0 2 2 0]
#############
predictions:  [2 1 0 0 0 0 1 1]
labels:       [0 2 0 2 0 0 1 1]
#############
predictions:  [1 2 0 1 2 0 0 0]
labels:       [1 0 1 1 0 2 2 2]
#############
predictions:  [0 0 0 0 2 2 1 2]
labels:       [2 2 2 1 1 1 1 0]
#############
predictions:  [1 2 0 1 0 0 2 0]
labels:       [2 2 1 1 1 0 2 0]
#############
predictions:  [2 0 0 0 0 2 1]
labels:       [0 1 1 2 2 0 2]
#############
======Average Training Loss: 1.11279======
======Average Training Accuracy: 33.77%======
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 1 1 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 0 2 1 0 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 2 2 1 2 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 1 2 0 1 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 2 0 0 1 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 1 2 1 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 1 2 0 2 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 1 2 2 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 2 2 0 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 0 2 0 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 1 1 2 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 1 2 2 0 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 0 0 1 2 2 1]
#############
predictions:  [0 0 0 1 0 0 0 0]
labels:       [0 0 1 1 0 2 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 2 2 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 1 2 2 2 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 2 1 2 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 0 0 2 2 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 0 1 0 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 2 2 2 2 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 2 1 1 0 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 1 1 2 0 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 2 1 2 2 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 1 0 2 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 2 1 1 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 0 1 2 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 1 1 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 1 0 0 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 0 0 0 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 1 1 2 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 1 2 1 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 2 0 1 1 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 1 0 1 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 2 2 1 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 2 0 2 0 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 1 1 1 0 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 1 2 2 0 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 2 0 0 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 0 1 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 2 1 1 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 2 2 2 2 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 2 2 1 0 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 2 2 2 1 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 0 0 1 0 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 0 0 0 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 1 2 0 2 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 2 0 1 2 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 2 0 0 0 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 1 0 0 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 0 1 1 2 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 0 0 2 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 2 1 1 1 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 0 0 2 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 2 1 0 2 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 1 2 2 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 0 0 2 1 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 2 0 2 1 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 2 0 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 0 0 1 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 2 2 0 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 1 1 1 0 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 1 2 2 1 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 0 2 0 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 1 1 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 1 1 1 1 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 2 1 0 0 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 2 1 0 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 2 2 0 0 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 2 2 0 0 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 0 2 2 2 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 0 1 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 1 2 0 1 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 0 0 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 1 2 0 2 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 1 0 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 0 1 0 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 0 0 2 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 1 1 2 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 2 2 0 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 2 0 1 1 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 0 0 1 2 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 2 1 2 0 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 1 1 1 0 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 1 2 0 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 1 0 1 1 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 0 2 1 0 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 0 0 0 2 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 0 1 2 2 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 2 0 1 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 2 1 0 2 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 1 2 0 2 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 2 2 2 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 1 2 0 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 1 1 0 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 2 2 2 2 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 0 0 1 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 2 1 2 1 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 0 0 0 2 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 1 1 1 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 1 0 2 2 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 1 1 1 2 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 2 0 1 0 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 2 2 0 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 1 2 2 2 1 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 0 1 0 2 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 2 1 0 2 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 2 0 2 2 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 2 0 0 1 0 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 1 0 0 0 2 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 2 0 1 2 1 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 2 2 2 2 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 0 1 2 0 2 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 2 1 1 1 1 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 0 0 0 1 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 1 2 0 1 2 2 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 1 1 1 2 1 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [1 0 1 1 1 0 0 2]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 2 0 0 0 0 1 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [0 0 1 1 2 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 1 1 1 0 1 0 0]
#############
predictions:  [0 0 0 0 0 0 0 0]
labels:       [2 0 2 2 2 0 0 1]
#############
predictions:  [0 0 0 0 0 0 0]
labels:       [2 2 1 1 0 0 1]
#############
======Average Validation Loss: 1.09527======
======Average Validation Accuracy: 35.53%======


title: What's the difference between using th and lua commands to run a lua script?

body:
Is there a difference in executing .lua files written for torch using th and lua?

On the server I'm using, I can't run the th command, but I can import torch inside the python interpreter (I'm assuming that this is because pytorch is installed but not torch itself).

I'm wondering if I can just run lua scriptname.lua instead of th scriptname.lua like instructed here: https://github.com/jwyang/JULE.torch


title: I want to use Conv1D and MaxPool1D in pytorch for a 3-d tensor to its third dimension

body:
For example, there is a 3-d tensor, I want to run the conv1d calculation on its third dimension,
import torch
import torch.nn as nn
x = torch.rand(4,5,6)
conv1d =nn.Conv1d(in_channels=1,out_channels=2,kernel_size=5,stride=3,padding=0)
y = conv1d(x)

I hope the shape of y is (4,5,2,-1), but I get an error

Given groups=1, weight of size [2, 1, 5], expected input[4, 5, 6] to have 1 channels, but got 5 channels instead

Then I modified the code,
import torch
import torch.nn as nn
x = torch.rand(4,5,6)
conv1d =nn.Conv1d(in_channels=1,out_channels=2,kernel_size=5,stride=3,padding=0)
x = x.unsqueeze(2)
y = conv1d(x)

There is another error:

Expected 3-dimensional input for 3-dimensional weight [2, 1, 5], but got 4-dimensional input of size [4, 5, 1, 6] instead

And if I want to run the maxpoo1d calulation in a tensor whose shape is (4,5,2,-1) ,in its last two dimension, what should I do?
I am searching for a long time on net. But no use. Please help or try to give some ideas how to achieve this. Thank you all for your help.
I made an attempt, but I felt it couldn’t meet the actual needs, I wanted to know if it's good practice to do that and what would be the best way to do that?
import torch
import torch.nn as nn
x = torch.rand(4,5,6)
conv1d =nn.Conv1d(in_channels=1,out_channels=2,kernel_size=2,stride=3,padding=0)
x = x.unsqueeze(2)
for i in range(4):
    y = conv1d(x[i,:,:,:])
    y = y.unsqueeze(0)
    if i==0:
        z = y
    else:
        z = torch.cat((z,y),0)
    print(y)
print(z.size())


title: Pytorch: How to format data before execution of machine learning

body:
I'm learning how to use pytorch and I was able to get a grasp on the overall process of construction and execution of ML models. However, what I am not able to grasp is how to "format" or "reshape" the data before executing the model. I keep getting errors like:

  RuntimeError: size mismatch, m1: [1 x 700], m2: [1 x 1] at c:\programdata\miniconda3\conda-bld\pytorch_1524543037166\work\aten\src\th\generic/THTensorMath.c:2033

Or,

  Expected object of type Variable[torch.DoubleTensor] but found type Variable[torch.FloatTensor] for argument #1 ‘mat2’

So, I have a csv file named "train.csv" with attributes called 'x' and 'y' and there are 700 samples in it, I want to perform a simple linear regression on the data, and I parse data from it using pandas, how do I format or reshape the data such that it will execute smoothly? How does pytorch iterate through input data?

The recent code i executed is:

import torch
import torch.nn as nn
from torch.autograd import Variable
import pandas as pd

class Linear_Reg(nn.Module):
    def __init__(self, inp_sz, out_sz):
        super(Linear_Reg, self).__init__()
        self.linear = nn.Linear(inp_sz, out_sz)

    def forward(self, x):
        out = self.linear(x)
        return out

train = pd.read_csv('C:\\Users\\hgstr\\Jupyter_Files\\Data_Sets\\linear_regression\\train.csv')
test = pd.read_csv('C:\\Users\\hgstr\\Jupyter_Files\\Data_Sets\\linear_regression\\test.csv')

x_train = torch.Tensor(train['x'])
y_train = torch.Tensor(train['y'])

x_test = torch.Tensor(test['x'])
y_test = torch.Tensor(test['y'])

x_train = torch.Tensor(x_train)
x_train = x_train.view(1,-1)

#================================
input_sz = 1;
output_sz = 1
epochs = 60
learning_rate = 0.001
#================================

model = Linear_Reg(input_sz, output_sz)
crit = nn.MSELoss()
opt = torch.optim.SGD(model.parameters(), learning_rate)

for e in range(epochs):

    opt.zero_grad()
    out = model(x_train)

    loss = crit(out, y_train)
    loss.backward()
    opt.step()

    print('epoch {}, loss {}'.format(e,loss.data[0]))

And it gave out the following:

  RuntimeError: size mismatch, m1: [1 x 700], m2: [1 x 1] at c:\programdata\miniconda3\conda-bld\pytorch_1524543037166\work\aten\src\th\generic/THTensorMath.c:2033

Solutions?


title: Torchvision.transforms implementation of Flatten()

body:
I have grayscale images, but I need transform it to a dataset of 1d vectors
How can I do this? I could not find a suitable method in transforms:
train_dataset = torchvision.datasets.ImageFolder(root='./data',train=True, transform=transforms.ToTensor())
test_dataset = torchvision.datasets.ImageFolder(root='./data',train=False, transform=transforms.ToTensor())

train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=4, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=4, shuffle=False)


title: What is the param `last_epoch` on Pytorch Optimizer's Schedulers is for?

body:
Regarding Pytorch Optimizer's Schedulers, what actually means the last_epoch argument?
It says

last_epoch (int) – The index of last epoch. Default: -1.

But it doesn't really explains much for those, like me, are just learning about these schedules.
I read most of that documentation, if not all, and I could understand what it does and why.


title: Loading a converted pytorch model in huggingface transformers properly

body:
I converted a pre-trained tf model to pytorch using the following function.
def convert_tf_checkpoint_to_pytorch(*, tf_checkpoint_path, albert_config_file, pytorch_dump_path):
    # Initialise PyTorch model
    config = AlbertConfig.from_json_file(albert_config_file)
    print("Building PyTorch model from configuration: {}".format(str(config)))
    model = AlbertForPreTraining(config)

    # Load weights from tf checkpoint
    load_tf_weights_in_albert(model, config, tf_checkpoint_path)

    # Save pytorch-model
    print("Save PyTorch model to {}".format(pytorch_dump_path))
    torch.save(model.state_dict(), pytorch_dump_path)

I am loading the converted model and encoding sentences in the following way:
def vectorize_sentence(text):
    albert_tokenizer = AlbertTokenizer.from_pretrained("albert-base-v2")
    config = AlbertConfig.from_pretrained(config_path, output_hidden_states=True)
    model = TFAlbertModel.from_pretrained(pytorch_dir, config=config, from_pt=True)
    e = albert_tokenizer.encode(text, max_length=512)
    model_input = tf.constant(e)[None, :]  # Batch size 1
    output = model(model_input)

    v = [0] * 768
    # generate sentence vectors by averaging the word vectors
    for i in range(1, len(model_input[0]) - 1):
        v = v + output[0][0][i].numpy()

    vector = v/len(model_input[0])
    return vector

However while loading the model, a warning comes up:

Some weights or buffers of the PyTorch model TFAlbertModel were not
initialized from the TF 2.0 model and are newly initialized:
['predictions.LayerNorm.bias', 'predictions.dense.weight',
'predictions.LayerNorm.weight', 'sop_classifier.classifier.bias',
'predictions.dense.bias', 'sop_classifier.classifier.weight',
'predictions.decoder.bias', 'predictions.bias',
'predictions.decoder.weight'] You should probably TRAIN this model on
a down-stream task to be able to use it for predictions and inference.

Can anyone tell me if I am doing anything wrong? What does the warning mean? I saw issue #5588 on the github repo of Transformers. Don't know if my issue is the same as this.


title: Save Pytorch 4D tensor as image

body:
I have a 4-d Pytorch tensor that I would like to save to disk as a .jpg
My tensor is the following size:
print(image_tensor.size())

>>>torch.Size([1, 3, 400, 711])

I can view the entire tensor as one image within my IDE:
ax1.imshow(im_convert(image_tensor))

Since I am able to view the entire tensor as one image, I am assuming there is a way to also save it as such. However, when I try to save the image, it looks like it only saves the blue color channel. I would like to save the entire tensor as a single image.
img1 = image_tensor[0]
save_image(img1, 'img1.jpg')


title: How does one use Pytorch (+ cuda) with an A100 GPU?

body:
I was trying to use my current code with an A100 gpu but I get this error:
---> backend='nccl'
/home/miranda9/miniconda3/envs/metalearningpy1.7.1c10.2/lib/python3.8/site-packages/torch/cuda/__init__.py:104: UserWarning:
A100-SXM4-40GB with CUDA capability sm_80 is not compatible with the current PyTorch installation.
The current PyTorch install supports CUDA capabilities sm_37 sm_50 sm_60 sm_61 sm_70 sm_75 compute_37.
If you want to use the A100-SXM4-40GB GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/

which is reather confusing because it points to the usual pytorch installation but doesn't tell me which combination of pytorch version + cuda version to use for my specific hardware (A100). What is the right way to install pytorch for an A100?

These are some versions I've tried:
# conda install -y pytorch==1.8.0 torchvision cudatoolkit=10.2 -c pytorch
# conda install -y pytorch torchvision cudatoolkit=10.2 -c pytorch
#conda install -y pytorch==1.7.1 torchvision torchaudio cudatoolkit=10.2 -c pytorch -c conda-forge
# conda install -y pytorch==1.6.0 torchvision cudatoolkit=10.2 -c pytorch
#conda install -y pytorch==1.7.1 torchvision torchaudio cudatoolkit=11.1 -c pytorch -c conda-forge

# conda install pytorch torchvision torchaudio cudatoolkit=11.0 -c pytorch
# conda install pytorch torchvision torchaudio cudatoolkit=11.1 -c pytorch -c conda-forge
# conda install -y pytorch torchvision cudatoolkit=9.2 -c pytorch # For Nano, CC
# conda install pytorch torchvision torchaudio cudatoolkit=11.1 -c pytorch -c conda-forge

note that this can be subtle because I've had this error with this machine + pytorch version in the past:
How to solve the famous `unhandled cuda error, NCCL version 2.7.8` error?


title: DLL files error in using pytorch

body:
I add pytorch via pip installation and now I'm trying to use it, but have this dll error:

Traceback (most recent call last):
  File "F:/Python/Projects/1.py", line 2, in <module>
    import torch
  File "C:\Users\Saeed\AppData\Local\Programs\Python\Python36\lib\site-packages\torch\__init__.py", line 78, in <module>
    from torch._C import *
ImportError: DLL load failed: The specified module could not be found.

I install msvcp71 and msvcr71 dll files but it's not worked!


title: pytorch scoring set of images and evaluating results

body:
In Pytorch by using GPU (cuda) need to score a set of images given trained NN.
The following code is meant to score a set of transformed images one by one.

model.to('cuda')
model.eval()

for ii, (inputs, classes) in enumerate(dataloaders['test']):
     inputs, classes = inputs, classes
     results = model.forward(inputs)
     ps = torch.exp(results)

Error Stack:

RuntimeError                              Traceback (most recent call last)
<ipython-input-24-948390e2b25a> in <module>()
      5 for ii, (inputs, classes) in enumerate(dataloaders['test']):
      6      inputs, classes = inputs, classes
----> 7      results = model(inputs)
      8      ps = torch.exp(results)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    489             result = self._slow_forward(*input, **kwargs)
    490         else:
--> 491             result = self.forward(*input, **kwargs)
    492         for hook in self._forward_hooks.values():
    493             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torchvision-0.2.1-py3.6.egg/torchvision/models/vgg.py in forward(self, x)
     40
     41     def forward(self, x):
---> 42         x = self.features(x)
     43         x = x.view(x.size(0), -1)
     44         x = self.classifier(x)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    489             result = self._slow_forward(*input, **kwargs)
    490         else:
--> 491             result = self.forward(*input, **kwargs)
    492         for hook in self._forward_hooks.values():
    493             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     89     def forward(self, input):
     90         for module in self._modules.values():
---> 91             input = module(input)
     92         return input
     93

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    489             result = self._slow_forward(*input, **kwargs)
    490         else:
--> 491             result = self.forward(*input, **kwargs)
    492         for hook in self._forward_hooks.values():
    493             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/conv.py in forward(self, input)
    299     def forward(self, input):
    300         return F.conv2d(input, self.weight, self.bias, self.stride,
--> 301                         self.padding, self.dilation, self.groups)
    302
    303

RuntimeError: Expected object of type torch.FloatTensor but found type torch.cuda.FloatTensor for argument #2 'weight'

Model made on a GPU (cuda).


title: ImportError: No module named nibabel in python 2.7

body:
I am using python 2.7 and I used

pip install nibabel

but I constantly get the error

ImportError: No module named nibabel.

when I type

pip show nibabel

I get this result so it seems that it is installed but still get the aforementioned error when using import nibabel as nib

    Name: nibabel
Version: 2.5.0
Summary: Access a multitude of neuroimaging data formats
Home-page: https://nipy.org/nibabel
Author: nibabel developers
Author-email: neuroimaging@python.org
License: MIT License
Location: /usr/local/lib/python3.5/dist-packages
Requires: six, numpy
Required-by:

Here the address shows it is installed in python3 path! How can I address this problem?


title: Pytorch: 1D target tensor expected, multi-target not supported

body:
I want to train a 1D CNN on time series. I get the following error message 1D target tensor expected, multi-target not supported
Here is the code with simulated data corresponding to the structures of my data as well as the error message
import torch
from torch.utils.data import DataLoader
import torch.utils.data as data
import torch.nn as nn

import numpy as np
import random
from tqdm.notebook import tqdm

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(device)

train_dataset = []
n_item = 20
for i in range(0,n_item):
    train_data = np.random.uniform(-10, 10, 500)
    train_dataset.append(train_data)
train_dataset = np.asarray(train_dataset)
train_dataset.shape
ecg_train = torch.from_numpy(train_dataset).float()
labels_train = np.random.randint(2, size=n_item)
labels_train = torch.from_numpy(labels_train).long()

val_dataset = []
n_item = 10
for i in range(0,n_item):
    val_data = np.random.uniform(-10, 10, 500)
    val_dataset.append(val_data)
val_dataset = np.asarray(val_dataset)
val_dataset.shape
ecg_validation = torch.from_numpy(val_dataset).float()
labels_validation = np.random.randint(2, size=n_item)
labels_validation = torch.from_numpy(labels_validation).long()

class ECGNet(data.Dataset):
    """ImageNet Limited dataset."""

    def __init__(self, ecgs, labls, transform=None):
            self.ecg = ecgs
            self.target = labls
            self.transform = transform

    def __getitem__(self, idx):
        ecgVec = self.ecg[idx] #.reshape(10, -1)
        labelID = self.target[idx].reshape(1)

        return ecgVec,labelID

    def __len__(self):
        return len(self.ecg)

train_data = ECGNet(ecg_train,
                             labels_train,
                   )
print("size of Training dataset: {}".format(len(train_data)))

validation_data = ECGNet(ecg_validation,
                             labels_validation,
                   )
print("size of Training dataset: {}".format(len(validation_data)))

batch_size = 1
train_dataloader = DataLoader(dataset = train_data,
                              batch_size=batch_size,
                              shuffle = True,
                              num_workers = 0)

val_dataloader = DataLoader(dataset = validation_data,
                              batch_size=batch_size,
                              shuffle = True,
                              num_workers = 0)

def train_epoch(model, train_dataloader, optimizer, loss_fn):
    losses = []
    correct_predictions = 0
    # Iterate mini batches over training dataset
    for images, labels in tqdm(train_dataloader):
        images = images.to(device)
    #labels = labels.squeeze_()
        labels = labels.to(device)

        #labels = labels.to(device=device, dtype=torch.int64)
        # Run predictions
        output = model(images)
        # Set gradients to zero
        optimizer.zero_grad()
        # Compute loss
        loss = loss_fn(output, labels)
        # Backpropagate (compute gradients)
        loss.backward()
        # Make an optimization step (update parameters)
        optimizer.step()
        # Log metrics
        losses.append(loss.item())
        predicted_labels = output.argmax(dim=1)
        correct_predictions += (predicted_labels == labels).sum().item()
    accuracy = 100.0 * correct_predictions / len(train_dataloader.dataset)
    # Return loss values for each iteration and accuracy
    mean_loss = np.array(losses).mean()
    return mean_loss, accuracy

def evaluate(model, dataloader, loss_fn):
    losses = []
    correct_predictions = 0
    with torch.no_grad():
        for images, labels in dataloader:
            images = images.to(device)
            #labels = labels.squeeze_()
            labels = labels.to(device=device, dtype=torch.int64)
            # Run predictions
            output = model(images)
            # Compute loss
            loss = loss_fn(output, labels)
            # Save metrics
            predicted_labels = output.argmax(dim=1)
            correct_predictions += (predicted_labels == labels).sum().item()
            losses.append(loss.item())
    mean_loss = np.array(losses).mean()
    accuracy = 100.0 * correct_predictions / len(dataloader.dataset)
    # Return mean loss and accuracy
    return mean_loss, accuracy

def train(model, train_dataloader, val_dataloader, optimizer, n_epochs, loss_function):
    # We will monitor loss functions as the training progresses
    train_losses = []
    val_losses = []
    train_accuracies = []
    val_accuracies = []

    for epoch in range(n_epochs):
        model.train()
        train_loss, train_accuracy = train_epoch(model, train_dataloader, optimizer, loss_fn)
        model.eval()
        val_loss, val_accuracy = evaluate(model, val_dataloader, loss_fn)
        train_losses.append(train_loss)
        val_losses.append(val_loss)
        train_accuracies.append(train_accuracy)
        val_accuracies.append(val_accuracy)
        print('Epoch {}/{}: train_loss: {:.4f},        train_accuracy: {:.4f}, val_loss: {:.4f},        val_accuracy: {:.4f}'.format(epoch+1, n_epochs,
                                     train_losses[-1],
                                     train_accuracies[-1],
                                     val_losses[-1],
                                     val_accuracies[-1]))
    return train_losses, val_losses, train_accuracies, val_accuracies

class Simple1DCNN(torch.nn.Module):
    def __init__(self):
        super(Simple1DCNN, self).__init__()
        self.layer1 = torch.nn.Conv1d(in_channels=50,
                                      out_channels=20,
                                      kernel_size=5,
                                      stride=2)
        self.act1 = torch.nn.ReLU()
        self.layer2 = torch.nn.Conv1d(in_channels=20,
                                      out_channels=10,
                                      kernel_size=1)

        self.fc1 = nn.Linear(10* 3, 2)
    def forward(self, x):
        print(x.shape)
        x = x.view(1, 50,-1)
        print(x.shape)
        x = self.layer1(x)
        print(x.shape)
        x = self.act1(x)
        print(x.shape)
        x = self.layer2(x)
        print(x.shape)
        x = x.view(1,-1)
        print(x.shape)
        x = self.fc1(x)
        print(x.shape)
        print(x)

        return x

model_a = Simple1DCNN()
model_a = model_a.to(device)

criterion = nn.CrossEntropyLoss()

loss_fn = torch.nn.CrossEntropyLoss()
n_epochs_a = 50
learning_rate_a = 0.01
alpha_a = 1e-5
momentum_a = 0.9
optimizer = torch.optim.SGD(model_a.parameters(),
                            momentum = momentum_a,
                            nesterov = True,
                            weight_decay = alpha_a,
                            lr=learning_rate_a)
train_losses_a, val_losses_a, train_acc_a, val_acc_a = train(model_a,
                                                             train_dataloader,
                                                             val_dataloader,
                                                             optimizer,
                                                             n_epochs_a,
                                                             loss_fn
                                                            )

Error message:
cpu
size of Training dataset: 20
size of Training dataset: 10
  0%|          | 0/20 [00:00<?, ?it/s]
torch.Size([1, 500])
torch.Size([1, 50, 10])
torch.Size([1, 20, 3])
torch.Size([1, 20, 3])
torch.Size([1, 10, 3])
torch.Size([1, 30])
torch.Size([1, 2])
tensor([[ 0.5785, -1.0169]], grad_fn=<AddmmBackward>)
Traceback (most recent call last):
  File "SO_question.py", line 219, in <module>
    train_losses_a, val_losses_a, train_acc_a, val_acc_a = train(model_a,
  File "SO_question.py", line 137, in train
    train_loss, train_accuracy = train_epoch(model, train_dataloader, optimizer, loss_fn)
  File "SO_question.py", line 93, in train_epoch
    loss = loss_fn(output, labels)
  File "/Users/mymac/Documents/programming/python/mainenv/lib/python3.8/site-packages/torch/nn/modules/module.py", line 727, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/Users/mymac/Documents/programming/python/mainenv/lib/python3.8/site-packages/torch/nn/modules/loss.py", line 961, in forward
    return F.cross_entropy(input, target, weight=self.weight,
  File "/Users/mymac/Documents/programming/python/mainenv/lib/python3.8/site-packages/torch/nn/functional.py", line 2468, in cross_entropy
    return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
  File "/Users/mymac/Documents/programming/python/mainenv/lib/python3.8/site-packages/torch/nn/functional.py", line 2264, in nll_loss
    ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: 1D target tensor expected, multi-target not supported

What am I doing wrong?


title: How do the same function for pytorch expand_as in keras

body:
pytorch code:

att_h = att_h.unsqueeze(1).expand_as(att)

att_h shape is (10,512)

att shape is (10,196,512)

Keras code:

K.expand_dims(att_h, 1).expand_as(att)

Got error:
    'Tensor' object has no attribute 'expand_as'

Not sure how to do the same thing in keras.


title: PyTorch equivalent of index_add_ that takes the maximum instead

body:
In PyTorch, the index_add_ method of a Tensor does a summation using a provided index tensor:

idx = torch.LongTensor([0,0,0,0,1,1])
child = torch.FloatTensor([1, 3, 5, 10, 8, 1])
parent = torch.FloatTensor([0, 0])
parent.index_add_(0, idx, child)

The first four child values sum into parent[0] and the next two go into parent[1], so the result is tensor([ 19.,   9.])

However, I need to do index_max_ instead, which doesn't exist in the API. Is there a way to do it efficiently (without having to loop or allocate more memory)? One (bad) loop solution would be:

for i in range(max(idx)+1):
    parent[i] = torch.max(child[idx == i])

This produces the desired result of tensor([ 10.,   8.]), but very slowly.


title: Issues with using cuda and float tensor

body:
I have some code, and when I run it, I get the following error: Expected object of type torch.cuda.FloatTensor but found type torch.FloatTensor for argument #2 'other'

From this error message, I assume there a problem with pushing my models to the GPU. However, I am not sure precisely where the problem lies.

I will place the code wherein I think the problem may lie at the end of this question. Could someone please describe what the error exactly means and how to fix it? Any help is much appreciated.

class VGG(nn.Module):
    '''
    VGG model
    '''
    def __init__(self, features): # features represents the layers array
        super(VGG, self).__init__()
        self.features = features
        self.classifier = nn.Sequential(
            nn.Dropout(),
            nn.Linear(512,512),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(512, 512),
            nn.ReLU(True),
            nn.Linear(512, 10),
        )
         # Initialize weights
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                m.bias.data.zero_()

    def forward(self, x): # x is the image, we run x through the layers
        print("Running through features")
        x = self.features(x) # runs through all features, where each feature is a function
        print("Finsihed features, going to classifier")
        x = x.view(x.size(0), -1)
        # after running through features, does sequential steps to finally classify
        x = self.classifier(x)
        return x

def make_layers(cfg, batch_norm=False):
   # print("Making layers!")
    layers = []
    in_channels = 3
    for v in cfg:
        if v == 'M':
            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
        else:
            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)
            if batch_norm:
                layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
            else:
                layers += [conv2d, nn.ReLU(inplace=True)]
            in_channels = v
            rlstm =RLSTM(v)
            rlstm.input_to_state = torch.nn.DataParallel(rlstm.input_to_state)
            rlstm.state_to_state = torch.nn.DataParallel(rlstm.state_to_state)
            rlstm=rlstm.cuda()
            layers+=[rlstm]

    return nn.Sequential(*layers)

class RLSTM(nn.Module):
    def __init__(self,ch):
       # torch.set_default_tensor_type('torch.cuda.FloatTensor')
        super(RLSTM,self).__init__()
        self.ch=ch
        self.input_to_state = torch.nn.Conv2d(self.ch,4*self.ch,kernel_size=(1,3),padding=(0,1))
        self.state_to_state = torch.nn.Conv2d(self.ch,4*self.ch,kernel_size=(1,3),padding=(0,1)) # error is here: hidPrev is an array - not a valid number of input channel
       # self.input_to_state = self.input_to_state.cuda()
        #self.state_to_state = self.state_to_state.cuda()

    def forward(self, image):
      #  print("going in row forward")
        global current
        global _layer
        global isgates
        size = image.size()
        print("size: "+str(size))
        b = size[0]
        indvs = list(image.split(1,0)) # split up the batch into individual images
        #print(indvs[0].size())
        tensor_array = []
        for i in range(b):
            current = 0
            _layer = []
            isgates = []
            print(len(tensor_array))
            tensor_array.append(self.RowLSTM(indvs[i]))

        seq=tuple(tensor_array)
        trans = torch.cat(seq,0)
        print(trans.size())
        return trans.cuda() # trying to make floattensor error go away
    def RowLSTM(self, image):
    #    print("going in rowlstm")
        global current
        global _layer
        global isgates

        # input-to-state (K_is * x_i) : 3x1 convolution. generate 4h x n x n tensor. 4hxnxn tensor contains all i -> s info

    # the input to state convolution should only be computed one time
        if current==0:
            n = image.size()[2]
            ch=image.size()[1]
           # input_to_state = torch.nn.Conv2d(ch,4*ch,kernel_size=(1,3),padding=(0,1))
          #  print("about to do convolution")
            isgates = self.splitIS(self.input_to_state(image)) # convolve, then split into gates (4 per row)

            cell=RowLSTMCell(0,torch.randn(ch,n,1),torch.randn(ch,n,1),torch.randn(ch,n,1),torch.randn(ch,n,1),torch.randn(ch,n,1),torch.randn(ch,n,1))
            # now have dummy, learnable variables for first row
            _layer.append(cell)
            print("layeres: "+str(len(_layer)))
        else:
            Cell_prev = _layer[current-1] # access previous row
            hidPrev = Cell_prev.getHiddenState()
            ch = image.size()[1]
        #   print("about to apply conv1d")
           # state_to_state = torch.nn.Conv2d(ch,4*ch,kernel_size=(1,3),padding=(0,1)) # error is here: hidPrev is an array - not a valid number of input channel
        #   print("applied conv1d")
            prevHid=Cell_prev.getHiddenState()
            ssgates = self.splitSS(self.state_to_state(prevHid.unsqueeze(0))) #need to unsqueeze (Ex: currently 16x5, need to make 1x16x5)
            gates = self.addGates(isgates,ssgates,current)
            # split gates
            ig, og, fg, gg = gates[0], gates[1], gates[2], gates[3] # into four, ADD SIGMOID!
            cell = RowLSTMCell(Cell_prev,ig,og,fg,gg,0,0)
            cell.compute()
            _layer.append(cell)
        # attempting to eliminate requirement of getting size

        #print(current)
        try:
            print("adding one to current")
            current+=1
            y=(isgates[0][0][1][current])

            return self.RowLSTM(image) #expecting floattensor, but gets cuda floattensor

        except Exception as error:
            print(error)
            concats=[]
            print(len(_layer))
            for cell in _layer:
                tensor=torch.unsqueeze(cell.h,0)
                concats.append(tensor)
            seq=tuple(concats)

            print("non catted tensor: "+str(tensor.size()))
            tense=torch.cat(seq,3)
            print("catted lstm tensor "+str(tense.size()))
            return tensor

The code runs, but when trying to go through the try/except block,  the error is thrown. I am guessing the mistake lies somewhere here?

Edit: Using print statements to see where the program exactly terminates, it seems that there is a mistake in code that I have note posted yet! I will post that now, it looks like the error is in the compute() function, since the statement "finished computing" never gets printed.

class RowLSTMCell(): #inherit torch.nn.LSTM?
    def __init__(self,prev_row, i, o, f, g, c, h):
        #super(RowLSTMCell,self).__init__()
        self.c=c

        #self.c = self.c.cuda()
        self.h=h
       # self.h = self.h.cuda()
        self.i=i
        self.i = self.i.cuda()
        self.o=o
        self.o = self.o.cuda()
        self.g=g
        self.g = self.g.cuda()
        self.f=f
        self.f = self.f.cuda()
        self.prev_row=prev_row
    def getStateSize(self):
        return self._state_size

    def getOutputSize(self):
        return self._output_size

    def compute(self):
        print("computing")
        c_prev = self.prev_row.getCellState()
        h_prev = self.prev_row.getHiddenState()

        self.c = self.f * c_prev + self.i * self.g
        self.h = torch.tanh(self.c) * self.o
        print("finished computing")
    def getHiddenState(self):
        return self.h

    def getCellState(self):
        return self.c


title: Backprop through Pytorch Element-Wise Operation if input contains NaNs

body:
Suppose I'd like to compute the element-wise quotient between two tensors. If one of the tensors contains NaNs, the resulting quotient will also contain NaNs, and this I understand. But why does the gradient becomes non-existent for the entire operation? And how can I preserve the gradient for the non-NaN entries?

For example:

>>> x = torch.tensor([1.0, np.NaN])
>>> y = torch.tensor([2.0, 3.0])
>>> z = torch.div(y, x)
>>> z
tensor([2., nan])
>>> z.backward()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.6/dist-packages/torch/tensor.py", line 107, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py", line 93, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn


title: Explanation of build_vocab in torch and it's association with pre-trained embeddings

body: Can some explain me what is build_vocab in torch, it is not clear from online documentation? Why do we need it and it's relation to pre-trained embeddings?


title: AttributeError: dataset object has no attribute 'c' FastAI

body:
I am trying to train a ResNet based UNet for image segmentation. I have the location of images and mask images in a csv file, that's why I have created my own dataloader, which is as follows:

X = list(df['input_img'])
y = list(df['mask_img'])

X_train, X_valid, y_train, y_valid = train_test_split(
     X, y, test_size=0.33, random_state=42)

class NumbersDataset():
    def __init__(self, inputs, labels):
        self.X = inputs
        self.y = labels

    def __len__(self):
        return len(self.X)

    def __getitem__(self, idx):
        img_train = cv2.imread(self.X[idx])
        img_mask = cv2.imread(self.y[idx])
        img_train = cv2.resize(img_train, (427,240), interpolation = cv2.INTER_LANCZOS4)
        img_mask = cv2.resize(img_mask, (427,240), interpolation = cv2.INTER_LANCZOS4)
        return img_train, img_mask

I then call this datagenerator in the __main__ function:

if __name__ == '__main__':
    dataset_train = NumbersDataset(X_train, y_train)
    dataloader_train = DataLoader(dataset_train, batch_size=4, shuffle=True, num_workers=2)

    dataset_valid = NumbersDataset(X_valid, y_valid)
    dataloader_valid = DataLoader(dataset_valid, batch_size=4, shuffle=True, num_workers=2)

    datas = DataBunch(train_dl = dataloader_train, valid_dl = dataloader_valid)
    leaner = unet_learner(data = datas, arch = models.resnet34)

But I end up getting the following error:

Traceback (most recent call last):
  File "dataset_test.py", line 70, in <module>
    leaner = unet_learner(data = datas, arch = models.resnet34)
  File "/home/sarvagya/miniconda3/envs/gr/lib/python3.6/site-packages/fastai/vision/learner.py", line 118, in unet_learner
    model = to_device(models.unet.DynamicUnet(body, n_classes=data.c, img_size=size, blur=blur, blur_final=blur_final,
  File "/home/sarvagya/miniconda3/envs/gr/lib/python3.6/site-packages/fastai/basic_data.py", line 122, in __getattr__
    def __getattr__(self,k:int)->Any: return getattr(self.train_dl, k)
  File "/home/sarvagya/miniconda3/envs/gr/lib/python3.6/site-packages/fastai/basic_data.py", line 38, in __getattr__
    def __getattr__(self,k:str)->Any: return getattr(self.dl, k)
  File "/home/sarvagya/miniconda3/envs/gr/lib/python3.6/site-packages/fastai/basic_data.py", line 20, in DataLoader___getattr__
    def DataLoader___getattr__(dl, k:str)->Any: return getattr(dl.dataset, k)

  AttributeError: 'NumbersDataset' object has no attribute 'c'

I tried searching and even tried using SegmentationItemList.from_df but nothing helped. What am I getting wrong here?


title: Pytorch - Indexing a range of multiple Indices?

body:
Lets say I have a tensor of size [100, 100] and I have a set of start_indices and end_indices of size [100]

I want to be able to do something like this:

tensor[start_indices:end_indices, :] = 0

Unfortunately, I get an error saying

TypeError: only integer tensors of a single element can be converted to an index

So is this actually possible without a for loop?


title: how pytorch nn.module save submodule

body:
I have some question about how pytorch nn.module works

import torch
import torch.nn as nn

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.sub_module = nn.Linear(10, 5)
        self.value = 3

net = Net()
print(net.__dict__)

output

{'_modules': OrderedDict([('sub_module', Linear (10 -> 5))]),  'value': 3, ...}

I know that every attribute of a class should be stored in __dict__, why value(a int value) is in it, but sub_module(a nn.Module) is not, instead, sub_module is stored in _modules

I read the code of nn.Module implementation, but I didn't figure that out. do anyone have any ideas?

thank you !!


using config:

       'CONFIG_NAME': 'stageI',
       'CUDA': True,
       'DATASET_NAME': 'coco',
       'DATA_DIR': '/home/anwar/Documents/StackGAN-Pytorch-
        master/data/coco',
       'EMBEDDING_TYPE': 'cnn-rnn',
       'GAN': {'CONDITION_DIM': 128, 'DF_DIM': 96, 'GF_DIM': 192, 'R_NUM':
        4},
       'GPU_ID': '1',
       'IMSIZE': 64,
       'NET_D': '',
       'NET_G': '',
       'STAGE': 1,
       'STAGE1_G': '',
       'TEXT': {'DIMENSION': 1024},
       'TRAIN': {'BATCH_SIZE': 128,
       'COEFF': {'KL': 2.0},
       'DISCRIMINATOR_LR': 0.0002,
       'FLAG': True,
       'GENERATOR_LR': 0.0002,
       'LR_DECAY_EPOCH': 20,
       'MAX_EPOCH': 120,
       'PRETRAINED_EPOCH': 600,
       'PRETRAINED_MODEL': '',
       'SNAPSHOT_INTERVAL': 10},
       'VIS_COUNT': 64,
       'WORKERS': 4,
       'Z_DIM': 100
Load filenames from: /home/anwar/Documents/StackGAN-Pytorch-
master/data/coco/train/filenames.pickle (82783)
embeddings:  (82783, 5, 1024)

  THCudaCheck FAIL file=torch/csrc/cuda/Module.cpp line=32 error=30 :
  unknown error Traceback (most recent call last):   File "main.py",
  line 74, in
      algo = GANTrainer(output_dir)   File "/home/anwar/Documents/StackGAN-Pytorch-master/code/trainer.py", line
  45, in init
      torch.cuda.set_device(self.gpus[0])   File "/home/anwar/.conda/envs/tensorflow/lib/python2.7/site-packages/torch/cuda/init.py",
  line 262, in set_device
      torch._C._cuda_setDevice(device)

  RuntimeError: cuda runtime error (30) : unknown error at
  torch/csrc/cuda/Module.cpp:32


title: How to extract features from a pytorch pretrained fine-tuned model

body:
I need to extract features from a pretrained (fine-tuned) BERT model.

I fine-tuned a pretrained BERT model in Pytorch using huggingface transformer. All the training/validation is done on a GPU in cloud.

At the end of the training, I save the model and tokenizer like below:

best_model.save_pretrained('./saved_model/')
tokenizer.save_pretrained('./saved_model/')

This creates below files in the saved_model directory:

config.json
added_token.json
special_tokens_map.json
tokenizer_config.json
vocab.txt
pytorch_model.bin

I save the saved_model directory in my computer and load the model and tokenizer like below

model = torch.load('./saved_model/pytorch_model.bin',map_location=torch.device('cpu'))
tokenizer = BertTokenizer.from_pretrained('./saved_model/')

Now to extract features, I do below

input_ids = torch.tensor([tokenizer.encode("Here is some text to encode", add_special_tokens=True)])
last_hidden_states = model(input_ids)[0][0]

But for the last line, it throws me error TypeError: 'collections.OrderedDict' object is not callable

It seems like I am not loading the model properly. Instead of loading the entire model in itself, I think my model=torch.load(....) line is loading a ordered dictionary.

What am I missing here? Am I even saving the model in the right way? Please suggest.


title: type dependence of 'backward ' in pytorch

body:
x = torch.ones(1, requires_grad=True)
print(x)
y = x + 2.
print(y)
y.backward()
print(x.grad)

-->result>>>>>
tensor([1.], requires_grad=True)
tensor([3.], grad_fn=<AddBackward0>)
tensor([1.])

Here is no problem. However, if I change the type, I get
x = torch.ones(1, requires_grad=True)
x = x.double()
print(x)
y = x + 2.
y = y.double()
print(y)
y.backward()
print(x.grad)

-->result>>>>>
tensor([1.], dtype=torch.float64, grad_fn=<CopyBackwards>)
tensor([3.], dtype=torch.float64, grad_fn=<AddBackward0>)
None

/usr/local/lib/python3.6/dist-packages/torch/tensor.py:746: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.
  warnings.warn("The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad "

=============================
What is the difference? Is there any restriction of 'type' on 'backward'?


title: Problems with CUDA 10.1 on K40c

body:
I am using PyTorch on a K40c GPU. I install PyTorch using conda with the following command:

conda install pytorch torchvision cudatoolkit=10.1 -c pytorch

The installation goes smoothly. However, I have problems when I run cuda prpgrams. Here is a minimum example:

import torch

print(torch.cuda.is_available())
print(torch.version.cuda)
x = torch.tensor(1.0).cuda()
y = torch.tensor(2.0).cuda()

print(x+y)

The output is:

True
10.1.243
Traceback (most recent call last):
  File "example.py", line 8, in <module>
    print(x+y)
RuntimeError: CUDA error: no kernel image is available for execution on the device

I have driver version 418.74. Following is the output of nvidia-smi:

+-----------------------------------------------------------------------------+
| NVIDIA-SMI 418.74       Driver Version: 418.74       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla K40c          On   | 00000000:04:00.0 Off |                    0 |
| 23%   39C    P8    23W / 235W |      0MiB / 11441MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

It looks like PyTorch correctly detects GPU and cuda, but fails when doing actual computation.


title: Is normalization necessary for regression problem in Neural network

body:
I am learning how to build a neural network using PyTorch.
This formula is the target of my code:
y =2X^3 + 7X^2 - 8*X + 120
It is a regression problem.
I used this because it is simple and the output can be calculated so that I can ensure my neural network is able to predict output with the given input.
However, I met some problem during training.
The problem occurs in this line of code:
loss = loss_func(prediction, outputs)

The loss computed in this line is NAN (not a number)
I am using MSEloss as the loss function. 100 datasets are used for training the ANN model. The input X_train is ranged from -1000 to 1000.
I believed that the problem is owing to the value of X_train and MSEloss. X_train should be scaled into some values between 0 and 1 so that MSEloss can compute the loss.
However, is it possible to train the ANN model without scaling the input into value between 0 and 1 in a regression problem?
Here is my code, it does not use MinMaxScaler and it print the loss with NAN:
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import torch.nn.functional as F
from torch.autograd import Variable

#Load datasets
dataset = pd.read_csv('test_100.csv')

x_temp_train = dataset.iloc[:79, :-1].values
y_temp_train = dataset.iloc[:79, -1:].values
x_temp_test = dataset.iloc[80:, :-1].values
y_temp_test = dataset.iloc[80:, -1:].values

#Turn into tensor
X_train = torch.FloatTensor(x_temp_train)
Y_train = torch.FloatTensor(y_temp_train)
X_test = torch.FloatTensor(x_temp_test)
Y_test = torch.FloatTensor(y_temp_test)

#Define a Artifical Neural Network
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.linear = nn.Linear(1,1)  #input=1, output=1, bias=True

    def forward(self, x):
        x = self.linear(x)
        return x
net = Net()
print(net)

#Define a Loss function and optimizer
optimizer = torch.optim.SGD(net.parameters(), lr=0.2)
loss_func = torch.nn.MSELoss()

#Training
inputs = Variable(X_train)
outputs = Variable(Y_train)
for i in range(100):      #epoch=100
    prediction = net(inputs)
    loss = loss_func(prediction, outputs)
    optimizer.zero_grad() #zero the parameter gradients
    loss.backward()       #compute gradients(dloss/dx)
    optimizer.step()      #updates the parameters

    if i % 10 == 9:        #print every 10 mini-batches
        #plot and show learning process
        plt.cla()
        plt.scatter(X_train.data.numpy(), Y_train.data.numpy())
        plt.plot(X_train.data.numpy(), prediction.data.numpy(), 'r-', lw=2)
        plt.text(0.5, 0, 'Loss=%.4f' % loss.data.numpy(), fontdict={'size': 10, 'color': 'red'})
        plt.pause(0.1)

plt.show()

Thanks for your time.


I am trying to normalize my targets (landmarks) here in which each image has 4 landmarks with x and y values for each landmark (keypoint). Here the batch size is 8.
network = Network()
network.cuda()

criterion = nn.MSELoss()
optimizer = optim.Adam(network.parameters(), lr=0.0001)

loss_min = np.inf
num_epochs = 1

start_time = time.time()
for epoch in range(1,num_epochs+1):

    loss_train = 0
    loss_test = 0
    running_loss = 0

    network.train()
    print('size of train loader is: ', len(train_loader))

    for step in range(1,len(train_loader)+1):

        batch = next(iter(train_loader))
        images, landmarks = batch['image'], batch['landmarks']
        #RuntimeError: Given groups=1, weight of size [64, 3, 7, 7], expected input[64, 600, 800, 3] to have 3 channels, but got 600 channels instead
        #using permute below to fix the above error
        images = images.permute(0,3,1,2)

        images = images.cuda()

        landmarks = landmarks.view(landmarks.size(0),-1).cuda()

        norm_image = transforms.Normalize([0.3809, 0.3810, 0.3810], [0.1127, 0.1129, 0.1130])
        for image in images:
            image = image.float()
            ##image = to_tensor(image) #TypeError: pic should be PIL Image or ndarray. Got <class 'torch.Tensor'>
            image = norm_image(image)

        norm_landmarks = transforms.Normalize(0.4949, 0.2165)
        landmarks = norm_landmarks(landmarks)
        ##landmarks = torchvision.transforms.Normalize(landmarks) #Do I need to normalize the target?

        predictions = network(images)

        # clear all the gradients before calculating them
        optimizer.zero_grad()

        print('predictions are: ', predictions.float())
        print('landmarks are: ', landmarks.float())
        # find the loss for the current step
        loss_train_step = criterion(predictions.float(), landmarks.float())

        loss_train_step = loss_train_step.to(torch.float32)
        print("loss_train_step before backward: ", loss_train_step)

        # calculate the gradients
        loss_train_step.backward()

        # update the parameters
        optimizer.step()

        print("loss_train_step after backward: ", loss_train_step)

        loss_train += loss_train_step.item()

        print("loss_train: ", loss_train)
        running_loss = loss_train/step
        print('step: ', step)
        print('running loss: ', running_loss)

        print_overwrite(step, len(train_loader), running_loss, 'train')

    network.eval()
    with torch.no_grad():

        for step in range(1,len(test_loader)+1):

            batch = next(iter(train_loader))
            images, landmarks = batch['image'], batch['landmarks']
            images = images.permute(0,3,1,2)
            images = images.cuda()
            landmarks = landmarks.view(landmarks.size(0),-1).cuda()

            predictions = network(images)

            # find the loss for the current step
            loss_test_step = criterion(predictions, landmarks)

            loss_test += loss_test_step.item()
            running_loss = loss_test/step

            print_overwrite(step, len(test_loader), running_loss, 'Validation')

    loss_train /= len(train_loader)
    loss_test /= len(test_loader)

    print('\n--------------------------------------------------')
    print('Epoch: {}  Train Loss: {:.4f} Valid Loss: {:.4f}'.format(epoch, loss_train, loss_test))
    print('--------------------------------------------------')

    if loss_test < loss_min:
        loss_min = loss_test
        torch.save(network.state_dict(), '../moth_landmarks.pth')
        print("\nMinimum Valid Loss of {:.4f} at epoch {}/{}".format(loss_min, epoch, num_epochs))
        print('Model Saved\n')

print('Training Complete')
print("Total Elapsed Time : {} s".format(time.time()-start_time))

But I get this error:
size of train loader is:  90

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-13-3e4770ad7109> in <module>
     40
     41         norm_landmarks = transforms.Normalize(0.4949, 0.2165)
---> 42         landmarks = norm_landmarks(landmarks)
     43         ##landmarks = torchvision.transforms.Normalize(landmarks) #Do I need to normalize the target?
     44

~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/transforms.py in __call__(self, tensor)
    210             Tensor: Normalized Tensor image.
    211         """
--> 212         return F.normalize(tensor, self.mean, self.std, self.inplace)
    213
    214     def __repr__(self):

~/anaconda3/lib/python3.7/site-packages/torchvision/transforms/functional.py in normalize(tensor, mean, std, inplace)
    282     if tensor.ndimension() != 3:
    283         raise ValueError('Expected tensor to be a tensor image of size (C, H, W). Got tensor.size() = '
--> 284                          '{}.'.format(tensor.size()))
    285
    286     if not inplace:

ValueError: Expected tensor to be a tensor image of size (C, H, W). Got tensor.size() = torch.Size([8, 8]).

How should I normalize my landmarks?


title: ValueError: only one element tensors can be converted to Python scalars while converting list to numpy array

body:
I have a dataset of images and I am using PIL and torch tensor to convert them in tensors in a loop and storing them in  a list.
def preprocess(string):
    return torch.FloatTensor(np.ascontiguousarray(np.array((PIL.Image.open(string)).convert('RGB'))[:, :, ::-1].transpose(2, 0, 1).astype(np.float32) * (1.0 / 255.0)))

import os
folder_name = 'folder name and path'
train1 = []
for item in folder_name:
    x = preprocess(item)
    train1.append(x)

Now before train I want to convert the list to np.array using:
train1 = np.array(train1)

This shows the error:
ValueError: only one element tensors can be converted to Python scalars

I also printed the shapes and types of the constituents of list train1 and found that they are tensors as expected.
Can anyone help me with it?


title: Generation of sequences from latent space [pytorch]

body:
I have several questions about best practice in using recurrent networks in pytorch for generation of sequences.

The first one, if I want to build decoder net should I use nn.GRU (or nn.LSTM) instead nn.LSTMCell (nn.GRUCell)? From my experience, if I work with LSTMCell the speed of calculations is drammatically lower (up to 100 times) than if I use nn.LSTM. Maybe it is related with cudnn optimisation for LSTM (and GRU) module? Is any way to speedup LSTMCell calculations?

I try to build an autoencoder, that accepts sequences of variable length. My autoencoder looks like:

class SimpleAutoencoder(nn.Module):
    def init(self, input_size, hidden_size, n_layers=3):
    super(SimpleAutoencoder, self).init()
    self.n_layers = n_layers
    self.hidden_size = hidden_size
    self.gru_encoder = nn.GRU(input_size, hidden_size,n_layers,batch_first=True)
    self.gru_decoder = nn.GRU(input_size, hidden_size, n_layers, batch_first=True)
    self.h2o = nn.Linear(hidden_size,input_size) # Hidden to output

def encode(self, input):
    output, hidden = self.gru_encoder(input, None)
    return output, hidden

def decode(self, input, hidden):
    output,hidden = self.gru_decoder(input,hidden)
    return output,hidden
def h2o_apply(self,input):
    return self.h2o(input)

My training loop looks like:

one_hot_batch = list(map(lambda x:Variable(torch.FloatTensor(x)),one_hot_batch))

packed_one_hot_batch = pack_padded_sequence(pad_sequence(one_hot_batch,batch_first=True).cuda(),batch_lens, batch_first=True)

 _, latent = vae.encode(packed_one_hot_batch)
 outputs, = vae.decode(packed_one_hot_batch,latent)
 packed = pad_packed_sequence(outputs,batch_first=True)

 for string,length,index in zip(*packed,range(batch_size)):
        decoded_string_without_sos_symbol = vae.h2o_apply(string[1:length])
        loss += criterion(decoded_string_without_sos_symbol,real_strings_batch[index][1:])
 loss /= len(batch)

The training in such manner, as I can understand, is teacher force. Because at the decoding stage the network feeds the real inputs (outputs,_ = vae.decode(packed_one_hot_batch,latent)). But, for my task it leads to the situation when, in the test stage, network can generate sequences very well only if I use the real symbols (as in training mode), but if I feed the output of the previous step, the network generates rubbish (just infinite repetition of one specific symbol).

I tried another one approach. I generated “fake” inputs( just ones), to make the model generate only from the hidden state.

one_hot_batch_fake = list(map(lambda x:torch.ones_like(x).cuda(),one_hot_batch))
packed_one_hot_batch_fake = pack_padded_sequence(pad_sequence(one_hot_batch_fake, batch_first=True).cuda(), batch_lens, batch_first=True)

_, latent = vae.encode(packed_one_hot_batch)
outputs, = vae.decode(packed_one_hot_batch_fake,latent)
packed = pad_packed_sequence(outputs,batch_first=True)

It works, but very inefficiently, the quality of reconstruction is very low. So the second question, what is the right way to generate sequences from latent representation?

I suppose, that good idea is to apply teacher forcing with some probability, but for that, how one can use nn.GRU layer so the output of the previous step should be the input for the next step?


title: Plot the derivative of a function with PyTorch?

body:
I have this code:

import torch
import matplotlib.pyplot as plt
x=torch.linspace(-10, 10, 10, requires_grad=True)
y = torch.sum(x**2)
y.backward()
plt.plot(x.detach().numpy(), y.detach().numpy(), label='function')
plt.legend()

But, I got this error:

ValueError: x and y must have same first dimension, but have shapes (10,) and (1,)


title: How to load images in the same folder in Pytorch?

body:
I want to load all of the images from the folder /img and /mask respectively. The data structure can be shown as follows:

data
    img
        0.png
        1.png
        2.png
        3.png
        ...
   mask
       label_0.png
       label_1.png
       label_2.png
       ...

Hopefully for help.


title: Manage memory differently on train and test time pytorch

body:
Currently I'm writing a segmentation model based on U-net with pytorch and I want to use something similar to inverted residual introduced on mobilenet v2 to improve the model's speed on cpu. pytorch code for mobile netv2

Then I realize that the model uses a lot more memory on train phase and test phase. While the model should use more memory on train phase because all the mid-step tensors(feature maps) are saved and with separable convolution there are more tensors created for each "convolution" operation.
But on run time, actually only few last step tensors must be saved to be used for skip connection and all the other tensors can be deleted once it's next step is created. The memory efficiency should be the same for u-net with normal convolution and u-net with separable convolution on test phase.

I'm newbee to pytorch so I don't know how to write code that prevents unnecessary memory cost on test time. Since pytorch is binded with python. I guess I can manually delete all the unnecessary tensors in forward function with del. But I guess that if I just delete variables on forward function, it will influence training stage.
Is here more advanced functionality on pytorch that is able to optimize test phase memory usage with a 'network graph'?
I'm also curious if tensorflow deals with those problems automatically since it has a more abstract and complex graph building logic.


title: Pytorch 1.0: what does net.to(device) do in nn.DataParallel?

body:
The following code from the tutorial to pytorch data paraleelism reads strange to me:

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

model = Model(input_size, output_size)
if torch.cuda.device_count() > 1:
  print("Let's use", torch.cuda.device_count(), "GPUs!")
  # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
  model = nn.DataParallel(model)

model.to(device)

According to my best knowledge, mode.to(device) copy the data to GPU.

  DataParallel splits your data automatically and sends job orders to multiple models on several GPUs. After each model finishes their job, DataParallel collects and merges the results before returning it to you.

If the DataParallel does the job of copying, what does the to(device) do here?


title: TypeError: 'NoneType' object is not iterable in loss.backward in pytorch

body:
I am trying to implement a Deep Q network (DQN) using a graph convolution network (GCN) using the dynamic graph library (DGL). The base code is taken from this repository. However, after I calculate the loss between the policy network and target network and run loss.backward(), I get TypeError: 'NoneType' object is not iterable. I have printed the loss value and it is not None.

I ran the original code from the repository and it is running perfectly. I have also implemented the GCN code in DGL and it seems to run. I have also visualized the graph using the torchviz, but I am unable to find why it is giving an error.

The code snippet is given below:

target = reward_tens + self.gamma * torch.max(self.model_target(observation_tens, self.G) + observation_tens * (-1e5), dim=1)[0]

current_q_values= self.model(last_observation_tens, self.G)
next_q_values=current_q_values.clone()
current_q_values[range(self.minibatch_length),action_tens,:] = target

L=self.criterion(current_q_values,next_q_values)
print('loss:',L.item())
self.optimizer.zero_grad()
L.backward(retain_graph=True)

self.optimizer.step()

  loss: 1461729.125

TypeError                                 Traceback (most recent call last)
<ipython-input-17-cd5e862dd609> in <module>()
     62
     63 if __name__ == "__main__":
---> 64     main()

7 frames
<ipython-input-17-cd5e862dd609> in main()
     55         print("Running a single instance simulation...")
     56         my_runner = Runner(env_class, agent_class, args.verbose)
---> 57         final_reward = my_runner.loop(graph_dic,args.ngames,args.epoch, args.niter)
     58         print("Obtained a final reward of {}".format(final_reward))
     59         agent_class.save_model()

<ipython-input-14-45cfc883a37b> in loop(self, graphs, games, nbr_epoch, max_iter)
     45                         # if self.verbose:
     46                         #   print("Simulation step {}:".format(i))
---> 47                         (obs, act, rew, done) = self.step()
     48                         action_list.append(act)
     49

<ipython-input-14-45cfc883a37b> in step(self)
     16         #reward = torch.tensor([reward], device=device)
     17
---> 18         self.agent.reward(observation, action, reward,done)
     19
     20         return (observation, action, reward, done)

<ipython-input-16-76d612e8663c> in reward(self, observation, action, reward, done)
    129               print('loss:',L.item())
    130               self.optimizer.zero_grad()
--> 131               L.backward(retain_graph=True)
    132               self.optimizer.step()
    133

/usr/local/lib/python3.6/dist-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph)
    148                 products. Defaults to ``False``.
    149         """
--> 150         torch.autograd.backward(self, gradient, retain_graph, create_graph)
    151
    152     def register_hook(self, hook):

/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables)
     97     Variable._execution_engine.run_backward(
     98         tensors, grad_tensors, retain_graph, create_graph,
---> 99         allow_unreachable=True)  # allow_unreachable flag
    100
    101

/usr/local/lib/python3.6/dist-packages/torch/autograd/function.py in apply(self, *args)
     75
     76     def apply(self, *args):
---> 77         return self._forward_cls.backward(self, *args)
     78
     79

/usr/local/lib/python3.6/dist-packages/dgl/backend/pytorch/tensor.py in backward(ctx, grad_out)
    394     def backward(ctx, grad_out):
    395         reducer, graph, target, in_map, out_map, in_data_nd, out_data_nd, degs \
--> 396             = ctx.backward_cache
    397         ctx.backward_cache = None
    398         grad_in = None

TypeError: 'NoneType' object is not iterable


title: PyTorch Multi-GPU K80s Batch fails for Tensors

body:
My training works for mini-batches when trained on single GPU (default).

if USE_CUDA:
    encoderchar = encoderchar.cuda()
    encoder = encoder.cuda()
    decoder = decoder.cuda()

But, when I train with all available GPUs, I get an error.

if USE_CUDA:
    encoderchar = torch.nn.DataParallel(encoderchar, device_ids=[0, 1, 2, 3, 4, 5, 6, 7])
    encoder =  torch.nn.DataParallel(encoder, device_ids=[0, 1, 2, 3, 4, 5, 6, 7])
    decoder = torch.nn.DataParallel(decoder, device_ids=[0, 1, 2, 3, 4, 5, 6, 7])
    encoderchar = encoderchar.cuda()
    encoder = encoder.cuda()
    decoder = decoder.cuda()

I get the following error during forward.

RuntimeError                              Traceback (most recent call last)
<ipython-input-10-227f3e86847c> in <module>()
18         loss, ar1, ar2 = train(data_input_batch_index, data_input_batch_length, data_target_batch_index, data_target_batch_length,
19                                encoderchar, encoder, decoder, encoderchar_optimizer, encoder_optimizer, decoder_optimizer,
---> 20                                criterion, batch_size)
21
22         # Keep track of loss
<ipython-input-8-21861d792653> in train(input_batch, input_batch_length, target_batch, target_batch_length, encoderchar, encoder, decoder, encoderchar_optimizer, encoder_optimizer, decoder_optimizer, criterion, batch_size)
21             #reshaped_input_length =  Variable(torch.LongTensor(reshaped_input_length)).cuda()
22         hidden_all, output = encoderchar(w, reshaped_input_length)
---> 23         encoder_input[ix] = output.transpose(0,1).contiguous().view(batch_size, -1)
24
25     temporary_target_batch_length = [15] * batch_size
/home/ubuntu/anaconda3/envs/tensorflow/lib/python3.6/site-packages/torch/autograd/variable.py in __setitem__(self, key, value)
78         else:
79             if isinstance(value, Variable):
---> 80                 return SetItem(key)(self, value)
81             else:
82                 return SetItem(key, value)(self)
/home/ubuntu/anaconda3/envs/tensorflow/lib/python3.6/site-packages/torch/autograd/_functions/tensor.py in forward(self, i, value)
37         else:  # value is Tensor
38             self.value_size = value.size()
---> 39         i._set_index(self.index, value)
40         return i
41

RuntimeError: sizes do not match at /py/conda-bld/pytorch_1493681908901/work/torch/lib/THC/THCTensorCopy.cu:31

one cuda long tensor and one list are the parameter types passed to the encoderchar feedforward.

hidden_all, output = encoderchar(w, reshaped_input_length)
encoder_input[ix] = output.transpose(0,1).contiguous().view(batch_size, -1)

nvidia-smi shows the following after the error is thrown.

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID  Type  Process name                               Usage
|    0     18320    C   python                                         453MiB |
|    1     18320    C   python                                         266MiB |
|    2     18320    C   python                                         266MiB |
|    3     18320    C   python                                         266MiB |
|    4     18320    C   python                                         266MiB |
|    5     18320    C   python                                         266MiB |
|    6     18320    C   python                                         266MiB |
|    7     18320    C   python                                         262MiB |
+-----------------------------------------------------------------------------+

What's wrong here?


title: An error occured while loading data by using dataset and dataloader of torch.utils.data. TypeError: object of type 'type' has no len()

body:
I am getting the following error:

TypeError: object of type 'type' has no len()

I think the error occured while loading image datasets. What does this error mean?

I have already searched many examples in github or surfing, there aren't any examples what I exactly want.

The code is:

class MakeDataSet(Dataset):
    def __init__(self, data_path, transform=None):
        self.file_names = glob(data_path)
        self.labels = []
        self.transform = transform
        for LA in self.file_names:
            if 'cat' in LA:
                self.labels.append(0)
            if 'dog' in LA:
                self.labels.append(1)

    def __getitem__(self, index):
        img = Image.open(self.file_names[index])
        if self.transform is not None:
            img = self.transform(img)

        label = torch.from_numpy(np.array(self.labels[index]))
        return img, label

    def __len__(self):
        return len(self.file_names)

transformation = transforms.Compose([transforms.ToTensor()])
DB = MakeDataSet(img_path, transform=transformation)
loader = DataLoader(MakeDataSet, batch_size=5, shuffle=True)

The full error message is:

File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.3\helpers\pydev\pydevd.py", line 1689, in <module>
    main()
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.3\helpers\pydev\pydevd.py", line 1683, in main
    globals = debugger.run(setup['file'], None, None, is_module)
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.3\helpers\pydev\pydevd.py", line 1083, in run
    pydev_imports.execfile(file, globals, locals)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm Community Edition 2018.3\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/BJ/PycharmProjects/AlexNet/test.py", line 45, in <module>
    loader = DataLoader(MakeDataSet, batch_size=5, shuffle=True)
  File "C:\Users\BJ\Anaconda3\envs\install_pytorch\lib\site-packages\torch\utils\data\dataloader.py", line 802, in __init__
    sampler = RandomSampler(dataset)
  File "C:\Users\BJ\Anaconda3\envs\install_pytorch\lib\site-packages\torch\utils\data\sampler.py", line 60, in __init__
    self.num_samples = len(self.data_source)
TypeError: object of type 'type' has no len()


title: How to change the values of a 2d tensor in certain rows and columns

body:
Suppose I have an all-zero mask tensor like this:

mask = torch.zeros(5,3, dtype=torch.bool)

Now I want to set the value of mask at the intersection of the following rows and cols indices to True:

rows = torch.tensor([0,2,4])
cols = torch.tensor([1,2])

I would like to produce the following result:

tensor([[False, True,  True ],
        [False, False, False],
        [False, True,  True ],
        [False, False, False],
        [False, True,  True ]])

When I try the following code, I receive an error:

mask[rows, cols] = True

IndexError: shape mismatch: indexing tensors could not be broadcast together with shapes [3], [2]

How can I do that efficiently in PyTorch?


title: Pytorch using Multi GPU / accuracy is too low(10%)

body:
if torch.cuda.is_available():
for epoch in range(epoch_num):
    for i,(images, labels) in enumerate(trainloader):
        images=images.to(device)
        labels=labels.to(device)

        optimizer.zero_grad()

        #Forward Backward Optimize
        outputs = net(images)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        # print statistics
        if i%1000==0:
            print('Number of epochs: %d, Mini Batch order: %d' %(epoch+1,i))

This is a train part. I use four GPUs to train. And after I run 5000 epochs (batch size is 128), the accuracy comes out "10%"!! It's too low!
Following is the test part:

with torch.no_grad():
num_correct = 0
total_data = 0
if torch.cuda.is_available():
    for images, labels in testloader:
        images=images.to(device)
        labels=labels.to(device)
        output = net(images)
        _, expected = torch.max(output.data, 1)

        total_data += labels.size(0)
        num_correct += (expected == labels).sum().item()

I have no idea what is wrong, how can I investigate this?