如何在NLP上用pytorch实现getting loss?
How to implement getting loss with pytorch on NLP?
我正在使用 pytorch 通过简单的玩具项目(只是生成文本)来研究 NLP。当我在网上引用一些示例代码时,遇到了一个我无法理解的问题。
代码如下(部分代码已省略未完成。):
- LSTM 模型(utils.py)。
def __init__(self, vocab_size, seq_size, embedding_size, hidden_size):
super(RNNModule, self).__init__()
self.seq_size = seq_size
self.hidden_size = hidden_size
self.embedding = nn.Embedding(vocab_size,
embedding_size)
self.lstm = nn.LSTM(input_size = embedding_size,
hidden_size = hidden_size,
num_layers = 2,
batch_first=True)
self.dense = nn.Linear(hidden_size, vocab_size)
def forward(self, x, prev_state):
embed = self.embedding(x)
output, state = self.lstm(embed, prev_state)
logits = self.dense(output)
print(logits.size())
return logits, state
# 첫 입력값을 위한 zero state를 출력.
def zero_state(self, batch_size):
return (torch.zeros(2, batch_size, self.hidden_size),
torch.zeros(2, batch_size, self.hidden_size))
def make_data_label(corpus) :
data = []
label = []
for c in corpus :
data.append(c[:-1])
label.append(c[1:])
data, label = torch.LongTensor(data), torch.LongTensor(label)
return data, label
- main.py
if __name__=="__main__":
""" 데이터 불러오기.
"""
corpus, word2id, id2word, weight = load_data()
corpus = torch.LongTensor(corpus)
""" 하이퍼 파라미터.
"""
# 훈련
epochs = 10
learning_rate = 0.003
batch_size = 16
hidden_size = 32 # lstm hidden 값 차원수
gradients_norm=5 # 기울기 클리핑.
# 문장
seq_size=len(corpus[0]) # 문장 1개 길이.
embedding_size=len(weight[0]) # 임베딩 벡터 사이즈.
vocab_size = len(word2id)
# 테스트
# initial_words=['I', 'am']
predict_top_k=5
checkpoint_path='./checkpoint'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('# corpus size : {} / vocab_size : {}'.format(len(corpus), vocab_size))
print('# batch size : {} / num of cell : {}'.format(batch_size, hidden_size))
print("# 디바이스 : ", device)
print('-'*30+"데이터 불러오기 및 하이퍼 파라미터 설정 분할 완료.")
""" data/label 분할
"""
c = corpus.numpy() # corpus가 Tensor 형태이므로 정상적인 slicing을 위해 numpy 형태로 바꾸어준다.
data, label = make_data_label(c)
dataset = CommentDataset(data, label)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
print('-'*30 + "Data Loader 준비 완료.")
""" Model 정의 및 생성.
"""
net = RNNModule(vocab_size, seq_size,
embedding_size, hidden_size)
net = net.to(device)
loss_f = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
for batch_idx, sample in enumerate(dataloader) :
data, label = sample
data = data.to(device)
label = label.to(device)
state_h, state_c = net.zero_state(batch_size) # initial h, c
state_h = state_h.to(device)
state_c = state_c.to(device)
logits, (state_h, state_c) = net.forward(data, (state_h, state_c))
print(logits.transpose(1, 2).size())
print(label.size())
loss = loss_f(logits.transpose(1, 2), label)
loss.backward()
optimizer.step()
break
所以,我不明白的是为什么张量 logits
(main.py 末尾的代码)必须转置。
logits 的形状和标签是:
logits : torch.Size([16, 19, 10002]) # [batch_size, setence_length, vocab_size]
label : torch.Size([16, 19]) # [batch_size, setence_length]
在我看来,要使用 CrossEntropy 计算损失,标签的形状和数据的形状必须是相同的维度,但事实并非如此。 (标签的形状:[batch_size、setence_length] -> [batch_size、setence_length、vocab_size])
我怎么理解这个?为什么有效?
ps。我参考了下面的网站!
: https://machinetalk.org/2019/02/08/text-generation-with-pytorch/
nn.CrossEntropyLoss()
不接受 one-hot 向量。相反,它接受 class 值。因此,您的 logits 和 targets 将不具有相同的维度。 Logit 的维度必须是 (num_examples, vocab_size)
,但您的标签只需要包含真正的 class 的索引,因此它的形状将是 (num_examples)
而不是 (num_examples, vocab_size)
。仅当您输入 one-hot 编码向量时才需要该形状。
至于为什么需要转置 logits 向量,nn.CrossEntropyLoss()
期望 logits 向量的维度为 (batch_size, num_classes,loss_dims)
,其中 loss_dims 是每个维度中的标记数批量。
我正在使用 pytorch 通过简单的玩具项目(只是生成文本)来研究 NLP。当我在网上引用一些示例代码时,遇到了一个我无法理解的问题。
代码如下(部分代码已省略未完成。):
- LSTM 模型(utils.py)。
def __init__(self, vocab_size, seq_size, embedding_size, hidden_size):
super(RNNModule, self).__init__()
self.seq_size = seq_size
self.hidden_size = hidden_size
self.embedding = nn.Embedding(vocab_size,
embedding_size)
self.lstm = nn.LSTM(input_size = embedding_size,
hidden_size = hidden_size,
num_layers = 2,
batch_first=True)
self.dense = nn.Linear(hidden_size, vocab_size)
def forward(self, x, prev_state):
embed = self.embedding(x)
output, state = self.lstm(embed, prev_state)
logits = self.dense(output)
print(logits.size())
return logits, state
# 첫 입력값을 위한 zero state를 출력.
def zero_state(self, batch_size):
return (torch.zeros(2, batch_size, self.hidden_size),
torch.zeros(2, batch_size, self.hidden_size))
def make_data_label(corpus) :
data = []
label = []
for c in corpus :
data.append(c[:-1])
label.append(c[1:])
data, label = torch.LongTensor(data), torch.LongTensor(label)
return data, label
- main.py
if __name__=="__main__":
""" 데이터 불러오기.
"""
corpus, word2id, id2word, weight = load_data()
corpus = torch.LongTensor(corpus)
""" 하이퍼 파라미터.
"""
# 훈련
epochs = 10
learning_rate = 0.003
batch_size = 16
hidden_size = 32 # lstm hidden 값 차원수
gradients_norm=5 # 기울기 클리핑.
# 문장
seq_size=len(corpus[0]) # 문장 1개 길이.
embedding_size=len(weight[0]) # 임베딩 벡터 사이즈.
vocab_size = len(word2id)
# 테스트
# initial_words=['I', 'am']
predict_top_k=5
checkpoint_path='./checkpoint'
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print('# corpus size : {} / vocab_size : {}'.format(len(corpus), vocab_size))
print('# batch size : {} / num of cell : {}'.format(batch_size, hidden_size))
print("# 디바이스 : ", device)
print('-'*30+"데이터 불러오기 및 하이퍼 파라미터 설정 분할 완료.")
""" data/label 분할
"""
c = corpus.numpy() # corpus가 Tensor 형태이므로 정상적인 slicing을 위해 numpy 형태로 바꾸어준다.
data, label = make_data_label(c)
dataset = CommentDataset(data, label)
dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)
print('-'*30 + "Data Loader 준비 완료.")
""" Model 정의 및 생성.
"""
net = RNNModule(vocab_size, seq_size,
embedding_size, hidden_size)
net = net.to(device)
loss_f = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)
for batch_idx, sample in enumerate(dataloader) :
data, label = sample
data = data.to(device)
label = label.to(device)
state_h, state_c = net.zero_state(batch_size) # initial h, c
state_h = state_h.to(device)
state_c = state_c.to(device)
logits, (state_h, state_c) = net.forward(data, (state_h, state_c))
print(logits.transpose(1, 2).size())
print(label.size())
loss = loss_f(logits.transpose(1, 2), label)
loss.backward()
optimizer.step()
break
所以,我不明白的是为什么张量 logits
(main.py 末尾的代码)必须转置。
logits 的形状和标签是:
logits : torch.Size([16, 19, 10002]) # [batch_size, setence_length, vocab_size]
label : torch.Size([16, 19]) # [batch_size, setence_length]
在我看来,要使用 CrossEntropy 计算损失,标签的形状和数据的形状必须是相同的维度,但事实并非如此。 (标签的形状:[batch_size、setence_length] -> [batch_size、setence_length、vocab_size])
我怎么理解这个?为什么有效?
ps。我参考了下面的网站!
: https://machinetalk.org/2019/02/08/text-generation-with-pytorch/
nn.CrossEntropyLoss()
不接受 one-hot 向量。相反,它接受 class 值。因此,您的 logits 和 targets 将不具有相同的维度。 Logit 的维度必须是 (num_examples, vocab_size)
,但您的标签只需要包含真正的 class 的索引,因此它的形状将是 (num_examples)
而不是 (num_examples, vocab_size)
。仅当您输入 one-hot 编码向量时才需要该形状。
至于为什么需要转置 logits 向量,nn.CrossEntropyLoss()
期望 logits 向量的维度为 (batch_size, num_classes,loss_dims)
,其中 loss_dims 是每个维度中的标记数批量。