PytorchでSeq2seqを使って文章の校正を行おうとしたのですが、
Decoderから同じ単語しか出力されません。
main.pyです。
これを実行すると訓練などもろもろを実行します。
#main.py
import copy
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import argparse
import sentencepiece as spm
from loader import Seq2seqDataset
from train import Operator
from models.EncoderDecoder import EncoderDecoder, Encoder, Decoder
from models.Decoder import Decoder as attentionGRU
from models.GRU import GRU
sp = spm.SentencePieceProcessor()
sp.load("./data/index.model")
def make_model(vocab, maxlen, d_model=512):
"Helper: Construct a model from hyperparameters."
c = copy.deepcopy
decoder_gru = attentionGRU(d_model, vocab, d_model, maxlen)
gru = GRU(d_model, vocab)
model = EncoderDecoder(Encoder(gru), Decoder(decoder_gru, maxlen))
return model.cuda()
def data_load(maxlen, source_size, batch_size):
data_set = Seq2seqDataset(maxlen=maxlen)
data_num = len(data_set)
train_ratio = int(data_num*0.8)
eval_ratio = int(data_num*0.1)
test_ratio = int(data_num*0.1)
res = int(data_num - (train_ratio + eval_ratio + test_ratio))
train_ratio += res
ratio=[train_ratio, eval_ratio, test_ratio]
train_dataset, val_dataset, test_dataset = torch.utils.data.random_split(data_set, ratio)
dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True)
del(train_dataset)
del(data_set)
return dataloader, val_dataloader
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Parse training parameters')
parser.add_argument('--batch_size', type=int, default=128,
help='number of examples in a batch')
parser.add_argument('--maxlen', type=int, default=20,
help='Sequences will be padded or truncated to this size.')
parser.add_argument('--epochs', type=int, default=10,
help='the number of epochs to train')
parser.add_argument('--hidden_size', type=int, default=512,
help='the number of hidden dim')
args = parser.parse_args()
vocab = sp.get_piece_size()
train_loader, val_loader = data_load(args.maxlen, vocab, args.batch_size)
model = make_model(vocab, args.maxlen, args.hidden_size)
criterion = nn.NLLLoss(ignore_index=0).cuda()
optimizer = optim.Adam(model.parameters(), lr=0.0001)
training_operator = Operator(model, optimizer, criterion)
for epoch in range(args.epochs):
training_operator.run_epoch(epoch, train_loader, val_loader)
train.pyです。
訓練や検証などを行います。
#train.py
from utils import to_np, trim_seqs
from torch.autograd import Variable
import torch
from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction
import sentencepiece as spm
sp = spm.SentencePieceProcessor()
sp.load("./data/index.model")
clip = 5.0
class Operator:
def __init__(self, model, optimizer, criterion):
self.model = model
self.optimizer = optimizer
self.criterion = criterion
def run_epoch(self, epoch, train_loader, eval_loader):
self.model.train()
losses = []
sampled_idxs = []
for idx, data in enumerate(train_loader):
self.optimizer.zero_grad()
input_data = Variable(data[0].cuda())
target = data[1].cuda()
target_y = Variable(target[:, 1:])
target = Variable(target[:, :-1])
out, _ = self.model(input_data, target)
loss = self.loss_compute(out, target_y, True)
losses.append(to_np(loss))
train_loss = sum(losses) / len(losses)
eval_loss, bleuscore = self.evaluate(out, eval_loader)
print("epochs: {} train_loss: {} eval_loss: {} val_bleuscore: {}".format(epoch+1, train_loss, eval_loss, bleuscore))
def loss_compute(self, out, target, flag=False):
loss = self.criterion(out.contiguous().view(-1, out.size(-1)), target.contiguous().view(-1))
if flag:
self.optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(self.model.parameters(), clip)
self.optimizer.step()
return loss
def evaluate(self, out, loader):
self.model.eval()
losses = []
all_output_seqs = []
all_target_seqs = []
for idx, data in enumerate(loader):
with torch.no_grad():
sampled_index = []
decoder_outputs = []
sampled_idxs = []
input_data = data[0].cuda()
target = data[1].cuda()
target_y = target[:, 1:]
_, hidden = self.model.encode(input_data)
start_symbol = [[sp.PieceToId("<s>")] for i in range(input_data.size(0))]
decoder_input = torch.tensor(start_symbol).cuda()
for i in range(input_data.size(1)):
decoder_output, hidden = self.model.decode(decoder_input, hidden)
_,topi = torch.topk(decoder_output, 1, dim=-1)
decoder_outputs.append(decoder_output)
sampled_idxs.append(topi)
decoder_input = topi.squeeze(1)
sampled_idxs = torch.stack(sampled_idxs, dim=1)
decoder_outputs = torch.stack(decoder_outputs, dim=1)
sampled_idxs = sampled_idxs.squeeze()
decoder_outputs = decoder_outputs.squeeze()
loss = self.loss_compute(decoder_outputs, target_y)
all_output_seqs.extend(trim_seqs(sampled_idxs))
all_target_seqs.extend([list(seq[seq > 0])] for seq in to_np(target))
losses.append(to_np(loss))
bleu_score = corpus_bleu(all_target_seqs, all_output_seqs, smoothing_function=SmoothingFunction().method1)
mean_loss = sum(losses) / len(losses)
self.generator(all_output_seqs, all_target_seqs, input_data.size(1))
return mean_loss, bleu_score
def generator(self, all_output_seqs, all_target_seqs, maxlen):
with open("./log/result.txt", "w", encoding="utf-8") as f:
for sentence in all_output_seqs:
for tok in sentence:
f.write(sp.IdToPiece(int(tok)))
f.write("\n")
with open("./log/target.txt", "w", encoding="utf-8") as f:
for sentence in all_target_seqs:
for tok in sentence[0]:
f.write(sp.IdToPiece(int(tok)))
f.write("\n")
loader.pyです。
コーパスをcsvファイルに格納したものを整形して返します。
#loader.py
import torch
import numpy as np
import csv
import sentencepiece as spm
class Seq2seqDataset(torch.utils.data.Dataset):
def __init__(self, maxlen):
self.sp = spm.SentencePieceProcessor()
self.sp.load("./data/index.model")
self.maxlen = maxlen
with open('./data/parallel_data.csv', mode='r', newline='', encoding='utf-8') as f:
#with open('./data/sample.csv', mode='r', newline='', encoding='utf-8') as f:
csv_file = csv.reader(f)
read_data = [row for row in csv_file]
self.data_num = len(read_data) - 1
jp_data = []
es_data = []
for i in range(1, self.data_num):
jp_data.append(read_data[i][1:2])
es_data.append(read_data[i][2:3])
self.en_data_idx = np.zeros((len(jp_data), maxlen))
self.de_data_idx = np.zeros((len(es_data), maxlen+1))
for i,sentence in enumerate(jp_data):
for j,idx in enumerate(self.sp.EncodeAsIds(sentence[0])[0:]):
self.en_data_idx[i][j] = idx
if j == maxlen-1:
break
if j < maxlen-1:
self.en_data_idx[i][j:] = self.sp.PieceToId("<unk>")
self.en_data_idx[i][-1] = self.sp.PieceToId("</s>")
for i,sentence in enumerate(es_data):
self.de_data_idx[i][0] = self.sp.PieceToId("<s>")
for j,idx in enumerate(self.sp.EncodeAsIds(sentence[0])[0:]):
self.de_data_idx[i][j+1] = idx
if j+1 == maxlen:
break
if j+1 < maxlen:
self.de_data_idx[i][j+1:] = self.sp.PieceToId("<unk>")
self.de_data_idx[i][-1] = self.sp.PieceToId("</s>")
def __len__(self):
return self.data_num
def __getitem__(self, idx):
en_data = torch.tensor(self.en_data_idx[idx-1][:], dtype=torch.long)
de_data = torch.tensor(self.de_data_idx[idx-1][:], dtype=torch.long)
target = torch.zeros((self.maxlen+1), dtype=torch.long)
for i, data in enumerate(de_data[:]):
target[i] = data
target[0] = self.sp.PieceToId("<s>")
return en_data, target
GRU.py
エンコーダです。
#GRU.py
import torch
import torch.nn as nn
class GRU(nn.Module):
def __init__(self, hidden_size, output_size, num_layers=1):
super(GRU, self).__init__()
self.hidden_size = hidden_size
self.num_layers = num_layers
self.output_size = output_size
self.embedding = nn.Embedding(self.output_size, self.hidden_size, padding_idx=0)
self.embedding.weight.data.normal_(0, 1 / self.hidden_size**0.5)
self.embedding.weight.data[0, :] = 0.0
self.gru_source = nn.GRU(hidden_size, hidden_size, num_layers=num_layers,
bidirectional=True, batch_first=True, dropout=0.2)
def forward(self, sentence_words):
self.gru_source.flatten_parameters()
embedded = self.embedding(sentence_words)
hx = self.init_hidden(sentence_words.size(0))
encoder_output, hx = self.gru_source(embedded, hx)
encoder_output = (encoder_output[:,:,:self.hidden_size] + encoder_output[:,:,self.hidden_size:]) / 2
hx = (hx[0] + hx[1]) / 2
hx = hx.unsqueeze(0)
return encoder_output, hx
def init_hidden(self, bc):
hx = torch.zeros((self.num_layers*2, bc, self.hidden_size))
hx = hx.cuda()
return hx
Decoder.pyです。
デコーダです。
#Decoder.py
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
import torchvision
class Decoder(nn.Module):
def __init__(self, hidden_dim, vocab_size, embedding_dim, max_length):
super(Decoder, self).__init__()
self.hidden_dim = hidden_dim
self.word_embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)
self.gru = nn.GRU(embedding_dim, hidden_dim, batch_first=True)
self.hidden2linear = nn.Linear(hidden_dim, vocab_size)
def forward(self, sequence, encoder_state):
embedding = self.word_embeddings(sequence)
embedding = F.relu(embedding)
output, state = self.gru(embedding, encoder_state)
output = self.hidden2linear(output)
output = F.log_softmax(output, dim=-1)
return output, state
EncoderDecoder.pyです。
EncoderとDecoderをひとまとめにして処理しやすくしています。
#EncoderDecoder.py
import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data
import torch.nn.functional as F
from torch.autograd import Variable
class EncoderDecoder(nn.Module):
def __init__(self, encoder, decoder):
super(EncoderDecoder, self).__init__()
self.encoder = encoder
self.decoder = decoder
def forward(self, data, target):
out, hx = self.encoder(data)
return self.decoder(target, hx)
def decode(self, target, hx):
return self.decoder(target, hx)
def encode(self, input_data):
return self.encoder(input_data)
class Encoder(nn.Module):
def __init__(self, base_module):
super(Encoder, self).__init__()
self.base_module = base_module
def forward(self, data):
return self.base_module(data)
class Decoder(nn.Module):
def __init__(self, base_module, maxlen):
super(Decoder, self).__init__()
self.base_module = base_module
self.maxlen = maxlen
def forward(self, data, hx):
return self.base_module(data, hx)
これを訓練して文章を生成すると、
全てpadで構成された文章になります。
lossも問題なく減少していきますが、生成される文章は×です。
以下出力例です。
epochs: 1 train_loss: 6.804564086012185 eval_loss: 6.8490905404090885 val_bleuscore: 0.00021425393895175578
epochs: 2 train_loss: 6.16480832815932 eval_loss: 7.3809502720832825 val_bleuscore: 5.9522479993669076e-05
epochs: 3 train_loss: 5.995282799291154 eval_loss: 7.583597528934479 val_bleuscore: 6.663744208921663e-05
epochs: 4 train_loss: 5.8298093747026245 eval_loss: 7.464733970165253 val_bleuscore: 0.00021450632944840199
epochs: 5 train_loss: 5.752497753777062 eval_loss: 7.193604528903961 val_bleuscore: 0.00013741750714360893
問題点などを教えていただけると幸いです。
個人的には、EncoderDecoderを統合して扱っているところ、modelをselfで渡して訓練しているところなどが問題になのではないかと思っています。
見ていただけるのであればすべてのファイルを共有します。