如何直接从列表或字典创建 torchtext.data.TabularDataset
How to create a torchtext.data.TabularDataset directly from a list or dict
torchtext.data.TabularDataset
可以从 TSV/JSON/CSV 文件创建,然后可用于从 Glove、FastText 或任何其他嵌入构建词汇表。但我的要求是直接从 list
或 dict
创建 torchtext.data.TabularDataset
。
当前通过读取 TSV 文件实现的代码
self.RAW = data.RawField()
self.TEXT = data.Field(batch_first=True)
self.LABEL = data.Field(sequential=False, unk_token=None)
self.train, self.dev, self.test = data.TabularDataset.splits(
path='.data/quora',
train='train.tsv',
validation='dev.tsv',
test='test.tsv',
format='tsv',
fields=[('label', self.LABEL),
('q1', self.TEXT),
('q2', self.TEXT),
('id', self.RAW)])
self.TEXT.build_vocab(self.train, self.dev, self.test, vectors=GloVe(name='840B', dim=300))
self.LABEL.build_vocab(self.train)
sort_key = lambda x: data.interleave_keys(len(x.q1), len(x.q2))
self.train_iter, self.dev_iter, self.test_iter = \
data.BucketIterator.splits((self.train, self.dev, self.test),
batch_sizes=[args.batch_size] * 3,
device=args.gpu,
sort_key=sort_key)
这是从文件中读取数据的当前工作代码。因此,为了直接从 List/Dict 创建数据集,我尝试了 Examples.fromDict
或 Examples.fromList 等内置函数,但在进入最后一个 for 循环时,它抛出了一个错误 AttributeError: 'BucketIterator' object has no attribute 'q1'
它要求我编写一个自己的 class 继承数据集 class 并在 torchtext.data.TabularDataset
class.
中进行少量修改
class TabularDataset_From_List(data.Dataset):
def __init__(self, input_list, format, fields, skip_header=False, **kwargs):
make_example = {
'json': Example.fromJSON, 'dict': Example.fromdict,
'tsv': Example.fromTSV, 'csv': Example.fromCSV}[format.lower()]
examples = [make_example(item, fields) for item in input_list]
if make_example in (Example.fromdict, Example.fromJSON):
fields, field_dict = [], fields
for field in field_dict.values():
if isinstance(field, list):
fields.extend(field)
else:
fields.append(field)
super(TabularDataset_From_List, self).__init__(examples, fields, **kwargs)
@classmethod
def splits(cls, path=None, root='.data', train=None, validation=None,
test=None, **kwargs):
if path is None:
path = cls.download(root)
train_data = None if train is None else cls(
train, **kwargs)
val_data = None if validation is None else cls(
validation, **kwargs)
test_data = None if test is None else cls(
test, **kwargs)
return tuple(d for d in (train_data, val_data, test_data)
if d is not None)
torchtext.data.TabularDataset
可以从 TSV/JSON/CSV 文件创建,然后可用于从 Glove、FastText 或任何其他嵌入构建词汇表。但我的要求是直接从 list
或 dict
创建 torchtext.data.TabularDataset
。
当前通过读取 TSV 文件实现的代码
self.RAW = data.RawField()
self.TEXT = data.Field(batch_first=True)
self.LABEL = data.Field(sequential=False, unk_token=None)
self.train, self.dev, self.test = data.TabularDataset.splits(
path='.data/quora',
train='train.tsv',
validation='dev.tsv',
test='test.tsv',
format='tsv',
fields=[('label', self.LABEL),
('q1', self.TEXT),
('q2', self.TEXT),
('id', self.RAW)])
self.TEXT.build_vocab(self.train, self.dev, self.test, vectors=GloVe(name='840B', dim=300))
self.LABEL.build_vocab(self.train)
sort_key = lambda x: data.interleave_keys(len(x.q1), len(x.q2))
self.train_iter, self.dev_iter, self.test_iter = \
data.BucketIterator.splits((self.train, self.dev, self.test),
batch_sizes=[args.batch_size] * 3,
device=args.gpu,
sort_key=sort_key)
这是从文件中读取数据的当前工作代码。因此,为了直接从 List/Dict 创建数据集,我尝试了 Examples.fromDict
或 Examples.fromList 等内置函数,但在进入最后一个 for 循环时,它抛出了一个错误 AttributeError: 'BucketIterator' object has no attribute 'q1'
它要求我编写一个自己的 class 继承数据集 class 并在 torchtext.data.TabularDataset
class.
class TabularDataset_From_List(data.Dataset):
def __init__(self, input_list, format, fields, skip_header=False, **kwargs):
make_example = {
'json': Example.fromJSON, 'dict': Example.fromdict,
'tsv': Example.fromTSV, 'csv': Example.fromCSV}[format.lower()]
examples = [make_example(item, fields) for item in input_list]
if make_example in (Example.fromdict, Example.fromJSON):
fields, field_dict = [], fields
for field in field_dict.values():
if isinstance(field, list):
fields.extend(field)
else:
fields.append(field)
super(TabularDataset_From_List, self).__init__(examples, fields, **kwargs)
@classmethod
def splits(cls, path=None, root='.data', train=None, validation=None,
test=None, **kwargs):
if path is None:
path = cls.download(root)
train_data = None if train is None else cls(
train, **kwargs)
val_data = None if validation is None else cls(
validation, **kwargs)
test_data = None if test is None else cls(
test, **kwargs)
return tuple(d for d in (train_data, val_data, test_data)
if d is not None)