fastai--fastai
wehub-resource-sync
290 行
14 KiB
Python
290 行
14 KiB
Python
"""Functions and transforms to help gather text data in a `Datasets`
|
|
|
|
Docs: https://docs.fast.ai/text.data.html.md"""
|
|
|
|
# AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/31_text.data.ipynb.
|
|
|
|
# %% auto #0
|
|
__all__ = ['pad_input', 'reverse_text', 'make_vocab', 'TensorText', 'LMTensorText', 'Numericalize', 'LMDataLoader', 'show_batch',
|
|
'Pad_Input', 'pad_chunk', 'pad_input_chunk', 'Pad_Chunk', 'SortedDL', 'TextBlock', 'TextDataLoaders']
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #21b87e7e
|
|
from ..torch_basics import *
|
|
from ..data.all import *
|
|
from .core import *
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #060cd127
|
|
def reverse_text(x): return x.flip(0)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #f58cfb2a
|
|
def make_vocab(count, min_freq=3, max_vocab=60000, special_toks=None):
|
|
"Create a vocab of `max_vocab` size from `Counter` `count` with items present more than `min_freq`"
|
|
vocab = [o for o,c in count.most_common(max_vocab) if c >= min_freq]
|
|
special_toks = ifnone(special_toks, defaults.text_spec_tok)
|
|
for o in reversed(special_toks): #Make sure all special tokens are in the vocab
|
|
if o in vocab: vocab.remove(o)
|
|
vocab.insert(0, o)
|
|
vocab = vocab[:max_vocab]
|
|
return vocab + [f'xxfake' for i in range(0, 8-len(vocab)%8)]
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #3b0e7f5c
|
|
class TensorText(TensorBase): pass
|
|
class LMTensorText(TensorText): pass
|
|
|
|
TensorText.__doc__ = "Semantic type for a tensor representing text"
|
|
LMTensorText.__doc__ = "Semantic type for a tensor representing text in language modeling"
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #0b36808c
|
|
class Numericalize(Transform):
|
|
"Reversible transform of tokenized texts to numericalized ids"
|
|
def __init__(self, vocab=None, min_freq=3, max_vocab=60000, special_toks=None):
|
|
store_attr('vocab,min_freq,max_vocab,special_toks')
|
|
self.o2i = None if vocab is None else defaultdict(int, {v:k for k,v in enumerate(vocab)})
|
|
|
|
def setups(self, dsets):
|
|
if dsets is None: return
|
|
if self.vocab is None:
|
|
count = dsets.counter if getattr(dsets, 'counter', None) is not None else Counter(p for o in dsets for p in o)
|
|
if self.special_toks is None and hasattr(dsets, 'special_toks'):
|
|
self.special_toks = dsets.special_toks
|
|
self.vocab = make_vocab(count, min_freq=self.min_freq, max_vocab=self.max_vocab, special_toks=self.special_toks)
|
|
self.o2i = defaultdict(int, {v:k for k,v in enumerate(self.vocab) if v != 'xxfake'})
|
|
|
|
def encodes(self, o): return TensorText(tensor([self.o2i [o_] for o_ in o]))
|
|
def decodes(self, o): return L(self.vocab[o_] for o_ in o)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #b27dd16b
|
|
def _maybe_first(o): return o[0] if isinstance(o, tuple) else o
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #7fd53e96
|
|
def _get_tokenizer(ds):
|
|
tok = getattr(ds, 'tokenizer', None)
|
|
if isinstance(tok, Tokenizer): return tok
|
|
if isinstance(tok, (list,L)):
|
|
for t in tok:
|
|
if isinstance(t, Tokenizer): return t
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #8e71a626
|
|
def _get_lengths(ds):
|
|
tok = _get_tokenizer(ds)
|
|
if tok is None: return
|
|
return tok.get_lengths(ds.items)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #1f66c43d
|
|
#TODO: add backward
|
|
@delegates()
|
|
class LMDataLoader(TfmdDL):
|
|
"A `DataLoader` suitable for language modeling"
|
|
def __init__(self, dataset, lens=None, cache=2, bs=64, seq_len=72, num_workers=0, **kwargs):
|
|
self.items = ReindexCollection(dataset, cache=cache, tfm=_maybe_first)
|
|
self.seq_len = seq_len
|
|
if lens is None: lens = _get_lengths(dataset)
|
|
if lens is None: lens = [len(o) for o in self.items]
|
|
self.lens = ReindexCollection(lens, idxs=self.items.idxs)
|
|
# The "-1" is to allow for final label, we throw away the end that's less than bs
|
|
corpus = round_multiple(sum(lens)-1, bs, round_down=True)
|
|
self.bl = corpus//bs #bl stands for batch length
|
|
self.n_batches = self.bl//(seq_len) + int(self.bl%seq_len!=0)
|
|
self.last_len = self.bl - (self.n_batches-1)*seq_len
|
|
self.make_chunks()
|
|
super().__init__(dataset=dataset, bs=bs, num_workers=num_workers, **kwargs)
|
|
self.n = self.n_batches*bs
|
|
|
|
def make_chunks(self): self.chunks = Chunks(self.items, self.lens)
|
|
def shuffle_fn(self,idxs):
|
|
self.items.shuffle()
|
|
self.make_chunks()
|
|
return idxs
|
|
|
|
def create_item(self, seq):
|
|
if seq is None: seq = 0
|
|
if seq>=self.n: raise IndexError
|
|
sl = self.last_len if seq//self.bs==self.n_batches-1 else self.seq_len
|
|
st = (seq%self.bs)*self.bl + (seq//self.bs)*self.seq_len
|
|
txt = self.chunks[st : st+sl+1]
|
|
return LMTensorText(txt[:-1]),txt[1:]
|
|
|
|
@delegates(TfmdDL.new)
|
|
def new(self, dataset=None, seq_len=None, **kwargs):
|
|
lens = self.lens.coll if dataset is None else None
|
|
seq_len = self.seq_len if seq_len is None else seq_len
|
|
return super().new(dataset=dataset, lens=lens, seq_len=seq_len, **kwargs)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #5e86ad40
|
|
@dispatch
|
|
def show_batch(x: TensorText, y, samples, ctxs=None, max_n=10, trunc_at=150, **kwargs):
|
|
if ctxs is None: ctxs = get_empty_df(min(len(samples), max_n))
|
|
if trunc_at is not None: samples = L((s[0].truncate(trunc_at),*s[1:]) for s in samples)
|
|
ctxs = get_show_batch_func(object)(x, y, samples, max_n=max_n, ctxs=ctxs, **kwargs)
|
|
display_df(pd.DataFrame(ctxs))
|
|
return ctxs
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #33cd4023
|
|
@dispatch
|
|
def show_batch(x: LMTensorText, y, samples, ctxs=None, max_n=10, trunc_at=150, **kwargs):
|
|
samples = L((s[0].truncate(trunc_at), s[1].truncate(trunc_at)) for s in samples)
|
|
return get_show_batch_func(TensorText)(x, None, samples, ctxs=ctxs, max_n=max_n, trunc_at=None, **kwargs)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #2d6f750e
|
|
class Pad_Input(ItemTransform):
|
|
def encodes(self,samples, pad_idx=1, pad_fields=0, pad_first=False, backwards=False):
|
|
"Function that collect `samples` and adds padding"
|
|
self.pad_idx = pad_idx
|
|
pad_fields = L(pad_fields)
|
|
max_len_l = pad_fields.map(lambda f: max([len(s[f]) for s in samples]))
|
|
if backwards: pad_first = not pad_first
|
|
def _f(field_idx, x):
|
|
if field_idx not in pad_fields: return x
|
|
idx = pad_fields.items.index(field_idx) #TODO: remove items if L.index is fixed
|
|
sl = slice(-len(x), sys.maxsize) if pad_first else slice(0, len(x))
|
|
pad = x.new_zeros(max_len_l[idx]-x.shape[0])+pad_idx
|
|
x1 = torch.cat([pad, x] if pad_first else [x, pad])
|
|
if backwards: x1 = x1.flip(0)
|
|
return retain_type(x1, x)
|
|
return [tuple(map(lambda idxx: _f(*idxx), enumerate(s))) for s in samples]
|
|
def decodes(self, o:TensorText):
|
|
pad_idx = self.pad_idx if hasattr(self,'pad_idx') else 1
|
|
return o[o != pad_idx]
|
|
pad_input=Pad_Input()
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #852c32d5
|
|
def pad_chunk(x,pad_idx=1, pad_first=True, seq_len=72, pad_len=10):
|
|
"Pad `x` by adding padding by chunks of size `seq_len`"
|
|
l = pad_len - x.shape[0]
|
|
pad_chunk = x.new_zeros((l//seq_len) * seq_len) + pad_idx
|
|
pad_res = x.new_zeros(l % seq_len) + pad_idx
|
|
x1 = torch.cat([pad_chunk, x, pad_res]) if pad_first else torch.cat([x, pad_chunk, pad_res])
|
|
return retain_type(x1, x)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #32faa119
|
|
@delegates(pad_chunk)
|
|
def pad_input_chunk(samples, n_inp=1,**kwargs):
|
|
"Pad `samples` by adding padding by chunks of size `seq_len`"
|
|
max_len = max([len(s[n]) for s in samples for n in range(n_inp)])
|
|
padeds = [[pad_chunk(s[n],pad_len=max_len,**kwargs) for n in range(n_inp) ] for s in samples]
|
|
return [(*p, *s[n_inp:]) for p,s in zip(padeds,samples)]
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #3563df07
|
|
class Pad_Chunk(DisplayedTransform):
|
|
"Pad `samples` by adding padding by chunks of size `seq_len`"
|
|
def __init__(self, pad_idx=1, pad_first=True, seq_len=72,decode=True,**kwargs):
|
|
store_attr('pad_idx, pad_first, seq_len,seq_len')
|
|
super().__init__(**kwargs)
|
|
def before_call(self, b):
|
|
"Set `self.max_len` before encodes"
|
|
self.max_len = max([x.shape[0] for xs in b for x in xs if isinstance(x,TensorText)])
|
|
def __call__(self, b, **kwargs):
|
|
self.before_call(b)
|
|
return super().__call__(tuple(b), **kwargs)
|
|
def encodes(self, x:TensorText):
|
|
return pad_chunk(x,pad_idx=self.pad_idx, pad_first=self.pad_first, seq_len=self.seq_len, pad_len=self.max_len)
|
|
def decodes(self, o:TensorText):
|
|
return o[o != self.pad_idx] if self.decode else o
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #c61cd6bb
|
|
def _default_sort(x): return len(x[0])
|
|
|
|
@delegates(TfmdDL)
|
|
class SortedDL(TfmdDL):
|
|
"A `DataLoader` that goes throught the item in the order given by `sort_func`"
|
|
def __init__(self, dataset, sort_func=None, res=None, **kwargs):
|
|
super().__init__(dataset, **kwargs)
|
|
self.sort_func = _default_sort if sort_func is None else sort_func
|
|
if res is None and self.sort_func == _default_sort: res = _get_lengths(dataset)
|
|
self.res = [self.sort_func(self.do_item(i)) for i in range_of(self.dataset)] if res is None else res
|
|
if len(self.res) > 0: self.idx_max = np.argmax(self.res)
|
|
|
|
def get_idxs(self):
|
|
idxs = super().get_idxs()
|
|
if self.shuffle: return idxs
|
|
return sorted(idxs, key=lambda i: self.res[i], reverse=True)
|
|
|
|
def shuffle_fn(self,idxs):
|
|
idxs = np.random.permutation(len(self.dataset))
|
|
idx_max = np.where(idxs==self.idx_max)[0][0]
|
|
idxs[0],idxs[idx_max] = idxs[idx_max],idxs[0]
|
|
sz = self.bs*50
|
|
chunks = [idxs[i:i+sz] for i in range(0, len(idxs), sz)]
|
|
chunks = [sorted(s, key=lambda i: self.res[i], reverse=True) for s in chunks]
|
|
sort_idx = np.concatenate(chunks)
|
|
|
|
sz = self.bs
|
|
batches = [sort_idx[i:i+sz] for i in range(0, len(sort_idx), sz)]
|
|
sort_idx = np.concatenate(np.random.permutation(batches[1:-1])) if len(batches) > 2 else np.array([],dtype=int)
|
|
sort_idx = np.concatenate((batches[0], sort_idx) if len(batches)==1 else (batches[0], sort_idx, batches[-1]))
|
|
return iter(sort_idx)
|
|
|
|
@delegates(TfmdDL.new)
|
|
def new(self, dataset=None, **kwargs):
|
|
if 'val_res' in kwargs and kwargs['val_res'] is not None: res = kwargs['val_res']
|
|
else: res = self.res if dataset is None else None
|
|
return super().new(dataset=dataset, res=res, **kwargs)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #99d34a59
|
|
class TextBlock(TransformBlock):
|
|
"A `TransformBlock` for texts"
|
|
@delegates(Numericalize.__init__)
|
|
def __init__(self, tok_tfm, vocab=None, is_lm=False, seq_len=72, backwards=False, **kwargs):
|
|
type_tfms = [tok_tfm, Numericalize(vocab, **kwargs)]
|
|
if backwards: type_tfms += [reverse_text]
|
|
return super().__init__(type_tfms=type_tfms,
|
|
dl_type=LMDataLoader if is_lm else SortedDL,
|
|
dls_kwargs={'seq_len': seq_len} if is_lm else {'before_batch': Pad_Chunk(seq_len=seq_len)})
|
|
|
|
@classmethod
|
|
@delegates(Tokenizer.from_df, keep=True)
|
|
def from_df(cls, text_cols, vocab=None, is_lm=False, seq_len=72, backwards=False, min_freq=3, max_vocab=60000, **kwargs):
|
|
"Build a `TextBlock` from a dataframe using `text_cols`"
|
|
return cls(Tokenizer.from_df(text_cols, **kwargs), vocab=vocab, is_lm=is_lm, seq_len=seq_len,
|
|
backwards=backwards, min_freq=min_freq, max_vocab=max_vocab)
|
|
|
|
@classmethod
|
|
@delegates(Tokenizer.from_folder, keep=True)
|
|
def from_folder(cls, path, vocab=None, is_lm=False, seq_len=72, backwards=False, min_freq=3, max_vocab=60000, **kwargs):
|
|
"Build a `TextBlock` from a `path`"
|
|
return cls(Tokenizer.from_folder(path, **kwargs), vocab=vocab, is_lm=is_lm, seq_len=seq_len,
|
|
backwards=backwards, min_freq=min_freq, max_vocab=max_vocab)
|
|
|
|
# %% ../../nbs/31_text.data.ipynb #c8bda6d7
|
|
class TextDataLoaders(DataLoaders):
|
|
"Basic wrapper around several `DataLoader`s with factory methods for NLP problems"
|
|
@classmethod
|
|
@delegates(DataLoaders.from_dblock)
|
|
def from_folder(cls, path, train='train', valid='valid', valid_pct=None, seed=None, vocab=None, text_vocab=None, is_lm=False,
|
|
tok_tfm=None, seq_len=72, splitter=None, backwards=False, **kwargs):
|
|
"Create from imagenet style dataset in `path` with `train` and `valid` subfolders (or provide `valid_pct`)"
|
|
if splitter is None:
|
|
splitter = GrandparentSplitter(train_name=train, valid_name=valid) if valid_pct is None else RandomSplitter(valid_pct, seed=seed)
|
|
blocks = [TextBlock.from_folder(path, text_vocab, is_lm, seq_len, backwards, tok=tok_tfm)]
|
|
if not is_lm: blocks.append(CategoryBlock(vocab=vocab))
|
|
get_items = partial(get_text_files, folders=[train,valid]) if valid_pct is None else get_text_files
|
|
dblock = DataBlock(blocks=blocks,
|
|
get_items=get_items,
|
|
splitter=splitter,
|
|
get_y=None if is_lm else parent_label)
|
|
return cls.from_dblock(dblock, path, path=path, seq_len=seq_len, **kwargs)
|
|
|
|
@classmethod
|
|
@delegates(DataLoaders.from_dblock)
|
|
def from_df(cls, df, path='.', valid_pct=0.2, seed=None, text_col=0, label_col=1, label_delim=None, y_block=None,
|
|
text_vocab=None, is_lm=False, valid_col=None, tok_tfm=None, tok_text_col="text", seq_len=72, backwards=False, **kwargs):
|
|
"Create from `df` in `path` with `valid_pct`"
|
|
blocks = [TextBlock.from_df(text_col, text_vocab, is_lm, seq_len, backwards, tok=tok_tfm)]
|
|
if y_block is None and not is_lm:
|
|
blocks.append(MultiCategoryBlock if is_listy(label_col) and len(label_col) > 1 else CategoryBlock)
|
|
if y_block is not None and not is_lm: blocks += (y_block if is_listy(y_block) else [y_block])
|
|
splitter = RandomSplitter(valid_pct, seed=seed) if valid_col is None else ColSplitter(valid_col)
|
|
dblock = DataBlock(blocks=blocks,
|
|
get_x=ColReader(tok_text_col),
|
|
get_y=None if is_lm else ColReader(label_col, label_delim=label_delim),
|
|
splitter=splitter)
|
|
return cls.from_dblock(dblock, df, path=path, seq_len=seq_len, **kwargs)
|
|
|
|
@classmethod
|
|
def from_csv(cls, path, csv_fname='labels.csv', header='infer', delimiter=None, quoting=csv.QUOTE_MINIMAL, **kwargs):
|
|
"Create from `csv` file in `path/csv_fname`"
|
|
df = pd.read_csv(Path(path)/csv_fname, header=header, delimiter=delimiter, quoting=quoting)
|
|
return cls.from_df(df, path=path, **kwargs)
|
|
|
|
TextDataLoaders.from_csv = delegates(to=TextDataLoaders.from_df)(TextDataLoaders.from_csv)
|