tokenizers-package	tokenizers
basic-tokenizers	basic-tokenizers
chunk_text	chunk_text
count_characters	word-counting
count_sentences	word-counting
count_words	word-counting
mobydick	mobydick
ngram-tokenizers	ngram-tokenizers
tokenizers	tokenizers
tokenize_characters	basic-tokenizers
tokenize_character_shingles	shingle-tokenizers
tokenize_lines	basic-tokenizers
tokenize_ngrams	ngram-tokenizers
tokenize_paragraphs	basic-tokenizers
tokenize_ptb	ptb-tokenizer
tokenize_regex	basic-tokenizers
tokenize_sentences	basic-tokenizers
tokenize_skip_ngrams	ngram-tokenizers
tokenize_tweets	basic-tokenizers
tokenize_words	basic-tokenizers
tokenize_word_stems	stem-tokenizers
