LICENSE.txt
MANIFEST.in
README.md
pyproject.toml
src/tokenizer/Abbrev.conf
src/tokenizer/__init__.py
src/tokenizer/abbrev.py
src/tokenizer/definitions.py
src/tokenizer/main.py
src/tokenizer/tokenizer.py
src/tokenizer.egg-info/PKG-INFO
src/tokenizer.egg-info/SOURCES.txt
src/tokenizer.egg-info/dependency_links.txt
src/tokenizer.egg-info/entry_points.txt
src/tokenizer.egg-info/not-zip-safe
src/tokenizer.egg-info/requires.txt
src/tokenizer.egg-info/top_level.txt
test/test_abbrev.py
test/test_cli.py
test/test_composite_glyphs.py
test/test_dashes.py
test/test_detokenize.py
test/test_helper_functions.py
test/test_index_calculation.py
test/test_tokenizer.py
test/test_tokenizer_tok.py