Cargo.toml
MANIFEST.in
README.md
pyproject.toml
rust-toolchain
setup.cfg
setup.py
py_src/tokenizers/__init__.py
py_src/tokenizers/__init__.pyi
py_src/tokenizers.egg-info/PKG-INFO
py_src/tokenizers.egg-info/SOURCES.txt
py_src/tokenizers.egg-info/dependency_links.txt
py_src/tokenizers.egg-info/not-zip-safe
py_src/tokenizers.egg-info/requires.txt
py_src/tokenizers.egg-info/top_level.txt
py_src/tokenizers/decoders/__init__.py
py_src/tokenizers/decoders/__init__.pyi
py_src/tokenizers/implementations/__init__.py
py_src/tokenizers/implementations/base_tokenizer.py
py_src/tokenizers/implementations/bert_wordpiece.py
py_src/tokenizers/implementations/byte_level_bpe.py
py_src/tokenizers/implementations/char_level_bpe.py
py_src/tokenizers/implementations/sentencepiece_bpe.py
py_src/tokenizers/implementations/sentencepiece_unigram.py
py_src/tokenizers/models/__init__.py
py_src/tokenizers/models/__init__.pyi
py_src/tokenizers/normalizers/__init__.py
py_src/tokenizers/normalizers/__init__.pyi
py_src/tokenizers/pre_tokenizers/__init__.py
py_src/tokenizers/pre_tokenizers/__init__.pyi
py_src/tokenizers/processors/__init__.py
py_src/tokenizers/processors/__init__.pyi
py_src/tokenizers/tools/__init__.py
py_src/tokenizers/tools/visualizer-styles.css
py_src/tokenizers/tools/visualizer.py
py_src/tokenizers/trainers/__init__.py
py_src/tokenizers/trainers/__init__.pyi
src/decoders.rs
src/encoding.rs
src/error.rs
src/lib.rs
src/models.rs
src/normalizers.rs
src/pre_tokenizers.rs
src/processors.rs
src/token.rs
src/tokenizer.rs
src/trainers.rs
src/utils/iterators.rs
src/utils/mod.rs
src/utils/normalization.rs
src/utils/pretokenization.rs
src/utils/regex.rs
tokenizers-lib/CHANGELOG.md
tokenizers-lib/Cargo.toml
tokenizers-lib/LICENSE
tokenizers-lib/Makefile
tokenizers-lib/README.md
tokenizers-lib/README.tpl
tokenizers-lib/rust-toolchain
tokenizers-lib/benches/bert_benchmark.rs
tokenizers-lib/benches/bpe_benchmark.rs
tokenizers-lib/benches/layout_benchmark.rs
tokenizers-lib/benches/common/mod.rs
tokenizers-lib/examples/serialization.rs
tokenizers-lib/examples/unstable_wasm/.gitignore
tokenizers-lib/examples/unstable_wasm/Cargo.toml
tokenizers-lib/examples/unstable_wasm/README.md
tokenizers-lib/examples/unstable_wasm/src/lib.rs
tokenizers-lib/examples/unstable_wasm/src/utils.rs
tokenizers-lib/examples/unstable_wasm/tests/web.rs
tokenizers-lib/examples/unstable_wasm/www/.gitignore
tokenizers-lib/examples/unstable_wasm/www/.travis.yml
tokenizers-lib/examples/unstable_wasm/www/LICENSE-APACHE
tokenizers-lib/examples/unstable_wasm/www/LICENSE-MIT
tokenizers-lib/examples/unstable_wasm/www/README.md
tokenizers-lib/examples/unstable_wasm/www/bootstrap.js
tokenizers-lib/examples/unstable_wasm/www/index.html
tokenizers-lib/examples/unstable_wasm/www/index.js
tokenizers-lib/examples/unstable_wasm/www/package-lock.json
tokenizers-lib/examples/unstable_wasm/www/package.json
tokenizers-lib/examples/unstable_wasm/www/webpack.config.js
tokenizers-lib/examples/unstable_wasm/www/.bin/create-wasm-app.js
tokenizers-lib/src/cli.rs
tokenizers-lib/src/lib.rs
tokenizers-lib/src/decoders/bpe.rs
tokenizers-lib/src/decoders/byte_fallback.rs
tokenizers-lib/src/decoders/ctc.rs
tokenizers-lib/src/decoders/fuse.rs
tokenizers-lib/src/decoders/mod.rs
tokenizers-lib/src/decoders/sequence.rs
tokenizers-lib/src/decoders/strip.rs
tokenizers-lib/src/decoders/wordpiece.rs
tokenizers-lib/src/models/mod.rs
tokenizers-lib/src/models/bpe/mod.rs
tokenizers-lib/src/models/bpe/model.rs
tokenizers-lib/src/models/bpe/serialization.rs
tokenizers-lib/src/models/bpe/trainer.rs
tokenizers-lib/src/models/bpe/word.rs
tokenizers-lib/src/models/unigram/lattice.rs
tokenizers-lib/src/models/unigram/mod.rs
tokenizers-lib/src/models/unigram/model.rs
tokenizers-lib/src/models/unigram/serialization.rs
tokenizers-lib/src/models/unigram/trainer.rs
tokenizers-lib/src/models/unigram/trie.rs
tokenizers-lib/src/models/wordlevel/mod.rs
tokenizers-lib/src/models/wordlevel/serialization.rs
tokenizers-lib/src/models/wordlevel/trainer.rs
tokenizers-lib/src/models/wordpiece/mod.rs
tokenizers-lib/src/models/wordpiece/serialization.rs
tokenizers-lib/src/models/wordpiece/trainer.rs
tokenizers-lib/src/normalizers/bert.rs
tokenizers-lib/src/normalizers/mod.rs
tokenizers-lib/src/normalizers/precompiled.rs
tokenizers-lib/src/normalizers/prepend.rs
tokenizers-lib/src/normalizers/replace.rs
tokenizers-lib/src/normalizers/strip.rs
tokenizers-lib/src/normalizers/unicode.rs
tokenizers-lib/src/normalizers/utils.rs
tokenizers-lib/src/pre_tokenizers/bert.rs
tokenizers-lib/src/pre_tokenizers/byte_level.rs
tokenizers-lib/src/pre_tokenizers/delimiter.rs
tokenizers-lib/src/pre_tokenizers/digits.rs
tokenizers-lib/src/pre_tokenizers/metaspace.rs
tokenizers-lib/src/pre_tokenizers/mod.rs
tokenizers-lib/src/pre_tokenizers/punctuation.rs
tokenizers-lib/src/pre_tokenizers/sequence.rs
tokenizers-lib/src/pre_tokenizers/split.rs
tokenizers-lib/src/pre_tokenizers/whitespace.rs
tokenizers-lib/src/pre_tokenizers/unicode_scripts/mod.rs
tokenizers-lib/src/pre_tokenizers/unicode_scripts/pre_tokenizer.rs
tokenizers-lib/src/pre_tokenizers/unicode_scripts/scripts.rs
tokenizers-lib/src/processors/bert.rs
tokenizers-lib/src/processors/mod.rs
tokenizers-lib/src/processors/roberta.rs
tokenizers-lib/src/processors/sequence.rs
tokenizers-lib/src/processors/template.rs
tokenizers-lib/src/tokenizer/added_vocabulary.rs
tokenizers-lib/src/tokenizer/encoding.rs
tokenizers-lib/src/tokenizer/mod.rs
tokenizers-lib/src/tokenizer/normalizer.rs
tokenizers-lib/src/tokenizer/pattern.rs
tokenizers-lib/src/tokenizer/pre_tokenizer.rs
tokenizers-lib/src/tokenizer/serialization.rs
tokenizers-lib/src/utils/cache.rs
tokenizers-lib/src/utils/fancy.rs
tokenizers-lib/src/utils/from_pretrained.rs
tokenizers-lib/src/utils/iter.rs
tokenizers-lib/src/utils/mod.rs
tokenizers-lib/src/utils/onig.rs
tokenizers-lib/src/utils/padding.rs
tokenizers-lib/src/utils/parallelism.rs
tokenizers-lib/src/utils/progress.rs
tokenizers-lib/src/utils/truncation.rs
tokenizers-lib/tests/added_tokens.rs
tokenizers-lib/tests/documentation.rs
tokenizers-lib/tests/from_pretrained.rs
tokenizers-lib/tests/offsets.rs
tokenizers-lib/tests/serialization.rs
tokenizers-lib/tests/training.rs
tokenizers-lib/tests/unigram.rs
tokenizers-lib/tests/common/mod.rs