| # Tokenizer Configuration - ViuMini-Dense-360M | |
| # Architecture: Byte-Level BPE with 48k vocabulary | |
| vocab_size: 48000 | |
| type: byte-level-bpe | |
| normalizer: NFC | |
| pre_tokenizer: byte-level | |
| special_tokens: | |
| - <pad> | |
| - <bos> | |
| - <eos> | |
| - <unk> | |
| - <mask> | |
| - <|hindi|> | |
| - <|english|> | |
| - <|hinglish|> | |
| - <soch> | |
| - </soch> | |
| - <|user|> | |
| - <|assistant|> | |
| - <|system|> | |
| data_mix: | |
| hinglish: 0.4 | |
| hindi: 0.3 | |
| english: 0.3 | |
| trainer: | |
| min_frequency: 2 | |
| max_lines: 500000 | |
| output: tokenizer/outputs/tokenizer.json | |