# Tokenizer Configuration - ViuMini-Dense-360M
# Architecture: Byte-Level BPE with 48k vocabulary
vocab_size: 48000
type: byte-level-bpe
normalizer: NFC
pre_tokenizer: byte-level
special_tokens:
  - <pad>
  - <bos>
  - <eos>
  - <unk>
  - <mask>
  - <|hindi|>
  - <|english|>
  - <|hinglish|>
  - <soch>
  - </soch>
  - <|user|>
  - <|assistant|>
  - <|system|>
data_mix:
  hinglish: 0.4
  hindi: 0.3
  english: 0.3
trainer:
  min_frequency: 2
  max_lines: 500000
output: tokenizer/outputs/tokenizer.json