craffel's picture
download
raw
6.32 kB
dump_dir: /fsx/craffel/lingua_logs/8k_v2_oracle
name: flexitok_8k_v2_oracle
steps: 100000
probe_freq: null
seed: 777
grad_acc_steps: 8
optim:
lr: 0.001
weight_decay: 0.1
epsilon: 1.0e-08
scheduler: cosine
warmup: 2000
lr_min_ratio: 1.0e-06
clip: 1.0
distributed:
fsdp_type: full_shard
model_dtype: bf16
matmul_allow_tf32: false
selective_activation_checkpointing: false
tp_size: 1
compile: true
model:
dim: 2048
n_layers: 25
n_heads: 16
weight_tying: false
vocab_size: 122447
data:
root_dir: /scratch/craffel/lingua/data/flexitok/
sources:
fw_edu: 0.4
dan_Latn: 0.0216582869670702
swe_Latn: 0.0216359765418466
vie_Latn: 0.0197485510268674
hun_Latn: 0.0247194573562308
fas_Arab: 0.0205634624231076
tur_Latn: 0.0235455794841729
ces_Latn: 0.0248024455266208
arb_Arab: 0.0234323706569333
ell_Grek: 0.0233670886888026
ind_Latn: 0.0269322054593488
nld_Latn: 0.0277796326621489
pol_Latn: 0.0294120104572311
por_Latn: 0.0301413168306825
ita_Latn: 0.0324056371021865
jpn_Jpan: 0.03553104151369
fra_Latn: 0.0381835560678536
spa_Latn: 0.0387222793083669
deu_Latn: 0.0419925340453022
cmn_Hani: 0.0454067521384114
rus_Cyrl: 0.0500198157431261
batch_size: 4
prefetch_size: 1024
seq_len: 4096
n_views: 2
load_async: true
add_eos: true
tokenizer:
name: supertokenizer
path: meta-llama/Llama-3.2-1B
seed: 42
superset_code_name: 8k_v2
n_words: 122447
tokenizers:
- name: huggingface
path: flexitok/bpe_ltr_arb_Arab_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_ces_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_cmn_Hani_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_dan_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_deu_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_ell_Grek_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_fas_Arab_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_fra_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_fw_edu_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_hun_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_ind_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_ita_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_jpn_Jpan_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_nld_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_pol_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_por_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_rus_Cyrl_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_spa_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_swe_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_tur_Latn_8000_v2
load_supermapping: true
- name: huggingface
path: flexitok/bpe_ltr_vie_Latn_8000_v2
load_supermapping: true
routing:
suitable_tokenizer_probability: 1.0
source_to_tokenizer:
arb_Arab: flexitok/bpe_ltr_arb_Arab_8000_v2
ces_Latn: flexitok/bpe_ltr_ces_Latn_8000_v2
cmn_Hani: flexitok/bpe_ltr_cmn_Hani_8000_v2
dan_Latn: flexitok/bpe_ltr_dan_Latn_8000_v2
deu_Latn: flexitok/bpe_ltr_deu_Latn_8000_v2
ell_Grek: flexitok/bpe_ltr_ell_Grek_8000_v2
fas_Arab: flexitok/bpe_ltr_fas_Arab_8000_v2
fra_Latn: flexitok/bpe_ltr_fra_Latn_8000_v2
fw_edu: flexitok/bpe_ltr_fw_edu_8000_v2
hun_Latn: flexitok/bpe_ltr_hun_Latn_8000_v2
ind_Latn: flexitok/bpe_ltr_ind_Latn_8000_v2
ita_Latn: flexitok/bpe_ltr_ita_Latn_8000_v2
jpn_Jpan: flexitok/bpe_ltr_jpn_Jpan_8000_v2
nld_Latn: flexitok/bpe_ltr_nld_Latn_8000_v2
pol_Latn: flexitok/bpe_ltr_pol_Latn_8000_v2
por_Latn: flexitok/bpe_ltr_por_Latn_8000_v2
rus_Cyrl: flexitok/bpe_ltr_rus_Cyrl_8000_v2
spa_Latn: flexitok/bpe_ltr_spa_Latn_8000_v2
swe_Latn: flexitok/bpe_ltr_swe_Latn_8000_v2
tur_Latn: flexitok/bpe_ltr_tur_Latn_8000_v2
vie_Latn: flexitok/bpe_ltr_vie_Latn_8000_v2
profiling:
run: true
mem_warmup: 0
mem_steps: 4
profile_warmup: 100
profile_steps: 4
checkpoint:
path: /fsx/craffel/lingua_logs/checkpoints/8k_v2_oracle/
init_ckpt_path: null
dump:
every: 10000
keep: -1
eval:
every: 10000
keep: -1
logging:
freq: 1
async_eval_gpus: 8
eval:
harness:
tasks:
- hellaswag
- piqa
- arc_easy
- arc_challenge
- include_base_44_arabic
- include_base_44_chinese
- include_base_44_german
- include_base_44_greek
- include_base_44_persian
- include_base_44_french
- include_base_44_hungarian
- include_base_44_indonesian
- include_base_44_italian
- include_base_44_japanese
- include_base_44_dutch
- include_base_44_portuguese
- include_base_44_russian
- include_base_44_spanish
- include_base_44_turkish
- include_base_44_vietnamese
- belebele_arb_Arab
- belebele_ces_Latn
- belebele_zho_Hans
- belebele_dan_Latn
- belebele_deu_Latn
- belebele_ell_Grek
- belebele_pes_Arab
- belebele_fra_Latn
- belebele_hun_Latn
- belebele_ind_Latn
- belebele_ita_Latn
- belebele_jpn_Jpan
- belebele_nld_Latn
- belebele_pol_Latn
- belebele_por_Latn
- belebele_rus_Cyrl
- belebele_spa_Latn
- belebele_swe_Latn
- belebele_tur_Latn
- belebele_vie_Latn
- belebele_eng_Latn
- xnli_ar
- xnli_zh
- xnli_de
- xnli_el
- xnli_en
- xnli_es
- xnli_fr
- xnli_ru
- xnli_tr
- xnli_vi
generator:
max_tokens: 16384
dtype: bf16
add_bos: false

Xet Storage Details

Size:
6.32 kB
·
Xet hash:
45511149a2c3d6caf0addfd7b358bf74054efa22bedc42be19295a9882631631

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.