| { |
| "best_global_step": 3000, |
| "best_metric": 3.7956512070627526, |
| "best_model_checkpoint": "checkpoints/gpt2_sparse_moe_wiki/checkpoint-3000", |
| "epoch": 0.45211788973974965, |
| "eval_steps": 300, |
| "global_step": 3000, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.007535298162329161, |
| "grad_norm": 1.2304445505142212, |
| "learning_rate": 8.166666666666667e-05, |
| "loss": 9.5844, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.015070596324658321, |
| "grad_norm": 0.5569408535957336, |
| "learning_rate": 0.000165, |
| "loss": 7.6009, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.022605894486987483, |
| "grad_norm": 0.4758251905441284, |
| "learning_rate": 0.0002483333333333333, |
| "loss": 6.7993, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.030141192649316643, |
| "grad_norm": 0.6503773331642151, |
| "learning_rate": 0.0003316666666666667, |
| "loss": 6.2649, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.037676490811645806, |
| "grad_norm": 0.5381407141685486, |
| "learning_rate": 0.000415, |
| "loss": 5.9237, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.045211788973974966, |
| "grad_norm": 0.280143141746521, |
| "learning_rate": 0.0004983333333333334, |
| "loss": 5.6423, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.045211788973974966, |
| "eval_loss": 5.46437172209171, |
| "eval_perplexity": 236.12745483202283, |
| "eval_runtime": 43.9995, |
| "eval_samples_per_second": 37.705, |
| "eval_steps_per_second": 0.591, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.052747087136304126, |
| "grad_norm": 0.25354620814323425, |
| "learning_rate": 0.0004909259259259259, |
| "loss": 5.3993, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.060282385298633286, |
| "grad_norm": 0.3524823486804962, |
| "learning_rate": 0.0004816666666666667, |
| "loss": 5.1954, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.06781768346096245, |
| "grad_norm": 0.3906818628311157, |
| "learning_rate": 0.0004724074074074074, |
| "loss": 5.025, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.07535298162329161, |
| "grad_norm": 0.456912636756897, |
| "learning_rate": 0.00046314814814814813, |
| "loss": 4.859, |
| "step": 500 |
| }, |
| { |
| "epoch": 0.08288827978562077, |
| "grad_norm": 0.3447301387786865, |
| "learning_rate": 0.00045388888888888893, |
| "loss": 4.728, |
| "step": 550 |
| }, |
| { |
| "epoch": 0.09042357794794993, |
| "grad_norm": 0.400978147983551, |
| "learning_rate": 0.00044462962962962967, |
| "loss": 4.6308, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.09042357794794993, |
| "eval_loss": 4.516210105340873, |
| "eval_perplexity": 91.48820943465822, |
| "eval_runtime": 43.4593, |
| "eval_samples_per_second": 38.174, |
| "eval_steps_per_second": 0.598, |
| "step": 600 |
| }, |
| { |
| "epoch": 0.09795887611027909, |
| "grad_norm": 0.30938830971717834, |
| "learning_rate": 0.00043537037037037036, |
| "loss": 4.5451, |
| "step": 650 |
| }, |
| { |
| "epoch": 0.10549417427260825, |
| "grad_norm": 0.3306984603404999, |
| "learning_rate": 0.0004261111111111111, |
| "loss": 4.4871, |
| "step": 700 |
| }, |
| { |
| "epoch": 0.11302947243493741, |
| "grad_norm": 0.30364739894866943, |
| "learning_rate": 0.00041685185185185184, |
| "loss": 4.4291, |
| "step": 750 |
| }, |
| { |
| "epoch": 0.12056477059726657, |
| "grad_norm": 0.32504332065582275, |
| "learning_rate": 0.00040759259259259264, |
| "loss": 4.3825, |
| "step": 800 |
| }, |
| { |
| "epoch": 0.12810006875959573, |
| "grad_norm": 0.32315653562545776, |
| "learning_rate": 0.00039833333333333333, |
| "loss": 4.3458, |
| "step": 850 |
| }, |
| { |
| "epoch": 0.1356353669219249, |
| "grad_norm": 0.2794428765773773, |
| "learning_rate": 0.00038907407407407407, |
| "loss": 4.3127, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.1356353669219249, |
| "eval_loss": 4.218866269811809, |
| "eval_perplexity": 67.95639638117432, |
| "eval_runtime": 43.6842, |
| "eval_samples_per_second": 37.977, |
| "eval_steps_per_second": 0.595, |
| "step": 900 |
| }, |
| { |
| "epoch": 0.14317066508425405, |
| "grad_norm": 0.2745271623134613, |
| "learning_rate": 0.0003798148148148148, |
| "loss": 4.2776, |
| "step": 950 |
| }, |
| { |
| "epoch": 0.15070596324658322, |
| "grad_norm": 0.3081075847148895, |
| "learning_rate": 0.0003705555555555556, |
| "loss": 4.2474, |
| "step": 1000 |
| }, |
| { |
| "epoch": 0.15824126140891237, |
| "grad_norm": 0.26861488819122314, |
| "learning_rate": 0.0003612962962962963, |
| "loss": 4.222, |
| "step": 1050 |
| }, |
| { |
| "epoch": 0.16577655957124154, |
| "grad_norm": 0.26902666687965393, |
| "learning_rate": 0.00035203703703703704, |
| "loss": 4.1946, |
| "step": 1100 |
| }, |
| { |
| "epoch": 0.1733118577335707, |
| "grad_norm": 0.24377495050430298, |
| "learning_rate": 0.0003427777777777778, |
| "loss": 4.1722, |
| "step": 1150 |
| }, |
| { |
| "epoch": 0.18084715589589986, |
| "grad_norm": 0.2884402573108673, |
| "learning_rate": 0.0003335185185185185, |
| "loss": 4.1505, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.18084715589589986, |
| "eval_loss": 4.069066753989171, |
| "eval_perplexity": 58.50234003225461, |
| "eval_runtime": 44.2137, |
| "eval_samples_per_second": 37.522, |
| "eval_steps_per_second": 0.588, |
| "step": 1200 |
| }, |
| { |
| "epoch": 0.188382454058229, |
| "grad_norm": 0.2788221836090088, |
| "learning_rate": 0.00032425925925925927, |
| "loss": 4.1284, |
| "step": 1250 |
| }, |
| { |
| "epoch": 0.19591775222055818, |
| "grad_norm": 0.25833043456077576, |
| "learning_rate": 0.000315, |
| "loss": 4.1179, |
| "step": 1300 |
| }, |
| { |
| "epoch": 0.20345305038288733, |
| "grad_norm": 0.24359293282032013, |
| "learning_rate": 0.00030574074074074076, |
| "loss": 4.0981, |
| "step": 1350 |
| }, |
| { |
| "epoch": 0.2109883485452165, |
| "grad_norm": 0.2737596929073334, |
| "learning_rate": 0.00029648148148148144, |
| "loss": 4.0855, |
| "step": 1400 |
| }, |
| { |
| "epoch": 0.21852364670754565, |
| "grad_norm": 0.24729043245315552, |
| "learning_rate": 0.00028722222222222224, |
| "loss": 4.0639, |
| "step": 1450 |
| }, |
| { |
| "epoch": 0.22605894486987482, |
| "grad_norm": 0.226772740483284, |
| "learning_rate": 0.000277962962962963, |
| "loss": 4.0538, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.22605894486987482, |
| "eval_loss": 3.974391115922098, |
| "eval_perplexity": 53.21770362637156, |
| "eval_runtime": 43.3887, |
| "eval_samples_per_second": 38.236, |
| "eval_steps_per_second": 0.599, |
| "step": 1500 |
| }, |
| { |
| "epoch": 0.23359424303220397, |
| "grad_norm": 0.2437480390071869, |
| "learning_rate": 0.0002687037037037037, |
| "loss": 4.0447, |
| "step": 1550 |
| }, |
| { |
| "epoch": 0.24112954119453314, |
| "grad_norm": 0.2321815937757492, |
| "learning_rate": 0.0002594444444444444, |
| "loss": 4.0357, |
| "step": 1600 |
| }, |
| { |
| "epoch": 0.2486648393568623, |
| "grad_norm": 0.23976509273052216, |
| "learning_rate": 0.00025018518518518516, |
| "loss": 4.0169, |
| "step": 1650 |
| }, |
| { |
| "epoch": 0.25620013751919146, |
| "grad_norm": 0.28224146366119385, |
| "learning_rate": 0.00024092592592592593, |
| "loss": 4.0091, |
| "step": 1700 |
| }, |
| { |
| "epoch": 0.26373543568152064, |
| "grad_norm": 0.23423795402050018, |
| "learning_rate": 0.00023166666666666667, |
| "loss": 4.0012, |
| "step": 1750 |
| }, |
| { |
| "epoch": 0.2712707338438498, |
| "grad_norm": 0.22995537519454956, |
| "learning_rate": 0.0002224074074074074, |
| "loss": 3.9909, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.2712707338438498, |
| "eval_loss": 3.910796159768955, |
| "eval_perplexity": 49.93869533041372, |
| "eval_runtime": 43.6043, |
| "eval_samples_per_second": 38.047, |
| "eval_steps_per_second": 0.596, |
| "step": 1800 |
| }, |
| { |
| "epoch": 0.2788060320061789, |
| "grad_norm": 0.23875726759433746, |
| "learning_rate": 0.00021314814814814815, |
| "loss": 3.9818, |
| "step": 1850 |
| }, |
| { |
| "epoch": 0.2863413301685081, |
| "grad_norm": 0.24082094430923462, |
| "learning_rate": 0.0002038888888888889, |
| "loss": 3.9765, |
| "step": 1900 |
| }, |
| { |
| "epoch": 0.2938766283308373, |
| "grad_norm": 0.24363255500793457, |
| "learning_rate": 0.00019462962962962964, |
| "loss": 3.9654, |
| "step": 1950 |
| }, |
| { |
| "epoch": 0.30141192649316645, |
| "grad_norm": 0.23058578372001648, |
| "learning_rate": 0.00018537037037037038, |
| "loss": 3.9612, |
| "step": 2000 |
| }, |
| { |
| "epoch": 0.30894722465549557, |
| "grad_norm": 0.22857844829559326, |
| "learning_rate": 0.00017611111111111112, |
| "loss": 3.952, |
| "step": 2050 |
| }, |
| { |
| "epoch": 0.31648252281782474, |
| "grad_norm": 0.223977729678154, |
| "learning_rate": 0.00016685185185185187, |
| "loss": 3.9433, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.31648252281782474, |
| "eval_loss": 3.863826939699745, |
| "eval_perplexity": 47.64734642630777, |
| "eval_runtime": 42.7477, |
| "eval_samples_per_second": 38.809, |
| "eval_steps_per_second": 0.608, |
| "step": 2100 |
| }, |
| { |
| "epoch": 0.3240178209801539, |
| "grad_norm": 0.21180413663387299, |
| "learning_rate": 0.00015759259259259258, |
| "loss": 3.9399, |
| "step": 2150 |
| }, |
| { |
| "epoch": 0.3315531191424831, |
| "grad_norm": 0.218739852309227, |
| "learning_rate": 0.00014833333333333335, |
| "loss": 3.9309, |
| "step": 2200 |
| }, |
| { |
| "epoch": 0.3390884173048122, |
| "grad_norm": 0.2420540750026703, |
| "learning_rate": 0.00013907407407407407, |
| "loss": 3.9235, |
| "step": 2250 |
| }, |
| { |
| "epoch": 0.3466237154671414, |
| "grad_norm": 0.23727546632289886, |
| "learning_rate": 0.00012981481481481484, |
| "loss": 3.9189, |
| "step": 2300 |
| }, |
| { |
| "epoch": 0.35415901362947055, |
| "grad_norm": 0.21484792232513428, |
| "learning_rate": 0.00012055555555555555, |
| "loss": 3.9085, |
| "step": 2350 |
| }, |
| { |
| "epoch": 0.3616943117917997, |
| "grad_norm": 0.20896899700164795, |
| "learning_rate": 0.0001112962962962963, |
| "loss": 3.9139, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.3616943117917997, |
| "eval_loss": 3.8293397981153188, |
| "eval_perplexity": 46.03213769600957, |
| "eval_runtime": 42.6597, |
| "eval_samples_per_second": 38.889, |
| "eval_steps_per_second": 0.609, |
| "step": 2400 |
| }, |
| { |
| "epoch": 0.36922960995412885, |
| "grad_norm": 0.21242469549179077, |
| "learning_rate": 0.00010203703703703704, |
| "loss": 3.9004, |
| "step": 2450 |
| }, |
| { |
| "epoch": 0.376764908116458, |
| "grad_norm": 0.2224598526954651, |
| "learning_rate": 9.277777777777778e-05, |
| "loss": 3.9026, |
| "step": 2500 |
| }, |
| { |
| "epoch": 0.3843002062787872, |
| "grad_norm": 0.19466078281402588, |
| "learning_rate": 8.351851851851852e-05, |
| "loss": 3.894, |
| "step": 2550 |
| }, |
| { |
| "epoch": 0.39183550444111637, |
| "grad_norm": 0.19186748564243317, |
| "learning_rate": 7.425925925925927e-05, |
| "loss": 3.8922, |
| "step": 2600 |
| }, |
| { |
| "epoch": 0.39937080260344554, |
| "grad_norm": 0.1871800720691681, |
| "learning_rate": 6.500000000000001e-05, |
| "loss": 3.8897, |
| "step": 2650 |
| }, |
| { |
| "epoch": 0.40690610076577466, |
| "grad_norm": 0.1799030303955078, |
| "learning_rate": 5.5740740740740744e-05, |
| "loss": 3.8854, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.40690610076577466, |
| "eval_loss": 3.8063672254540974, |
| "eval_perplexity": 44.98671506714982, |
| "eval_runtime": 42.5792, |
| "eval_samples_per_second": 38.963, |
| "eval_steps_per_second": 0.611, |
| "step": 2700 |
| }, |
| { |
| "epoch": 0.41444139892810383, |
| "grad_norm": 0.17992734909057617, |
| "learning_rate": 4.6481481481481486e-05, |
| "loss": 3.8828, |
| "step": 2750 |
| }, |
| { |
| "epoch": 0.421976697090433, |
| "grad_norm": 0.1795385181903839, |
| "learning_rate": 3.722222222222222e-05, |
| "loss": 3.88, |
| "step": 2800 |
| }, |
| { |
| "epoch": 0.4295119952527622, |
| "grad_norm": 0.17680570483207703, |
| "learning_rate": 2.7962962962962965e-05, |
| "loss": 3.8807, |
| "step": 2850 |
| }, |
| { |
| "epoch": 0.4370472934150913, |
| "grad_norm": 0.16620458662509918, |
| "learning_rate": 1.8703703703703707e-05, |
| "loss": 3.8766, |
| "step": 2900 |
| }, |
| { |
| "epoch": 0.44458259157742047, |
| "grad_norm": 0.227438822388649, |
| "learning_rate": 9.444444444444445e-06, |
| "loss": 3.8783, |
| "step": 2950 |
| }, |
| { |
| "epoch": 0.45211788973974965, |
| "grad_norm": 0.1560600996017456, |
| "learning_rate": 1.8518518518518518e-07, |
| "loss": 3.876, |
| "step": 3000 |
| }, |
| { |
| "epoch": 0.45211788973974965, |
| "eval_loss": 3.7956512070627526, |
| "eval_perplexity": 44.507210380219355, |
| "eval_runtime": 42.4711, |
| "eval_samples_per_second": 39.062, |
| "eval_steps_per_second": 0.612, |
| "step": 3000 |
| } |
| ], |
| "logging_steps": 50, |
| "max_steps": 3000, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 300, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 6.244546904064e+17, |
| "train_batch_size": 32, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
