Add files using upload-large-folder tool

.gitattributes

@@ -41,3 +41,22 @@ mosesdecoder/moses/TranslationModel/UG/util/ibm1-align filter=lfs diff=lfs merge
 mosesdecoder/contrib/iSenWeb/Introduction/iSenWeb[[:space:]]A[[:space:]]Web-based[[:space:]]Machine[[:space:]]Translation[[:space:]]System[[:space:]]to[[:space:]]Translate[[:space:]]Sentences.docx filter=lfs diff=lfs merge=lfs -text
 FacebookAI/xlm-roberta-large/onnx/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 mosesdecoder/contrib/promix/test_data/esen.nc.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/iSenWeb/themes/images/common/Logo[[:space:]](2000x2000).png filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/iSenWeb/Introduction/iSenWeb[[:space:]]A[[:space:]]Web-based[[:space:]]Machine[[:space:]]Translation[[:space:]]System[[:space:]]to[[:space:]]Translate[[:space:]]Sentences.pdf filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1 filter=lfs diff=lfs merge=lfs -text
+mosesdecoder/contrib/promix/test_data/esen.ep.model.filtered/phrase-table.0-0.1.1.binphr.tgtdata.wa filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.6/trainable_data/train.de-en.de.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.6/trainable_data/train.de-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.4/trainable_data/train.de-en.de.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.4/trainable_data/train.zh-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.6/trainable_data/train.zh-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.5/trainable_data/train.de-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.7/trainable_data/train.zh-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.4/trainable_data/train.zh-en.zh.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.5/trainable_data/train.de-en.de.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.7/trainable_data/train.zh-en.zh.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.5/trainable_data/train.zh-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.4/trainable_data/train.de-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.7/trainable_data/train.de-en.de.idx filter=lfs diff=lfs merge=lfs -text
+data/de2en/0.7/trainable_data/train.de-en.en.idx filter=lfs diff=lfs merge=lfs -text
+data/zh2en/0.6/trainable_data/train.zh-en.zh.idx filter=lfs diff=lfs merge=lfs -text

data/de2en/0.4/trainable_data/preprocess.log

@@ -0,0 +1,7 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', optimizer=None, lr_scheduler='fixed', tokenizer=None, bpe=None, scoring='bleu', task='translation', source_lang='de', target_lang='en', trainpref='/home/luoyf/xzq/data/de2en/0.4/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/de2en/0.4/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de2en/0.4/trainable_data/dict.en.txt', srcdict='/home/luoyf/xzq/data/de2en/0.4/trainable_data/dict.de.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', optimizer=None, lr_scheduler='fixed', tokenizer=None, bpe=None, scoring='bleu', task='translation', source_lang='de', target_lang='en', trainpref='/home/luoyf/xzq/data/de2en/0.4/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/de2en/0.4/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.en.txt', srcdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.de.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[de] Dictionary: 47776 types
+[de] /home/luoyf/xzq/data/de2en/0.4/tokenized/bpe.train.de: 46388489 sents, 1161403088 tokens, 0.0% replaced by <unk>
+[en] Dictionary: 47776 types
+[en] /home/luoyf/xzq/data/de2en/0.4/tokenized/bpe.train.en: 46388489 sents, 1093341883 tokens, 3.37e-05% replaced by <unk>
+Wrote preprocessed data to /home/luoyf/xzq/data/de2en/0.4/trainable_data

data/de2en/0.4/trainable_data/train.de-en.de.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45630adf99dab73623937030327c96aad2a8173be734d9290264bb343c400ada
+size 556661894

data/de2en/0.4/trainable_data/train.de-en.en.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa589765b69df43d7101556633b330d917797f1a6dca5bb780e4d2f36cabad74
+size 556661894

data/de2en/0.4/trainable_data/valid.de-en.de.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bffa8af7d0a005a67d5bab8e6bfbc14b9abd27898d509a28d2ff8d3f17b556ed
+size 116454

data/de2en/0.4/trainable_data/valid.de-en.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c1ee37e89aafb067d9c7528fd7502ef13268b3554f5232a244d54156846e5c
+size 108124

data/de2en/0.5/trainable_data/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', optimizer=None, lr_scheduler='fixed', tokenizer=None, bpe=None, scoring='bleu', task='translation', source_lang='de', target_lang='en', trainpref='/home/luoyf/xzq/data/de2en/0.5/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/de2en/0.5/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.en.txt', srcdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.de.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[de] Dictionary: 47776 types
+[de] /home/luoyf/xzq/data/de2en/0.5/tokenized/bpe.train.de: 46388489 sents, 1161403088 tokens, 0.0% replaced by <unk>
+[en] Dictionary: 47776 types
+[en] /home/luoyf/xzq/data/de2en/0.5/tokenized/bpe.train.en: 46388489 sents, 1091759627 tokens, 6.59e-05% replaced by <unk>
+Wrote preprocessed data to /home/luoyf/xzq/data/de2en/0.5/trainable_data

data/de2en/0.5/trainable_data/train.de-en.de.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45630adf99dab73623937030327c96aad2a8173be734d9290264bb343c400ada
+size 556661894

data/de2en/0.5/trainable_data/train.de-en.en.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d8cac62ca77ef3256226853e31b02dd0f3d618969a95b44b5073b63a555475f
+size 556661894

data/de2en/0.5/trainable_data/valid.de-en.de.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bffa8af7d0a005a67d5bab8e6bfbc14b9abd27898d509a28d2ff8d3f17b556ed
+size 116454

data/de2en/0.5/trainable_data/valid.de-en.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c1ee37e89aafb067d9c7528fd7502ef13268b3554f5232a244d54156846e5c
+size 108124

data/de2en/0.6/trainable_data/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', optimizer=None, lr_scheduler='fixed', tokenizer=None, bpe=None, scoring='bleu', task='translation', source_lang='de', target_lang='en', trainpref='/home/luoyf/xzq/data/de2en/0.6/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/de2en/0.6/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.en.txt', srcdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.de.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[de] Dictionary: 47776 types
+[de] /home/luoyf/xzq/data/de2en/0.6/tokenized/bpe.train.de: 46388489 sents, 1161403088 tokens, 0.0% replaced by <unk>
+[en] Dictionary: 47776 types
+[en] /home/luoyf/xzq/data/de2en/0.6/tokenized/bpe.train.en: 46388489 sents, 1088982249 tokens, 9e-05% replaced by <unk>
+Wrote preprocessed data to /home/luoyf/xzq/data/de2en/0.6/trainable_data

data/de2en/0.6/trainable_data/train.de-en.de.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45630adf99dab73623937030327c96aad2a8173be734d9290264bb343c400ada
+size 556661894

data/de2en/0.6/trainable_data/train.de-en.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fee349d545908c8d1b7d595c46dc0732978da2d04018371a2d67ee643145f83
+size 2177964498

data/de2en/0.6/trainable_data/train.de-en.en.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:135a8960c4d6ad8304017ca9b5e2356aedf3e610c4160f8362728723028f75e2
+size 556661894

data/de2en/0.6/trainable_data/valid.de-en.de.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bffa8af7d0a005a67d5bab8e6bfbc14b9abd27898d509a28d2ff8d3f17b556ed
+size 116454

data/de2en/0.6/trainable_data/valid.de-en.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c1ee37e89aafb067d9c7528fd7502ef13268b3554f5232a244d54156846e5c
+size 108124

data/de2en/0.7/trainable_data/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', optimizer=None, lr_scheduler='fixed', tokenizer=None, bpe=None, scoring='bleu', task='translation', source_lang='de', target_lang='en', trainpref='/home/luoyf/xzq/data/de2en/0.7/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/de2en/0.7/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de2en/0.7/trainable_data/dict.en.txt', srcdict='/home/luoyf/xzq/data/de2en/0.7/trainable_data/dict.de.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[de] Dictionary: 47776 types
+[de] /home/luoyf/xzq/data/de2en/0.7/tokenized/bpe.train.de: 46388489 sents, 1161403088 tokens, 0.0% replaced by <unk>
+[en] Dictionary: 47776 types
+[en] /home/luoyf/xzq/data/de2en/0.7/tokenized/bpe.train.en: 46388489 sents, 1086119495 tokens, 0.000119% replaced by <unk>
+Wrote preprocessed data to /home/luoyf/xzq/data/de2en/0.7/trainable_data

data/de2en/0.7/trainable_data/train.de-en.de.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45630adf99dab73623937030327c96aad2a8173be734d9290264bb343c400ada
+size 556661894

data/de2en/0.7/trainable_data/train.de-en.en.idx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a96e991f89953b98e5f61a89c8e58f7e9f53856104b0e09dbac21f39bdd49e26
+size 556661894

data/de2en/0.7/trainable_data/valid.de-en.de.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bffa8af7d0a005a67d5bab8e6bfbc14b9abd27898d509a28d2ff8d3f17b556ed
+size 116454

data/de2en/0.7/trainable_data/valid.de-en.en.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77c1ee37e89aafb067d9c7528fd7502ef13268b3554f5232a244d54156846e5c
+size 108124

data/en2de/0.4/trainable_data/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', tokenizer=None, bpe=None, optimizer=None, lr_scheduler='fixed', scoring='bleu', task='translation', source_lang='en', target_lang='de', trainpref='/home/luoyf/xzq/data/en2de/0.4/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/en2de/0.4/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.de.txt', srcdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.en.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[en] Dictionary: 47776 types
+[en] /home/luoyf/xzq/data/en2de/0.4/tokenized/bpe.train.en: 46388489 sents, 1094684830 tokens, 0.0% replaced by <unk>
+[de] Dictionary: 47776 types
+[de] /home/luoyf/xzq/data/en2de/0.4/tokenized/bpe.train.de: 46388489 sents, 1160669688 tokens, 2.21e-05% replaced by <unk>
+Wrote preprocessed data to /home/luoyf/xzq/data/en2de/0.4/trainable_data

data/en2de/0.6/trainable_data/preprocess.log

@@ -0,0 +1,6 @@
+Namespace(no_progress_bar=False, log_interval=100, log_format=None, tensorboard_logdir=None, seed=42, cpu=False, tpu=False, bf16=False, memory_efficient_bf16=False, fp16=False, memory_efficient_fp16=False, fp16_no_flatten_grads=False, fp16_init_scale=128, fp16_scale_window=None, fp16_scale_tolerance=0.0, min_loss_scale=0.0001, threshold_loss_scale=None, user_dir=None, empty_cache_freq=0, all_gather_list_size=16384, model_parallel_size=1, checkpoint_suffix='', checkpoint_shard_count=1, quantization_config_path=None, profile=False, criterion='cross_entropy', optimizer=None, lr_scheduler='fixed', tokenizer=None, bpe=None, scoring='bleu', task='translation', source_lang='en', target_lang='de', trainpref='/home/luoyf/xzq/data/en2de/0.6/tokenized/bpe.train', validpref=None, testpref=None, align_suffix=None, destdir='/home/luoyf/xzq/data/en2de/0.6/trainable_data', thresholdtgt=0, thresholdsrc=0, tgtdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.de.txt', srcdict='/home/luoyf/xzq/data/de-en/wmt23/trainable_data/dict.en.txt', nwordstgt=-1, nwordssrc=-1, alignfile=None, dataset_impl='mmap', joined_dictionary=False, only_source=False, padding_factor=8, workers=32)
+[en] Dictionary: 47776 types
+[en] /home/luoyf/xzq/data/en2de/0.6/tokenized/bpe.train.en: 46388489 sents, 1094684830 tokens, 0.0% replaced by <unk>
+[de] Dictionary: 47776 types
+[de] /home/luoyf/xzq/data/en2de/0.6/tokenized/bpe.train.de: 46388489 sents, 1160113236 tokens, 0.000117% replaced by <unk>
+Wrote preprocessed data to /home/luoyf/xzq/data/en2de/0.6/trainable_data