JustinLin610
update
8437114
raw
history blame
23.8 kB
#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
if [ -z $WORKDIR_ROOT ] ;
then
echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..."
exit
fi
set -x -e
# TODO update the workdir and dest dir name
# put fasttext model
WORKDIR=$WORKDIR_ROOT
# put intermediate files
TMP_DIR=$WORKDIR_ROOT/tmp/tmp_wmt20_lowres_download
# output {train,valid,test} files to dest
DEST=$WORKDIR_ROOT/ML50/raw
UTILS=$PWD/utils
# per dataset locations
COMMONCRAWL_DIR=$TMP_DIR/commoncrawl
YANDEX_CORPUS=$WORKDIR_ROOT/wmt20/official/ru/yandex/1mcorpus.zip
# unzipped
CZENG_CORPUS=$WORKDIR_ROOT/wmt20/official/cs/czeng/czeng20-train
CCMT_DIR=$WORKDIR_ROOT/wmt20/official/zh/ccmt/parallel
download_and_select() {
SUBFOLDER=$1
URL=$2
UNCOMPRESS_CMD=$3
LANG=$4
INPUT_FILEPATH=$5
if [[ $# -gt 5 ]]; then
LANG_COL=$6
EN_COL=$7
fi
mkdir -p $SUBFOLDER
cd $SUBFOLDER
wget -nc --content-disposition $URL
$UNCOMPRESS_CMD
if [[ $# -gt 5 ]]; then
cut -f$LANG_COL $INPUT_FILEPATH > $INPUT_FILEPATH.$LANG
cut -f$EN_COL $INPUT_FILEPATH > $INPUT_FILEPATH.en
fi
cd ..
ln -sf $SUBFOLDER/$INPUT_FILEPATH.$LANG $SUBFOLDER.$LANG
ln -sf $SUBFOLDER/$INPUT_FILEPATH.en $SUBFOLDER.en
}
prepare_lid() {
pip install fasttext
# TODO specify global workdir
MODEL=$WORKDIR/fasttext/lid.176.bin
LID_MULTI=$UTILS/fasttext_multi_filter.py
if [ ! -f "$MODEL" ]; then
echo "downloading fasttext lid model..."
mkdir -p $WORKDIR/fasttext
wget -nc https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -O $MODEL
fi
}
prepare_moses() {
pushd $UTILS
echo 'Cloning Moses github repository (for tokenization scripts)...'
git clone https://github.com/moses-smt/mosesdecoder.git
popd
}
lid_filter() {
# TODO specify global workdir
MODEL=$WORKDIR/fasttext/lid.176.bin
LID_MULTI=$UTILS/fasttext_multi_filter.py
prepare_lid
SRC=$1
SRC_FILE=$2
SRC_OUTPUT=$3
TGT=$4
TGT_FILE=$5
TGT_OUTPUT=$6
python $LID_MULTI --model $MODEL --inputs $SRC_FILE $TGT_FILE --langs $SRC $TGT --outputs $SRC_OUTPUT $TGT_OUTPUT
}
prepare_ja_ted() {
mkdir -p ted
cd ted
wget -nc https://wit3.fbk.eu/archive/2017-01-trnted//texts/en/ja/en-ja.tgz
tar -zxvf en-ja.tgz
cat en-ja/train.tags.en-ja.en | grep -v -P "^[ ]*\<" | sed 's/^[ \t]*//g' | sed 's/[ \t]*$//g' > en-ja/train.en-ja.en
cat en-ja/train.tags.en-ja.ja | grep -v -P "^[ ]*\<" | sed 's/^[ \t]*//g' | sed 's/[ \t]*$//g' > en-ja/train.en-ja.ja
cd ..
ln -sf ted/en-ja/train.en-ja.ja ted.ja
ln -sf ted/en-ja/train.en-ja.en ted.en
}
prepare_ja() {
OUTPUT_DIR=$TMP_DIR/ja
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select paracrawl "http://www.kecl.ntt.co.jp/icl/lirg/jparacrawl/release/2.0/bitext/en-ja.tar.gz" "tar -zxvf en-ja.tar.gz" ja en-ja/en-ja.bicleaner05.txt 4 3 &
download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-ja.tsv.gz" "gunzip -f news-commentary-v15.en-ja.tsv.gz" ja news-commentary-v15.en-ja.tsv 2 1 &
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ja-en.tsv.gz" "gunzip -f wikititles-v2.ja-en.tsv.gz" ja wikititles-v2.ja-en.tsv 1 2 &
download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ja.langid.tsv.gz" "gunzip -f WikiMatrix.v1.en-ja.langid.tsv.gz" ja WikiMatrix.v1.en-ja.langid.tsv 3 2 &
download_and_select subtitle "https://nlp.stanford.edu/projects/jesc/data/split.tar.gz" "tar -zxvf split.tar.gz" ja split/train 2 1 &
download_and_select kftt "http://www.phontron.com/kftt/download/kftt-data-1.0.tar.gz" "tar -zxvf kftt-data-1.0.tar.gz" ja kftt-data-1.0/data/orig/kyoto-train &
prepare_ja_ted &
# ted data needs to
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.ja" | sort -V | xargs cat > all.ja
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter ja all.ja $DEST/train.ja_XX-en_XX.ja_XX en all.en $DEST/train.ja_XX-en_XX.en_XX
}
prepare_ta() {
OUTPUT_DIR=$TMP_DIR/ta
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ta-en.tsv.gz" "gunzip -f wikititles-v2.ta-en.tsv.gz" ta wikititles-v2.ta-en.tsv 1 2 &
download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ta.langid.tsv.gz" "gunzip -f WikiMatrix.v1.en-ta.langid.tsv.gz" ta WikiMatrix.v1.en-ta.langid.tsv 3 2 &
download_and_select pmindia "http://data.statmt.org/pmindia/v1/parallel/pmindia.v1.ta-en.tsv" "" ta pmindia.v1.ta-en.tsv 2 1 &
download_and_select tanzil "https://object.pouta.csc.fi/OPUS-Tanzil/v1/moses/en-ta.txt.zip" "unzip en-ta.txt.zip" ta Tanzil.en-ta &
download_and_select pib "http://preon.iiit.ac.in/~jerin/resources/datasets/pib-v0.tar" "tar -xvf pib-v0.tar" ta pib/en-ta/train &
download_and_select mkb "http://preon.iiit.ac.in/~jerin/resources/datasets/mkb-v0.tar" "tar -xvf mkb-v0.tar" ta mkb/en-ta/mkb &
download_and_select ufal "http://ufal.mff.cuni.cz/~ramasamy/parallel/data/v2/en-ta-parallel-v2.tar.gz" "tar -zxvf en-ta-parallel-v2.tar.gz" ta en-ta-parallel-v2/corpus.bcn.train &
wait
# need special handling for nlpc
mkdir -p nlpc
cd nlpc
wget -nc https://raw.githubusercontent.com/nlpc-uom/English-Tamil-Parallel-Corpus/master/En-Ta%20Corpus/En-Ta%20English.txt
wget -nc https://github.com/nlpc-uom/English-Tamil-Parallel-Corpus/raw/master/En-Ta%20Corpus/En-Ta%20Tamil.txt
tail -n +4 "En-Ta English.txt" > en-ta.en
tail -n +4 "En-Ta Tamil.txt" > en-ta.ta
cd ..
ln -sf nlpc/en-ta.en nlpc.en
ln -sf nlpc/en-ta.ta nlpc.ta
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.ta" | sort -V | xargs cat > all.ta
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter ta all.ta $DEST/train.ta_IN-en_XX.ta_IN en all.en $DEST/train.ta_IN-en_XX.en_XX
}
prepare_iu() {
OUTPUT_DIR=$TMP_DIR/iu
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select nh "https://nrc-digital-repository.canada.ca/eng/view/dataset/?id=c7e34fa7-7629-43c2-bd6d-19b32bf64f60" "tar -zxvf Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0.1.tgz" iu Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0/NunavutHansard > /dev/null &
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.iu-en.tsv.gz" "gunzip -f wikititles-v2.iu-en.tsv.gz" iu wikititles-v2.iu-en.tsv 1 2 &
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.iu" | sort -V | xargs cat | nh/Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0/scripts/normalize-iu-spelling.pl > all.iu
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
paste all.iu all.en | awk -F $'\t' '$1!=""&&$2!=""' > all.iuen
cut -f1 all.iuen > $DEST/train.iu_CA-en_XX.iu_CA
cut -f2 all.iuen > $DEST/train.iu_CA-en_XX.en_XX
}
prepare_km() {
OUTPUT_DIR=$TMP_DIR/km
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select paracrawl "http://data.statmt.org/wmt20/translation-task/ps-km/wmt20-sent.en-km.xz" "unxz wmt20-sent.en-km.zx" km wmt20-sent.en-km 2 1 &
# km-parallel has multiple sets, concat all of them together
mkdir -p opus
cd opus
wget -nc "http://data.statmt.org/wmt20/translation-task/ps-km/km-parallel.tgz"
tar -zxvf km-parallel.tgz
find ./km-parallel -maxdepth 1 -name "*.km" | sort -V | xargs cat > opus.km
find ./km-parallel -maxdepth 1 -name "*.en" | sort -V | xargs cat > opus.en
cd ..
ln -sf opus/opus.km .
ln -sf opus/opus.en .
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.km" | sort -V | xargs cat > all.km
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter km all.km $DEST/train.km_KH-en_XX.km_KH en all.en $DEST/train.km_KH-en_XX.en_XX
}
prepare_ps() {
OUTPUT_DIR=$TMP_DIR/ps
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select paracrawl "http://data.statmt.org/wmt20/translation-task/ps-km/wmt20-sent.en-ps.xz" "unxz wmt20-sent.en-ps.xz" ps wmt20-sent.en-ps 2 1 &
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ps-en.tsv.gz" "gunzip -f wikititles-v2.ps-en.tsv.gz" ps wikititles-v2.ps-en.tsv 1 2 &
# ps-parallel has multiple sets, concat all of them together
mkdir -p opus
cd opus
wget -nc "http://data.statmt.org/wmt20/translation-task/ps-km/ps-parallel.tgz"
tar -zxvf ps-parallel.tgz
find ./ps-parallel -maxdepth 1 -name "*.ps" | sort -V | xargs cat > opus.ps
find ./ps-parallel -maxdepth 1 -name "*.en" | sort -V | xargs cat > opus.en
cd ..
ln -sf opus/opus.ps opus.ps
ln -sf opus/opus.en opus.en
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.ps" | sort -V | xargs cat > all.ps
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter ps all.ps $DEST/train.ps_AF-en_XX.ps_AF en all.en $DEST/train.ps_AF-en_XX.en_XX
}
download_commoncrawl() {
mkdir -p $COMMONCRAWL_DIR
cd $COMMONCRAWL_DIR
wget -nc "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz"
tar -zxvf training-parallel-commoncrawl.tgz
}
link_commoncrawl() {
LANG=$1
ln -sf $COMMONCRAWL_DIR/commoncrawl.$LANG-en.en commoncrawl.en
ln -sf $COMMONCRAWL_DIR/commoncrawl.$LANG-en.$LANG commoncrawl.$LANG
}
strip_xlf() {
INPUT_FILE=$1
SRC=$2
TGT=$3
grep '<source xml:lang=' $INPUT_FILE | sed 's/^<[^<>]*>//g' | sed 's/<[^<>]*>$//g' > $INPUT_FILE.$SRC
grep '<target xml:lang=' $INPUT_FILE | sed 's/^<[^<>]*>//g' | sed 's/<[^<>]*>$//g' > $INPUT_FILE.$TGT
}
download_and_process_tilde() {
URL=$1
UNCOMPRESS_CMD=$2
FILENAME=$3
LANG=$4
PROCESS_CMD=$5
mkdir -p tilde
cd tilde
wget -nc $URL
$UNCOMPRESS_CMD
echo "executing cmd"
echo $PROCESS_CMD
$PROCESS_CMD
cd ..
ln -sf tilde/$FILENAME.$LANG tilde.$LANG
ln -sf tilde/$FILENAME.en tilde.en
}
prepare_cs() {
OUTPUT_DIR=$TMP_DIR/cs
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
#download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.cs-en.tsv.gz" "gunzip europarl-v10.cs-en.tsv.gz" cs europarl-v10.cs-en.tsv 1 2 &
#download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-cs.txt.gz" "gunzip en-cs.txt.gz" cs en-cs.txt 2 1 &
#link_commoncrawl cs
#download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.cs-en.tsv.gz" "gunzip news-commentary-v15.cs-en.tsv.gz" cs news-commentary-v15.cs-en.tsv 1 2 &
#download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.cs-en.tsv.gz" "gunzip wikititles-v2.cs-en.tsv.gz" cs wikititles-v2.cs-en.tsv 1 2 &
#download_and_process_tilde "http://data.statmt.org/wmt20/translation-task/rapid/RAPID_2019.cs-en.xlf.gz" "gunzip RAPID_2019.cs-en.xlf.gz" RAPID_2019.cs-en.xlf cs "strip_xlf RAPID_2019.cs-en.xlf cs en" &
#download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.cs-en.langid.tsv.gz" "gunzip WikiMatrix.v1.cs-en.langid.tsv.gz" cs WikiMatrix.v1.cs-en.langid.tsv 2 3 &
#wait
# remove previous results
#rm -f all.??
#find ./ -maxdepth 1 -name "*.cs" | sort -V | xargs cat > all.cs
#find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
if [ -z $CZENG_CORPUS ] ;
then
echo "Please download CZENG_CORPUS manually and place them at $CZENG_CORPUS. Exitting..."
exit
fi
cat $CZENG_CORPUS | sed '/^$/d' | cut -f5 > all.cs
cat $CZENG_CORPUS | sed '/^$/d' | cut -f6 > all.en
lid_filter cs all.cs $DEST/train.cs_CZ-en_XX.cs_CZ en all.en $DEST/train.cs_CZ-en_XX.en_XX
}
prepare_de() {
OUTPUT_DIR=$TMP_DIR/de
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.de-en.tsv.gz" "gunzip europarl-v10.de-en.tsv.gz" de europarl-v10.de-en.tsv 1 2 &
download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-de.txt.gz" "gunzip en-de.txt.gz" de en-de.txt 2 1 &
link_commoncrawl de
download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.de-en.tsv.gz" "gunzip news-commentary-v15.de-en.tsv.gz" de news-commentary-v15.de-en.tsv 1 2 &
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.de-en.tsv.gz" "gunzip wikititles-v2.de-en.tsv.gz" de wikititles-v2.de-en.tsv 1 2 &
download_and_process_tilde "http://data.statmt.org/wmt20/translation-task/rapid/RAPID_2019.de-en.xlf.gz" "gunzip RAPID_2019.de-en.xlf.gz" RAPID_2019.de-en.xlf de "strip_xlf RAPID_2019.de-en.xlf de en" &
download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.de-en.langid.tsv.gz" "gunzip WikiMatrix.v1.de-en.langid.tsv.gz" de WikiMatrix.v1.de-en.langid.tsv 2 3 &
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.de" | sort -V | xargs cat > all.de
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter de all.de $DEST/train.de_DE-en_XX.de_DE en all.en $DEST/train.de_DE-en_XX.en_XX
}
prepare_tmx() {
TMX_FILE=$1
git clone https://github.com/amake/TMX2Corpus $UTILS/tmx2corpus
pip install tinysegmenter
python $UTILS/tmx2corpus/tmx2corpus.py $TMX_FILE
}
prepare_pl() {
OUTPUT_DIR=$TMP_DIR/pl
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
# download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.pl-en.tsv.gz" "gunzip europarl-v10.pl-en.tsv.gz" pl europarl-v10.pl-en.tsv 1 2 &
# download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-pl.txt.gz" "gunzip en-pl.txt.gz" pl en-pl.txt 2 1 &
# download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.pl-en.tsv.gz" "gunzip wikititles-v2.pl-en.tsv.gz" pl wikititles-v2.pl-en.tsv 1 2 &
download_and_select tilde "https://tilde-model.s3-eu-west-1.amazonaws.com/rapid2019.en-pl.tmx.zip" "gunzip rapid2019.en-pl.tmx.zip" bitext pl "prepare_tmx RAPID_2019.UNIQUE.en-pl.tmx" &
# download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-pl.langid.tsv.gz" "gunzip WikiMatrix.v1.en-pl.langid.tsv.gz" pl WikiMatrix.v1.en-pl.langid.tsv 3 2 &
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.pl" | sort -V | xargs cat > all.pl
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter pl all.pl $DEST/train.pl_PL-en_XX.pl_PL en all.en $DEST/train.pl_PL-en_XX.en_XX
}
prepare_uncorpus() {
$URLS=$1
$FILES=$2
mkdir -p uncorpus
cd uncorpus
for URL in $URLS; do
wget -nc $URL
done
cat $FILES > uncorpus.tar.gz
tar -zxvf uncorpus.tar.gz
cd ..
ln -sf uncorpus/en-$LANG/UNv1.0.en-$LANG.$LANG uncorpus.$LANG
ln -sf uncorpus/en-$LANG/UNv1.0.en-$LANG.en uncorpus.en
}
prepare_yandex() {
mkdir -p yandex
cd yandex
unzip $YANDEX_CORPUS ./
cd ..
ln -s yandex/corpus.en_ru.1m.en yandex.en
ln -s yandex/corpus.en_ru.1m.ru yandex.ru
}
prepare_ru() {
OUTPUT_DIR=$TMP_DIR/ru
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz" "tar -zxvf paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz" ru paracrawl-release1.en-ru.zipporah0-dedup-clean &
link_commoncrawl ru
download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-ru.tsv.gz" "gunzip news-commentary-v15.en-ru.tsv.gz" ru news-commentary-v15.en-ru.tsv 2 1 &
prepare_yandex &
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ru-en.tsv.gz" "gunzip wikititles-v2.ru-en.tsv.gz" ru wikititles-v2.ru-en.tsv 1 2 &
prepare_uncorpus "https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.00 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.01 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.02" "UNv1.0.en-ru.tar.gz.00 UNv1.0.en-ru.tar.gz.01 UNv1.0.en-ru.tar.gz.02" &
download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ru.langid.tsv.gz" "gunzip WikiMatrix.v1.en-ru.langid.tsv.gz" ru WikiMatrix.v1.en-ru.langid.tsv 3 2 &
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.ru" | sort -V | xargs cat > all.ru
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter ru all.ru $DEST/train.ru_RU-en_XX.ru_RU en all.en $DEST/train.ru_RU-en_XX.en_XX
}
prepare_ccmt() {
mkdir -p ccmt
cd ccmt
# assume ccmt data is already unzipped under CCMT_DIR folder
cat $CCMT_DIR/datum2017/Book*_cn.txt | sed 's/ //g' > datum2017.detok.zh
cat $CCMT_DIR/datum2017/Book*_en.txt > datum2017.detok.en
cat $CCMT_DIR/casict2011/casict-A_ch.txt $CCMT_DIR/casict2011/casict-B_ch.txt $CCMT_DIR/casict2015/casict2015_ch.txt $CCMT_DIR/datum2015/datum_ch.txt $CCMT_DIR/neu2017/NEU_cn.txt datum2017.detok.zh > ccmt.zh
cat $CCMT_DIR/casict2011/casict-A_en.txt $CCMT_DIR/casict2011/casict-B_en.txt $CCMT_DIR/casict2015/casict2015_en.txt $CCMT_DIR/datum2015/datum_en.txt $CCMT_DIR/neu2017/NEU_en.txt datum2017.detok.en > ccmt.en
cd ..
ln -sf ccmt/ccmt.zh ccmt.zh
ln -sf ccmt/ccmt.en ccmt.en
}
prepare_zh() {
OUTPUT_DIR=$TMP_DIR/zh
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-zh.tsv.gz" "gunzip news-commentary-v15.en-zh.tsv.gz" zh news-commentary-v15.en-zh.tsv 2 1 &
download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.zh-en.tsv.gz" "gunzip wikititles-v2.zh-en.tsv.gz" zh wikititles-v2.zh-en.tsv 1 2 &
prepare_uncorpus "https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.00 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.01" "UNv1.0.en-zh.tar.gz.00 UNv1.0.en-zh.tar.gz.01" &
prepare_ccmt &
download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-zh.langid.tsv.gz" "gunzip WikiMatrix.v1.en-zh.langid.tsv.gz" zh WikiMatrix.v1.en-zh.langid.tsv 3 2 &
wait
# remove previous results
rm -f all.??
find ./ -maxdepth 1 -name "*.zh" | sort -V | xargs cat > all.zh
find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
lid_filter zh all.zh $DEST/train.zh_CN-en_XX.zh_CN en all.en $DEST/train.zh_CN-en_XX.en_XX
}
prepare_tests() {
OUTPUT_DIR=$TMP_DIR
mkdir -p $OUTPUT_DIR
cd $OUTPUT_DIR
wget -nc http://data.statmt.org/wmt20/translation-task/dev.tgz
tar -zxvf dev.tgz
cd dev
cat newsdev2020-jaen-src.ja.sgm | $UTILS/strip_sgm.sh > newsdev2020-jaen.ja
cat newsdev2020-jaen-ref.en.sgm | $UTILS/strip_sgm.sh > newsdev2020-jaen.en
split newsdev2020-jaen.ja -a 0 -n r/1/2 > $DEST/valid.ja_XX-en_XX.ja_XX
split newsdev2020-jaen.en -a 0 -n r/1/2 > $DEST/valid.ja_XX-en_XX.en_XX
split newsdev2020-jaen.ja -a 0 -n r/2/2 > $DEST/test.ja_XX-en_XX.ja_XX
split newsdev2020-jaen.en -a 0 -n r/2/2 > $DEST/test.ja_XX-en_XX.en_XX
cat newsdev2020-iuen-src.iu.sgm | strip_sgm.sh > newsdev2020-iuen.iu
cat newsdev2020-iuen-ref.en.sgm | strip_sgm.sh > newsdev2020-iuen.en
split newsdev2020-iuen.iu -a 0 -n r/1/2 > $DEST/valid.iu_CA-en_XX.iu_CA
split newsdev2020-iuen.en -a 0 -n r/1/2 > $DEST/valid.iu_CA-en_XX.en_XX
split newsdev2020-iuen.iu -a 0 -n r/2/2 > $DEST/test.iu_CA-en_XX.iu_CA
split newsdev2020-iuen.en -a 0 -n r/2/2 > $DEST/test.iu_CA-en_XX.en_XX
cat newsdev2020-taen-src.ta.sgm | strip_sgm.sh > newsdev2020-taen.ta
cat newsdev2020-taen-ref.en.sgm | strip_sgm.sh > newsdev2020-taen.en
split newsdev2020-taen.ta -a 0 -n r/1/2 > $DEST/valid.ta_IN-en_XX.ta_IN
split newsdev2020-taen.en -a 0 -n r/1/2 > $DEST/valid.ta_IN-en_XX.en_XX
split newsdev2020-taen.ta -a 0 -n r/2/2 > $DEST/test.ta_IN-en_XX.ta_IN
split newsdev2020-taen.en -a 0 -n r/2/2 > $DEST/test.ta_IN-en_XX.en_XX
cp wikipedia.dev.km-en.km $DEST/valid.km_KH-en_XX.km_KH
cp wikipedia.dev.km-en.en $DEST/valid.km_KH-en_XX.en_XX
cp wikipedia.devtest.km-en.km $DEST/test.km_KH-en_XX.km_KH
cp wikipedia.devtest.km-en.en $DEST/test.km_KH-en_XX.en_XX
cp wikipedia.dev.ps-en.ps $DEST/valid.ps_AF-en_XX.ps_AF
cp wikipedia.dev.ps-en.en $DEST/valid.ps_AF-en_XX.en_XX
cp wikipedia.devtest.ps-en.ps $DEST/test.ps_AF-en_XX.ps_AF
cp wikipedia.devtest.ps-en.en $DEST/test.ps_AF-en_XX.en_XX
cat newsdev2020-plen-src.pl.sgm | strip_sgm.sh > newsdev2020-plen.pl
cat newsdev2020-plen-ref.en.sgm | strip_sgm.sh > newsdev2020-plen.en
split newsdev2020-plen.pl -a 0 -n r/1/2 > $DEST/valid.pl_PL-en_XX.pl_PL
split newsdev2020-plen.en -a 0 -n r/1/2 > $DEST/valid.pl_PL-en_XX.en_XX
split newsdev2020-plen.pl -a 0 -n r/2/2 > $DEST/test.pl_PL-en_XX.pl_PL
split newsdev2020-plen.en -a 0 -n r/2/2 > $DEST/test.pl_PL-en_XX.en_XX
cat newstest2018-encs-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-cs_CZ.en_XX
cat newstest2018-encs-ref.cs.sgm | strip_sgm.sh > $DEST/valid.en_XX-cs_CZ.cs_CZ
cat newstest2019-encs-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-cs_CZ.en_XX
cat newstest2019-encs-ref.cs.sgm | strip_sgm.sh > $DEST/test.en_XX-cs_CZ.cs_CZ
cat newstest2018-deen-src.de.sgm | strip_sgm.sh > $DEST/valid.de_DE-en_XX.de_DE
cat newstest2018-deen-ref.en.sgm | strip_sgm.sh > $DEST/valid.de_DE-en_XX.en_XX
cat newstest2018-ende-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-de_DE.en_XX
cat newstest2018-ende-ref.de.sgm | strip_sgm.sh > $DEST/valid.en_XX-de_DE.de_DE
cat newstest2019-deen-src.de.sgm | strip_sgm.sh > $DEST/test.de_DE-en_XX.de_DE
cat newstest2019-deen-ref.en.sgm | strip_sgm.sh > $DEST/test.de_DE-en_XX.en_XX
cat newstest2019-ende-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-de_DE.en_XX
cat newstest2019-ende-ref.de.sgm | strip_sgm.sh > $DEST/test.en_XX-de_DE.de_DE
cat newstest2018-ruen-src.ru.sgm | strip_sgm.sh > $DEST/valid.ru_RU-en_XX.ru_RU
cat newstest2018-ruen-ref.en.sgm | strip_sgm.sh > $DEST/valid.ru_RU-en_XX.en_XX
cat newstest2018-enru-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-ru_RU.en_XX
cat newstest2018-enru-ref.ru.sgm | strip_sgm.sh > $DEST/valid.en_XX-ru_RU.ru_RU
cat newstest2019-ruen-src.ru.sgm | strip_sgm.sh > $DEST/test.ru_RU-en_XX.ru_RU
cat newstest2019-ruen-ref.en.sgm | strip_sgm.sh > $DEST/test.ru_RU-en_XX.en_XX
cat newstest2019-enru-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-ru_RU.en_XX
cat newstest2019-enru-ref.ru.sgm | strip_sgm.sh > $DEST/test.en_XX-ru_RU.ru_RU
cat newstest2018-zhen-src.zh.sgm | strip_sgm.sh > $DEST/valid.zh_CN-en_XX.zh_CN
cat newstest2018-zhen-ref.en.sgm | strip_sgm.sh > $DEST/valid.zh_CN-en_XX.en_XX
cat newstest2018-enzh-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-zh_CN.en_XX
cat newstest2018-enzh-ref.zh.sgm | strip_sgm.sh > $DEST/valid.en_XX-zh_CN.zh_CN
cat newstest2019-zhen-src.zh.sgm | strip_sgm.sh > $DEST/test.zh_CN-en_XX.zh_CN
cat newstest2019-zhen-ref.en.sgm | strip_sgm.sh > $DEST/test.zh_CN-en_XX.en_XX
cat newstest2019-enzh-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-zh_CN.en_XX
cat newstest2019-enzh-ref.zh.sgm | strip_sgm.sh > $DEST/test.en_XX-zh_CN.zh_CN
}
mkdir -p $DEST
prepare_lid
prepare_moses
download_commoncrawl
prepare_ja &
prepare_ta &
prepare_km &
prepare_ps &
prepare_iu &
prepare_cs &
prepare_de &
prepare_pl &
prepare_ru &
prepare_zh &
# prepare valid/test set
prepare_tests &
# wait
# TODO remove intermediate files
# rm -rf $TMP_DIR