Created
August 12, 2021 06:06
-
-
Save a9QrX3Lu/6d2305ffd0b49a0c749be88d27a24c40 to your computer and use it in GitHub Desktop.
Check md5 of multiple files on Aliyun OSS
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| src_dir=/home/wzl/OneDrive/quant/data/data_md5 | |
| dst_dir=oss://edaoquant | |
| check_file() { | |
| src_file=$1 | |
| dst_file=$2 | |
| echo $dst_file | |
| echo $src_file | |
| # Check integrity of the md5 file | |
| diff <(ossutil cat "${dst_file}" | cut -d ' ' -f 1 | head -n -3) \ | |
| <(cut -d ' ' -f 1 "${src_file}") > /dev/null 2>&1 | |
| if [[ $? -eq 1 ]]; then | |
| echo "Integrity check failed" | |
| exit 1 | |
| fi | |
| echo "Integrity check: OK" | |
| } | |
| md5_file() { | |
| local file_suffix=$1 | |
| local file=${dst_dir}/${file_suffix} | |
| echo "$file" | |
| filename=$(echo ${file_suffix} | rev | cut -d'/' -f1 | rev) | |
| echo $filename | |
| ossutil cp $file /tmp/${filename} | |
| md5=$(md5sum /tmp/${filename}) | |
| echo "${md5} ${file}" >> check.md5 | |
| } | |
| main() { | |
| check_file ${src_dir}/data_dst_2018.md5 ${dst_dir}/data_dst_2018.md5 | |
| check_file ${src_dir}/data_dst_2020.md5 ${dst_dir}/data_dst_2020.md5 | |
| # If no local md5 for remote files, fetch and calculate | |
| if [ ! -e check.md5 ]; then | |
| touch check.md5 | |
| while read -r line; do | |
| md5_file "$line" | |
| done < <(sort -k2 ${src_dir}/data_dst_2018.md5 | cut -d ' ' -f 3 | \ | |
| cut -d'/' -f5-) | |
| fi | |
| } | |
| main |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment