3
#!/bin/bash
. ~/.bash_profile
source /etc/profile;
echo "start outer $(date)"
su - hdfs <<EOF
echo "start inner $(date)"
hadoop fs -rmr /data/path
sh /a/spark/task/run.sh
rm -rf /data/product/hello.txt
hadoop fs -getmerge /data/product/* /data/product/hello.txt
rm -rf /data/product/hello.txt
echo "end inner $(date)"
exit;
EOF
tail -1 /data/product/hello.txt > /data/product/empty.txt
scp -r /data/product/empty.txt 10.0.1.80:/data/hello.json
sleep 2
scp -r /data/product/hello.txt 10.0.1.80:/data/hello.json
echo "end outer $(date)"
#date
echo 'end'
版本 1.9.1
期望:任务顺利执行
实际情况:executor崩溃,log页面卡住