#!/bin/bash # ${ClickHouseVersion} 1 # ${ClickHouseNodeCount} 2 # ${RootStackName} 3 # ${AWS::Region} 4 # http://s3.${AWS::Region}.${AWS::URLSuffix}/${ClickHouseBucketName}/quickstart-clickhouse-data/ 5 # ${MoveFactor} 6 # ${ClickHouseTimezone} 7 # ${ZookeeperPrivateIp1} 8 # ${ZookeeperPrivateIp2} 9 # ${ZookeeperPrivateIp3} 10 # `python3 ./find-secret.py secretfile ` 11 # ${MaxThreads} 12 # ${MaxInsertThreads} 13 # ${DistributedProductMode} 14 # ${MaxMemoryUsage} 15 # ${LoadBalancing} 16 # ${MaxDataPartSize} 17 # ShardNum 18 # ReplicaNum 19 # ${SourceCodeStorage} 20 sudo useradd -m -s /bin/bash clickhouse passwd clickhouse< ninja.out cd .. #wget https://repo.yandex.ru/clickhouse/tgz/stable/clickhouse-client-$1.tgz #wget https://repo.yandex.ru/clickhouse/tgz/stable/clickhouse-server-$1.tgz wget https://repo.yandex.ru/clickhouse/tgz/lts/clickhouse-server-$1.tgz wget https://repo.yandex.ru/clickhouse/tgz/lts/clickhouse-client-$1.tgz tar -xzvf clickhouse-client-$1.tgz tar -xzvf clickhouse-server-$1.tgz find /var/lib/clickhouse/clickhouse-server-$1/install/ -name 'doinst.sh' | xargs perl -pi -e "s|done|done;rm -f /usr/bin/clickhouse-*;cp -r -f /var/lib/clickhouse/ClickHouse/build-arm64/programs/clickhouse-* /usr/bin/|g" chmod +x clickhouse-client-$1/install/doinst.sh chmod +x clickhouse-server-$1/install/doinst.sh mkdir -p /var/log/clickhouse-server chown clickhouse.clickhouse -R /var/log/clickhouse-server chown clickhouse.clickhouse -R /var/lib/clickhouse clickhouse-client-$1/install/doinst.sh clickhouse-server-$1/install/doinst.sh mkdir /etc/clickhouse-server/config.d/ chown -R clickhouse.clickhouse /etc/clickhouse-server/config.d/ cd /root/ echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " true" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode1" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode2" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml if [ $2 -ge 4 ] then echo " " >> /etc/clickhouse-server/metrika.xml echo " true" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode3" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode4" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml fi if [ $2 -ge 6 ] then echo " " >> /etc/clickhouse-server/metrika.xml echo " true" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode5" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode6" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml fi if [ $2 -ge 8 ] then echo " " >> /etc/clickhouse-server/metrika.xml echo " true" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode7" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ClickHouseNode8" >> /etc/clickhouse-server/metrika.xml echo " 9000" >> /etc/clickhouse-server/metrika.xml echo " default" >> /etc/clickhouse-server/metrika.xml echo " ${11}" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml fi echo " " >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " $8" >> /etc/clickhouse-server/metrika.xml echo " 2181" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " $9" >> /etc/clickhouse-server/metrika.xml echo " 2181" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo " ${10}" >> /etc/clickhouse-server/metrika.xml echo " 2181" >> /etc/clickhouse-server/metrika.xml echo " " >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo " ::/0" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo " 10000000000" >> /etc/clickhouse-server/metrika.xml echo " 0.01" >> /etc/clickhouse-server/metrika.xml echo " lz4" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml echo "" >> /etc/clickhouse-server/metrika.xml flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard01-replica01 --region $4` > instancelist-1 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-1` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-1 result-1 node1=`sed -n '1p' result-1` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode1|${node1}|g" break fi echo $flag let flag-- sleep 1 done flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard01-replica02 --region $4` > instancelist-2 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-2` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-2 result-2 node2=`sed -n '1p' result-2` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode2|${node2}|g" break fi echo $flag let flag-- sleep 1 done if (( $2 >= 4)) then flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard02-replica01 --region $4` > instancelist-3 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-3` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-3 result-3 node3=`sed -n '1p' result-3` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode3|${node3}|g" break fi let flag-- sleep 1 done flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard02-replica02 --region $4` > instancelist-4 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-4` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-4 result-4 node4=`sed -n '1p' result-4` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode4|${node4}|g" break fi let flag-- sleep 1 done fi if (( $2 >= 6)) then flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard03-replica01 --region $4` > instancelist-5 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-5` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-5 result-5 node5=`sed -n '1p' result-5` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode5|${node5}|g" break fi let flag-- sleep 1 done flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard03-replica02 --region $4` > instancelist-6 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-6` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-6 result-6 node6=`sed -n '1p' result-6` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode6|${node6}|g" break fi let flag-- sleep 1 done fi if (( $2 >= 8)) then flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard04-replica01 --region $4` > instancelist-7 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-7` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-7 result-7 node7=`sed -n '1p' result-7` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode7|${node7}|g" break fi let flag-- sleep 1 done flag=600 while((flag > 0)) do echo `aws ec2 describe-tags --filters Name=key,Values=$3-clickhouse-shard04-replica02 --region $4` > instancelist-8 count=`awk -v RS="@#$j" '{print gsub(/instance/,"&")}' instancelist-8` if (( $count >= 1 )) then python3 find-clickhouse-node.py instancelist-8 result-8 node8=`sed -n '1p' result-8` find /etc/clickhouse-server/ -name 'metrika.xml' | xargs perl -pi -e "s|ClickHouseNode8|${node8}|g" break fi let flag-- sleep 1 done fi if [ $1 = 21.4.5.46 ]; then echo "Update the config.xml of $1" sed -i '508, 617d' /etc/clickhouse-server/config.xml elif [ $1 = 21.5.5.12 ]; then echo "Update the config.xml of $1" sed -i '520, 630d' /etc/clickhouse-server/config.xml elif [ $1 = 21.8.7.22 ]; then echo "Update the config.xml of $1" sed -i '590, 695d' /etc/clickhouse-server/config.xml fi find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e 's||/etc/clickhouse-server/metrika.xml|g' find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s|trace|information|g" find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s|/var/log/clickhouse-server/clickhouse-server.log|/home/clickhouse/data/log/clickhouse-server.log|g" find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s|/var/log/clickhouse-server/clickhouse-server.err.log|/home/clickhouse/data/log/clickhouse-server.err.log|g" find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s|/var/lib/clickhouse/|/home/clickhouse/data/clickhouse-data/|g" find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s|/var/lib/clickhouse/|/home/clickhouse/data/|g" find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s||$7|g" find /etc/clickhouse-server/ -name 'config.xml' | xargs perl -pi -e "s||0.0.0.0|g" find /etc/clickhouse-server/ -name 'users.xml' | xargs perl -pi -e "s||${11}|g" #password_sha256_hex=`echo -n '${11}' | sha256sum | tr -d '-' | sed 's/ //g'` #find /etc/clickhouse-server/ -name 'users.xml' | xargs perl -pi -e "s||${password_sha256_hex}|g" sudo sed -i "9a ${12}" /etc/clickhouse-server/users.xml sudo sed -i "9a ${13}" /etc/clickhouse-server/users.xml sudo sed -i "9a ${14}" /etc/clickhouse-server/users.xml sudo sed -i "s|10000000000|${15}|" /etc/clickhouse-server/users.xml sudo sed -i "s|random|${16}|" /etc/clickhouse-server/users.xml sudo sed -i 's||1|' /etc/clickhouse-server/users.xml echo "" >> /etc/clickhouse-server/config.d/storage.xml echo " " >> /etc/clickhouse-server/config.d/storage.xml echo " " >> /etc/clickhouse-server/config.d/storage.xml echo " " >> /etc/clickhouse-server/config.d/storage.xml echo " " >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo " default" >> /etc/clickhouse-server/config.d/storage.xml echo " ${17}" >> /etc/clickhouse-server/config.d/storage.xml echo " false" >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo "" >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " s3" >> /etc/clickhouse-server/config.d/storage.xml #echo " $5" >> /etc/clickhouse-server/config.d/storage.xml #echo " true" >> /etc/clickhouse-server/config.d/storage.xml #echo " 10000" >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo " default" >> /etc/clickhouse-server/config.d/storage.xml #echo " ${17}" >> /etc/clickhouse-server/config.d/storage.xml #echo " false" >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " s3" >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo " ${6}" >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " s3" >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo " " >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml #echo "
" >> /etc/clickhouse-server/config.d/storage.xml echo "" >> /etc/clickhouse-server/config.d/macros.xml echo " " >> /etc/clickhouse-server/config.d/macros.xml echo " cluster01-${18}-${19}" >> /etc/clickhouse-server/config.d/macros.xml echo " ${18}" >> /etc/clickhouse-server/config.d/macros.xml echo " 01" >> /etc/clickhouse-server/config.d/macros.xml echo " " >> /etc/clickhouse-server/config.d/macros.xml echo "" >> /etc/clickhouse-server/config.d/macros.xml chown -R clickhouse.clickhouse /home/clickhouse/ chown -R clickhouse.clickhouse /etc/clickhouse-server/ echo "* soft nofile 65536" >> /etc/security/limits.conf echo "* hard nofile 65536" >> /etc/security/limits.conf echo "* soft nproc 131072" >> /etc/security/limits.conf echo "* hard nproc 131072" >> /etc/security/limits.conf systemctl stop clickhouse-server systemctl start clickhouse-server systemctl status clickhouse-server # Restart ClickHouse sleep 5 systemctl stop clickhouse-server systemctl start clickhouse-server systemctl status clickhouse-server sleep 1 echo "[Install]" >> /lib/systemd/system/rc-local.service echo "WantedBy=multi-user.target" >> /lib/systemd/system/rc-local.service echo "Alias=rc-local.service" >> /lib/systemd/system/rc-local.service ln -s /lib/systemd/system/rc-local.service /etc/systemd/system/ # No default rc.local on Ubuntu 18+ cat << EOF > /etc/rc.local #!/bin/bash -e systemctl start clickhouse-server EOF chmod +x /etc/rc.local