From a497a9fc471de411aab17e51f6ebc0ad9c768e1f Mon Sep 17 00:00:00 2001
From: Yifan Gao <git@yfgao.com>
Date: Thu, 19 Dec 2019 16:00:25 +0800
Subject: [PATCH] Support parallel generating

---
 common.sh   | 19 ++++++++++++++-----
 generate.sh |  6 ++++--
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/common.sh b/common.sh
index f357d6b8..075c9483 100755
--- a/common.sh
+++ b/common.sh
@@ -19,15 +19,24 @@ get_asn(){
 	awk '{gsub(/AS/, ""); print $1 }'
 }
 
-prepare_data(){
-	curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt
+prepare_data_v4(){
 	curl -sSLo rib.bz2 http://archive.routeviews.org/dnszones/rib.bz2
+	log_info "runing bgpdump v4 ..."
+	docker run -it --rm -v `pwd`:/bgpdump -w /bgpdump gaoyifan/bgpdump bgpdump -m -O rib.txt rib.bz2
+	log_info "finish bgpdump v4"
+}
+prepare_data_v6(){
 	IP6UPSTREAM="http://archive.routeviews.org/route-views6/bgpdata"
 	MONTH6=$(lftp -e 'cls -1;exit' $IP6UPSTREAM  2>/dev/null | sort | tail -n 1)
 	LATEST6=$(lftp -e 'cls -1;exit' $IP6UPSTREAM/$MONTH6/RIBS/  2>/dev/null | sort | tail -n 1)
 	curl -sSLo rib6.bz2 "$IP6UPSTREAM/$MONTH6/RIBS/$LATEST6"
-	log_info "runing bgpdump ..."
-	docker run -it --rm -v `pwd`:/bgpdump -w /bgpdump gaoyifan/bgpdump bgpdump -m -O rib.txt rib.bz2
+	log_info "runing bgpdump v6 ..."
 	docker run -it --rm -v `pwd`:/bgpdump -w /bgpdump gaoyifan/bgpdump bgpdump -m -O rib6.txt rib6.bz2
-	log_info "done"
+	log_info "finish bgpdump v6"
+}
+prepare_data(){
+    curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt &
+    prepare_data_v4 &
+    prepare_data_v6 &
+    wait
 }
diff --git a/generate.sh b/generate.sh
index d0dd8a3f..f003af68 100755
--- a/generate.sh
+++ b/generate.sh
@@ -3,13 +3,15 @@
 source common.sh
 set -e
 [[ $SKIP_DATA_PREPARATION != true ]] && prepare_data
+reset
 mkdir -p result
 for file in operator/*.conf; do
 	operator=${file%.*}
 	operator=${operator##*/}
 	log_info "generating IP list of $operator ..."
 	get_asn $file
-	get_asn $file | xargs bgptools -b rib.txt  | sort | uniq | docker run -i --rm yangzhaofengsteven/cidr-merge > result/${operator}.txt
-	get_asn $file | xargs bgptools -b rib6.txt | sort | uniq | docker run -i --rm yangzhaofengsteven/cidr-merge > result/${operator}6.txt
+	get_asn $file | xargs bgptools -b rib.txt  | sort | uniq | docker run -i --rm yangzhaofengsteven/cidr-merge > result/${operator}.txt &
+	get_asn $file | xargs bgptools -b rib6.txt | sort | uniq | docker run -i --rm yangzhaofengsteven/cidr-merge > result/${operator}6.txt &
 done
 
+wait