Unify the preprocessing process of IPv4 and IPv6

common.sh

@@ -19,38 +19,25 @@ get_asn(){
 	awk '{gsub(/AS/, ""); print $1 }'
 }
 
-prepare_data_v4(){
+prepare_data(){
+	curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt
 	bgpkit-broker latest -c rrc00 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib.gz
 	stat rib.gz
-	log_info "runing bgpdump v4 ..."
+	log_info "runing bgpdump..."
 	bgpdump -m -O rib.txt rib.gz
 	stat rib.txt
-	log_info "finish bgpdump v4"
+	log_info "finish bgpdump"
-}
-prepare_data_v6(){
-	bgpkit-broker latest -c route-views6 --json | jq -c '.[] | select( .data_type | contains("rib")) | .url' | head -n 1 | xargs axel -q -o rib6.bz2
-	stat rib6.bz2
-	log_info "runing bgpdump v6 ..."
-	bgpdump -m -O rib6.txt rib6.bz2
-	stat rib6.txt
-	log_info "finish bgpdump v6"
-}
-prepare_data(){
-	curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt &
-	prepare_data_v4 &
-	prepare_data_v6 &
-	wait_exit
 }
 
 wait_exit(){
-	local oldstate=$(set +o)
+       local oldstate=$(set +o)
-	set +e
+       set +e
-	local s=0
+       local s=0
-	while [[ $s -ne 127 ]]; do
+       while [[ $s -ne 127 ]]; do
-		[[ $s -ne 0 ]] && exit $s
+               [[ $s -ne 0 ]] && exit $s
-		wait -n
+               wait -n
-		s=$?
+               s=$?
-	done
+       done
-	eval "$oldstate"
+       eval "$oldstate"
-	return 0
+       return 0
 }

generate.sh

@@ -9,8 +9,13 @@ for file in operator/*.conf; do
 	operator=${operator##*/}
 	log_info "generating IP list of $operator ..."
 	get_asn $file
-	get_asn $file | xargs bgptools -b rib.txt  | cidr-merger -s | grep -Fv : | cat > result/${operator}.txt  &
+	(
-	get_asn $file | xargs bgptools -b rib6.txt | grep -v '^::/0$' | cidr-merger -s | grep -F  : | cat > result/${operator}6.txt &
+	    raw_result=$(mktemp)
+		get_asn $file | xargs bgptools -b rib.txt > ${raw_result}
+		cidr-merger -s < ${raw_result} | grep -Fv : | cat > result/${operator}.txt
+		grep -v '^::/0$' < ${raw_result} | cidr-merger -s | grep -F  : | cat > result/${operator}6.txt
+		rm -f "${raw_result}"
+	) &
 done
 
 wait_exit