add ipv6 list (#12)

* change asname source

* change rib source

* generate for ipv6

* fix typo

* update travis data

* change docker image

* delete duplicate lines before merge cidr

* change rib source

* change style of upstream

* modify arg for uniq

* change some conf files to symlink

* update stat

* add sudo for apt in travis config

* add stat for 0

* fix bug for stat

* set -x

* curl -vv

* change dist to bionic

* remove verbose options

.travis.yml

@@ -1,14 +1,19 @@
 sudo: required
+dist: bionic
+addons:
+    apt:
+        packages:
+          - lftp
 services:
     - docker
 language: rust
 env:
   global:
-  - BGPTOOLS_VERSION=0.0.1
+  - BGPTOOLS_VERSION=0.0.2
 before_script:
   - cargo install --vers $BGPTOOLS_VERSION bgptools
   - export PATH=$PATH:/home/travis/.cargo/bin
-  - docker pull gaoyifan/cidrmerge
+  - docker pull yangzhaofengsteven/cidr-merge
   - docker pull gaoyifan/bgpdump
 script:
   - ./generate.sh

common.sh

@@ -20,9 +20,14 @@ get_asn(){
 }
 
 prepare_data(){
-	curl -sSLo asnames.txt http://bgp.potaroo.net/as1221/asnames.txt
+	curl -sSL https://bgp.potaroo.net/cidr/autnums.html | awk '-F[<>]' '{print $3,$5}' | grep '^AS' > asnames.txt
 	curl -sSLo rib.bz2 http://archive.routeviews.org/dnszones/rib.bz2
+	IP6UPSTREAM="http://archive.routeviews.org/route-views6/bgpdata"
+	MONTH6=$(lftp -e 'cls -1;exit' $IP6UPSTREAM  2>/dev/null | sort | tail -n 1)
+	LATEST6=$(lftp -e 'cls -1;exit' $IP6UPSTREAM/$MONTH6/RIBS/  2>/dev/null | sort | tail -n 1)
+	curl -sSLo rib6.bz2 "$IP6UPSTREAM/$MONTH6/RIBS/$LATEST6"
 	log_info "runing bgpdump ..."
 	docker run -it --rm -v `pwd`:/bgpdump -w /bgpdump gaoyifan/bgpdump bgpdump -m -O rib.txt rib.bz2
+	docker run -it --rm -v `pwd`:/bgpdump -w /bgpdump gaoyifan/bgpdump bgpdump -m -O rib6.txt rib6.bz2
 	log_info "done"
 }

generate.sh

@@ -8,6 +8,14 @@ for file in operator/*.conf; do
 	operator=${file%.*}
 	operator=${operator##*/}
 	log_info "generating IP list of $operator ..."
-	get_asn $file | tee /dev/stderr | xargs bgptools | docker run -i --rm gaoyifan/cidrmerge > result/$operator.txt
+	get_asn $file | tee /dev/stderr | xargs bgptools -b rib.txt | sort | uniq | docker run -i --rm yangzhaofengsteven/cidr-merge > result/$operator.txt
+	log_info "done"
+done
+
+for file in operator6/*.conf; do
+	operator=${file%.*}
+	operator=${operator##*/}
+	log_info "generating IPv6 list of $operator ..."
+	get_asn $file | tee /dev/stderr | xargs bgptools -b rib6.txt | sort | uniq | docker run -i --rm yangzhaofengsteven/cidr-merge > result/${operator}6.txt
 	log_info "done"
 done

operator6/cernet.conf

@@ -0,0 +1,2 @@
+PATTERN='(cngi|cernet)'
+COUNTRY='CN'

operator6/china.conf

@@ -0,0 +1 @@
+../operator/china.conf

operator6/chinanet.conf

@@ -0,0 +1 @@
+../operator/chinanet.conf

operator6/cmcc.conf

@@ -0,0 +1 @@
+../operator/cmcc.conf

operator6/cstnet.conf

@@ -0,0 +1 @@
+../operator/cstnet.conf

operator6/drpeng.conf

@@ -0,0 +1 @@
+../operator/drpeng.conf

operator6/tietong.conf

@@ -0,0 +1 @@
+../operator/tietong.conf

operator6/unicom.conf

@@ -0,0 +1 @@
+../operator/unicom.conf

stat.sh

@@ -4,15 +4,32 @@ source common.sh
 cd result
 for file in *.txt; do 
 	echo ${file%.*}
-	cat $file | 
-	awk -F\/ '{print $2}' | 
-	(
-		sum=0
-		while read n; do
-			((s=32-n))
-			((sum+=1<<s))
-		done
-		echo $sum
-	)
+	if [[ $file == *6.txt ]]; then
+		cat $file |
+		awk -F\/ '{print $2}' |
+		(
+			sum=0
+			while read n; do
+				if [[ -n $n ]]; then
+					((s=64-n))
+					((sum+=1<<s))
+				fi
+			done
+			echo $sum
+			)
+	else
+		cat $file | 
+		awk -F\/ '{print $2}' | 
+		(
+			sum=0
+			while read n; do
+				if [[ -n $n ]]; then
+					((s=32-n))
+					((sum+=1<<s))
+				fi
+			done
+			echo $sum
+		)
+	fi
 	echo
 done | tee stat