update webtitle

Plugins/webtitle.go

@@ -1,16 +1,13 @@
 package Plugins
 
 import (
-	"bytes"
 	"compress/gzip"
 	"crypto/tls"
 	"fmt"
-	"github.com/saintfish/chardet"
 	"github.com/shadow1ng/fscan/WebScan"
 	"github.com/shadow1ng/fscan/WebScan/lib"
 	"github.com/shadow1ng/fscan/common"
 	"golang.org/x/text/encoding/simplifiedchinese"
-	"golang.org/x/text/transform"
 	"io"
 	"io/ioutil"
 	"net"
@@ -19,98 +16,80 @@ import (
 	"regexp"
 	"strings"
 	"time"
-)
+	"unicode/utf8"
-
-var (
-	Charsets = []string{"utf-8", "gbk", "gb2312"}
 )
 
 func WebTitle(info *common.HostInfo) error {
-	err := GOWebTitle(info)
+	err, CheckData := GOWebTitle(info)
-	if err != nil {
+	info.Infostr = WebScan.InfoCheck(info.Url, &CheckData)
+	if common.IsWebCan == false && err == nil {
+		WebScan.WebScan(info)
+	} else {
 		errlog := fmt.Sprintf("[-] webtitle %v %v", info.Url, err)
 		common.LogError(errlog)
 	}
 	return err
 }
-
+func GOWebTitle(info *common.HostInfo) (err error, CheckData []WebScan.CheckDatas) {
-//flag 1 first try
-//flag 2 /favicon.ico
-//flag 3 302
-//flag 4 400 -> https
-
-func GOWebTitle(info *common.HostInfo) error {
-	var CheckData []WebScan.CheckDatas
 	if info.Url == "" {
-		if info.Ports == "80" {
+		switch info.Ports {
+		case "80":
 			info.Url = fmt.Sprintf("http://%s", info.Host)
-		} else if info.Ports == "443" {
+		case "443":
 			info.Url = fmt.Sprintf("https://%s", info.Host)
-		} else {
+		default:
 			host := fmt.Sprintf("%s:%s", info.Host, info.Ports)
 			protocol := GetProtocol(host, info.Timeout)
 			info.Url = fmt.Sprintf("%s://%s:%s", protocol, info.Host, info.Ports)
 		}
 	} else {
 		if !strings.Contains(info.Url, "://") {
-			protocol := GetProtocol(info.Url, info.Timeout)
+			host := strings.Split(info.Url, "/")[0]
+			protocol := GetProtocol(host, info.Timeout)
 			info.Url = fmt.Sprintf("%s://%s", protocol, info.Url)
 		}
 	}
 
 	err, result, CheckData := geturl(info, 1, CheckData)
 	if err != nil && !strings.Contains(err.Error(), "EOF") {
-		return err
+		return
 	}
 
+	//有跳转
 	if strings.Contains(result, "://") {
-		//有跳转
+		info.Url = result
-		redirecturl, err := url.Parse(result)
+		err, result, CheckData = geturl(info, 3, CheckData)
-		if err == nil {
+		if err != nil {
-			info.Url = redirecturl.String()
+			return
-			err, result, CheckData = geturl(info, 3, CheckData)
-			if err != nil {
-				return err
-			}
 		}
 	}
 
 	if result == "https" && !strings.HasPrefix(info.Url, "https://") {
 		info.Url = strings.Replace(info.Url, "http://", "https://", 1)
 		err, result, CheckData = geturl(info, 1, CheckData)
+		//有跳转
 		if strings.Contains(result, "://") {
-			//有跳转
+			info.Url = result
-			redirecturl, err := url.Parse(result)
+			err, result, CheckData = geturl(info, 3, CheckData)
-			if err == nil {
-				info.Url = redirecturl.String()
-				err, result, CheckData = geturl(info, 3, CheckData)
-				if err != nil {
-					return err
-				}
-			}
-		} else {
 			if err != nil {
-				return err
+				return
 			}
 		}
-	} else if err != nil {
-		return err
 	}
 
 	err, _, CheckData = geturl(info, 2, CheckData)
 	if err != nil {
-		return err
+		return
 	}
-
+	return
-	info.Infostr = WebScan.InfoCheck(info.Url, CheckData)
-
-	if common.IsWebCan == false {
-		WebScan.WebScan(info)
-	}
-	return err
 }
 
 func geturl(info *common.HostInfo, flag int, CheckData []WebScan.CheckDatas) (error, string, []WebScan.CheckDatas) {
+	//flag 1 first try
+	//flag 2 /favicon.ico
+	//flag 3 302
+	//flag 4 400 -> https
+
 	Url := info.Url
 	if flag == 2 {
 		URL, err := url.Parse(Url)
@@ -120,127 +99,65 @@ func geturl(info *common.HostInfo, flag int, CheckData []WebScan.CheckDatas) (er
 			Url += "/favicon.ico"
 		}
 	}
-
 	req, err := http.NewRequest("GET", Url, nil)
-	if err == nil {
+	if err != nil {
-		req.Header.Set("User-agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36")
+		return err, "", CheckData
-		req.Header.Set("Accept", "*/*")
+	}
-		req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
+	req.Header.Set("User-agent", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36")
-		if common.Pocinfo.Cookie != "" {
+	req.Header.Set("Accept", "*/*")
-			req.Header.Set("Cookie", "rememberMe=1;"+common.Pocinfo.Cookie)
+	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9")
-		} else {
+	if common.Pocinfo.Cookie != "" {
-			req.Header.Set("Cookie", "rememberMe=1")
+		req.Header.Set("Cookie", "rememberMe=1;"+common.Pocinfo.Cookie)
-		}
+	} else {
-		req.Header.Set("Connection", "close")
+		req.Header.Set("Cookie", "rememberMe=1")
+	}
+	req.Header.Set("Connection", "close")
+	var client *http.Client
+	if flag == 1 {
+		client = lib.ClientNoRedirect
+	} else {
+		client = lib.Client
+	}
 
-		var client *http.Client
+	resp, err := client.Do(req)
-		if flag == 1 {
+	if err != nil {
-			client = lib.ClientNoRedirect
-		} else {
-			client = lib.Client
-		}
-
-		resp, err := client.Do(req)
-		if err == nil {
-			defer resp.Body.Close()
-			var title string
-			var text []byte
-			body, err := getRespBody(resp)
-			if err != nil {
-				return err, "https", CheckData
-			}
-			if flag != 2 {
-				re := regexp.MustCompile("(?ims)<title>(.*)</title>")
-				find := re.FindSubmatch(body)
-				if len(find) > 1 {
-					text = find[1]
-					GetEncoding := func() string { // 判断Content-Type
-						r1, err := regexp.Compile(`(?im)charset=\s*?([\w-]+)`)
-						if err != nil {
-							return ""
-						}
-						headerCharset := r1.FindString(resp.Header.Get("Content-Type"))
-						if headerCharset != "" {
-							for _, v := range Charsets { // headers 编码优先，所以放在前面
-								if strings.Contains(strings.ToLower(headerCharset), v) == true {
-									return v
-								}
-							}
-						}
-
-						r2, err := regexp.Compile(`(?im)<meta.*?charset=['"]?([\w-]+)["']?.*?>`)
-						if err != nil {
-							return ""
-						}
-						htmlCharset := r2.FindString(string(body))
-						if htmlCharset != "" {
-							for _, v := range Charsets {
-								if strings.Contains(strings.ToLower(htmlCharset), v) == true {
-									return v
-								}
-							}
-						}
-						return ""
-					}
-					encode := GetEncoding()
-					//_, encode1, _ := charset.DetermineEncoding(body, "")
-					var encode2 string
-					detector := chardet.NewTextDetector()
-					detectorstr, _ := detector.DetectBest(body)
-					if detectorstr != nil {
-						encode2 = detectorstr.Charset
-					}
-					if encode == "gbk" || encode == "gb2312" || strings.Contains(strings.ToLower(encode2), "gb") {
-						titleGBK, err := Decodegbk(text)
-						if err == nil {
-							title = string(titleGBK)
-						}
-					} else {
-						title = string(text)
-					}
-				} else {
-					title = "None"
-				}
-				title = strings.Trim(title, "\r\n \t")
-				title = strings.Replace(title, "\n", "", -1)
-				title = strings.Replace(title, "\r", "", -1)
-				title = strings.Replace(title, "&nbsp;", " ", -1)
-				if len(title) > 100 {
-					title = title[:100]
-				}
-				if title == "" {
-					title = "None"
-				}
-				length := resp.Header.Get("Content-Length")
-				if length == "" {
-					length = fmt.Sprintf("%v", len(body))
-				}
-				result := fmt.Sprintf("[*] WebTitle:%-25v code:%-3v len:%-6v title:%v", resp.Request.URL, resp.StatusCode, length, title)
-				common.LogSuccess(result)
-			}
-			CheckData = append(CheckData, WebScan.CheckDatas{body, fmt.Sprintf("%s", resp.Header)})
-			redirURL, err1 := resp.Location()
-			if err1 == nil {
-				return nil, redirURL.String(), CheckData
-			}
-			if resp.StatusCode == 400 && !strings.HasPrefix(info.Url, "https") {
-				return err, "https", CheckData
-			}
-			return err, "", CheckData
-		}
 		return err, "https", CheckData
 	}
-	return err, "", CheckData
-}
 
-func Decodegbk(s []byte) ([]byte, error) { // GBK解码
+	defer resp.Body.Close()
-	I := bytes.NewReader(s)
+	var title string
-	O := transform.NewReader(I, simplifiedchinese.GBK.NewDecoder())
+	body, err := getRespBody(resp)
-	d, e := ioutil.ReadAll(O)
+	if err != nil {
-	if e != nil {
+		return err, "https", CheckData
-		return nil, e
 	}
-	return d, nil
+	if !utf8.Valid(body) {
+		body, _ = simplifiedchinese.GBK.NewDecoder().Bytes(body)
+	}
+	CheckData = append(CheckData, WebScan.CheckDatas{body, fmt.Sprintf("%s", resp.Header)})
+	var reurl string
+	if flag != 2 {
+		title = gettitle(body)
+		length := resp.Header.Get("Content-Length")
+		if length == "" {
+			length = fmt.Sprintf("%v", len(body))
+		}
+		redirURL, err1 := resp.Location()
+		if err1 == nil {
+			reurl = redirURL.String()
+		}
+		result := fmt.Sprintf("[*] WebTitle:%-25v code:%-3v len:%-6v title:%v", resp.Request.URL, resp.StatusCode, length, title)
+		if reurl != "" {
+			result += fmt.Sprintf(" 跳转url: %s", reurl)
+		}
+		common.LogSuccess(result)
+	}
+	if reurl != "" {
+		return nil, reurl, CheckData
+	}
+	if resp.StatusCode == 400 && !strings.HasPrefix(info.Url, "https") {
+		return nil, "https", CheckData
+	}
+	return nil, "", CheckData
 }
 
 func getRespBody(oResp *http.Response) ([]byte, error) {
@@ -272,14 +189,42 @@ func getRespBody(oResp *http.Response) ([]byte, error) {
 	return body, nil
 }
 
-func GetProtocol(host string, Timeout int64) string {
+func gettitle(body []byte) (title string) {
+	re := regexp.MustCompile("(?ims)<title>(.*)</title>")
+	find := re.FindSubmatch(body)
+	if len(find) > 1 {
+		title = string(find[1])
+		title = strings.TrimSpace(title)
+		title = strings.Replace(title, "\n", "", -1)
+		title = strings.Replace(title, "\r", "", -1)
+		title = strings.Replace(title, "&nbsp;", " ", -1)
+		if len(title) > 100 {
+			title = title[:100]
+		}
+	}
+	if title == "" {
+		title = "None"
+	}
+	return
+}
+
+func GetProtocol(host string, Timeout int64) (protocol string) {
+	protocol = "http"
+	//如果端口是80或443,跳过Protocol判断
+	if strings.HasSuffix(host, ":80") || !strings.Contains(host, ":") {
+		return
+	} else if strings.HasSuffix(host, ":443") {
+		protocol = "https"
+		return
+	}
+
 	conn, err := tls.DialWithDialer(&net.Dialer{Timeout: time.Duration(Timeout) * time.Second}, "tcp", host, &tls.Config{InsecureSkipVerify: true})
 	defer func() {
 		if conn != nil {
 			conn.Close()
 		}
 	}()
-	protocol := "http"
+
 	if err == nil || strings.Contains(err.Error(), "handshake failure") {
 		protocol = "https"
 	}

WebScan/InfoScan.go

@@ -13,11 +13,11 @@ type CheckDatas struct {
 	Headers string
 }
 
-func InfoCheck(Url string, CheckData []CheckDatas) []string {
+func InfoCheck(Url string, CheckData *[]CheckDatas) []string {
 	var matched bool
 	var infoname []string
 
-	for _, data := range CheckData {
+	for _, data := range *CheckData {
 		for _, rule := range info.RuleDatas {
 			if rule.Type == "code" {
 				matched, _ = regexp.MatchString(rule.Rule, string(data.Body))
@@ -67,4 +67,3 @@ func removeDuplicateElement(languages []string) []string {
 	}
 	return result
 }
-