diff --git a/pkg/util/fetcher.go b/pkg/util/fetcher.go deleted file mode 100644 index 6082fd2..0000000 --- a/pkg/util/fetcher.go +++ /dev/null @@ -1,195 +0,0 @@ -package util - -import ( - "bufio" - "bytes" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "net/http" - "net/url" - "time" - - "golang.org/x/net/html/charset" - "golang.org/x/text/encoding" - "golang.org/x/text/encoding/unicode" - "golang.org/x/text/transform" -) - -var rateLimiter = time.Tick(20 * time.Millisecond) - -func Fetch(url string) ([]byte, error) { - <-rateLimiter - resp, err := http.Get(url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - //fmt.Println("Error status code", resp.StatusCode) - return nil, fmt.Errorf("error status code: %d", resp.StatusCode) - } - bodyReader := bufio.NewReader(resp.Body) - e := determinEncoding(bodyReader) - utf8Reader := transform.NewReader(bodyReader, e.NewDecoder()) - - return io.ReadAll(utf8Reader) -} - -func FetchHost(reqUrl, proxyIP string) ([]byte, error) { - <-rateLimiter - - client := &http.Client{} - - //是否使用代理IP - if proxyIP != "" { - proxy, err := url.Parse(proxyIP) - if err != nil { - log.Fatalf("1:%v\n", err) - } - netTransport := &http.Transport{ - Proxy: http.ProxyURL(proxy), - MaxIdleConnsPerHost: 10, - ResponseHeaderTimeout: time.Second * time.Duration(5), - } - client = &http.Client{ - Timeout: time.Second * 10, - Transport: netTransport, - } - } - - req, err := http.NewRequest("GET", reqUrl, nil) - if err != nil { - log.Fatalln(err) - } - req.Header.Set("Host", "www.stats.gov.cn") - //req.Header.Set("Cookie", "_trs_uv=kma79tru_6_1yw6; SF_cookie_1=37059734; wzws_cid=ed0db2d09a630ccd20292459e4bfc8091d460d5990f415e3d928761ee970d90f7ba6254f9c4b8e9b79ad456094d4a1381305d0b065ff9cc539ee1775ec262f946af61ddbed371e2a6dae2bc2041a30fc") - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.54") - req.Header.Set("Upgrade-Insecure-Requests", "1") - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - //fmt.Println("Error status code", resp.StatusCode) - return nil, fmt.Errorf("error status code: %d", resp.StatusCode) - } - - bodyReader := bufio.NewReader(resp.Body) - e := determinEncoding(bodyReader) - utf8Reader := transform.NewReader(bodyReader, e.NewDecoder()) - - return io.ReadAll(utf8Reader) -} - -// HttpPost 模拟请求方法 -func HttpPost(postUrl string, headers map[string]string, jsonMap map[string]interface{}, proxyIP string) ([]byte, error) { - client := &http.Client{} - //转换成postBody - bytesData, err := json.Marshal(jsonMap) - if err != nil { - fmt.Println(err.Error()) - return nil, err - } - postBody := bytes.NewReader(bytesData) - - //是否使用代理IP - if proxyIP != "" { - //proxy := func(_ *http.Request) (*url.URL, error) { - // return url.Parse(proxyIP) - //} - transport := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - proxyUrl, err := url.Parse(proxyIP) - if err == nil { // 使用传入代理 - transport.Proxy = http.ProxyURL(proxyUrl) - } - //&http.Transport{Proxy: proxy} - client = &http.Client{Transport: transport} - } else { - client = &http.Client{} - } - - // get请求 - req, err := http.NewRequest("GET", postUrl, postBody) - if err != nil { - log.Fatalln(err) - return nil, err - } - for k, v := range headers { - req.Header.Add(k, v) - } - resp, err := client.Do(req) - if err != nil { - log.Fatalln(err) - return nil, err - } - - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - //fmt.Println("Error status code", resp.StatusCode) - return nil, fmt.Errorf("error status code: %d", resp.StatusCode) - } - - bodyReader := bufio.NewReader(resp.Body) - e := determinEncoding(bodyReader) - utf8Reader := transform.NewReader(bodyReader, e.NewDecoder()) - - return io.ReadAll(utf8Reader) -} - -func determinEncoding(r *bufio.Reader) encoding.Encoding { - bytes, err := r.Peek(1024) - if err != nil && err != io.EOF { - log.Printf("Fetcher error: %v", err) - return unicode.UTF8 - } - - e, _, _ := charset.DetermineEncoding(bytes, "") - return e -} - -// PostRequest 请求 -func PostRequest(url, parameter string) ([]byte, error) { - client := &http.Client{} - byteParameter := bytes.NewBuffer([]byte(parameter)) - request, _ := http.NewRequest("POST", url, byteParameter) - request.Header.Set("Content-type", "application/json") - response, _ := client.Do(request) - if response.StatusCode != 200 { - return nil, errors.New("网络请求失败") - } - all, err := io.ReadAll(response.Body) - if err != nil { - return nil, errors.New("读取网络内容失败") - } - return all, nil -} - -// PostRequestString 请求 -func PostRequestString(url, parameter string) ([]byte, error) { - client := &http.Client{} - byteParameter := bytes.NewBuffer([]byte(parameter)) - request, _ := http.NewRequest("POST", url, byteParameter) - request.Header.Set("Content-type", "application/x-www-form-urlencoded") - response, _ := client.Do(request) - if response.StatusCode != 200 { - return nil, errors.New("网络请求失败") - } - all, err := io.ReadAll(response.Body) - if err != nil { - return nil, errors.New("读取网络内容失败") - } - return all, nil -} diff --git a/pkg/util/location.go b/pkg/util/location.go deleted file mode 100644 index 2849693..0000000 --- a/pkg/util/location.go +++ /dev/null @@ -1,140 +0,0 @@ -package util - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "time" -) - -type LocationResponse struct { - ShowApiResCode int `json:"showapi_res_code"` - ShowApiResError string `json:"showapi_res_error"` - ShowApiResBody LocationBodyResponse `json:"showapi_res_body"` -} - -type LocationBodyResponse struct { - - // 0为成功，其他失败。失败时不扣点数 - RetCode int `json:"ret_code"` - - // 1移动 2电信 3联通 - Type int `json:"type"` - - // 号段 1890871 - Num int `json:"num"` - - // 网络 服务商 - Name string `json:"name"` - - // 此地区身份证号开头几位 530000 - ProvCode string `json:"prov_code"` - - // 邮编 650000 - PostCode string `json:"post_code"` - - // 省份 云南 - Prov string `json:"prov"` - - // 城市 昆明 - City string `json:"city"` - - // 区号 0871 - AreaCode string `json:"area_code"` -} - -func GetLocationWithPhone(phone string) (LocationResponse, error) { - url := "http://showphone.market.alicloudapi.com/6-1?num=" + phone - appCode := "ee6e2d724f3a42f4881c36ec90b9ef4c" - client := &http.Client{Timeout: time.Second * 5} - req, err := http.NewRequest("GET", url, nil) - if err != nil { - log.Fatalln(err) - } - req.Header.Set("Authorization", "APPCODE "+appCode) - - var out LocationResponse - get, err := client.Do(req) - if err != nil { - return out, fmt.Errorf("接口请求失败：%v", err) - } - defer get.Body.Close() - - res, err := ioutil.ReadAll(get.Body) - if err != nil { - return out, fmt.Errorf("body读取失败：%v", err) - } - - log.Printf("1: %#v", res) - - err = json.Unmarshal(res, &out) - if err != nil { - return out, fmt.Errorf("json反序列化失败：%v", err) - } - - return out, nil -} - -type RegionResponse struct { - ShowApiResCode int `json:"showapi_res_code"` - ShowApiResError string `json:"showapi_res_error"` - ShowApiResBody RegionBodyResponse `json:"showapi_res_body"` -} - -type RegionBodyResponse struct { - - // 0为成功，其他失败。失败时不扣点数 - RetCode int `json:"ret_code"` - - // 县 - county string `json:"county"` - - // 网络 服务商 - ISP string `json:"isp"` - - // IP - IP string `json:"ip"` - - // 国家 - Country string `json:"country"` - - // 省份 云南 - Region string `json:"region"` - - // 城市 昆明 - City string `json:"city"` -} - -func GetRegionWithIP(ip string) (RegionResponse, error) { - url := "http://saip.market.alicloudapi.com/ip?ip=" + ip - appCode := "ee6e2d724f3a42f4881c36ec90b9ef4c" - client := &http.Client{Timeout: time.Second * 5} - req, err := http.NewRequest("GET", url, nil) - if err != nil { - log.Fatalln(err) - } - req.Header.Set("Authorization", "APPCODE "+appCode) - - var out RegionResponse - get, err := client.Do(req) - if err != nil { - return out, fmt.Errorf("接口请求失败：%v", err) - } - defer get.Body.Close() - - res, err := ioutil.ReadAll(get.Body) - if err != nil { - return out, fmt.Errorf("body读取失败：%v", err) - } - - log.Printf("2: %#v", res) - - err = json.Unmarshal(res, &out) - if err != nil { - return out, fmt.Errorf("json反序列化失败：%v", err) - } - - return out, nil -}