update
This commit is contained in:
parent
7833a2234c
commit
19ea9c5a16
@ -1,195 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html/charset"
|
||||
"golang.org/x/text/encoding"
|
||||
"golang.org/x/text/encoding/unicode"
|
||||
"golang.org/x/text/transform"
|
||||
)
|
||||
|
||||
var rateLimiter = time.Tick(20 * time.Millisecond)
|
||||
|
||||
func Fetch(url string) ([]byte, error) {
|
||||
<-rateLimiter
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
//fmt.Println("Error status code", resp.StatusCode)
|
||||
return nil, fmt.Errorf("error status code: %d", resp.StatusCode)
|
||||
}
|
||||
bodyReader := bufio.NewReader(resp.Body)
|
||||
e := determinEncoding(bodyReader)
|
||||
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
|
||||
|
||||
return io.ReadAll(utf8Reader)
|
||||
}
|
||||
|
||||
func FetchHost(reqUrl, proxyIP string) ([]byte, error) {
|
||||
<-rateLimiter
|
||||
|
||||
client := &http.Client{}
|
||||
|
||||
//是否使用代理IP
|
||||
if proxyIP != "" {
|
||||
proxy, err := url.Parse(proxyIP)
|
||||
if err != nil {
|
||||
log.Fatalf("1:%v\n", err)
|
||||
}
|
||||
netTransport := &http.Transport{
|
||||
Proxy: http.ProxyURL(proxy),
|
||||
MaxIdleConnsPerHost: 10,
|
||||
ResponseHeaderTimeout: time.Second * time.Duration(5),
|
||||
}
|
||||
client = &http.Client{
|
||||
Timeout: time.Second * 10,
|
||||
Transport: netTransport,
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("GET", reqUrl, nil)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
req.Header.Set("Host", "www.stats.gov.cn")
|
||||
//req.Header.Set("Cookie", "_trs_uv=kma79tru_6_1yw6; SF_cookie_1=37059734; wzws_cid=ed0db2d09a630ccd20292459e4bfc8091d460d5990f415e3d928761ee970d90f7ba6254f9c4b8e9b79ad456094d4a1381305d0b065ff9cc539ee1775ec262f946af61ddbed371e2a6dae2bc2041a30fc")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.54")
|
||||
req.Header.Set("Upgrade-Insecure-Requests", "1")
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
//fmt.Println("Error status code", resp.StatusCode)
|
||||
return nil, fmt.Errorf("error status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
bodyReader := bufio.NewReader(resp.Body)
|
||||
e := determinEncoding(bodyReader)
|
||||
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
|
||||
|
||||
return io.ReadAll(utf8Reader)
|
||||
}
|
||||
|
||||
// HttpPost 模拟请求方法
|
||||
func HttpPost(postUrl string, headers map[string]string, jsonMap map[string]interface{}, proxyIP string) ([]byte, error) {
|
||||
client := &http.Client{}
|
||||
//转换成postBody
|
||||
bytesData, err := json.Marshal(jsonMap)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return nil, err
|
||||
}
|
||||
postBody := bytes.NewReader(bytesData)
|
||||
|
||||
//是否使用代理IP
|
||||
if proxyIP != "" {
|
||||
//proxy := func(_ *http.Request) (*url.URL, error) {
|
||||
// return url.Parse(proxyIP)
|
||||
//}
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
},
|
||||
}
|
||||
proxyUrl, err := url.Parse(proxyIP)
|
||||
if err == nil { // 使用传入代理
|
||||
transport.Proxy = http.ProxyURL(proxyUrl)
|
||||
}
|
||||
//&http.Transport{Proxy: proxy}
|
||||
client = &http.Client{Transport: transport}
|
||||
} else {
|
||||
client = &http.Client{}
|
||||
}
|
||||
|
||||
// get请求
|
||||
req, err := http.NewRequest("GET", postUrl, postBody)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range headers {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
//fmt.Println("Error status code", resp.StatusCode)
|
||||
return nil, fmt.Errorf("error status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
bodyReader := bufio.NewReader(resp.Body)
|
||||
e := determinEncoding(bodyReader)
|
||||
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
|
||||
|
||||
return io.ReadAll(utf8Reader)
|
||||
}
|
||||
|
||||
func determinEncoding(r *bufio.Reader) encoding.Encoding {
|
||||
bytes, err := r.Peek(1024)
|
||||
if err != nil && err != io.EOF {
|
||||
log.Printf("Fetcher error: %v", err)
|
||||
return unicode.UTF8
|
||||
}
|
||||
|
||||
e, _, _ := charset.DetermineEncoding(bytes, "")
|
||||
return e
|
||||
}
|
||||
|
||||
// PostRequest 请求
|
||||
func PostRequest(url, parameter string) ([]byte, error) {
|
||||
client := &http.Client{}
|
||||
byteParameter := bytes.NewBuffer([]byte(parameter))
|
||||
request, _ := http.NewRequest("POST", url, byteParameter)
|
||||
request.Header.Set("Content-type", "application/json")
|
||||
response, _ := client.Do(request)
|
||||
if response.StatusCode != 200 {
|
||||
return nil, errors.New("网络请求失败")
|
||||
}
|
||||
all, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, errors.New("读取网络内容失败")
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
// PostRequestString 请求
|
||||
func PostRequestString(url, parameter string) ([]byte, error) {
|
||||
client := &http.Client{}
|
||||
byteParameter := bytes.NewBuffer([]byte(parameter))
|
||||
request, _ := http.NewRequest("POST", url, byteParameter)
|
||||
request.Header.Set("Content-type", "application/x-www-form-urlencoded")
|
||||
response, _ := client.Do(request)
|
||||
if response.StatusCode != 200 {
|
||||
return nil, errors.New("网络请求失败")
|
||||
}
|
||||
all, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, errors.New("读取网络内容失败")
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
@ -1,140 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type LocationResponse struct {
|
||||
ShowApiResCode int `json:"showapi_res_code"`
|
||||
ShowApiResError string `json:"showapi_res_error"`
|
||||
ShowApiResBody LocationBodyResponse `json:"showapi_res_body"`
|
||||
}
|
||||
|
||||
type LocationBodyResponse struct {
|
||||
|
||||
// 0为成功，其他失败。失败时不扣点数
|
||||
RetCode int `json:"ret_code"`
|
||||
|
||||
// 1移动 2电信 3联通
|
||||
Type int `json:"type"`
|
||||
|
||||
// 号段 1890871
|
||||
Num int `json:"num"`
|
||||
|
||||
// 网络 服务商
|
||||
Name string `json:"name"`
|
||||
|
||||
// 此地区身份证号开头几位 530000
|
||||
ProvCode string `json:"prov_code"`
|
||||
|
||||
// 邮编 650000
|
||||
PostCode string `json:"post_code"`
|
||||
|
||||
// 省份 云南
|
||||
Prov string `json:"prov"`
|
||||
|
||||
// 城市 昆明
|
||||
City string `json:"city"`
|
||||
|
||||
// 区号 0871
|
||||
AreaCode string `json:"area_code"`
|
||||
}
|
||||
|
||||
func GetLocationWithPhone(phone string) (LocationResponse, error) {
|
||||
url := "http://showphone.market.alicloudapi.com/6-1?num=" + phone
|
||||
appCode := "ee6e2d724f3a42f4881c36ec90b9ef4c"
|
||||
client := &http.Client{Timeout: time.Second * 5}
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "APPCODE "+appCode)
|
||||
|
||||
var out LocationResponse
|
||||
get, err := client.Do(req)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("接口请求失败：%v", err)
|
||||
}
|
||||
defer get.Body.Close()
|
||||
|
||||
res, err := ioutil.ReadAll(get.Body)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("body读取失败：%v", err)
|
||||
}
|
||||
|
||||
log.Printf("1: %#v", res)
|
||||
|
||||
err = json.Unmarshal(res, &out)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("json反序列化失败：%v", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type RegionResponse struct {
|
||||
ShowApiResCode int `json:"showapi_res_code"`
|
||||
ShowApiResError string `json:"showapi_res_error"`
|
||||
ShowApiResBody RegionBodyResponse `json:"showapi_res_body"`
|
||||
}
|
||||
|
||||
type RegionBodyResponse struct {
|
||||
|
||||
// 0为成功，其他失败。失败时不扣点数
|
||||
RetCode int `json:"ret_code"`
|
||||
|
||||
// 县
|
||||
county string `json:"county"`
|
||||
|
||||
// 网络 服务商
|
||||
ISP string `json:"isp"`
|
||||
|
||||
// IP
|
||||
IP string `json:"ip"`
|
||||
|
||||
// 国家
|
||||
Country string `json:"country"`
|
||||
|
||||
// 省份 云南
|
||||
Region string `json:"region"`
|
||||
|
||||
// 城市 昆明
|
||||
City string `json:"city"`
|
||||
}
|
||||
|
||||
func GetRegionWithIP(ip string) (RegionResponse, error) {
|
||||
url := "http://saip.market.alicloudapi.com/ip?ip=" + ip
|
||||
appCode := "ee6e2d724f3a42f4881c36ec90b9ef4c"
|
||||
client := &http.Client{Timeout: time.Second * 5}
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "APPCODE "+appCode)
|
||||
|
||||
var out RegionResponse
|
||||
get, err := client.Do(req)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("接口请求失败：%v", err)
|
||||
}
|
||||
defer get.Body.Close()
|
||||
|
||||
res, err := ioutil.ReadAll(get.Body)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("body读取失败：%v", err)
|
||||
}
|
||||
|
||||
log.Printf("2: %#v", res)
|
||||
|
||||
err = json.Unmarshal(res, &out)
|
||||
if err != nil {
|
||||
return out, fmt.Errorf("json反序列化失败：%v", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user