kenneth/gohelpers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
kenneth 2021-09-17 11:26:41 +08:00
parent 7833a2234c
commit 19ea9c5a16
2 changed files with 0 additions and 335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
pkg/util/fetcher.go
View File

@ -1,195 +0,0 @@
package util
import (
"bufio"
"bytes"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"time"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
var rateLimiter = time.Tick(20 * time.Millisecond)
func Fetch(url string) ([]byte, error) {
<-rateLimiter
resp, err := http.Get(url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//fmt.Println("Error status code", resp.StatusCode)
return nil, fmt.Errorf("error status code: %d", resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determinEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
return io.ReadAll(utf8Reader)
}
func FetchHost(reqUrl, proxyIP string) ([]byte, error) {
<-rateLimiter
client := &http.Client{}
//是否使用代理IP
if proxyIP != "" {
proxy, err := url.Parse(proxyIP)
if err != nil {
log.Fatalf("1:%v\n", err)
}
netTransport := &http.Transport{
Proxy: http.ProxyURL(proxy),
MaxIdleConnsPerHost: 10,
ResponseHeaderTimeout: time.Second * time.Duration(5),
}
client = &http.Client{
Timeout: time.Second * 10,
Transport: netTransport,
}
}
req, err := http.NewRequest("GET", reqUrl, nil)
if err != nil {
log.Fatalln(err)
}
req.Header.Set("Host", "www.stats.gov.cn")
//req.Header.Set("Cookie", "_trs_uv=kma79tru_6_1yw6; SF_cookie_1=37059734; wzws_cid=ed0db2d09a630ccd20292459e4bfc8091d460d5990f415e3d928761ee970d90f7ba6254f9c4b8e9b79ad456094d4a1381305d0b065ff9cc539ee1775ec262f946af61ddbed371e2a6dae2bc2041a30fc")
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36 Edg/89.0.774.54")
req.Header.Set("Upgrade-Insecure-Requests", "1")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//fmt.Println("Error status code", resp.StatusCode)
return nil, fmt.Errorf("error status code: %d", resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determinEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
return io.ReadAll(utf8Reader)
}
// HttpPost 模拟请求方法
func HttpPost(postUrl string, headers map[string]string, jsonMap map[string]interface{}, proxyIP string) ([]byte, error) {
client := &http.Client{}
//转换成postBody
bytesData, err := json.Marshal(jsonMap)
if err != nil {
fmt.Println(err.Error())
return nil, err
}
postBody := bytes.NewReader(bytesData)
//是否使用代理IP
if proxyIP != "" {
//proxy := func(_ *http.Request) (*url.URL, error) {
// return url.Parse(proxyIP)
//}
transport := &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
proxyUrl, err := url.Parse(proxyIP)
if err == nil { // 使用传入代理
transport.Proxy = http.ProxyURL(proxyUrl)
}
//&http.Transport{Proxy: proxy}
client = &http.Client{Transport: transport}
} else {
client = &http.Client{}
}
// get请求
req, err := http.NewRequest("GET", postUrl, postBody)
if err != nil {
log.Fatalln(err)
return nil, err
}
for k, v := range headers {
req.Header.Add(k, v)
}
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
//fmt.Println("Error status code", resp.StatusCode)
return nil, fmt.Errorf("error status code: %d", resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determinEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
return io.ReadAll(utf8Reader)
}
func determinEncoding(r *bufio.Reader) encoding.Encoding {
bytes, err := r.Peek(1024)
if err != nil && err != io.EOF {
log.Printf("Fetcher error: %v", err)
return unicode.UTF8
}
e, _, _ := charset.DetermineEncoding(bytes, "")
return e
}
// PostRequest 请求
func PostRequest(url, parameter string) ([]byte, error) {
client := &http.Client{}
byteParameter := bytes.NewBuffer([]byte(parameter))
request, _ := http.NewRequest("POST", url, byteParameter)
request.Header.Set("Content-type", "application/json")
response, _ := client.Do(request)
if response.StatusCode != 200 {
return nil, errors.New("网络请求失败")
}
all, err := io.ReadAll(response.Body)
if err != nil {
return nil, errors.New("读取网络内容失败")
}
return all, nil
}
// PostRequestString 请求
func PostRequestString(url, parameter string) ([]byte, error) {
client := &http.Client{}
byteParameter := bytes.NewBuffer([]byte(parameter))
request, _ := http.NewRequest("POST", url, byteParameter)
request.Header.Set("Content-type", "application/x-www-form-urlencoded")
response, _ := client.Do(request)
if response.StatusCode != 200 {
return nil, errors.New("网络请求失败")
}
all, err := io.ReadAll(response.Body)
if err != nil {
return nil, errors.New("读取网络内容失败")
}
return all, nil
}

140
pkg/util/location.go
View File

@ -1,140 +0,0 @@
package util
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
)
type LocationResponse struct {
ShowApiResCode int `json:"showapi_res_code"`
ShowApiResError string `json:"showapi_res_error"`
ShowApiResBody LocationBodyResponse `json:"showapi_res_body"`
}
type LocationBodyResponse struct {
// 0为成功，其他失败。失败时不扣点数
RetCode int `json:"ret_code"`
// 1移动 2电信 3联通
Type int `json:"type"`
// 号段 1890871
Num int `json:"num"`
// 网络 服务商
Name string `json:"name"`
// 此地区身份证号开头几位 530000
ProvCode string `json:"prov_code"`
// 邮编 650000
PostCode string `json:"post_code"`
// 省份 云南
Prov string `json:"prov"`
// 城市 昆明
City string `json:"city"`
// 区号 0871
AreaCode string `json:"area_code"`
}
func GetLocationWithPhone(phone string) (LocationResponse, error) {
url := "http://showphone.market.alicloudapi.com/6-1?num=" + phone
appCode := "ee6e2d724f3a42f4881c36ec90b9ef4c"
client := &http.Client{Timeout: time.Second * 5}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalln(err)
}
req.Header.Set("Authorization", "APPCODE "+appCode)
var out LocationResponse
get, err := client.Do(req)
if err != nil {
return out, fmt.Errorf("接口请求失败：%v", err)
}
defer get.Body.Close()
res, err := ioutil.ReadAll(get.Body)
if err != nil {
return out, fmt.Errorf("body读取失败：%v", err)
}
log.Printf("1: %#v", res)
err = json.Unmarshal(res, &out)
if err != nil {
return out, fmt.Errorf("json反序列化失败：%v", err)
}
return out, nil
}
type RegionResponse struct {
ShowApiResCode int `json:"showapi_res_code"`
ShowApiResError string `json:"showapi_res_error"`
ShowApiResBody RegionBodyResponse `json:"showapi_res_body"`
}
type RegionBodyResponse struct {
// 0为成功，其他失败。失败时不扣点数
RetCode int `json:"ret_code"`
// 县
county string `json:"county"`
// 网络 服务商
ISP string `json:"isp"`
// IP
IP string `json:"ip"`
// 国家
Country string `json:"country"`
// 省份 云南
Region string `json:"region"`
// 城市 昆明
City string `json:"city"`
}
func GetRegionWithIP(ip string) (RegionResponse, error) {
url := "http://saip.market.alicloudapi.com/ip?ip=" + ip
appCode := "ee6e2d724f3a42f4881c36ec90b9ef4c"
client := &http.Client{Timeout: time.Second * 5}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Fatalln(err)
}
req.Header.Set("Authorization", "APPCODE "+appCode)
var out RegionResponse
get, err := client.Do(req)
if err != nil {
return out, fmt.Errorf("接口请求失败：%v", err)
}
defer get.Body.Close()
res, err := ioutil.ReadAll(get.Body)
if err != nil {
return out, fmt.Errorf("body读取失败：%v", err)
}
log.Printf("2: %#v", res)
err = json.Unmarshal(res, &out)
if err != nil {
return out, fmt.Errorf("json反序列化失败：%v", err)
}
return out, nil
}