kenneth/gohelpers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

0.0.3

Browse Source
This commit is contained in:
kenneth
2022-04-07 15:20:21 +08:00
parent 19ea9c5a16
commit 811536bea4
35 changed files with 466 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
pkg/ip2region/ip2Region.go
View File

@@ -13,19 +13,17 @@ const (
TOTAL_HEADER_LENGTH = 8192
)
var err error
var ipInfo IpInfo
var IpRegion *Ip2Region
type Ip2Region struct {
// db file handler
dbFileHandler *os.File
//header block info
headerSip []int64
headerPtr []int64
headerLen int64
headerSip []int64
headerPtr []int64
headerLen int64
// super block index info
firstIndexPtr int64
@@ -35,20 +33,20 @@ type Ip2Region struct {
// for memory mode only
// the original db binary string
dbBinStr []byte
dbFile string
dbBinStr []byte
dbFile string
}
type IpInfo struct {
CityId int64
Country string
Region string
Province string
City string
ISP string
CityId int64 `json:"city_id"`
Country string `json:"country"`
Region string `json:"region"`
Province string `json:"province"`
City string `json:"city"`
ISP string `json:"isp"`
}
func (ip IpInfo)String() string {
func (ip IpInfo) String() string {
return strconv.FormatInt(ip.CityId, 10) + "|" + ip.Country + "|" + ip.Region + "|" + ip.Province + "|" + ip.City + "|" + ip.ISP
}
@@ -59,7 +57,7 @@ func getIpInfo(cityId int64, line []byte) IpInfo {
length := len(lineSlice)
ipInfo.CityId = cityId
if length < 5 {
for i := 0; i <= 5 - length; i++ {
for i := 0; i <= 5-length; i++ {
lineSlice = append(lineSlice, "")
}
}
@@ -72,37 +70,39 @@ func getIpInfo(cityId int64, line []byte) IpInfo {
return ipInfo
}
func New(path string) (*Ip2Region, error) {
func New(path string) error {
file, err := os.Open(path)
if err != nil {
return nil, err
return err
}
return &Ip2Region{
dbFile:path,
dbFileHandler:file,
}, nil
IpRegion = &Ip2Region{
dbFile: path,
dbFileHandler: file,
}
return nil
}
func (this *Ip2Region) Close() {
this.dbFileHandler.Close()
func (region *Ip2Region) Close() error {
return region.dbFileHandler.Close()
}
func (this *Ip2Region) MemorySearch(ipStr string) (ipInfo IpInfo, err error) {
func (region *Ip2Region) MemorySearch(ipStr string) (ipInfo IpInfo, err error) {
ipInfo = IpInfo{}
if this.totalBlocks == 0 {
this.dbBinStr, err = ioutil.ReadFile(this.dbFile)
if region.totalBlocks == 0 {
region.dbBinStr, err = ioutil.ReadFile(region.dbFile)
if err != nil {
return ipInfo, err
}
this.firstIndexPtr = getLong(this.dbBinStr, 0)
this.lastIndexPtr = getLong(this.dbBinStr, 4)
this.totalBlocks = (this.lastIndexPtr - this.firstIndexPtr) /INDEX_BLOCK_LENGTH + 1
region.firstIndexPtr = getLong(region.dbBinStr, 0)
region.lastIndexPtr = getLong(region.dbBinStr, 4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/INDEX_BLOCK_LENGTH + 1
}
ip, err := ip2long(ipStr)
@@ -110,22 +110,22 @@ func (this *Ip2Region) MemorySearch(ipStr string) (ipInfo IpInfo, err error) {
return ipInfo, err
}
h := this.totalBlocks
var dataPtr, l int64;
for (l <= h) {
h := region.totalBlocks
var dataPtr, l int64
for l <= h {
m := (l + h) >> 1
p := this.firstIndexPtr + m *INDEX_BLOCK_LENGTH
sip := getLong(this.dbBinStr, p)
p := region.firstIndexPtr + m*INDEX_BLOCK_LENGTH
sip := getLong(region.dbBinStr, p)
if ip < sip {
h = m - 1
} else {
eip := getLong(this.dbBinStr, p + 4)
eip := getLong(region.dbBinStr, p+4)
if ip > eip {
l = m + 1
} else {
dataPtr = getLong(this.dbBinStr, p + 8)
break;
dataPtr = getLong(region.dbBinStr, p+8)
break
}
}
}
@@ -134,26 +134,26 @@ func (this *Ip2Region) MemorySearch(ipStr string) (ipInfo IpInfo, err error) {
}
dataLen := ((dataPtr >> 24) & 0xFF)
dataPtr = (dataPtr & 0x00FFFFFF);
ipInfo = getIpInfo(getLong(this.dbBinStr, dataPtr), this.dbBinStr[(dataPtr) + 4:dataPtr + dataLen])
dataPtr = (dataPtr & 0x00FFFFFF)
ipInfo = getIpInfo(getLong(region.dbBinStr, dataPtr), region.dbBinStr[(dataPtr)+4:dataPtr+dataLen])
return ipInfo, nil
}
func (this *Ip2Region)BinarySearch(ipStr string) (ipInfo IpInfo, err error) {
func (region *Ip2Region) BinarySearch(ipStr string) (ipInfo IpInfo, err error) {
ipInfo = IpInfo{}
if this.totalBlocks == 0 {
this.dbFileHandler.Seek(0, 0)
if region.totalBlocks == 0 {
region.dbFileHandler.Seek(0, 0)
superBlock := make([]byte, 8)
this.dbFileHandler.Read(superBlock)
this.firstIndexPtr = getLong(superBlock, 0)
this.lastIndexPtr = getLong(superBlock, 4)
this.totalBlocks = (this.lastIndexPtr - this.firstIndexPtr) /INDEX_BLOCK_LENGTH + 1
region.dbFileHandler.Read(superBlock)
region.firstIndexPtr = getLong(superBlock, 0)
region.lastIndexPtr = getLong(superBlock, 4)
region.totalBlocks = (region.lastIndexPtr-region.firstIndexPtr)/INDEX_BLOCK_LENGTH + 1
}
var l, dataPtr, p int64
h := this.totalBlocks
h := region.totalBlocks
ip, err := ip2long(ipStr)
@@ -161,21 +161,21 @@ func (this *Ip2Region)BinarySearch(ipStr string) (ipInfo IpInfo, err error) {
return
}
for (l <= h) {
for l <= h {
m := (l + h) >> 1
p = m * INDEX_BLOCK_LENGTH
_, err = this.dbFileHandler.Seek(this.firstIndexPtr + p, 0)
_, err = region.dbFileHandler.Seek(region.firstIndexPtr+p, 0)
if err != nil {
return
}
buffer := make([]byte, INDEX_BLOCK_LENGTH)
_, err = this.dbFileHandler.Read(buffer)
_, err = region.dbFileHandler.Read(buffer)
if err != nil {
return ipInfo, err
}
sip := getLong(buffer, 0)
if ip < sip {
@@ -186,7 +186,7 @@ func (this *Ip2Region)BinarySearch(ipStr string) (ipInfo IpInfo, err error) {
l = m + 1
} else {
dataPtr = getLong(buffer, 8)
break;
break
}
}
@@ -198,76 +198,76 @@ func (this *Ip2Region)BinarySearch(ipStr string) (ipInfo IpInfo, err error) {
}
dataLen := ((dataPtr >> 24) & 0xFF)
dataPtr = (dataPtr & 0x00FFFFFF);
dataPtr = (dataPtr & 0x00FFFFFF)
this.dbFileHandler.Seek(dataPtr, 0)
region.dbFileHandler.Seek(dataPtr, 0)
data := make([]byte, dataLen)
this.dbFileHandler.Read(data)
region.dbFileHandler.Read(data)
ipInfo = getIpInfo(getLong(data, 0), data[4:dataLen])
err = nil
return
}
func (this *Ip2Region) BtreeSearch(ipStr string) (ipInfo IpInfo, err error) {
func (region *Ip2Region) BtreeSearch(ipStr string) (ipInfo IpInfo, err error) {
ipInfo = IpInfo{}
ip, err := ip2long(ipStr)
if this.headerLen == 0 {
this.dbFileHandler.Seek(8, 0)
if region.headerLen == 0 {
region.dbFileHandler.Seek(8, 0)
buffer := make([]byte, TOTAL_HEADER_LENGTH)
this.dbFileHandler.Read(buffer)
var idx int64;
region.dbFileHandler.Read(buffer)
var idx int64
for i := 0; i < TOTAL_HEADER_LENGTH; i += 8 {
startIp := getLong(buffer, int64(i))
dataPar := getLong(buffer, int64(i + 4))
dataPar := getLong(buffer, int64(i+4))
if dataPar == 0 {
break
}
this.headerSip = append(this.headerSip, startIp)
this.headerPtr = append(this.headerPtr, dataPar)
idx ++;
region.headerSip = append(region.headerSip, startIp)
region.headerPtr = append(region.headerPtr, dataPar)
idx++
}
this.headerLen = idx
region.headerLen = idx
}
var l, sptr, eptr int64
h := this.headerLen
h := region.headerLen
for l <= h {
m := int64(l + h) >> 1
if m < this.headerLen {
if ip == this.headerSip[m] {
m := int64(l+h) >> 1
if m < region.headerLen {
if ip == region.headerSip[m] {
if m > 0 {
sptr = this.headerPtr[m - 1]
eptr = this.headerPtr[m]
sptr = region.headerPtr[m-1]
eptr = region.headerPtr[m]
} else {
sptr = this.headerPtr[m]
eptr = this.headerPtr[m + 1]
sptr = region.headerPtr[m]
eptr = region.headerPtr[m+1]
}
break
}
if ip < this.headerSip[m] {
if ip < region.headerSip[m] {
if m == 0 {
sptr = this.headerPtr[m]
eptr = this.headerPtr[m + 1]
sptr = region.headerPtr[m]
eptr = region.headerPtr[m+1]
break
} else if ip > this.headerSip[m - 1] {
sptr = this.headerPtr[m - 1]
eptr = this.headerPtr[m]
} else if ip > region.headerSip[m-1] {
sptr = region.headerPtr[m-1]
eptr = region.headerPtr[m]
break
}
h = m - 1
} else {
if m == this.headerLen - 1 {
sptr = this.headerPtr[m - 1]
eptr = this.headerPtr[m]
if m == region.headerLen-1 {
sptr = region.headerPtr[m-1]
eptr = region.headerPtr[m]
break
} else if ip <= this.headerSip[m + 1] {
sptr = this.headerPtr[m ]
eptr = this.headerPtr[m + 1]
} else if ip <= region.headerSip[m+1] {
sptr = region.headerPtr[m]
eptr = region.headerPtr[m+1]
break
}
l = m + 1
@@ -282,25 +282,25 @@ func (this *Ip2Region) BtreeSearch(ipStr string) (ipInfo IpInfo, err error) {
}
blockLen := eptr - sptr
this.dbFileHandler.Seek(sptr, 0)
index := make([]byte, blockLen +INDEX_BLOCK_LENGTH)
this.dbFileHandler.Read(index)
region.dbFileHandler.Seek(sptr, 0)
index := make([]byte, blockLen+INDEX_BLOCK_LENGTH)
region.dbFileHandler.Read(index)
var dataptr int64
h = blockLen / INDEX_BLOCK_LENGTH
l = 0
for l <= h {
m := int64(l + h) >> 1
m := int64(l+h) >> 1
p := m * INDEX_BLOCK_LENGTH
sip := getLong(index, p)
if ip < sip {
h = m - 1;
h = m - 1
} else {
eip := getLong(index, p + 4)
eip := getLong(index, p+4)
if ip > eip {
l = m + 1
} else {
dataptr = getLong(index, p + 8)
dataptr = getLong(index, p+8)
break
}
}
@@ -314,19 +314,19 @@ func (this *Ip2Region) BtreeSearch(ipStr string) (ipInfo IpInfo, err error) {
dataLen := (dataptr >> 24) & 0xFF
dataPtr := dataptr & 0x00FFFFFF
this.dbFileHandler.Seek(dataPtr, 0)
region.dbFileHandler.Seek(dataPtr, 0)
data := make([]byte, dataLen)
this.dbFileHandler.Read(data)
region.dbFileHandler.Read(data)
ipInfo = getIpInfo(getLong(data, 0), data[4:])
return
}
func getLong(b []byte, offset int64) int64 {
val := (int64(b[offset ]) |
int64(b[offset + 1]) << 8 |
int64(b[offset + 2]) << 16 |
int64(b[offset + 3]) << 24)
val := (int64(b[offset]) |
int64(b[offset+1])<<8 |
int64(b[offset+2])<<16 |
int64(b[offset+3])<<24)
return val
@@ -341,9 +341,8 @@ func ip2long(IpStr string) (int64, error) {
var sum int64
for i, n := range bits {
bit, _ := strconv.ParseInt(n, 10, 64)
sum += bit << uint(24 - 8 * i)
sum += bit << uint(24-8*i)
}
return sum, nil
}

49
pkg/ip2region/ip2Region_test.go
View File

@@ -1,49 +0,0 @@
package ip2region
import (
"testing"
)
func BenchmarkBtreeSearch(B *testing.B) {
region, err := New("../../data/ip2region.db ")
if err != nil {
B.Error(err)
}
for i:=0;i<B.N;i++{
region.BtreeSearch("127.0.0.1")
}
}
func BenchmarkMemorySearch(B *testing.B) {
region, err := New("../../data/ip2region.db ")
if err != nil {
B.Error(err)
}
for i:=0;i<B.N;i++{
region.MemorySearch("127.0.0.1")
}
}
func BenchmarkBinarySearch(B *testing.B) {
region, err := New("../../data/ip2region.db ")
if err != nil {
B.Error(err)
}
for i:=0;i<B.N;i++{
region.BinarySearch("127.0.0.1")
}
}
func TestIp2long(t *testing.T) {
ip, err := ip2long("127.0.0.1")
if err != nil {
t.Error(err)
}
if ip != 2130706433 {
t.Error("result error")
}
t.Log(ip)
}