DAT format access code

IPv4 access code：

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;

public class Ip {

    private int[] prefStart = new int[256];
    private int[] prefEnd = new int[256];
    private byte[] data;
    private static Ip instance = null;

    private Ip(String fileName) {
        Path path = Paths.get(fileName);

        try {
            data = Files.readAllBytes(path);
        } catch (IOException e) {
            e.printStackTrace();
        }

        for (int k = 0; k < 256; k++) {
            int i = k * 8 + 4;
            prefStart[k] = (int) unpackInt4byte(data[i], data[i + 1], data[i + 2], data[i + 3]);
            prefEnd[k] = (int) unpackInt4byte(data[i + 4], data[i + 5], data[i + 6], data[i + 7]);
        }
    }

    public static synchronized Ip getInstance(String fileName) {
        if (instance == null)
            instance = new Ip(fileName);
        return instance;
    }

    public String get(String ip) {

        String[] ips = ip.split("\\.");
        int pref = Integer.valueOf(ips[0]);
        long val = ipToLong(ip);
        int low = prefStart[pref], high = prefEnd[pref];
        long cur = low == high ? low : search(low, high, val);
        return getAddr((int) cur);

    }

    private String getAddr(int cur){
        int p = 2052 + (cur * 9);
        int offset = (int)unpackInt4byte(data[4 + p], data[5 + p], data[6 + p],data[7+p]);
        int length = data[8 + p] & 0xff;
        return new String(Arrays.copyOfRange(data,offset,offset + length));
    }

    private int search(int low, int high, long k) {
        int M = 0;
        while (low <= high) {
            int mid = (low + high) / 2;

            int p = 2052 + (mid * 9);
            long endipNum = unpackInt4byte(data[p], data[1 + p], data[2 + p], data[3 + p]);

            if (endipNum >= k) {
                M = mid;
                if (mid == 0) {
                    break;
                }
                high = mid - 1;
            } else
                low = mid + 1;
        }
        return M;
    }

    private long unpackInt4byte(byte a, byte b, byte c, byte d) {
        return (a & 0xFFL) | ((b << 8) & 0xFF00L) | ((c << 16) & 0xFF0000L) | ((d << 24) & 0xFF000000L);

    }

    private long ipToLong(String ip) {
        long result = 0;
        String[] d = ip.split("\\.");
        for (String b : d) {
            result <<= 8;
            result |= Long.parseLong(b) & 0xff;
        }
        return result;
    }


}

import mmap
import struct
import socket


class IPV4Find:
    def __init__(self, file_name):

        self.buchang = 9
        self._handle = open(file_name, "rb")
        self.data = mmap.mmap(self._handle.fileno(), 0, access=mmap.ACCESS_READ)
        self.prefArr = []
        record_size = self.unpack_int_4byte(0)
        i = 0
        while i < 256:
            p = i * 8 + 4
            self.prefArr.append([self.unpack_int_4byte(p), self.unpack_int_4byte(p + 4)])
            i += 1
        self.endArr = []

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, exc_tb):
        self.close()

    def close(self):
        self._handle.close()

    def get(self, ip):

        ipdot = ip.split('.')
        prefix = int(ipdot[0])
        if prefix < 0 or prefix > 255 or len(ipdot) != 4:
            raise ValueError("invalid ip address")
        intIP = self.ip_to_int(ip)
        low = self.prefArr[prefix][0]
        high = self.prefArr[prefix][1]
        cur = low if low == high else self.search(low, high, intIP)
        # return self.addrArr[cur]
        return self.get_addr(cur)

    def search(self, low, high, k):
        M = 0
        while low <= high:
            mid = (low + high) // 2
            end_ip_num = self.unpack_int_4byte(2052 + (mid * self.buchang))
            if end_ip_num >= k:
                M = mid
                if mid == 0:
                    break
                high = mid - 1
            else:
                low = mid + 1
        return M

    def ip_to_int(self, ip):
        _ip = socket.inet_aton(ip)
        return struct.unpack("!L", _ip)[0]

    def unpack_int_4byte(self, offset):
        return struct.unpack('<L', self.data[offset:offset + 4])[0]

    def unpack_int_1byte(self, offset):
        return struct.unpack('B', self.data[offset:offset + 1])[0]

    def unpack_int_8byte(self, offset):
        return struct.unpack('<Q', self.data[offset:offset + 8])[0]

    def unpack_int_2byte(self, offset):
        return struct.unpack('<H', self.data[offset:offset + 2])[0]

    def get_addr(self, j):
        p = 2052 + (j * self.buchang)

        offset = self.unpack_int_4byte(4 + p)
        length = self.unpack_int_1byte(8 + p)
        return self.data[offset:offset + length].decode('utf-8')

using System.Net;
using System.Text;

namespace ipnews_dat
{

    public class IPSearch
    {
        private static readonly Lazy<IPSearch> lazy = new Lazy<IPSearch>(() => new IPSearch());
        public static IPSearch Instance { get { return lazy.Value; } }
        private IPSearch()
        {
            LoadDat();
        }

        private string datPath = "D:\\databases\\ipnews_scenes_mob.dat";

        private DateTime lastRead = DateTime.MinValue;

        private long[,] prefmap = new long[256, 2];

        private string[] addrArr;
        private byte[] data;



        /// <summary>
        /// initialize binary  ipnews.dat 
        /// </summary>

        private void LoadDat()
        {
            data = File.ReadAllBytes(datPath);

            for (int k = 0; k < 256; k++)
            {
                int i = k * 8 + 4;
                int prefix = k;
                long startIndex = ReadLittleEndian32(data[i], data[i + 1], data[i + 2], data[i + 3]);
                long endIndex = ReadLittleEndian32(data[i + 4], data[i + 5], data[i + 6], data[i + 7]);
                prefmap[k, 0] = startIndex; prefmap[k, 1] = endIndex;
            }


        }



        public string Find(string ip)
        {
            String[] ips = ip.Split(".");
            long pref = long.Parse(ips[0]);
            long val = IpToLong(ip);
            long low = prefmap[pref, 0], high = prefmap[pref, 1];
            long cur = low == high ? low : BinarySearch(low, high, val);

            return cur > -1 ? GetAddr(cur) : "||||||||||||||||||";
        }


        public string GetAddr(long cur)
        {
            long p = 2052 + (cur * 9);
            int offset = data[4 + p] + ((data[5 + p]) << 8) + ((data[6 + p]) << 16) + ((data[7 + p]) << 24);
            int length = data[8 + p];
            return Encoding.UTF8.GetString(data, offset, length);
        }


        
        private long BinarySearch(long low, long high, long k)
        {
            long M = 0, mid = 0;
            while (low <= high)
            {
                mid = (low + high) / 2;

                long p = 2052 + (mid * 9);
                uint endipnum = ReadLittleEndian32(data[p], data[1 + p], data[2 + p], data[3 + p]);

                if (endipnum >= k)
                {
                    M = mid;
                    if (mid == 0)
                    {
                        break;
                    }
                    high = mid - 1;
                }
                else
                    low = mid + 1;
            }
            return M;
        }


        static long IpToLong(string ipAddress)
        {
            IPAddress ip = IPAddress.Parse(ipAddress);

            byte[] bytes = ip.GetAddressBytes();

            Array.Reverse(bytes); 
            return BitConverter.ToUInt32(bytes, 0);
        }



        private uint ReadBigEndian32(byte a, byte b, byte c, byte d)
        {
            return (uint)((a << 24) | (b << 16) | (c << 8) | d);
        }


        private uint ReadLittleEndian32(byte a, byte b, byte c, byte d)
        {
            return (uint)((a | (b << 8) | (c << 16) | (d << 24)));
        }

      
    }


}

ipnews.cpp

#include "ipnews.h"
#include <iostream>
#include "string.h"
#include <stdlib.h>

//char* IP_FILENAME = "F:\\ipv4_city.dat";


geo_ip *ip_instance(const char *fileName) {
    static geo_ip *instance = NULL;
    if (instance == NULL) {
        instance = new geo_ip();
        if (ip_loadDat(instance, fileName) >= 0) {
            return instance;
        }
        if (instance == NULL) {
            delete instance;
        }
        return NULL;
    }
    return instance;
}

int32_t ip_loadDat(geo_ip *p, const char *fileName) {
    FILE *file;

    long len = 0;
    int k, i, j;
    file = fopen(fileName, "rb");
    if (file == NULL) {
        printf("%s", "Open file error");
        return -2;
    }
    fseek(file, 0, SEEK_END);
    len = ftell(file);
    fseek(file, 0, SEEK_SET);
    p->buffer = (uint8_t *) malloc(len * sizeof(uint8_t));
    fread(p->buffer, 1, len, file);
    fclose(file);

    for (k = 0; k < 256; k++) {
        i = k * 8 + 4;
        p->prefStart[k] = ip_read_int32(p, i);
        p->prefEnd[k] = ip_read_int32(p, i + 4);
    }

    return 0;
}

char *ip_query(geo_ip *p, const char *ip) {
    uint32_t pref, cur, intIP, low, high;
    if (NULL == p) {
        return NULL;
    }
    intIP = ip_ip2long(p, ip, &pref);;
    low = p->prefStart[pref];
    high = p->prefEnd[pref];
    cur = (low == high) ? low : ip_binary_search(p, low, high, intIP);
    if (cur == 100000000) {
        char *nil = (char *) malloc(2 * sizeof(char));
        nil[0] = '|';
        nil[1] = '\0';
        return nil;
    }
    return get_addr(p, cur);
}

long ip_binary_search(geo_ip *p, long low, long high, long k) {
    long M = 0;
    while (low <= high) {
        long mid = (low + high) / 2;
        int j = 2052 + (mid * 9);
        uint32_t endipNum = ip_read_int32(p,  j);
        if (endipNum >= k) {
            M = mid;
            if (mid == 0) {
                break;
            }
            high = mid - 1;
        } else
            low = mid + 1;
    }
    return M;
}

char *get_addr(geo_ip *p, uint32_t j) {
    uint32_t index = 2052 + j * 9;
    uint32_t offset = ip_read_int32(p,  index + 4);
    uint32_t length = p->buffer[index + 8];
    try {
        char *result = (char *) malloc((length + 1) * sizeof(char));
        memcpy(result, p->buffer + offset, length);
        result[length] = '\0';
        return result;
    }catch (...) {
        char* result = (char *) malloc((18 + 1) * sizeof(char));
        strcpy(result, "||||||||||||||||||");
        return result;
    }

}

uint32_t ip_ip2long(geo_ip *p, const char *addr, uint32_t *prefix) {
    uint32_t c, octet, t;
    uint32_t ipnum;
    int i = 3;

    octet = ipnum = 0;
    while ((c = *addr++)) {
        if (c == '.') {
            ipnum <<= 8;
            ipnum += octet;
            i--;
            octet = 0;
        } else {
            t = octet;
            octet <<= 3;
            octet += t;
            octet += t;
            c -= '0';

            octet += c;
            if (i == 3) {
                *prefix = octet;
            }
        }
    }

    ipnum <<= 8;

    return ipnum + octet;
}

uint32_t ip_read_int32(geo_ip *p, int pos) {
    uint32_t result;
    result = (uint32_t) ((p->buffer[pos]) | (p->buffer[pos + 1] << 8) | (p->buffer[pos + 2] << 16) | (p->buffer[pos + 3] << 24));
    return result;
}

ipnews.h

#ifndef __IP_H4_
#define __IP_H4_

#include <stdint.h>

#ifdef __cplusplus
extern "C" {
#endif

typedef struct tag_ip {
    uint32_t prefStart[256];
    uint32_t prefEnd[256];
    uint8_t *buffer;
} geo_ip;

geo_ip *ip_instance(const char *fileName);

int32_t ip_loadDat(geo_ip *p, const char *fileName);

char *ip_query(geo_ip *p, const char *ip);

long ip_binary_search(geo_ip *p, long low, long high, long k);

uint32_t ip_ip2long(geo_ip *p, const char *addr, uint32_t *prefix);

uint32_t ip_read_int32(geo_ip *p, int pos);


char *get_addr(geo_ip *p, uint32_t j);
;
#ifdef __cplusplus
}
#endif
#endif

main.cpp

#include<iostream>
#include<memory>
#include "ipDataCloud.h"


int main()
{
    std::unique_ptr<geo_ip> finder(ip_instance("file_path"));
    const char* ip = "ip_address";
    char* result = ip_query(finder.get(), ip);
    std::cout << result << std::endl;
    free(result);
    result = NULL;
	
}

package ipnews

import (
	"encoding/binary"
	"errors"
	"io/ioutil"
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
)

type IpInfo struct {
	prefStart [256]uint32
	prefEnd   [256]uint32
	data      []byte
}

var obj *IpInfo
var once sync.Once

func GetObject(fileName string) *IpInfo {
	once.Do(func() {
		obj = &IpInfo{}
		var err error
		obj, err = LoadFile(fileName)
		if err != nil {
			log.Fatal("the IP Dat loaded failed!")
		}
	})
	return obj
}

func LoadFile(file string) (*IpInfo, error) {
	p := IpInfo{}
	data, err := ioutil.ReadFile(file)
	p.data = data
	if err != nil {
		return nil, err
	}

	for k := 0; k < 256; k++ {
		i := k*8 + 4
		p.prefStart[k] = unpackInt4byte(data[i], data[i+1], data[i+2], data[i+3])
		p.prefEnd[k] = unpackInt4byte(data[i+4], data[i+5], data[i+6], data[i+7])
	}

	return &p, err

}

func (p *IpInfo) Get(ip string) (string, error) {
	ips := strings.Split(ip, ".")
	x, _ := strconv.Atoi(ips[0])
	prefix := uint32(x)
	intIP, err := ipToInt(ip)
	if err != nil {
		return "", err
	}

	low := p.prefStart[prefix]
	high := p.prefEnd[prefix]

	var cur uint32
	if low == high {
		cur = low
	} else {
		cur = p.Search(low, high, intIP)
	}
	if cur == 100000000 {
		return "无信息", errors.New("无信息")
	} else {
		return p.getAddr(cur), nil
	}

}

func (p *IpInfo) Search(low uint32, high uint32, k uint32) uint32 {
	var M uint32 = 0
	for low <= high {
		mid := (low + high) / 2

		j := 2052 + (mid * 9)
		endipNum := unpackInt4byte(p.data[j], p.data[1+j], p.data[2+j], p.data[3+j])

		if endipNum >= k {
			M = mid
			if mid == 0 {
				break
			}
			high = mid - 1
		} else {
			low = mid + 1
		}
	}

	return M
}

func (p *IpInfo) getAddr(cur uint32) string {
	j := 2052 + (cur * 9)
	offset := unpackInt4byte(p.data[4+j], p.data[5+j], p.data[6+j], p.data[7+j])
	length := uint32(p.data[8+j])
	return string(p.data[offset:int(offset+length)])
}

func ipToInt(ipstr string) (uint32, error) {
	ip := net.ParseIP(ipstr)
	ip = ip.To4()
	if ip == nil {
		return 0, errors.New("ip 不合法")
	}
	return binary.BigEndian.Uint32(ip), nil
}

func unpackInt4byte(a, b, c, d byte) uint32 {
	return (uint32(a) & 0xFF) | ((uint32(b) << 8) & 0xFF00) | ((uint32(c) << 16) & 0xFF0000) | ((uint32(d) << 24) & 0xFF000000)
}

ipdata.lua

local ipdata = { prefStart = {}, prefEnd = {},
                 endArr = {}, addrArr = {},
                 file = {}, _version = "0.1.2" }

function unpackInt4byte(a, b, c, d)
    return (a & 0xFF) | ((b << 8) & 0xFF00) | ((c << 16) & 0xFF0000) | ((d << 24) & 0xFF000000)
end

function fsize(file)
    local current = file:seek() 
    local size = file:seek("end") 
    file:seek("set", current) 
    return size
end

function redaFileBytes(file, length, off)
    file:seek("set", off)
    local str = file:read(length)
    return { str:byte(1, #str) }
end

function ipdata.LoadFile(path)
    local sizeData = {}

    ipdata.file = io.open(path, "rb")

    for k = 1, 256 do
        local i = k * 8 + 4
        sizeData = redaFileBytes(ipdata.file, 8, i)
        ipdata.prefStart[k] = unpackInt4byte(sizeData[1], sizeData[2], sizeData[3], sizeData[4])
        ipdata.prefEnd[k] = unpackInt4byte(sizeData[5], sizeData[6], sizeData[7], sizeData[8])
    end
    -- 文件大小
    sizeData = redaFileBytes(ipdata.file, 4, 0)
    local recordSize = unpackInt4byte(sizeData[1], sizeData[2], sizeData[3], sizeData[4])
    
    for i = 1, recordSize do
        local j = 2052 + (i * 9)
        sizeData = redaFileBytes(ipdata.file, 9, j)
        local endipnum = unpackInt4byte(sizeData[1], sizeData[2], sizeData[3], sizeData[4])
        ipdata.endArr[i] = endipnum
       
    end
end

function ipdata.getAddr(row)
    local j = 2052 + (row * 9)
    ipdata.file:seek("set", j + 4)
    local tempIndexData = ipdata.file:read(5)
    local offset = unpackInt4byte(tempIndexData:byte(1, 4))
    local length = tempIndexData:byte(5)
    ipdata.file:seek("set", offset)
    local tempData = ipdata.file:read(length)
    return tempData
end


function ip2int(ip)
    local o1, o2, o3, o4 = ip:match("(%d+)%.(%d+)%.(%d+)%.(%d+)")
    local num = 2 ^ 24 * o1 + 2 ^ 16 * o2 + 2 ^ 8 * o3 + o4
    return math.floor(num)
end

function ipdata.Search(low, high, k)
    local M = 0
    while low <= high
    do
        local mid = math.floor((low + high) / 2)
        local endipNum = ipdata.endArr[mid]
        if endipNum == nil then
            break
        end
        if endipNum >= k then
            M = mid
            if mid == 0 then
                break
            end
            high = mid - 1
        else
            low = mid + 1
        end
    end
    return M
end

function ipdata.FindIP(ip)
    local ips = { string.match(ip, "([^.]+).?") }
    local prefix = tonumber(ips[1])

    local low = ipdata.prefStart[prefix]
    local high = ipdata.prefEnd[prefix]
    local intIP = ip2int(ip)

    -- print("FindIP", ip, prefix, intIP)
    local cur = 0
    if low == high then
        cur = low
    else
        cur = ipdata.Search(low, high, intIP)
    end
    if cur == 100000000 then
        return "0.0.0.0"
    else
        -- return ipdata.addrArr[cur]
        return ipdata.getAddr(cur)
    end
end

return ipdata

ipdata_5.1.4.lua

ipdata = { prefStart = {}, prefEnd = {},
    endArr = {}, addrArr = {},
    file = {}, _version = "0.1.2" }

function unpackInt4byte(a, b, c, d)
    return (a % 256) + (b * 256) + (c * 65536) + (d * 16777216)
end

function fsize(file)
    local current = file:seek()
    local size = file:seek("end")
    file:seek("set", current)
    return size
end

function redaFileBytes(file, length, off)
    file:seek("set", off)
    local str = file:read(length)
    local bytes = {}
    for i = 1, #str do
        table.insert(bytes, string.byte(str, i))
    end
    return bytes
end

function ipdata.LoadFile(path)
    local sizeData = {}

    ipdata.file = io.open(path, "rb")

    for k = 1, 256 do
        local i = k * 8 + 4
        sizeData = redaFileBytes(ipdata.file, 8, i)
        ipdata.prefStart[k] = unpackInt4byte(sizeData[1], sizeData[2], sizeData[3], sizeData[4])
        ipdata.prefEnd[k] = unpackInt4byte(sizeData[5], sizeData[6], sizeData[7], sizeData[8])
    end

    sizeData = redaFileBytes(ipdata.file, 4, 0)
    local recordSize = unpackInt4byte(sizeData[1], sizeData[2], sizeData[3], sizeData[4])

    local allsizeData = redaFileBytes(ipdata.file, recordSize*9+9, 2052)
    for i = 1, recordSize do
        local x = i * 9
		local a = 1 + x
		local b = 2 + x
		local c = 3 + x
		local d = 4 + x
		local endipnum2 = unpackInt4byte(allsizeData[a], allsizeData[b], allsizeData[c], allsizeData[d])
		ipdata.endArr[i] = endipnum2

    end
end

function ipdata.getAddr(row)
    local j = 2052 + (row * 9)
    ipdata.file:seek("set", j + 4)
    local tempIndexData = ipdata.file:read(5)
    local offset = unpackInt4byte(string.byte(tempIndexData, 1), string.byte(tempIndexData, 2), string.byte(tempIndexData, 3), string.byte(tempIndexData, 4))
    local length = string.byte(tempIndexData, 5)
    ipdata.file:seek("set", offset)
    local tempData = ipdata.file:read(length)
    return tempData
end

function ip2int(ip)
    local o1, o2, o3, o4 = string.match(ip, "(%d+)%.(%d+)%.(%d+)%.(%d+)")
    local num = 2 ^ 24 * o1 + 2 ^ 16 * o2 + 2 ^ 8 * o3 + o4
    return math.floor(num)
end

function ipdata.Search(low, high, k)
    local M = 0
    while low <= high do
        local mid = math.floor((low + high) / 2)
        local endipNum = ipdata.endArr[mid]
        if endipNum == nil then
            break
        end
        if endipNum >= k then
            M = mid
            if mid == 0 then
                break
            end
            high = mid - 1
        else
            low = mid + 1
        end
    end
    return M
end

function ipdata.FindIP(ip)
    local ips = { string.match(ip, "([^.]+).?") }
    local prefix = tonumber(ips[1])

    local low = ipdata.prefStart[prefix]
    local high = ipdata.prefEnd[prefix]
    local intIP = ip2int(ip)

    local cur = 0
    if low == high then
        cur = low
    else
        cur = ipdata.Search(low, high, intIP)
    end

    if cur == 100000000 then
        return "0.0.0.0"
    else
        return ipdata.getAddr(cur)
    end
end

return ipdata

<?php
/* 
PHP Version 7+
*/
class Ip
{

    private $prefStart = array();
    private $prefEnd = array();
    private $endArr = array();
    private $data;

    static private $instance = null;

    private function __construct()
    {
        $this->loadFile();
    }

    private function loadFile()
    {
        $path = 'ipv4.dat';
        $fp = fopen($path, 'rb');
        $fsize = filesize($path);

        $this->data = fread($fp, $fsize);

        for ($k = 0; $k < 256; $k++) {
            $i = $k * 8 + 4;
            $this->prefStart[$k] = unpack("V", $this->data, $i)[1];
            $this->prefEnd[$k] = unpack("V", $this->data, $i + 4)[1];
        }
        fclose($fp);
    }

    

    private function __clone()
    {
    }

    private function __wakeup()
    {
    }

    public static function getInstance()
    {
        if (self::$instance instanceof Ip) {
            return self::$instance;
        } else {

            return self::$instance = new Ip();

        }

    }

    private function getByCur($i)
    {
        $p = 2052 + (intval($i) * 9);
        $offset = unpack("V", $this->data[4 + $p] . $this->data[5 + $p] . $this->data[6 + $p] . $this->data[7 + $p])[1];
        $length = unpack("C", $this->data[8 + $p])[1];
        return substr($this->data, $offset, $length);
    }

    public function get($ip)
    {
        $val = sprintf("%u", ip2long($ip));
        $ip_arr = explode('.', $ip);
        $pref = $ip_arr[0];
        $low = $this->prefStart[$pref];
        $high = $this->prefEnd[$pref];
        $cur = $low == $high ? $low : $this->Search($low, $high, $val);
        if ($cur == 100000000) {
            return "无信息";
        }
        return $this->getByCur($cur);
    }

    private function Search($low, $high, $k)
    {
        $M = 0;
        while ($low <= $high) {
            $mid = floor(($low + $high) / 2);
            $p = 2052 + ($mid * 9);

            $endipNum = unpack("V", $this->data, $p)[1];
            if ($endipNum >= $k) {
                $M = $mid;
                if ($mid == 0) {
                    break;
                }
                $high = $mid - 1;
            } else $low = $mid + 1;
        }
        return $M;
    }
}

require_once 'index.php'; // 引入包含MyClass定义的文件
$ipInstance = Ip::getInstance();
$startTime = microtime(true);

$ipInstance->get('1.0.3.0');

$endTime = microtime(true);
$executionTime = $endTime - $startTime;
echo "程序运行时间：" . number_format($executionTime, 4) . " 秒";

echo $ipInstance->get('1.0.3.0');

PreviousIPv4 NextIPv6

Last updated 8 months ago