純真IP地址數(shù)據(jù)庫qqwry.dat解析

liang1234_ 2020-07-17

展開全文

ip地址數(shù)據(jù)庫，在現(xiàn)在互聯(lián)網(wǎng)時(shí)代非常有用，比如大型網(wǎng)站的用戶安全保護(hù)系統(tǒng)，就常常會根據(jù)ip反查的信息，甄別賬號的一些不安全登錄行為，比如跨區(qū)域登錄問題等。ip其實(shí)關(guān)聯(lián)了一些有信息，比如區(qū)域，所在運(yùn)營商，一些收錄全的，甚至包括具體經(jīng)緯度，像百度的IP定位api就比較全。下面來介紹一下“ 純真IP地址數(shù)據(jù)庫qqwry”的格式以及解析

以下是“ 純真IP地址數(shù)據(jù)庫qqwry”官網(wǎng)對其的介紹。

純真版IP地址數(shù)據(jù)庫是當(dāng)前網(wǎng)絡(luò)上最權(quán)威、地址最精確、IP記錄以及網(wǎng)吧數(shù)據(jù)最多的IP地址數(shù)據(jù)庫。收集了包括中國電信、中國移動(dòng)、中國聯(lián)通、鐵通、長城寬帶等各 ISP 的最新準(zhǔn)確 IP 地址數(shù)據(jù)。通過大家的共同努力打造一個(gè)沒有未知數(shù)據(jù)，沒有錯(cuò)誤數(shù)據(jù)的QQ IP。IP數(shù)據(jù)庫每5天更新一次，請大家定期更新最新的IP數(shù)據(jù)庫！

格式

+———-+
| 文件頭 | (8字節(jié))
+———-+
| 記錄區(qū) | (不定長)
+———-+
| 索引區(qū) | (大小由文件頭決定)
+———-+

使用java語言解析的兩種思路：

使用內(nèi)存映射文件方式讀取，使用java的MappedByteBuffer 將原數(shù)據(jù)文件映射到MappedByteBuffer對象中，然后通過MappedByteBuffer 提供的字節(jié)讀取方式實(shí)現(xiàn)ip的查找。搜索是在索引區(qū)使用二分法
使用byte數(shù)組讀取，及將二進(jìn)制的數(shù)據(jù)庫信息全都按順序讀入到一個(gè)數(shù)組中，由于數(shù)據(jù)是有格式的，我們便可計(jì)算根據(jù)索引區(qū)和記錄區(qū)在數(shù)組中的位置，當(dāng)查詢ip時(shí)，從數(shù)組中的索引區(qū)開始通過二分查找方式找到IP地址對應(yīng)的國家和區(qū)域的位置，然后從數(shù)組中取出地區(qū)信息。

熱升級思路：

使用一個(gè)可調(diào)度的單線程的線程池，線程定時(shí)檢測qqwry.dat文件是否修改，若修改則重新將數(shù)據(jù)重新載入，載入過程可使用可重入鎖ReentrantLock來鎖住資源，避免在更新的過程中臟查詢

兩種解析方式的實(shí)現(xiàn)源碼如下：
方式一（MappedByteBuffer ）：

package com.difeng.qqwry1;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
 * @Description:ip定位查找工具(使用內(nèi)存映射文件方式讀取，線程安全)
 * @author:difeng
 * @date:2016年12月11日
 */
public class IPLocation {

    private static final int IP_RECORD_LENGTH = 7;

    private static final byte REDIRECT_MODE_1 = 0x01;

    private static final byte REDIRECT_MODE_2 = 0x02;

    private MappedByteBuffer mbbFile;

    private static Long lastModifyTime = 0L;

    public static boolean enableFileWatch = false;

    private static ReentrantLock lock = new ReentrantLock();

    private File qqwryFile;

    private long firstIndexOffset;

    private long lastIndexOffset;

    private long totalIndexCount;

    public IPLocation(String filePath) throws Exception {
        this.qqwryFile = new File(filePath);
        load();
        if (enableFileWatch) {
            watch();
        }
    }

    private void watch(){
        Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                long time = qqwryFile.lastModified();
                if (time > lastModifyTime) {
                    lastModifyTime = time;
                    try {
                        load();
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        }, 1000L, 30000L, TimeUnit.MILLISECONDS);
    }

    public long read4ByteAsLong(long pos) {
        mbbFile.position((int)pos);
        return 0xFFFFFFFFL & mbbFile.getInt();
    }

    public long read3ByteAsLong(long pos){
        mbbFile.position((int)pos);
        return 0xFFFFFFL & mbbFile.getInt();
    }


    @SuppressWarnings("resource")
    private void load() throws Exception {
        lastModifyTime = qqwryFile.lastModified();
        lock.lock();
        try {
            mbbFile =  new RandomAccessFile(qqwryFile, "r")
                    .getChannel()
                    .map(FileChannel.MapMode.READ_ONLY, 0, qqwryFile.length());
            mbbFile.order(ByteOrder.LITTLE_ENDIAN);
            firstIndexOffset = read4ByteAsLong(0);
            lastIndexOffset = read4ByteAsLong(4);
            totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;
        } finally {
            lock.unlock();
        }
    }
    
    /**
     * @Description:將“.”號分隔的字符串轉(zhuǎn)換為long類型的數(shù)字，字節(jié)序例如:
     *   ip:182.92.240.48  16進(jìn)制表示(B6.5C.F0.30)
     *   轉(zhuǎn)換后ip的16進(jìn)制表示:0xB65CF030
     * @param ipStr
     * @return:long
     */
    private static long inet_pton(String ipStr) {
        if(ipStr == null){
            throw new NullPointerException("ip不能為空");
        }
        String [] arr = ipStr.split("\\.");
        long ip = (Long.parseLong(arr[0])  & 0xFFL) << 24 & 0xFF000000L;
        ip |=  (Long.parseLong(arr[1])  & 0xFFL) << 16 & 0xFF0000L;
        ip |=  (Long.parseLong(arr[2])  & 0xFFL) << 8 & 0xFF00L;
        ip |=  (Long.parseLong(arr[3])  & 0xFFL);
        return ip;
    }

    private long search(long ip) {
        long low = 0;
        long high = totalIndexCount;
        long mid = 0;
        while(low <= high) {
            mid = (low + high) >>> 1 ;
            long indexIP = read4ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH);
            long nextIndexIP =  read4ByteAsLong(firstIndexOffset + mid * IP_RECORD_LENGTH);
            if(indexIP <= ip && ip < nextIndexIP) {
                return read3ByteAsLong(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4);
            } else {
                if(ip > indexIP) {
                    low = mid + 1;
                } else if(ip < indexIP) {
                    high = mid - 1;
                }
            }
        }
        return -1;
    }

    private Location readIPLocation(long offset) {
        try {
            mbbFile.position((int)offset + 4);
            Location loc = new Location();
            byte redirectMode = mbbFile.get();
            if (redirectMode == REDIRECT_MODE_1) {
                long countryOffset = read3ByteAsLong((int)offset + 5);
                mbbFile.position((int)countryOffset);
                redirectMode = mbbFile.get();
                if (redirectMode == REDIRECT_MODE_2) {
                    loc.country = readString(read3ByteAsLong(countryOffset + 1));
                    mbbFile.position((int)countryOffset + 4);
                } else {
                    loc.country = readString(countryOffset);
                }
                loc.area = readArea(mbbFile.position());
            } else if (redirectMode == REDIRECT_MODE_2) {
                loc.country = readString(read3ByteAsLong((int)offset + 5));
                loc.area = readArea((int)offset + 8);
            } else {
                loc.country = readString(mbbFile.position() - 1);
                loc.area = readArea(mbbFile.position());
            }
            return loc;
        } catch (Exception e) {
            return null;
        }
    }

    private String readArea(int offset) {
        mbbFile.position(offset);
        byte redirectMode = mbbFile.get();
        if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {
            long areaOffset = read3ByteAsLong((int)offset + 1);
            if (areaOffset == 0){
                return "";
            } else {
                return readString(areaOffset);
            }
        } else {
            return readString(offset);
        }
    }

    private String readString(long offset) {
        try {
            mbbFile.position((int)offset);
            byte[] buf = new byte[128];
            int i;
            for (i = 0, buf[i] = mbbFile.get(); buf[i] != 0; buf[++i] = mbbFile.get());
            
            if (i != 0){
                return new String(buf, 0, i, "GBK");
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        return "";
    }

    public  Location fetchIPLocation(String ip) {
        lock.lock();
        try {
            long offset = search(inet_pton(ip));
            if(offset != -1){
                return readIPLocation(offset);
            }
        } finally {
            lock.unlock();
        }
        return null;
    }
}

方式二（數(shù)組方式）：

package com.difeng.qqwry2;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
/**
 * @Description:ip定位(使用byte數(shù)據(jù)方式讀取)
 * @author:difeng
 * @date:2016年12月13日
 */
public class IPLocation {
    
    private  byte[] data;
    
    private  long firstIndexOffset;
    
    private  long lastIndexOffset;
    
    private  long totalIndexCount;
    
    private static final byte REDIRECT_MODE_1 = 0x01;
    
    private static final byte REDIRECT_MODE_2 = 0x02;
    
    static   final long IP_RECORD_LENGTH = 7;
    
    private static ReentrantLock lock = new ReentrantLock();
    
    private static Long lastModifyTime = 0L;

    public static boolean enableFileWatch = false;
    
    private File qqwryFile;
    
    public IPLocation(String  filePath) throws Exception {
        this.qqwryFile = new File(filePath);
        load();
        if(enableFileWatch){
            watch();
        }
    }
    
    private void watch() {
        Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                long time = qqwryFile.lastModified();
                if (time > lastModifyTime) {
                    lastModifyTime = time;
                    try {
                        load();
                        System.out.println("reload");
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            }
        }, 1000L, 5000L, TimeUnit.MILLISECONDS);
    }
    
    private void load() throws Exception {
        lastModifyTime = qqwryFile.lastModified();
        ByteArrayOutputStream out = null;
        FileInputStream in = null;
        lock.lock();
        try {
            out = new ByteArrayOutputStream();
            byte[] b = new byte[1024];
            in = new FileInputStream(qqwryFile);
            while(in.read(b) != -1){
                out.write(b);
            }
            data = out.toByteArray();
            firstIndexOffset = read4ByteAsLong(0);
            lastIndexOffset = read4ByteAsLong(4);
            totalIndexCount = (lastIndexOffset - firstIndexOffset) / IP_RECORD_LENGTH + 1;
            in.close();
            out.close();
        } finally {
            try {
                if(out != null) {
                    out.close();
                }
                if(in != null) {
                    in.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
            lock.unlock();
        }
    }
    
    private long read4ByteAsLong(final int  offset) {
        long val = data[offset] & 0xFF;
        val |= (data[offset + 1] << 8L) & 0xFF00L;
        val |= (data[offset + 2] << 16L) & 0xFF0000L;
        val |= (data[offset + 3] << 24L) & 0xFF000000L;
        return val;
    }

    private long read3ByteAsLong(final int offset) {
        long val = data[offset] & 0xFF;
        val |= (data[offset + 1] << 8) & 0xFF00;
        val |= (data[offset + 2] << 16) & 0xFF0000;
        return val;
    }
    
    private long search(long ip) {
        long low = 0;
        long high = totalIndexCount;
        long mid = 0;
        while(low <= high){
            mid = (low + high) >>> 1 ;
            long indexIP = read4ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH));
            long indexIPNext = read4ByteAsLong((int)(firstIndexOffset + mid * IP_RECORD_LENGTH));
            if(indexIP <= ip && ip < indexIPNext) {
                return read3ByteAsLong((int)(firstIndexOffset + (mid - 1) * IP_RECORD_LENGTH + 4));
            } else {
                if(ip > indexIP) {
                    low = mid + 1;
                } else if (ip < indexIP) {
                    high = mid - 1;
                }
            }
        }
        return -1;
    }
    
    public Location fetchIPLocation(String ip) {
        long numericIp = inet_pton(ip);
        lock.lock();
        long offset = search(numericIp);
        try{
            if(offset != -1) {
                return readIPLocation((int)offset);
            }
        } finally {
            lock.unlock();
        }
        return null;
    }
    
    private Location readIPLocation(final int offset) {
        final Location loc = new Location();
        try {
            byte redirectMode = data[offset + 4];
            if (redirectMode == REDIRECT_MODE_1) {
                long countryOffset = read3ByteAsLong((int)offset + 5);
                redirectMode = data[(int)countryOffset];
                if (redirectMode == REDIRECT_MODE_2) {
                    final QQwryString country = readString((int)read3ByteAsLong((int)countryOffset + 1));
                    loc.country = country.string;
                    countryOffset = countryOffset + 4;
                } else {
                    final QQwryString country = readString((int)countryOffset);
                    loc.country = country.string;
                    countryOffset += country.byteCountWithEnd;
                }
                loc.area = readArea((int)countryOffset);
            } else if (redirectMode == REDIRECT_MODE_2) {
                loc.country = readString((int)read3ByteAsLong((int)offset + 5)).string;
                loc.area = readArea((int)offset + 8);
            } else {
                final QQwryString country = readString((int)offset + 4);
                loc.country = country.string;
                loc.area = readArea((int)offset + 4 + country.byteCountWithEnd);
            }
            return loc;
        } catch (Exception e) {
            return null;
        }
    }

    private String readArea(final int offset) {
        byte redirectMode = data[offset];
        if (redirectMode == REDIRECT_MODE_1 || redirectMode == REDIRECT_MODE_2) {
            long areaOffset = read3ByteAsLong((int)offset + 1);
            if (areaOffset == 0) {
                return "";
            } else {
                return readString((int)areaOffset).string;
            }
        } else {
            return readString(offset).string;
        }
    }
    
    private QQwryString readString(int offset) {
        int pos = offset;
        final byte[] b = new byte[128];
        int i;
        for (i = 0, b[i] = data[pos++]; b[i] != 0; b[++i] = data[pos++]);
        try{
               return new QQwryString(new String(b,0,i,"GBK"),i + 1);
        } catch(UnsupportedEncodingException e) {
            return new QQwryString("",0);
        }
    }
    
     /**
     * @Description:“.”號分隔的字符串轉(zhuǎn)換為long類型的數(shù)字
     * @param ipStr 
     * @return:long
     */
    private static long inet_pton(String ipStr) {
        if(ipStr == null){
            throw new NullPointerException("ip不能為空");
        }
        String [] arr = ipStr.split("\\.");
        long ip = (Long.parseLong(arr[0])  & 0xFFL) << 24 & 0xFF000000L;
        ip |=  (Long.parseLong(arr[1])  & 0xFFL) << 16 & 0xFF0000L;
        ip |=  (Long.parseLong(arr[2])  & 0xFFL) << 8 & 0xFF00L;
        ip |=  (Long.parseLong(arr[3])  & 0xFFL);
        return ip;
    }
    
    private class QQwryString{
        
        public final String string;
        
        public final int byteCountWithEnd;
        
        public QQwryString(final String string,final int byteCountWithEnd) {
            this.string = string;
            this.byteCountWithEnd = byteCountWithEnd;
        }
        
        @Override
        public String toString() {
            return string;
        }
        
    }
}

以上為主要代碼，獲取全部代碼請點(diǎn)擊全部代碼

使用

final IPLocation ipLocation = new IPLocation(filePath);
Location loc = ipl.fetchIPLocation("182.92.240.50");
System.out.printf("%s %s",loc.country,loc.area);

格式改進(jìn)

由于原格式中讀取地區(qū)記錄時(shí)采用重定向，有些繁瑣。去掉之后格式更簡單，國家和地區(qū)單獨(dú)存放,索引里分別記錄的國家和地區(qū)的地址。
新格式如下：

final IPFileConvertor convertor = new 
IPFileConvertor(IPFileConvertor.class.getResource("/qqwry.dat").getPath(),"./qqwry.dat");
convertor.convert();
新格式使用方法和之前的一致，使用com.difeng.convert包下的解析類IPLocation解析即可。

本站是提供個(gè)人知識管理的網(wǎng)絡(luò)存儲空間，所有內(nèi)容均由用戶發(fā)布，不代表本站觀點(diǎn)。請注意甄別內(nèi)容中的聯(lián)系方式、誘導(dǎo)購買等信息，謹(jǐn)防詐騙。如發(fā)現(xiàn)有害或侵權(quán)內(nèi)容，請點(diǎn)擊一鍵舉報(bào)。

轉(zhuǎn)藏 分享

QQ空間 QQ好友新浪微博微信

獻(xiàn)花（0） +1

來自： liang1234_ > 《ip數(shù)據(jù)庫》

舉報(bào)/認(rèn)領(lǐng)