大数据去重复

package com.zsins.risk.util;

import java.io.Serializable;

public class LongMap implements BitMap, Serializable  {
    
    private static final long serialVersionUID = 1L;



    private final long[] longs;



    /**

     * 构造

     */

    public LongMap() {

        longs = new long[93750000];

    }



    /**

     * 构造

     * 

     * @param size 容量

     */

    public LongMap(int size) {

        longs = new long[size];

    }



    @Override

    public void add(long i) {

        int r = (int) (i / BitMap.MACHINE128);

        long c = i % BitMap.MACHINE128;

        longs[r] = longs[r] | (1L << c);

    }



    @Override

    public boolean contains(long i) {

        int r = (int) (i / BitMap.MACHINE128);

        long c = i % BitMap.MACHINE128;

        return ((longs[r] >>> c) & 1) == 1;

    }



    @Override

    public void remove(long i) {

        int r = (int) (i / BitMap.MACHINE128);

        long c = i % BitMap.MACHINE128;

        longs[r] &= ~(1L << c);

    }



}

package com.zsins.risk.util;



/**

 * BitMap接口，用于将某个int或long值映射到一个数组中，从而判定某个值是否存在

 * 

 * @author looly

 *

 */

public interface BitMap{



    int MACHINE32 = 32;

    int MACHINE64 = 64;
    
    int MACHINE128 = 128;



    /**

     * 加入值

     * 

     * @param i 值

     */

    void add(long i);



    /**

     * 检查是否包含值

     * 

     * @param i 值

     * @return 是否包含

     */

    boolean contains(long i);



    /**

     * 移除值

     * 

     * @param i 值

     */

    void remove(long i);

}

public static void main(String[] args) {
		try {
			//第一种：借助BitMap,LongMap；位图去重
			//存在局限性：这个方法只适合long类型；
			System.out.println("----------1:"+System.currentTimeMillis());
			BitMap bitMap = new LongMap();
			for(long i = 0 ;i<1000000L;i++){
				bitMap.add(i);
			}
			System.out.println("----------2:"+System.currentTimeMillis());
			for(long j = 0 ;j<1000000L;j++){
				bitMap.contains(j);
				if(j < 10){
					System.out.println("----------3:"+bitMap.contains(j));
				}
			}
			
			//第二种：Set<Strig>方法去重；效率同样很快
			/*
			final Set<String> des = new HashSet<String>();
			final List<String> sourse = new ArrayList<String>();
			for(int i=0;i<1000000;i++){
				des.add("10000000000"+i);
				if(i<10000){
					sourse.add("10000000000"+i);
				}else{
					sourse.add("20000000000"+i);
				}
			}
			
			if (sourse == null || sourse.size() <= 0) {
	            return;
	        }
			System.out.println("------111:"+sourse.size());
			System.out.println("----------1:"+System.currentTimeMillis());
	        Iterator<String> listStr = sourse.iterator();
	        while (listStr.hasNext()) {
	            String item = listStr.next();
	            if (des.contains(item)) {
	                listStr.remove();
	            }
	        }
	        System.out.println("------222:"+sourse.size());
			System.out.println("----------3:"+System.currentTimeMillis());*/
		} catch (Exception e) {
			System.out.println(e.getStackTrace());
			// TODO: handle exception
			System.out.println(e);
		}
		
	}