遍历一个文件夹下的所有文件，并对其中内容进行去重。

package test;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;

public class xiong {
	public static void main(String[] args) {
		File root = new File("F:\\doc\\Android");
		try {
			List<String> files = showAllFiles(root);
			for(String s:files){
//				System.out.println(s);
				readTxtFile(s);
			}
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		// String filePath = "F:\\doc\\Android\\腾讯.txt";
		// readTxtFile(filePath);
	}

	final static List<String> showAllFiles(File dir) throws Exception {
		File[] fs = dir.listFiles();
		List<String> files = new ArrayList<String>();
		for (int i = 0; i < fs.length; i++) {
//			System.out.println(fs[i].getAbsolutePath());
			files.add(fs[i].getAbsolutePath());
			if (fs[i].isDirectory()) {
				try {
					showAllFiles(fs[i]);
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		}
		return files;
	}

	public static void readTxtFile(String filePath) {
		try {
			String encoding = "UTF-8";
			File file = new File(filePath);
			Set<String> set = new HashSet<String>();
			FileWriter writer = new FileWriter("F:\\doc\\xiao.txt", true);
			if (file.isFile() && file.exists()) {
				InputStreamReader reader = new InputStreamReader(
						new FileInputStream(file), encoding);
				BufferedReader bufferedReader = new BufferedReader(reader);
				String lineTxt = null;
				String filename = file.getName();
				String tempfile = filename.substring(0, filename.lastIndexOf("."));
				System.out.print(tempfile + ":\t");
				writer.write(tempfile + ":\t");
				while ((lineTxt = bufferedReader.readLine()) != null) {
					// System.out.print(lineTxt);
					String[] sArray = lineTxt.split(",");
					for (String s : sArray) {
						set.add(s);
					}
				}
				// System.out.println("");
				reader.close();
			} else {
				System.out.println("file not found");
			}
			for(Iterator<String> it = set.iterator();it.hasNext();) {
				String temp = it.next();
				System.out.print(temp.toString() + ",");
				writer.write(temp.toString() + ",");
			}
			writer.write("\r\n");
			writer.flush();
			System.out.println();
		} catch (Exception e) {
			System.out.println("file error");
			e.printStackTrace();
		}
	}
}

 

 

写法2

 

package test;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Scanner;
import java.util.Set;

public class xiong {
	public static void main(String[] args) {
		Scanner sc=new Scanner(System.in);
		String str=null;
		String targetPath = null; 
		System.out.print("请输入文件夹名:"); //   F:\\doc\\Android
		str=sc.nextLine();
		System.out.print("请输入输出文件名:"); // F:\\doc\\xiao.txt
		targetPath=sc.nextLine();
		File root = new File(str);
//		File root = new File("F:\\doc\\Android");
		try {
			List<String> files = showAllFiles(root);
			for(String s:files){
				readTxtFile(s, targetPath);
			}
			System.out.println("task finished");
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	final static List<String> showAllFiles(File dir) throws Exception {
		File[] fs = dir.listFiles();
		List<String> files = new ArrayList<String>();
		for (int i = 0; i < fs.length; i++) {
			files.add(fs[i].getAbsolutePath());
			if (fs[i].isDirectory()) {
				try {
					showAllFiles(fs[i]);
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		}
		return files;
	}

	public static void readTxtFile(String filePath, String tagertPath) {
		try {
			String encoding = "UTF-8";
			File file = new File(filePath);
			Set<String> set = new HashSet<String>();
			FileWriter writer = new FileWriter(tagertPath, true);
			if (file.isFile() && file.exists()) {
				InputStreamReader reader = new InputStreamReader(
						new FileInputStream(file), encoding);
				BufferedReader bufferedReader = new BufferedReader(reader);
				String lineTxt = null;
				String filename = file.getName();
				String tempfile = filename.substring(0, filename.lastIndexOf("."));
				writer.write(tempfile + ":\t");
				while ((lineTxt = bufferedReader.readLine()) != null) {
					String[] sArray = lineTxt.split(",");
					for (String s : sArray) {
						set.add(s);
					}
				}
				reader.close();
			} else {
				System.out.println("file not found");
			}
			for(Iterator<String> it = set.iterator();it.hasNext();) {
				String temp = it.next();
				writer.write(temp.toString() + ",");
			}
			writer.write("\r\n");
			writer.flush();
//			System.out.println();
		} catch (Exception e) {
			System.out.println("file error");
			e.printStackTrace();
		}
	}
}