扫描FTP，保存文件
1、需求：某公司ftp服务器中一个文件夹中有30个文件（文件名字是不同的），每五分钟产生一个新的文件，同时删除这三十个文件中最早产生的文件，该文件夹中始终保持30个文件。
现在需要采集一周的数据做研究。
解决思路，用java扫描该文件夹，把所有产生的新文件保存到本地一个目录下，文件名持久化一个文本中，防止FTP出问题。一周后可以得到这一周的数据。
下面是代码：
package cim; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.net.SocketException; import java.util.List; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPReply; import org.apache.log4j.Logger; public class FtpUtil { static Logger logger = Logger.getLogger(FtpUtil.class); /** * 获取FTPClient对象 * * @param ftpHost * FTP主机服务器 * @param ftpPassword * FTP 登录密码 * @param ftpUserName * FTP登录用户名 * @param ftpPort * FTP端口 默认为21 * @return */ static FTPClient ftpClient = null; public static FTPClient getFTPClient(String ftpHost, String ftpUserName, String ftpPassword, int ftpPort) { try { ftpClient = new FTPClient(); ftpClient.setControlEncoding("GBK"); ftpClient.connect(ftpHost, ftpPort);// 连接FTP服务器 ftpClient.login(ftpUserName, ftpPassword);// 登陆FTP服务器 if (!FTPReply.isPositiveCompletion(ftpClient.getReplyCode())) { logger.info("未连接到FTP，用户名或密码错误。"); close(); } else { // logger.info("FTP连接成功。"); } } catch (SocketException e) { e.printStackTrace(); logger.info("FTP连接错误，请正确配置IP地址，账号和密码。"); } catch (IOException e) { e.printStackTrace(); logger.info("FTP的端口错误,请正确配置。"); } return ftpClient; } /* * 从FTP服务器下载文件 * * @param ftpHost FTP IP地址 * * @param ftpUserName FTP 用户名 * * @param ftpPassword FTP用户名密码 * * @param ftpPort FTP端口 * * @param ftpPath FTP服务器中文件所在路径 格式： ftptest/aa * * @param localPath 下载到本地的位置 格式：H:/download * * @param txtAddress 本地文件名 * * @param time 扫描时间间隔 */ public static void downloadFtpFile(String ftpHost, String ftpUserName, String ftpPassword, int ftpPort, String ftpPath, String localPath, String txtAddress, int time) { while(ftpPort!=21){ logger.info("检测到用户输入的端口号为:"+ftpPort+"，现已修改为默认值21."); ftpPort = 21; } int fs0 = 0;// 定义初始化文件的个数为零 for (int a = 0; a < 500000; a++) { FTPClient ftpClient = null; try { ftpClient = getFTPClient(ftpHost, ftpUserName, ftpPassword,ftpPort); ftpClient.setFileType(FTPClient.BINARY_FILE_TYPE);// 设置文件类型为二进制否则可能导致乱码文件无法打开 ftpClient.enterLocalPassiveMode();// 设置被动模式 boolean dir = ftpClient.changeWorkingDirectory(ftpPath); // System.out.println("进入指定FTP文件夹==>"+dir); txtUtil txt = new txtUtil(); List<String> listStrings = null; try { listStrings = txt.readTxtFile(txtAddress); } catch (Exception e1) { System.out.println("警告：发现问题读取文本的时候，程序错误"); e1.printStackTrace(); } System.out.println("第" + (a + 1) + "次循环的listStrings长度为："+ listStrings.size()); if (dir) { File file2 = null; FTPFile[] fs = ftpClient.listFiles(); if (a == 0) { for (int i = 0; i < fs.length; i++) { boolean bo = listStrings.contains(fs[i].getName()); if (!bo) { if(fs[i].getName().endsWith("xml")){ File file = new File(localPath); if (!file.exists() && !file.isDirectory()) { file.mkdir(); logger.info("本地cim_download文件夹不存在，创建成功"); } File localFile = new File(localPath + File.separatorChar + fs[i].getName()); OutputStream os = new FileOutputStream(localFile); ftpClient.retrieveFile(fs[i].getName(), os); os.close(); logger.info("首次启动文件复制成功：" + fs[i].getName()+ " 大小为：" + localFile.length() + "字节"); txt.recordName(txtAddress, fs[i].getName()); listStrings.add(fs[i].getName()); } } else { logger.info("首次启动发现：<" + fs[i].getName()+ ">，本地已存在，不再复制"); } } }else{ for (int i = 0; i < fs.length; i++) { boolean bo = listStrings.contains(fs[i].getName()); if (!bo) { if(fs[i].getName().endsWith("xml")){ System.out.println("发现新文件：" + fs[i].getName()); try { Thread.sleep(500); } catch (InterruptedException e) { e.printStackTrace(); } file2 = new File(localPath); if (!file2.exists() && !file2.isDirectory()) { file2.mkdir(); logger.info("本地cim_download文件夹可能被误删，现已创建成功"); } File localFile = new File(localPath+ File.separatorChar + fs[i].getName()); OutputStream os = new FileOutputStream(localFile); ftpClient.retrieveFile(fs[i].getName(), os); os.close(); logger.info("新文件：" + fs[i].getName() + " 复制成功"); // listStrings.add(fs[i].getName()); txt.recordName(txtAddress, fs[i].getName()); } } } } fs0 = fs.length; try { System.out.println("查询第" + (a + 1) + "次时，有" + fs0+ "个文件，" + "下次扫描时间为" + time + "毫秒后。"); Thread.sleep(time); } catch (InterruptedException e) { e.printStackTrace(); } } } catch(NullPointerException e){ logger.error("NullPointerException,创建连接为空。请检查FTP服务器"+"!!!"+e.getMessage()); }catch (FileNotFoundException e) { logger.error("FileNotFoundException"+e.getMessage()); e.printStackTrace(); } catch (SocketException e) { logger.error("连接FTP失败."); e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); logger.error("文件读取错误。"); e.printStackTrace(); } catch (Exception e) { logger.error("未知异常！！！"); } finally { try { close(); } catch (Exception e) { logger.error("关闭FTP错误！"+e.getMessage()); e.printStackTrace(); } } } } /** * 转码[GBK -> ISO-8859-1] 不同的平台需要不同的转码 * * @param obj * @return */ private static String gbkToIso8859(Object obj) { try { if (obj == null) return ""; else return new String(obj.toString().getBytes("GBK"), "iso-8859-1"); } catch (Exception e) { return ""; } } /** * 转码[ISO-8859-1 -> GBK] 不同的平台需要不同的转码 * * @param obj * @return */ private static String iso8859ToGbk(Object obj) { try { if (obj == null) return ""; else { String str = new String(obj.toString().getBytes("iso-8859-1"), "GBK"); return str; } } catch (Exception e) { return ""; } } /** * 关闭当前连接 */ public static void close() { try { ftpClient.logout(); ftpClient.disconnect(); } catch (IOException e) { logger.error("ftp ftpserver close error : " + e.getMessage()); } } }
package cim; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; public class txtUtil { public void recordName(String txtAddress,String name) throws IOException{ File file = new File(txtAddress); if(!file.exists()){ file.createNewFile(); } FileWriter fw = new FileWriter(file, true); PrintWriter pw = new PrintWriter(fw); pw.print(name); pw.print(","); pw.flush(); try { fw.flush(); pw.close(); fw.close(); } catch (IOException e) { e.printStackTrace(); } } public List<String> readTxtFile(String txtAddress)throws Exception{ List<String> list = new ArrayList<>(); try { String encoding="UTF-8"; File file=new File(txtAddress); if(!file.exists()){ file.createNewFile(); } if(file.isFile() && file.exists()){ //判断文件是否存在 InputStreamReader read = new InputStreamReader(new FileInputStream(file),encoding);//考虑到编码格式 BufferedReader bufferedReader = new BufferedReader(read); String lineTxt = null; while((lineTxt = bufferedReader.readLine()) != null){ String[] sourceStrArray =lineTxt.split(","); for(int i = 0 ;i<sourceStrArray.length;i++){ list.add(sourceStrArray[i]); } } read.close(); }else{ System.out.println("读取文件名集合出错"); } }catch (Exception e) { System.out.println("读取文件内容出错"); e.printStackTrace(); } return list; } }
1 package cim; 2 3 import java.io.File; 4 import java.util.HashMap; 5 import java.util.Map; 6 7 import javax.xml.parsers.DocumentBuilder; 8 import javax.xml.parsers.DocumentBuilderFactory; 9 10 import org.w3c.dom.Document; 11 import org.w3c.dom.Element; 12 import org.w3c.dom.Node; 13 import org.w3c.dom.NodeList; 14 15 public class ftpTest { 16 17 public static void main(String[] args) { 18 19 ftpTest test = new ftpTest(); 20 Map<String, Object> map = test.XMLReader(); 21 String ftpHost = (String) map.get("ftpHost"); 22 String ftpUserName = (String) map.get("ftpUserName"); 23 String ftpPassword = (String) map.get("ftpPassword"); 24 int ftpPort = Integer.parseInt((String) map.get("ftpPort")) ; 25 String ftpPath = (String) map.get("ftpPath"); 26 String localPath = (String) map.get("localPath"); 27 String txtAddress = (String) map.get("txtAddress"); 28 int time = Integer.parseInt((String)map.get("time")); 29 30 FtpUtil.downloadFtpFile(ftpHost, ftpUserName, ftpPassword, ftpPort, ftpPath, localPath, txtAddress,time); 31 } 32 33 public Map<String,Object> XMLReader(){ 34 Element element = null; 35 File f = new File("test.xml"); 36 DocumentBuilder db = null; 37 DocumentBuilderFactory dbf = null; 38 Map<String, Object> map = new HashMap<>(); 39 try { 40 dbf = DocumentBuilderFactory.newInstance(); 41 db = dbf.newDocumentBuilder(); 42 Document dt = db.parse(f); 43 element = dt.getDocumentElement(); 44 NodeList childNodes = element.getChildNodes(); 45 for (int i = 0; i < childNodes.getLength(); i++) { 46 Node node1 = childNodes.item(i); 47 if ("Account".equals(node1.getNodeName())) { 48 NodeList nodeDetail = node1.getChildNodes(); 49 for (int j = 0; j < nodeDetail.getLength(); j++) { 50 Node detail = nodeDetail.item(j); 51 if ("ftpHost".equals(detail.getNodeName())) 52 map.put("ftpHost", detail.getTextContent()); 53 else if ("ftpUserName".equals(detail.getNodeName())) 54 map.put("ftpUserName", detail.getTextContent()); 55 else if ("ftpPassword".equals(detail.getNodeName())) 56 map.put("ftpPassword", detail.getTextContent()); 57 else if ("ftpPort".equals(detail.getNodeName())) 58 map.put("ftpPort", detail.getTextContent()); 59 else if ("ftpPath".equals(detail.getNodeName())) 60 map.put("ftpPath", detail.getTextContent()); 61 else if ("localPath".equals(detail.getNodeName())) 62 map.put("localPath", detail.getTextContent()); 63 else if ("txtAddress".equals(detail.getNodeName())) 64 map.put("txtAddress", detail.getTextContent()); 65 else if ("time".equals(detail.getNodeName())) 66 map.put("time", detail.getTextContent()); 67 } 68 } 69 } 70 } catch (Exception e) { 71 e.printStackTrace(); 72 } 73 return map; 74 } 75 }
log4j.rootLogger=info,A1,A2 # \u8F93\u51FA\u5230\u63A7\u5236\u53F0 log4j.appender.A1=org.apache.log4j.ConsoleAppender log4j.appender.A1.layout=org.apache.log4j.PatternLayout log4j.appender.A1.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [\u4fe1\u606f] %m%n # \u8F93\u51FA\u5230\u6587\u4EF6\u5F53\u4E2D log4j.appender.A2=org.apache.log4j.FileAppender log4j.appender.A2.File=cim_logging.log log4j.appender.A2.Append=true log4j.appender.A2.layout=org.apache.log4j.PatternLayout log4j.appender.A2.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [\u4fe1\u606f] %m%n
<?xml version="1.0" encoding="UTF-8"?> <Accounts> <Account type="type1"> <!-- FTP端口号 --> <ftpHost>127.0.0.1</ftpHost> <!-- FTP账号 --> <ftpUserName>1</ftpUserName> <!-- FTP密码 --> <ftpPassword>1</ftpPassword> <!-- FTP端口号 --> <ftpPort>22</ftpPort> <!-- FTP路径 --> <ftpPath>users/ems/open2000e/data/xmldat/nari/all/</ftpPath> <!-- 本地保存文件路径 --> <localPath>cim_download</localPath> <!-- 本地保存文件名集合路径 --> <txtAddress>cim_name.txt</txtAddress> <!-- 设置扫描文件的时间间隔,单位：毫秒 --> <time>5000</time> </Account> </Accounts>
是用的main方法启动。
Fat Jar打包插件方法 http://jingyan.baidu.com/article/da1091fbd7dae1027849d63b.html
下面是运行jar包
