读取nutch爬取内容方法

读取nutch内容有如下两种方法：

1 通过Nutch api SegmentReader读取。

         public Content readSegment(String segPath,String url){  

              
            Text key= new Text(url);  
            Path path= new Path(segPath);  
            Content content = null;  
      
            ArrayList<Writable> parsedLst = null;  
            Map<String,List<Writable>> results=new HashMap<String, List<Writable>>();  
            SegmentReader reader= new SegmentReader(configuration,true,true,true,true,true,true);  
            try {  
                reader.get(path, key, new StringWriter(), results);  
                parsedLst=(ArrayList<Writable>) results.get("co");  
                Iterator<Writable> parseIter=parsedLst.iterator();  
                while(parseIter.hasNext()){  
                    content=(Content) parseIter.next();  
                }  
            } catch (Exception e) {  
                e.printStackTrace();  
            }  
      
            return content;  

        }  

 

  2 通过SequenceFile 读取

        public static void main(String[] args) throws IOException { 

            args=new String[]{"D:\\nutchv\\nutch12\\apache-nutch-1.2\\data\\csdn2\\segments\\20140904104348"};   

            Configuration conf = NutchConfiguration.create();       
            Options opts = new Options();       
            GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);       
            String[] remainingArgs = parser.getRemainingArgs();     
            FileSystem fs = FileSystem.get(conf);
            String segment = remainingArgs[0];
            Path file = new Path(segment, Content.DIR_NAME + "/part-00000/data");
            SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
            Text key = new Text();
            Content content = new Content();
            // Loop through sequence files
            while (reader.next(key, content)) {
                try {
                    System.out.write(content.getContent(), 0,
                            content.getContent().length);
                } catch (Exception e) {
                }
            }

 

        }