Hadoop的FlieSystem类的使用

1.使用FileSystem类需要导入jar包

　　解压hadoop-2.7.7.tar.gz

　　复制如下三个jar包和lib下所有jar包到项目文件下的lib文件

　　

2.查看文件信息

@Test
public void readListFiles() throws Exception {
    // 1 创建配置信息对象
    Configuration configuration = new Configuration();
        
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),configuration, "root");
        
    // 思考：为什么返回迭代器，而不是List之类的容器
    RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

    while (listFiles.hasNext()) {
        LocatedFileStatus fileStatus = listFiles.next();            
        System.out.println(fileStatus.getPath().getName()); //路径
        System.out.println(fileStatus.getBlockSize());  //块
        System.out.println(fileStatus.getPermission()); //权限
        System.out.println(fileStatus.getLen()); //文件大小
        System.out.println(fileStatus.isFile()); //是不是一个文件
            System.out.println(fileStatus.isDirectory()); //是不是一个目录
           
        BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            
        for (BlockLocation bl : blockLocations) {
                
            System.out.println("block-offset:" + bl.getOffset());
                
            String[] hosts = bl.getHosts();
                
            for (String host : hosts) {
                System.out.println(host);
            }
        }
            
        System.out.println("----------------------------");
    }
    }

 

3.文件下载(get)

@Test
    public void download() {
        Configuration conf=new Configuration();
        try
        {
            FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),conf);
            FSDataInputStream in = fileSystem.open(new Path("/upload.txt"));
            FileOutputStream out = new FileOutputStream(new File("d://lib//updoad.txt"));
            byte[]b=new byte[1024];
            int len=0;
            while((len=in.read(b))!=-1) {
                out.write(b,0,len);
            }
            in.close();
            out.close();
        } catch (IOException | URISyntaxException e)
        {
            // TODO 自动生成的 catch 块
            e.printStackTrace();
        }
        
        
    }

 4.上传文件(create)

@Test
    public void upload() {
        Configuration conf=new Configuration();
        try
        {
            FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),conf);
            FSDataOutputStream out = fileSystem.create(new Path("/jetbrains-agent.jar"));
            FileInputStream in=new FileInputStream(new File("d:\\jetbrains-agent.jar"));
            byte[]b=new byte[10240];
            int len=0;
            while((len=in.read(b))!=-1) {
                out.write(b,0,len);
            }
            in.close();
            out.close();
        } catch (IOException | URISyntaxException e)
        {
            // TODO 自动生成的 catch 块
            e.printStackTrace();
        }
    }

5.重命名(rename)

@Test
    public void mv() {
        Configuration conf=new Configuration();
        try
        {
            FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),conf);
            fileSystem.rename(new Path("/hdp01"), new Path("/HDP01"));
            fileSystem.close();
        } catch (IOException | URISyntaxException e)
        {
            // TODO 自动生成的 catch 块
            e.printStackTrace();
        }
    }

6.文件夹删除

@Test
public void deleteAtHDFS() throws Exception{
    // 1 创建配置信息对象
    Configuration configuration = new Configuration();
        
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),configuration, "root"); 

　　//2 删除文件夹 ，如果是非空文件夹，参数2是否递归删除，true递归 
　　fs.delete(new Path("hdfs://192.168.0.xxx:9000/upload/output"), true);

 }

7.创建文件夹

@Test
public void mkdirAtHDFS() throws Exception{
    // 1 创建配置信息对象
    Configuration configuration = new Configuration();
        
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),configuration, "root");    
        
    //2 创建目录
    fs.mkdirs(new Path("hdfs://192.168.0.xxx:9000/upload/output"));
    }

8.遍历所有文件状态

@Test
public void findAtHDFS() throws Exception, IllegalArgumentException, IOException{
        
    // 1 创建配置信息对象
    Configuration configuration = new Configuration();
        
    FileSystem fs = FileSystem.get(new URI("hdfs://192.168.0.xxx:9000"),configuration, "root");
        
    // 2 获取查询路径下的文件状态信息
    FileStatus[] listStatus = fs.listStatus(new Path("/"));

    // 3 遍历所有文件状态
    for (FileStatus status : listStatus) {
        if (status.isFile()) {
            System.out.println("f--" + status.getPath().getName());
        } else {
            System.out.println("d--" + status.getPath().getName());
        }
    }
}