Hadoop HDFS文件操作

 

import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;

//利用JavaAPI来访问HDFS的文件与目录

public class HDFSTest {

    public static void main(String[] args) throws Exception {
        try {
            getHDFSNode();
            makeDir();
            writeToHDFS();
            uploadToHdfs();
            listHdfs();
            readFromHdfs();
            deleteFromHdfs();
            deleteDir();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            System.out.println("SUCCESS");
        }
    }

    // HDFS集群上所有节点名称信息
    public static void getHDFSNode() throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000/"),
                conf);

        DistributedFileSystem dfs = (DistributedFileSystem) fs;
        DatanodeInfo[] dataNodeStats = dfs.getDataNodeStats();

        for (int i = 0; i < dataNodeStats.length; i++) {
            System.out.println("DataNode_" + i + "_Node:"
                    + dataNodeStats[i].getHostName());
        }
    }

    /** 遍历HDFS上的文件和目录 */
    private static void listHdfs() throws FileNotFoundException, IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000/"),
                conf);

        FileStatus fileList[] = fs.listStatus(new Path("/"));
        for (int i = 0; i < fileList.length; i++) {
            System.out.println("name:" + fileList[i].getPath().getName()
                    + "\t\tsize:" + fileList[i].getLen());
        }
        fs.close();
    }

    public static void writeToHDFS() throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000"),
                conf);

        Path path = new Path("/test/write");
        FSDataOutputStream out = fs.create(path);
        out.writeUTF("Hello HDFS!");
        out.close();
        fs.close();
    }

    // 创建HDFS目录
    public static void makeDir() throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000"),
                conf);

        Path path = new Path("/test");
        fs.mkdirs(path);
        fs.close();
    }

    // 删除HDFS目录
    public static void deleteDir() throws IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(conf);

        Path path = new Path("/test");
        fs.delete(path, true);
        fs.close();
    }

    /** 上传文件到HDFS上去 */

    private static void uploadToHdfs() throws FileNotFoundException,
            IOException {

        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000/"),
                conf);

        String localSrc = "/etc/profile";
        String hdfsDst = "/profile";

        Path src = new Path(localSrc);
        Path dst = new Path(hdfsDst);
        fs.copyFromLocalFile(src, dst);
        fs.close();
    }

    /** 从HDFS上读取文件 */
    private static void readFromHdfs() throws FileNotFoundException,
            IOException {

        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000/"),
                conf);

        String dst = "/profile";
        FSDataInputStream hdfsInStream = fs.open(new Path(dst));

        OutputStream out = new FileOutputStream("/home/manhua/profile");
        byte[] ioBuffer = new byte[1024];
        int readLen = hdfsInStream.read(ioBuffer);

        if (fs.exists(new Path(dst))) {
            while (-1 != readLen) {
                out.write(ioBuffer, 0, readLen);
                readLen = hdfsInStream.read(ioBuffer);
            }
            out.close();
            hdfsInStream.close();
            fs.close();
        }
    }

    /** 从HDFS上删除文件 */
    private static void deleteFromHdfs() throws FileNotFoundException,
            IOException {
        Configuration conf = new Configuration();
        FileSystem fs = FileSystem.get(URI.create("hdfs://localhost:9000"),
                conf);

        String dst = "/profile";
        fs.deleteOnExit(new Path(dst));
        fs.close();
    }

}