HDFSManager

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.security.UserGroupInformation;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;

@Slf4j
@Component
public class HDFSManager {

    private FileSystem fileSystem;

    @Value("${hdfs.root.path}")
    private String rootPath;

    @Value("${hdfs.user}")
    private String user;

    @Value("${hdfs.kerberos.enable}")
    private Boolean kerberosEnable;

    @Value("${hdfs.config.path}")
    private String hdfsConfigPath;

    @Value("${hdfs.keytab.name:''}")
    private String hdfsKeytabName;

    @Value("${hdfs.krb5.conf.name:''}")
    private String hdfsKrb5ConfName;

    @PostConstruct
    private void init() throws IOException {
        Configuration configuration = new Configuration();
        if (!hdfsConfigPath.endsWith("/")) {
            hdfsConfigPath = hdfsConfigPath + "/";
        }
        configuration.addResource(new Path(hdfsConfigPath + "core-site.xml"));
        configuration.addResource(new Path(hdfsConfigPath + "hdfs-site.xml"));
        if (kerberosEnable) {
            System.setProperty("java.security.krb5.conf", hdfsConfigPath + hdfsKrb5ConfName);
            UserGroupInformation.setConfiguration(configuration);
            UserGroupInformation.loginUserFromKeytab(user, hdfsConfigPath + hdfsKeytabName);
        } else {
            UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(user));
        }
        log.info("当前HDFS user ：{}", UserGroupInformation.getCurrentUser().getUserName());
        fileSystem = FileSystem.newInstance(configuration);
        log.info("fsDefault ：{}", configuration.get(FileSystem.FS_DEFAULT_NAME_KEY));
        mkdirs(rootPath);
    }

    public FileStatus[] list(String path) throws IOException {
        return fileSystem.listStatus(new Path(path));
    }

    private void mkdirs(String path) throws IOException {
        Path srcPath = new Path(path);
        if (fileSystem.exists(srcPath)) {
            log.info("目录已存在");
            return;
        }
        fileSystem.mkdirs(srcPath);
    }

    public String createFile(String path, byte[] b) throws IOException {
        String filePath = rootPath + path;
        try (FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(filePath))) {
            fsDataOutputStream.write(b);
            fsDataOutputStream.close();
            return filePath;
        }
    }

    public String rename(String oldName, String newName) throws IOException {
        String newPath = rootPath + newName;
        if (fileSystem.rename(new Path(oldName), new Path(newPath))) {
            return newPath;
        }
        throw new IOException("重命名失败");
    }


    public InputStream readFile(String path) throws IOException {
        return fileSystem.open(new Path(path));
    }


    public boolean delete(String path) throws IOException {
        return fileSystem.delete(new Path(path), false);
    }

    public String readContent(String path) throws IOException {
        try (InputStream inputStream = readFile(path);
             StringWriter writer = new StringWriter()) {
            IOUtils.copy(inputStream, writer, StandardCharsets.UTF_8.name());
            return writer.toString();
        }
    }

    public String modifyContent(String location, String content) throws IOException {
        try (FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(location))) {
            fsDataOutputStream.write(content.getBytes());
            return location;
        }
    }

    public String replace(String filePath, byte[] b) throws IOException {
        try (FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path(filePath))) {
            fsDataOutputStream.write(b);
            return filePath;
        }
    }
}