为CDP7.1.7添加Spark History Server认证功能

在CDP7.1.7中，对于非Kerberos环境下的Spark History Server可以直接访问。有部分客户的安全团队要求增加账户/密码认证功能，以满足内部的安全合规要求。

本安全认证方案的核心基于Spark的扩展配置属性spark.ui.filters实现。该属性支持集成标准的Java Servlet Filter。由于Spark History Server内部集成了轻量级Jetty服务器作为其Web服务引擎，而Jetty作为标准的Web 容器，原生具备对Servlet规范的完美支持。因此能够通过向Jetty动态注入自定义过滤器，实现对所有UI请求的强制拦截与安全审计。以下为该方案具体实现：

image

 

1. pom配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.cloudera.spark</groupId>
    <artifactId>spark-ui-auth</artifactId>
    <version>1.0-SNAPSHOT</version>

    <repositories>
        <repository>
            <id>mvn-repo</id>
            <name>mvn-repo</name>
            <url>https://mvnrepository.com/</url>
        </repository>
        <repository>
            <id>cloudera-repo</id>
            <name>cloudera-repo</name>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
            <layout>default</layout>
        </repository>
    </repositories>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>javax.servlet</groupId>
            <artifactId>javax.servlet-api</artifactId>
            <version>3.1.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.4.7.7.1.7.1065-1</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-test</artifactId>
            <version>5.2.25.RELEASE</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.springframework</groupId>
            <artifactId>spring-web</artifactId>
            <version>5.2.25.RELEASE</version>
            <scope>provided</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.8.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

 

2. 代码实现

package com.cloudera.spark.authentication;


import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.servlet.*;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;

/**
 * 1. 打包上传到spark history server的/opt/cloudera/parcels/CDH/lib/spark/jars目录下
 * 2. 在cm > spark > configuration > spark-conf/spark-history-server.conf添加下面配置
 *  spark.ui.filters=com.cloudera.spark.authentication.SparkAuthFilter                //配置SparkAuthFilter类进行认证
 *  spark.com.cloudera.spark.authentication.SparkAuthFilter.param.username=foo        //配置spark-webUI界面登陆的用户
 *  spark.com.cloudera.spark.authentication.SparkAuthFilter.param.password=foo        //配置spark-webUI界面登陆的密码
 * 3. 重启spark history server
 * 4. 访问spark history server的webui就需要输入账号密码了
 *
 * @Description TODO
 * @Created by mengyao
 * @Date 2026/6/4 10:12
 * @Version 1.0
 */
public class SparkAuthFilter implements Filter {

    private static final String FILTER_NAME = SparkAuthFilter.class.getSimpleName();

    private static final Logger LOG = LoggerFactory.getLogger(SparkAuthFilter.class);
    private String username;
    private String password;

    @Override
    public void init(FilterConfig filterConfig) throws ServletException {
        username = filterConfig.getInitParameter("username");
        password = filterConfig.getInitParameter("password");
        if (this.username == null || this.password == null) {
            throw new ServletException(FILTER_NAME+" initialization failed: 'username' or 'password' is missing in spark-default.properties.");
        }
        LOG.info("Initialization of {} successful, username is: {}", FILTER_NAME, username);
    }

    @Override
    public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain)
            throws IOException, ServletException {
        HttpServletRequest request = (HttpServletRequest) servletRequest;
        HttpServletResponse response = (HttpServletResponse) servletResponse;
        String authHeader = request.getHeader("Authorization");

        if (authHeader != null && authHeader.startsWith("Basic ")) {
            String base64Credentials = authHeader.substring("Basic ".length()).trim();
            String credentials = new String(Base64.getDecoder().decode(base64Credentials), StandardCharsets.UTF_8);
            LOG.debug("Log in using account and password: {}", credentials);
            final String[] values = credentials.split(":", 2);
            if (values.length == 2) {
                String inputUser = values[0];
                String inputPass = convertToMd5(values[1]);
                if (username.equals(inputUser) && password.equalsIgnoreCase(inputPass)) {
                    filterChain.doFilter(request, response);
                    return;
                }
            }
        }
        response.setHeader("WWW-Authenticate", "Basic realm=\"Spark UI Secure Area\"");
        response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
        response.getWriter().println("401 Unauthorized: Invalid username or password.");
    }

    @Override
    public void destroy() {
    }

    private static String convertToMd5(String source) {
        try {
            MessageDigest md = MessageDigest.getInstance("MD5");
            byte[] array = md.digest(source.getBytes(StandardCharsets.UTF_8));
            StringBuilder sb = new StringBuilder();
            for (byte b : array) {
                sb.append(Integer.toHexString((b & 0xFF) | 0x100), 1, 3);
            }
            return sb.toString();
        } catch (NoSuchAlgorithmException e) {
            throw new RuntimeException("MD5 algorithm not found", e);
        }
    }

}

3. 打包和上传

mvn clean package
mengyao@C02G7451MD6R target % scp spark-ui-auth-1.0-SNAPSHOT.jar root@ccycloud-6.cdp717ceb.root.comops.site:/opt/cloudera/parcels/CDH-7.1.7-1.cdh7.1.7.p1064.46686528/lib/spark/jars/
root@ccycloud-6.cdp717ceb.root.comops.site's password:
spark-ui-auth-1.0-SNAPSHOT.jar                                                                                                                                                 100% 5403    16.6KB/s   00:00

 

4. 修改Spark配置

在CM > Spark > Configuration > History Server Advanced Configuration Snippet (Safety Valve) for spark-conf/spark-history-server.conf下新增：
spark.ui.filters=com.cloudera.spark.authentication.SparkAuthFilter
# 访问spark history server的账号为admin
spark.com.cloudera.spark.authentication.SparkAuthFilter.param.username=admin
# 访问spark history server的密码也是admin，值为admin的md5字符串
spark.com.cloudera.spark.authentication.SparkAuthFilter.param.password=21232f297a57a5a743894a0e4a801fc3

5. 重启Spark History Server

image

6. 验证登陆功能

image

 输入admin/admin登陆即可访问所有的spark作业

image

 

 

 

 

posted @ 2026-06-04 21:29  孟尧  阅读(9)  评论(0)    收藏  举报