openmetadata es 存储安全认证添加

openmetadata 默认是没有安全控制的（比较危险），所以简单添加了下对于用户密码的支持（已经pr官方了，应该0.6版本会发布）

需要修改的部分

python 的elasticsearch sink 部分以及rest server 的es 部分

代码修改

  • rest api 部分
    主要是配置以及search
    配置
    ElasticSearchConfiguration.java
 
package org.openmetadata.catalog;
 
import javax.validation.constraints.NotEmpty;
 
public class ElasticSearchConfiguration {
 
    @NotEmpty
    private String host;
 
    @NotEmpty
    private Integer port;
 
    private String username;
 
    private String password;
 
    public String getHost() {
        return host;
    }
 
    public void setHost(String host) {
        this.host = host;
    }
 
    public Integer getPort() {
        return port;
    }
 
    public void setPort(Integer port) {
        this.port = port;
    }
 
    public String getUsername() {
        return username;
    }
 
    public void setUsername(String username) {
        this.username = username;
    }
 
    public String getPassword() {
        return password;
    }
 
    public void setPassword(String password) {
        this.password = password;
    }
 
    @Override
    public String toString() {
        return "ElasticSearchConfiguration{" +
                "host='" + host + '\'' +
                ", port=" + port +
                ", username='" + username + '\'' +
                ", password='" + password + '\'' +
                '}';
    }
}
  • search api 部分
    SearchResource.java
 
public class SearchResource {
  private final RestHighLevelClient client;
  private static final Logger LOG = LoggerFactory.getLogger(SearchResource.class);
 
  public SearchResource(ElasticSearchConfiguration esConfig) {
    RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(esConfig.getHost(), esConfig.getPort(), "http"));
    if(StringUtils.isNotEmpty(esConfig.getUsername())){
      CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
      credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(esConfig.getUsername(), esConfig.getPassword()));
      restClientBuilder.setHttpClientConfigCallback(httpAsyncClientBuilder -> {
        httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        return  httpAsyncClientBuilder;
      });
    }
    this.client = new RestHighLevelClient(restClientBuilder);
  }
  • python sink 部分
    ingestion/src/metadata/ingestion/sink/elasticsearch.py
 
class ElasticSearchConfig(ConfigModel):
    es_host: str
    es_port: int = 9200
    es_username: str
    es_password: str
    index_tables: Optional[bool] = True
    index_topics: Optional[bool] = True
    index_dashboards: Optional[bool] = True
    index_pipelines: Optional[bool] = True
    table_index_name: str = "table_search_index"
    topic_index_name: str = "topic_search_index"
    dashboard_index_name: str = "dashboard_search_index"
    pipeline_index_name: str = "pipeline_search_index"
class ElasticsearchSink(Sink):
    """ """
 
    DEFAULT_ELASTICSEARCH_INDEX_MAPPING = TABLE_ELASTICSEARCH_INDEX_MAPPING
 
    @classmethod
    def create(
        cls, config_dict: dict, metadata_config_dict: dict, ctx: WorkflowContext
    ):
        config = ElasticSearchConfig.parse_obj(config_dict)
        metadata_config = MetadataServerConfig.parse_obj(metadata_config_dict)
        return cls(ctx, config, metadata_config)
 
    def __init__(
        self,
        ctx: WorkflowContext,
        config: ElasticSearchConfig,
        metadata_config: MetadataServerConfig,
    ) -> None:
 
        self.config = config
        self.metadata_config = metadata_config
        self.ctx = ctx
        self.status = SinkStatus()
        self.rest = OpenMetadataAPIClient(self.metadata_config)
        self.elasticsearch_doc_type = "_doc"
        http_auth = None
        if self.config.es_username:
            http_auth = (self.config.es_username, self.config.es_password)
        self.elasticsearch_client = Elasticsearch(
            [
                {"host": self.config.es_host, "port": self.config.es_port},
            ],
            http_auth=http_auth,
        )

使用

使用主要是对于es 开启basic auth 然后api 以及sink 部分配置就可以了
rest api server
openmetadata.yaml (使用了默认密码，可以自己调整)

 
elasticsearch:
  host: localhost
  port: 9200
  username: admin
  password: admin

sink
pipeline 中的metadata_to_es.json 文件

 
{
  "source": {
    "type": "metadata",
    "config": {
      "include_tables": "true",
      "include_topics": "true",
      "include_dashboards": "true",
      "limit_records": 10
    }
  },
  "sink": {
    "type": "elasticsearch",
    "config": {
      "index_tables": "true",
      "index_topics": "true",
      "index_dashboards": "true",
      "es_host": "localhost",
      "es_username":"admin",
      "es_password":"admin",
      "es_port": 9200
    }
  },
  "metadata_server": {
    "type": "metadata-server",
    "config": {
      "api_endpoint": "http://localhost:8585/api",
      "auth_provider_type": "no-auth"
    }
  }
}

说明

因为python 模块还没有发布，所以不知直接使用，但是可以先安装，然后通过修改源码的模式打补丁，这样就可以使用了
参考效果

 

 


 

 

参考资料

https://open-metadata.org/
https://docs.open-metadata.org/install/metadata-ingestion/ingest-sample-data
https://github.com/open-metadata/OpenMetadata/pull/894

posted on 2021-10-23 23:02  荣锋亮  阅读(218)  评论(0)  编辑  收藏  举报

导航