duckdb读取s3

 

package com.nvxclouds;

import java.sql.*;

/**
 * Copyright (c) 2024 hello.Co.Ltd. All rights reserved.
 *
 * @author muzhi
 * @version 3.0.0
 * @date 2025-04-28 09:50
 */
public class DuckDBReader {
    public static void main(String[] args) throws ClassNotFoundException {
        readFromHttpServer();
        System.out.println("------");
        readFromHttpsServer();
    }

    private static void readFromHttpsServer() throws ClassNotFoundException {
        String accessKey = "nvx1";
        String secretKey = "nvx1";
        String host = "192.168.50.97";
        int port = 38333;
        String bucketName = "testbucket";
        String objectKey = "data_export.csv";
        String url = "s3://" + bucketName + "/" + objectKey;
        Class.forName("org.duckdb.DuckDBDriver");

        try (Connection conn = DriverManager.getConnection("jdbc:duckdb:")) {
            try (Statement stmt = conn.createStatement()) {
//                stmt.execute("INSTALL httpfs");
//                stmt.execute("LOAD httpfs");

                stmt.execute("SET s3_access_key_id='"+accessKey+"';");
                stmt.execute("SET s3_secret_access_key='"+secretKey+"';");
                stmt.execute("SET s3_region='us-east-1';");
                stmt.execute("SET s3_endpoint='192.168.50.97:32021';");
                stmt.execute("SET s3_url_style='path';");
                stmt.execute("SET s3_use_ssl=true;");

                System.out.println(url);
                String sql = "SELECT * FROM read_csv('" + url + "', header=true)";
                ResultSet rs = stmt.executeQuery(sql);
                while (rs.next()) {
                    for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
                        System.out.print(rs.getString(i) + "\t");
                    }
                    System.out.println();
                }
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    private static void readFromHttpServer() throws ClassNotFoundException {
        {
            String accessKey = "nvx1";
            String secretKey = "nvx1";
            String host = "192.168.50.97";
            int port = 38333;
            String bucketName = "testbucket";
            String objectKey = "data_export.csv";
            String url = "s3://" + bucketName + "/" + objectKey;
            Class.forName("org.duckdb.DuckDBDriver");

            try (Connection conn = DriverManager.getConnection("jdbc:duckdb:")) {
                try (Statement stmt = conn.createStatement()) {
//                stmt.execute("INSTALL httpfs");
//                stmt.execute("LOAD httpfs");

                    stmt.execute("SET s3_access_key_id='"+accessKey+"';");
                    stmt.execute("SET s3_secret_access_key='"+secretKey+"';");
                    stmt.execute("SET s3_region='us-east-1';");
                    stmt.execute("SET s3_endpoint='192.168.50.97:38333';");
                    stmt.execute("SET s3_url_style='path';");
                    stmt.execute("SET s3_use_ssl=false;");

                    System.out.println(url);
                    String sql = "SELECT * FROM read_csv('" + url + "', header=true)";
                    ResultSet rs = stmt.executeQuery(sql);
                    while (rs.next()) {
                        for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
                            System.out.print(rs.getString(i) + "\t");
                        }
                        System.out.println();
                    }
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
}

package com.nvxclouds;

import java.sql.*;

/**
 * Copyright (c) 2024 NVXClouds.Co.Ltd. All rights reserved.
 *
 * @author muzhi
 * @version 3.0.0
 * @date 2025-04-28 09:50
 */
public class DuckDBReader {
    public static void main(String[] args) throws ClassNotFoundException {
        readFromHttpServer();
        System.out.println("------");
        readFromHttpsServer();
    }

    private static void readFromHttpsServer() throws ClassNotFoundException {
        String accessKey = "nvx1";
        String secretKey = "nvx1";
        String host = "192.168.50.97";
        int port = 38333;
        String bucketName = "testbucket";
        String objectKey = "data_export.csv";
        String url = "s3://" + bucketName + "/" + objectKey;
        Class.forName("org.duckdb.DuckDBDriver");

        try (Connection conn = DriverManager.getConnection("jdbc:duckdb:")) {
            try (Statement stmt = conn.createStatement()) {
//                stmt.execute("INSTALL httpfs");
//                stmt.execute("LOAD httpfs");

                stmt.execute("SET s3_access_key_id='"+accessKey+"';");
                stmt.execute("SET s3_secret_access_key='"+secretKey+"';");
                stmt.execute("SET s3_region='us-east-1';");
                stmt.execute("SET s3_endpoint='192.168.50.97:32021';");
                stmt.execute("SET s3_url_style='path';");
                stmt.execute("SET s3_use_ssl=true;");

                System.out.println(url);
                String sql = "SELECT * FROM read_csv('" + url + "', header=true)";
                ResultSet rs = stmt.executeQuery(sql);
                while (rs.next()) {
                    for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
                        System.out.print(rs.getString(i) + "\t");
                    }
                    System.out.println();
                }
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    private static void readFromHttpServer() throws ClassNotFoundException {
        {
            String accessKey = "nvx1";
            String secretKey = "nvx1";
            String host = "192.168.50.97";
            int port = 38333;
            String bucketName = "testbucket";
            String objectKey = "data_export.csv";
            String url = "s3://" + bucketName + "/" + objectKey;
            Class.forName("org.duckdb.DuckDBDriver");

            try (Connection conn = DriverManager.getConnection("jdbc:duckdb:")) {
                try (Statement stmt = conn.createStatement()) {
//                stmt.execute("INSTALL httpfs");
//                stmt.execute("LOAD httpfs");

                    stmt.execute("SET s3_access_key_id='"+accessKey+"';");
                    stmt.execute("SET s3_secret_access_key='"+secretKey+"';");
                    stmt.execute("SET s3_region='us-east-1';");
                    stmt.execute("SET s3_endpoint='192.168.50.97:38333';");
                    stmt.execute("SET s3_url_style='path';");
                    stmt.execute("SET s3_use_ssl=false;");

                    System.out.println(url);
                    String sql = "SELECT * FROM read_csv('" + url + "', header=true)";
                    ResultSet rs = stmt.executeQuery(sql);
                    while (rs.next()) {
                        for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
                            System.out.print(rs.getString(i) + "\t");
                        }
                        System.out.println();
                    }
                }
            } catch (SQLException e) {
                e.printStackTrace();
            }
        }
    }
}