Flink系列--扩展FLink table connector

官网地址： https://nightlies.apache.org/flink/flink-docs-master/docs/dev/table/sourcessinks/#extension-points

1 扩展FLink table connector

org.apache.flink.table.factories.DynamicTableSourceFactory可以实现构造一个DynamicTableSource.

org.apache.flink.table.factories.DynamicTableSinkFactory可以实现构造一个DynamicTableSink.

在 JAR 文件中，可以将对新实现的引用添加到服务文件中：

META-INF/services/org.apache.flink.table.factories.Factory

2 示例

　　实现一个最简单的 word 生成器

2.1 工厂类

factoryIdentifier 工厂类表示，对应sql中的属性“connector”，见示例

requiredOptions 必选参数

optionalOptions 可选参数

import org.apache.flink.configuration.ConfigOption;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.factories.DynamicTableSourceFactory;

import java.util.HashSet;
import java.util.Set;

public class WordTableSourceFactory implements DynamicTableSourceFactory {
    public static final String WORD_TYPE = "word";
    @Override
    public DynamicTableSource createDynamicTableSource(Context context) {
        return new WordTableSource();
    }
    @Override
    public String factoryIdentifier() {
        return WORD_TYPE;
    }
    @Override
    public Set<ConfigOption<?>> requiredOptions() {
        return new HashSet();
    }
    @Override
    public Set<ConfigOption<?>> optionalOptions() {
        return new HashSet<>();
    }
}

2.2 Table source实现

 

import org.apache.flink.table.connector.ChangelogMode;
import org.apache.flink.table.connector.source.DynamicTableSource;
import org.apache.flink.table.connector.source.ScanTableSource;
import org.apache.flink.table.connector.source.SourceFunctionProvider;


public class WordTableSource implements ScanTableSource {

    @Override
    public ChangelogMode getChangelogMode() {
        return ChangelogMode.insertOnly();
    }

    @Override
    public ScanRuntimeProvider getScanRuntimeProvider(ScanContext runtimeProviderContext) {
        return SourceFunctionProvider.of(new WordSourceFunction(),false);
    }
    @Override
    public DynamicTableSource copy() {
        return new WordTableSource();
    }
    @Override
    public String asSummaryString() {
        return  this.getClass().getSimpleName();
    }
}

 

2.3 Source Function实现

 

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.RichSourceFunction;
import org.apache.flink.table.data.GenericRowData;
import org.apache.flink.table.data.StringData;

import java.util.Random;


public class WordSourceFunction extends RichSourceFunction {

    private final String[] words = new String[]{
            "Welcome", "to", "the", "world", "of", "flink"
    };
    @Override
    public void run(SourceContext ctx) throws Exception {
        Random random = new Random();
        while (true) {
            int index = random.nextInt(words.length);
            GenericRowData row = new GenericRowData(1);
            row.setField(0,StringData.fromString(words[index]));
            ctx.collect(row);
            Thread.sleep(100);
        }
    }
    @Override
    public void cancel() {
    }
    @Override
    public void open(Configuration parameters) throws Exception {
        super.open(parameters);
    }
}

 

2.4 工厂类放到resource下指定目录下的文件中

META-INF/services/org.apache.flink.table.factories.Factory

 

 3 测试类

import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.state.filesystem.FsStateBackend;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.data.RowData;


public class SqlKeyedStateDemo {
    public static void main(String[] args) {

        Configuration configuration = new Configuration();
        configuration.setInteger("rest.port", 8081);
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(configuration);
        env.enableCheckpointing(5 * 1000);
        env.setStateBackend(new FsStateBackend("file:///D:/IDE/FlinkTableApiTest/demo/checkpoint/sql"));
        env.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
        StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

        tEnv.executeSql("CREATE TABLE wordSource (\n" +
                "    word  STRING\n" +
                ") WITH (\n" +
                "    'connector' = 'word'" +
                ")");

        tEnv.executeSql("CREATE TABLE print (\n" +
                "    word           STRING,\n" +
                "    wordCount      BIGINT,\n" +
                "    PRIMARY KEY (word) NOT ENFORCED" +
                ") WITH (\n" +
                "   'connector'  = 'print'\n" +
                ")");

        tEnv.executeSql("insert into print select word ,count(word) as wordCount from wordSource group by word");


    }
}