Flink - Asynchronous I/O
https://docs.google.com/document/d/1Lr9UYXEz6s6R_3PWg3bZQLF3upGaNEkc0rQCFSzaYDI/edit
// create the original stream DataStream<String> stream = ...; // apply the async I/O transformation DataStream<Tuple2<String, String>> resultStream = AsyncDataStream.unorderedWait(stream, new AsyncDatabaseRequest(), 1000, TimeUnit.MILLISECONDS, 100);
AsyncDataStream
有一组接口，
unorderedWait
orderedWait
最终都是调用到，
addOperator(in, func, timeUnit.toMillis(timeout), capacity, OutputMode.ORDERED)
是否是ordered，只是最后一个参数不同
private static <IN, OUT> SingleOutputStreamOperator<OUT> addOperator( DataStream<IN> in, AsyncFunction<IN, OUT> func, long timeout, int bufSize, OutputMode mode) { TypeInformation<OUT> outTypeInfo = TypeExtractor.getUnaryOperatorReturnType(func, AsyncFunction.class, false, true, in.getType(), Utils.getCallLocationName(), true); // create transform AsyncWaitOperator<IN, OUT> operator = new AsyncWaitOperator<>( in.getExecutionEnvironment().clean(func), timeout, bufSize, mode); return in.transform("async wait operator", outTypeInfo, operator); }
AsyncWaitOperator
setup主要是初始化，任务队列
@Override public void setup(StreamTask<?, ?> containingTask, StreamConfig config, Output<StreamRecord<OUT>> output) { super.setup(containingTask, config, output); // create the operators executor for the complete operations of the queue entries this.executor = Executors.newSingleThreadExecutor(); //单线程的Executor，用于处理队列 switch (outputMode) { case ORDERED: queue = new OrderedStreamElementQueue( capacity, executor, this); break; case UNORDERED: queue = new UnorderedStreamElementQueue( capacity, executor, this); break; default: throw new IllegalStateException("Unknown async mode: " + outputMode + '.'); } }
看下，OrderedStreamElementQueue
public class OrderedStreamElementQueue implements StreamElementQueue { /** Queue for the inserted StreamElementQueueEntries. */ private final ArrayDeque<StreamElementQueueEntry<?>> queue; //放所有的element @Override public AsyncResult peekBlockingly() throws InterruptedException { //取 lock.lockInterruptibly(); try { while (queue.isEmpty() || !queue.peek().isDone()) { //如果queue的第一个element没有完成 headIsCompleted.await(); //等锁，等他完成 } return queue.peek(); //如果完成就peek出来，注意peek是不会移除这个element的，所以需要poll } finally { lock.unlock(); } } @Override public AsyncResult poll() throws InterruptedException { //单独做poll lock.lockInterruptibly(); try { while (queue.isEmpty() || !queue.peek().isDone()) { //如果第一个没完成，等待 headIsCompleted.await(); } notFull.signalAll(); //poll后，队列一定不满，所以解锁notFull return queue.poll(); } finally { lock.unlock(); } } private <T> void addEntry(StreamElementQueueEntry<T> streamElementQueueEntry) { //put，tryput都是调用这个 queue.addLast(streamElementQueueEntry); //加到queue里面 streamElementQueueEntry.onComplete(new AcceptFunction<StreamElementQueueEntry<T>>() { //给element加上complete的callback，调用onCompleteHandler @Override public void accept(StreamElementQueueEntry<T> value) { try { onCompleteHandler(value); } } }, executor); } private void onCompleteHandler(StreamElementQueueEntry<?> streamElementQueueEntry) throws InterruptedException { lock.lockInterruptibly(); try { if (!queue.isEmpty() && queue.peek().isDone()) { headIsCompleted.signalAll(); //放开锁，告诉大家我完成了 } } finally { lock.unlock(); } } }
对于queue主要就是，读取操作
这里取是分两步，先peek，再poll
open，主要是处理从snapshot中恢复的数据
并启动emiter
@Override public void open() throws Exception { super.open(); // process stream elements from state, since the Emit thread will start as soon as all // elements from previous state are in the StreamElementQueue, we have to make sure that the // order to open all operators in the operator chain proceeds from the tail operator to the // head operator. if (recoveredStreamElements != null) { for (StreamElement element : recoveredStreamElements.get()) { //处理从snapshot中恢复出的element if (element.isRecord()) { processElement(element.<IN>asRecord()); } else if (element.isWatermark()) { processWatermark(element.asWatermark()); } else if (element.isLatencyMarker()) { processLatencyMarker(element.asLatencyMarker()); } else { throw new IllegalStateException("Unknown record type " + element.getClass() + " encountered while opening the operator."); } } recoveredStreamElements = null; } // create the emitter this.emitter = new Emitter<>(checkpointingLock, output, queue, this); //创建Emitter // start the emitter thread this.emitterThread = new Thread(emitter, "AsyncIO-Emitter-Thread (" + getOperatorName() + ')'); emitterThread.setDaemon(true); emitterThread.start(); }
Emitter
@Override public void run() { try { while (running) { LOG.debug("Wait for next completed async stream element result."); AsyncResult streamElementEntry = streamElementQueue.peekBlockingly(); output(streamElementEntry); }
从queue中peek数据，对于上面OrderedStreamElementQueue，只有完成的数据会被peek到
private void output(AsyncResult asyncResult) throws InterruptedException { if (asyncResult.isWatermark()) { //...... } else { AsyncCollectionResult<OUT> streamRecordResult = asyncResult.asResultCollection(); synchronized (checkpointLock) { //collect数据需要加checkpoint锁 LOG.debug("Output async stream element collection result."); try { Collection<OUT> resultCollection = streamRecordResult.get(); if (resultCollection != null) { for (OUT result : resultCollection) { timestampedCollector.collect(result); //真正emit数据 } } } // remove the peeked element from the async collector buffer so that it is no longer // checkpointed streamElementQueue.poll(); //emit完可以将数据从queue中删除 // notify the main thread that there is again space left in the async collector // buffer checkpointLock.notifyAll(); } } }
可以看到当数据被emit后，才会从queue删除掉
processElement
@Override public void processElement(StreamRecord<IN> element) throws Exception { final StreamRecordQueueEntry<OUT> streamRecordBufferEntry = new StreamRecordQueueEntry<>(element); //封装成StreamRecordQueueEntry if (timeout > 0L) { // register a timeout for this AsyncStreamRecordBufferEntry long timeoutTimestamp = timeout + getProcessingTimeService().getCurrentProcessingTime(); final ScheduledFuture<?> timerFuture = getProcessingTimeService().registerTimer( //开个定时器，到时间就会colloct一个超时异常 timeoutTimestamp, new ProcessingTimeCallback() { @Override public void onProcessingTime(long timestamp) throws Exception { streamRecordBufferEntry.collect( new TimeoutException("Async function call has timed out.")); } }); // Cancel the timer once we've completed the stream record buffer entry. This will remove // the register trigger task streamRecordBufferEntry.onComplete(new AcceptFunction<StreamElementQueueEntry<Collection<OUT>>>() { //在StreamRecordQueueEntry完成是触发删除这个定时器，这样就只有未完成的会触发定时器 @Override public void accept(StreamElementQueueEntry<Collection<OUT>> value) { timerFuture.cancel(true); } }, executor); } addAsyncBufferEntry(streamRecordBufferEntry); //把StreamRecordQueueEntry加到queue中去 userFunction.asyncInvoke(element.getValue(), streamRecordBufferEntry); //调用用户定义的asyncInvoke }
StreamRecordQueueEntry
public class StreamRecordQueueEntry<OUT> extends StreamElementQueueEntry<Collection<OUT>> implements AsyncCollectionResult<OUT>, AsyncCollector<OUT> { /** Future containing the collection result. */ private final CompletableFuture<Collection<OUT>> resultFuture; @Override public void collect(Collection<OUT> result) { resultFuture.complete(result); } @Override public void collect(Throwable error) { resultFuture.completeExceptionally(error); } }
前面在emitter里面判断，entry是否做完就看，resultFuture是否isDone
可以看到resultFuture只有在collect的时候才会被complete
当resultFuture.complete时，onComplete callback会被触发，
这个callback在OrderedStreamElementQueue.addEntry被注册上来，做的事也就是告诉大家headIsCompleted；这样随后Emitter可以把结果数据emit出去
最终调用到用户定义的，
userFunction.asyncInvoke
@Override public void asyncInvoke(final String str, final AsyncCollector<Tuple2<String, String>> asyncCollector) throws Exception { // issue the asynchronous request, receive a future for result Future<String> resultFuture = client.query(str); // set the callback to be executed once the request by the client is complete // the callback simply forwards the result to the collector resultFuture.thenAccept( (String result) -> { asyncCollector.collect(Collections.singleton(new Tuple2<>(str, result))); }); } }
首先client必须是异步的，如果不是，没法返回Future，那需要自己用连接池实现
主要逻辑就是在resultFuture完成后，调用asyncCollector.collect把结果返回给element
