最近，为了支持部门团队的项目，通过H5实现摄像头的视频流数据的捕获，抓取到视频流后，传输到视频识别服务器进行后续的逻辑处理。
视频数据的采集过程，其实是比较没有谱的过程，因为之前没有研究过HTML5操控摄像头并取视频流。
研究了下网络上的所谓的经验帖子，大都说基于WebRTC的方案，没有错，但是也不对，我们这里涉及到的技术，确切的说是基于H5的navigator以及MediaRecorder API实现，辅助的工具是FileReader以及Blob。
参考的资料：（相关的内容，在这里就不详细描述）
navigator的getUserMedia： https://developer.mozilla.org/en-US/docs/Web/API/Navigator/getUserMedia
MediaRecorder：https://developer.mozilla.org/en-US/docs/Web/API/MediaStream_Recording_API
FileReader：https://developer.mozilla.org/en-US/docs/Web/API/FileReader
Blob：https://developer.mozilla.org/en-US/docs/Web/API/Blob
这个任务的实现逻辑，前端搭建一个Java的小Web应用，H5视频采集之后，通过WebSocket的方式，将视频流数据传递到Java的web小应用后台，然后从后台向视频识别服务器通过UDP传递视频数据。基本的架构如下图：
说明一下，本博文，视频采集的部分，参考了一个老外的帖子，从他的帖子，改造后，得到我们的项目需要的效果。参考的帖子地址：https://addpipe.com/blog/mediarecorder-api/
接下来，上前端页面以及代码. 大体说下，我的软件架构，jersey2 + freemarker + spring。
前端页面：
<!DOCTYPE html> <html> <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes"> <meta name="mobile-web-app-capable" content="yes"> <meta id="theme-color" name="theme-color" content="#fff"> <base target="_blank"> <title>Media Recorder API Demo</title> <link rel="stylesheet" href="${basePath}/css/video/main.css" /> #basePath是Web项目的根地址，例如 http://10.90.9.20:9080/RDConsumer <style> a#downloadLink { display: block; margin: 0 0 1em 0; min-height: 1.2em; } p#data { min-height: 6em; } </style> </head> <body> <div id="container"> <div style = "text-align:center;"> <h1>Media Recorder API Demo </h1> <h2>Record a 640x480 video using the media recorder API implemented in Firefox and Chrome</h2> <video controls autoplay></video><br> <button id="rec" onclick="onBtnRecordClicked()">Record</button> <button id="pauseRes" onclick="onPauseResumeClicked()" disabled>Pause</button> <button id="stop" onclick="onBtnStopClicked()" disabled>Stop</button> </div> <a id="downloadLink" download="mediarecorder.webm" name="mediarecorder.webm" href></a> <p id="data"></p> <script src="${basePath}/js/jquery-1.11.1.min.js"></script> <script src="${basePath}/js/video/main.js"></script> <h2>Works on:</h2> <p><ul><li>Firefox 30 and up</li><li>Chrome 47,48 (video only, enable <em>experimental Web Platform features</em> at <a href="chrome://flags/#enable-experimental-web-platform-features">chrome://flags</a>)</li><li>Chrome 49+</li></ul></p> <h2> <span style="color:red">Issues:</span> <p><ul><li>Pause does not stop audio recording on Chrome 49,50</li></ul></p> <h2>Containers & codecs:</h2> <p><table style="width:100%"> <thead> <tr> <th> </th><th>Chrome 47</th><th>Chrome 48</th><th>Chrome 49+</th><th>Chrome 52+</th><th>Firefox 30+</th> </tr> </thead> <tbody> <tr> <td><strong>Container</strong></td><td>webm</td><td>webm</td><td>webm</td><td>webm</td><td>webm</td> </tr> <tr> <td><strong>Video</strong></td><td>VP8</td><td>VP8</td><td>VP8/VP9</td><td>VP8/VP9/H264</td><td>VP8</td> </tr> <tr> <td><strong>Audio</strong></td><td>none</td><td>none</td><td>Opus @ 48kHz</td><td>Opus @ 48kHz</td><td>Vorbis @ 44.1 kHz</td> </tr> </tbody> </table> </p> <h2>Links:</h2> <p> <ul> <li>Article: <a target="_blank" href="https://addpipe.com/blog/mediarecorder-api/">https://addpipe.com/blog/mediarecorder-api/</a></li> <li>GitHub: <a target="_blank" href="https://github.com/addpipe/Media-Recorder-API-Demo">https://github.com/addpipe/Media-Recorder-API-Demo</a></li> <li>W3C Draft: <a target="_blank" href="http://w3c.github.io/mediacapture-record/MediaRecorder.html">http://w3c.github.io/mediacapture-record/MediaRecorder.html</a></li> <li>Media Recorder API at 65% penetration thanks to Chrome: <a target="_blank" href="https://addpipe.com/blog/media-recorder-api-is-now-supported-by-65-of-all-desktop-internet-users/">https://addpipe.com/blog/media-recorder-api-is-now-supported-by-65-of-all-desktop-internet-users/</a></li> </ul> </p> </div> </body> </html>
前端界面的效果图：
JS的代码（重点之一在这个JS里面的红色部分，下面代码是main.js的正文内容）：
'use strict'; /* globals MediaRecorder */ // Spec is at http://dvcs.w3.org/hg/dap/raw-file/tip/media-stream-capture/RecordingProposal.html navigator.getUserMedia = navigator.getUserMedia || navigator.webkitGetUserMedia || navigator.mozGetUserMedia || navigator.msGetUserMedia; if(getBrowser() == "Chrome"){ var constraints = {"audio": true, "video": { "mandatory": { "minWidth": 640, "maxWidth": 640, "minHeight": 480,"maxHeight": 480 }, "optional": [] } };//Chrome }else if(getBrowser() == "Firefox"){ var constraints = {audio: false, video: { width: { min: 640, ideal: 640, max: 640 }, height: { min: 480, ideal: 480, max: 480 }}}; //Firefox } var recBtn = document.querySelector('button#rec'); var pauseResBtn = document.querySelector('button#pauseRes'); var stopBtn = document.querySelector('button#stop'); var videoElement = document.querySelector('video'); var dataElement = document.querySelector('#data'); var downloadLink = document.querySelector('a#downloadLink'); videoElement.controls = false; function errorCallback(error){ console.log('navigator.getUserMedia error: ', error); } /* var mediaSource = new MediaSource(); mediaSource.addEventListener('sourceopen', handleSourceOpen, false); var sourceBuffer; */ var mediaRecorder; var chunks = []; var count = 0; var wsurl = "ws://10.90.9.20:9080/RDConsumer/websocket" var ws = null;
function createWs(){ var url = wsurl; if ('WebSocket' in window) { ws = new WebSocket(url); } else if ('MozWebSocket' in window) { ws = new MozWebSocket(url); } else { console.log("您的浏览器不支持WebSocket。"); return ; } } function init() { if (ws != null) { console.log("现已连接"); return ; } createWs(); ws.onopen = function() { //设置发信息送类型为：ArrayBuffer ws.binaryType = "arraybuffer"; } ws.onmessage = function(e) { console.log(e.data.toString()); } ws.onclose = function(e) { console.log("onclose: closed"); ws = null; createWs(); //这个函数在这里之所以再次调用，是为了解决视频传输的过程中突发的连接断开问题。 } ws.onerror = function(e) { console.log("onerror: error"); ws = null; createWs(); //同上面的解释 } } $(document).ready(function(){ init(); }) function startRecording(stream) { log('Start recording...'); if (typeof MediaRecorder.isTypeSupported == 'function'){ /* MediaRecorder.isTypeSupported is a function announced in https://developers.google.com/web/updates/2016/01/mediarecorder and later introduced in the MediaRecorder API spec http://www.w3.org/TR/mediastream-recording/ */
//这里涉及到视频的容器以及编解码参数，这个与浏览器有密切的关系 if (MediaRecorder.isTypeSupported('video/webm;codecs=vp9')) { var options = {mimeType: 'video/webm;codecs=h264'}; } else if (MediaRecorder.isTypeSupported('video/webm;codecs=h264')) { var options = {mimeType: 'video/webm;codecs=h264'}; } else if (MediaRecorder.isTypeSupported('video/webm;codecs=vp8')) { var options = {mimeType: 'video/webm;codecs=vp8'}; } log('Using '+options.mimeType); mediaRecorder = new MediaRecorder(stream, options); }else{ log('isTypeSupported is not supported, using default codecs for browser'); mediaRecorder = new MediaRecorder(stream); } pauseResBtn.textContent = "Pause"; mediaRecorder.start(10); var url = window.URL || window.webkitURL; videoElement.src = url ? url.createObjectURL(stream) : stream; videoElement.play();
//这个地方，是视频数据捕获好了后，会触发MediaRecorder一个dataavailable的Event，在这里做视频数据的采集工作，主要是基于Blob进行转写，利用FileReader进行读取。FileReader一定
//要注册loadend的监听器，或者写onload的函数。在loadend的监听函数里面，进行格式转换，方便websocket进行数据传输，因为websocket的数据类型支持blob以及arrayBuffer，我们这里用
//的是arrayBuffer，所以，将视频数据的Blob转写为Unit8Buffer，便于websocket的后台服务用ByteBuffer接收。 mediaRecorder.ondataavailable = function(e) { //log('Data available...'); //console.log(e.data); //console.log(e.data.type); //console.log(e); chunks.push(e.data); var reader = new FileReader(); reader.addEventListener("loadend", function() { //reader.result是一个含有视频数据流的Blob对象 var buf = new Uint8Array(reader.result); console.log(reader.result); if(reader.result.byteLength > 0){ //加这个判断，是因为有很多数据是空的，这个没有必要发到后台服务器，减轻网络开销，提升性能吧。 ws.send(buf); } }); reader.readAsArrayBuffer(e.data); }; mediaRecorder.onerror = function(e){ log('Error: ' + e); }; mediaRecorder.onstart = function(){ log('Started & state = ' + mediaRecorder.state); }; mediaRecorder.onstop = function(){ log('Stopped & state = ' + mediaRecorder.state); var blob = new Blob(chunks, {type: "video/webm"}); chunks = []; var videoURL = window.URL.createObjectURL(blob); downloadLink.href = videoURL; videoElement.src = videoURL; downloadLink.innerHTML = 'Download video file'; var rand = Math.floor((Math.random() * 10000000)); var name = "video_"+rand+".webm" ; downloadLink.setAttribute( "download", name); downloadLink.setAttribute( "name", name); }; mediaRecorder.onpause = function(){ log('Paused & state = ' + mediaRecorder.state); } mediaRecorder.onresume = function(){ log('Resumed & state = ' + mediaRecorder.state); } mediaRecorder.onwarning = function(e){ log('Warning: ' + e); }; } //function handleSourceOpen(event) { // console.log('MediaSource opened'); // sourceBuffer = mediaSource.addSourceBuffer('video/webm; codecs="vp9"'); // console.log('Source buffer: ', sourceBuffer); //}
//点击按钮，启动视频流的采集。重点是getUserMedia函数使用。本案例中，视频采集的入口，是点击页面上的record按钮，也就是下面这个函数的逻辑。 function onBtnRecordClicked (){ if (typeof MediaRecorder === 'undefined' || !navigator.getUserMedia) { alert('MediaRecorder not supported on your browser, use Firefox 30 or Chrome 49 instead.'); }else { navigator.getUserMedia(constraints, startRecording, errorCallback); recBtn.disabled = true; pauseResBtn.disabled = false; stopBtn.disabled = false; } } function onBtnStopClicked(){ mediaRecorder.stop(); videoElement.controls = true; recBtn.disabled = false; pauseResBtn.disabled = true; stopBtn.disabled = true; } function onPauseResumeClicked(){ if(pauseResBtn.textContent === "Pause"){ console.log("pause"); pauseResBtn.textContent = "Resume"; mediaRecorder.pause(); stopBtn.disabled = true; }else{ console.log("resume"); pauseResBtn.textContent = "Pause"; mediaRecorder.resume(); stopBtn.disabled = false; } recBtn.disabled = true; pauseResBtn.disabled = false; } function log(message){ dataElement.innerHTML = dataElement.innerHTML+'<br>'+message ; } //browser ID function getBrowser(){ var nVer = navigator.appVersion; var nAgt = navigator.userAgent; var browserName = navigator.appName; var fullVersion = ''+parseFloat(navigator.appVersion); var majorVersion = parseInt(navigator.appVersion,10); var nameOffset,verOffset,ix; // In Opera, the true version is after "Opera" or after "Version" if ((verOffset=nAgt.indexOf("Opera"))!=-1) { browserName = "Opera"; fullVersion = nAgt.substring(verOffset+6); if ((verOffset=nAgt.indexOf("Version"))!=-1) fullVersion = nAgt.substring(verOffset+8); } // In MSIE, the true version is after "MSIE" in userAgent else if ((verOffset=nAgt.indexOf("MSIE"))!=-1) { browserName = "Microsoft Internet Explorer"; fullVersion = nAgt.substring(verOffset+5); } // In Chrome, the true version is after "Chrome" else if ((verOffset=nAgt.indexOf("Chrome"))!=-1) { browserName = "Chrome"; fullVersion = nAgt.substring(verOffset+7); } // In Safari, the true version is after "Safari" or after "Version" else if ((verOffset=nAgt.indexOf("Safari"))!=-1) { browserName = "Safari"; fullVersion = nAgt.substring(verOffset+7); if ((verOffset=nAgt.indexOf("Version"))!=-1) fullVersion = nAgt.substring(verOffset+8); } // In Firefox, the true version is after "Firefox" else if ((verOffset=nAgt.indexOf("Firefox"))!=-1) { browserName = "Firefox"; fullVersion = nAgt.substring(verOffset+8); } // In most other browsers, "name/version" is at the end of userAgent else if ( (nameOffset=nAgt.lastIndexOf(' ')+1) < (verOffset=nAgt.lastIndexOf('/')) ) { browserName = nAgt.substring(nameOffset,verOffset); fullVersion = nAgt.substring(verOffset+1); if (browserName.toLowerCase()==browserName.toUpperCase()) { browserName = navigator.appName; } } // trim the fullVersion string at semicolon/space if present if ((ix=fullVersion.indexOf(";"))!=-1) fullVersion=fullVersion.substring(0,ix); if ((ix=fullVersion.indexOf(" "))!=-1) fullVersion=fullVersion.substring(0,ix); majorVersion = parseInt(''+fullVersion,10); if (isNaN(majorVersion)) { fullVersion = ''+parseFloat(navigator.appVersion); majorVersion = parseInt(navigator.appVersion,10); } return browserName; }
其中的byteLength的判断，是有原因的，前端打印的日志可以看出：
我的这个案例，用的是Firefox的浏览器，因为我本地的Chrome的版本比较新，在应用启动的时候爆出错误：
时间紧，没有深入研究这个错误，所以一直都是Firefox基础上进行验证的。
下面剩下的就是Java后台的Websocket的服务了。直接上代码：
/* * Copyright © reserved by roomdis.com, service for tgn company whose important business is rural e-commerce. */ package com.roomdis.mqr.infra.core; import java.io.IOException; import java.net.DatagramPacket; import java.net.DatagramSocket; import java.net.InetAddress; import java.net.SocketException; import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.websocket.OnClose; import javax.websocket.OnError; import javax.websocket.OnMessage; import javax.websocket.OnOpen; import javax.websocket.Session; import javax.websocket.server.ServerEndpoint; import org.apache.log4j.Logger; import org.springframework.web.context.ContextLoader; import com.google.gson.Gson; import com.roomdis.mqr.infra.msg.KefuMessage; /** * @author shihuc * @date 2017年8月22日 下午2:20:18 */ @ServerEndpoint("/websocket") public class WebsocketService { private static Logger logger = Logger.getLogger(WebsocketService.class); private HttpSendService httpSendService; private String videoRecServerHost = "10.90.7.10"; private int videoRecServerPort = 7667; /* * 当存在多个客户端访问时，为了保证会话继续保持，将连接缓存。 */ private static Map<String, WebsocketService> webSocketMap = new ConcurrentHashMap<String, WebsocketService>(); private Session session; private static final WebsocketService instance = new WebsocketService(); public static final WebsocketService getInstance() { return instance; } @OnMessage public void onTextMessage(String message, Session session) throws IOException, InterruptedException { // Print the client message for testing purposes logger.info("Received: " + message); //TODO: 调用接口将消息发送给客户端后台服务系统 Gson gson = new Gson(); KefuMessage kfMsg = gson.fromJson(message, KefuMessage.class); httpSendService = ContextLoader.getCurrentWebApplicationContext().getBean(HttpSendService.class); } /** * 主要用来接受二进制数据。 * * @author shihuc * @param message * @param session * @throws IOException * @throws InterruptedException */ @OnMessage public void onBinaryMessage(ByteBuffer message, Session session, boolean last) throws IOException, InterruptedException { byte [] sentBuf = message.array(); logger.info("Binary Received: " + sentBuf.length + ", last: " + last); //下面的代码逻辑，是用UDP协议发送视频流数据到视频处理服务器做后续逻辑处理 //sendToVideoRecognizer(sentBuf); } /** * @author shihuc * @param sentBuf * @throws SocketException * @throws UnknownHostException * @throws IOException */ private void sendToVideoRecognizer(byte[] sentBuf) throws SocketException, UnknownHostException, IOException { DatagramSocket client = new DatagramSocket(); InetAddress addr = InetAddress.getByName(videoRecServerHost); DatagramPacket sendPacket = new DatagramPacket(sentBuf, sentBuf.length, addr, videoRecServerPort); client.send(sendPacket); client.close(); } // @OnOpen // public void onOpen(Session session){ // this.session = session; // String staffId = session.getQueryString(); // webSocketMap.put(staffId, this); // logger.info(staffId + " client opened"); // } @OnOpen public void onOpen(Session session){ logger.info("client opened: " + session.toString()); } @OnClose public void onClose() { logger.info("client onclose"); } @OnError public void onError(Session session, Throwable error){ logger.info("connection onError"); logger.info(error.getCause()); } public boolean sendMessage(String message, String staffId) throws IOException{ WebsocketService client = webSocketMap.get(staffId); if (client == null) { return false; } boolean result=false; try { client.session.getBasicRemote().sendText(message); result=true; } catch (IOException e) { try { client.session.close(); } catch (IOException e1) { e1.printStackTrace(); } } return result; } }
这里，重点要注意的是，@OnMessage注解对应的函数，入参非常有讲究的。对于arrayBuffer的二进制数据类型，参数个数必须是三个，最后的boolean的必须有，否则前端发送数据的时候，浏览器上会抛出错误：
最后，看看后台运行的日志：
[2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false [2017-09-27 20:03:47] [ INFO] [http-nio-9080-exec-7] [com.roomdis.mqr.infra.core.WebsocketService.onBinaryMessage(WebsocketService.java:80)] - Binary Received: 7494, last: true
并附上一副前端运行的效果截图：
总结：
1. 重点研究getUserMedia。
2.重点研究MediaRecorder。
3.重点研究Blob以及FileReader。
4.重点研究Websocket的@OnMessage的注解函数的参数，以及数据传输中连接可能会断掉的处理方案。
2018-05-03
PS：既然有人对我这个研究有兴趣，我就将源码共享出来，帮助有需要的技术爱好者。源码地址在github上面：https://github.com/shihuc/VideoConverter
希望有兴趣的朋友，通过关注我的博客，共同互动，研究一些特别的应用！
