WGAI项目图像视频语音识别功能 - 实践
1、图像识别
<template>
<a-card class=
"contablelist" :bordered="false">
<
!-- 查询区域 -->
<div class=
"table-page-search-wrapper">
<a-form layout="inline" @keyup.enter.native="searchQuery">
<
/a-form>
<
/div>
<
!-- 查询区域-END -->
<div class=
"contable">
<
!-- 操作按钮区域 -->
<div class=
"table-operator">
<a-button @click="handleAdd" type="primary" class=
"xz" icon="plus">新增<
/a-button>
<a-button type="primary" icon="download" class=
"dc" @click="handleExportXls('模型绑定')">导出<
/a-button>
<a-upload name="file" :showUploadList="false" :multiple="false" :headers="tokenHeader" :action="importExcelUrl" @change="handleImportExcel">
<a-button type="primary"class=
"dr" icon="import">导入<
/a-button>
<
/a-upload>
<
!-- 高级查询区域 -->
<a-dropdown v-if="selectedRowKeys.length > 0">
<a-menu slot="overlay">
<a-menu-item key="1" @click="batchDel">
<a-icon type="delete"/>删除<
/a-menu-item>
<
/a-menu>
<a-button style="margin-left: 8px"> 批量操作 <a-icon type="down" />
<
/a-button>
<
/a-dropdown>
<
/div>
<
!-- table区域-begin -->
<div>
<div class=
"ant-alert ant-alert-info" style="margin-bottom: 16px;">
<i class=
"anticon anticon-info-circle ant-alert-icon">
<
/i> 已选择 <a
style="font-weight: 600">
{
{ selectedRowKeys.length
}
}<
/a>项
<a style="margin-left: 24px" @click="onClearSelected">清空<
/a>
<
/div>
<a-table ref="table" size="middle" :scroll="{x:true}" bordered rowKey="id" :columns="columns"
:dataSource="dataSource" :pagination="ipagination" :loading="loading"
:rowSelection="{selectedRowKeys: selectedRowKeys, onChange: onSelectChange}" class=
"j-table-force-nowrap"
@change="handleTableChange">
<template slot="htmlSlot" slot-scope="text">
<div v-html="text">
<
/div>
<
/template>
<template slot="imgSlot" slot-scope="text,record">
<span v-if="!text" style="font-size: 12px;font-style: italic;">无图片<
/span>
<img v-else :src="getImgView(text)" :preview="record.id" height="25px" alt=""
style="max-width:80px;font-size: 12px;font-style: italic;" />
<
/template>
<template slot="fileSlot" slot-scope="text">
<span v-if="!text" style="font-size: 12px;font-style: italic;">无文件<
/span>
<a-button v-else :ghost="true" type="primary" icon="download" size="small" @click="downloadFile(text)">
下载
<
/a-button>
<
/template>
<span slot="action" slot-scope="text, record">
<a @click="handleEdit(record)">编辑<
/a>
<a-divider type="vertical" />
<a @click="handleIdentify(record)">
AI识别<
/a>
<
!-- <a-divider type="vertical" v-if="record.spaceOne==='1'" />
<a v-if="record.spaceOne==='1'" @click="handleIdentifyClose(record)">视频识别结束<
/a>
<a-divider type="vertical" v-if="record.spaceOne==='1'" />
<a v-if="record.spaceOne==='1'" @click="handleOpenVideo(record)">视频区域报警配置<
/a>
-->
<a-divider type="vertical" />
<a-dropdown>
<a class=
"ant-dropdown-link">更多
<a-icon type="down" />
<
/a>
<a-menu slot="overlay">
<a-menu-item>
<a v-if="record.spaceOne==='1'" @click="handleOpenVideo(record)">视频区域报警配置<
/a>
<
/a-menu-item>
<a-menu-item>
<a v-if="record.spaceOne==='1'" @click="handleIdentifyClose(record)">视频识别结束<
/a>
<
/a-menu-item>
<a-menu-item>
<a @click="handleDetail(record)">详情<
/a>
<
/a-menu-item>
<a-menu-item>
<a-popconfirm title="确定删除吗?" @confirm="() => handleDelete(record.id)">
<a>删除<
/a>
<
/a-popconfirm>
<
/a-menu-item>
<
/a-menu>
<
/a-dropdown>
<
/span>
<
/a-table>
<
/div>
<tab-ai-model-bund-modal class=
"contc" :width="1200" ref="modalForm" @ok="modalFormOk">
<
/tab-ai-model-bund-modal>
<
/div>
<
/a-card>
<
/template>
<script>
import {
httpAction,
getAction
} from '@/api/manage'
import '@/assets/less/TableExpand.less'
import {
mixinDevice
} from '@/utils/mixin'
import {
JeecgListMixin
} from '@/mixins/JeecgListMixin'
import TabAiModelBundModal from './modules/TabAiModelBundModal'
import { filterObj
} from '@/utils/util';
export default {
name: 'TabAiModelBundList',
mixins: [JeecgListMixin, mixinDevice],
components: {
TabAiModelBundModal
},
data() {
return {
description: '模型绑定管理页面',
// 表头
columns: [{
title: '#',
dataIndex: '',
key: 'rowIndex',
width: 60,
align: "center",
customRender: function(t, r, index) {
return parseInt(index) + 1;
}
},
{
title: '识别名称',
align: "center",
dataIndex: 'spaceTwo'
},
{
title: '模型名称',
align: "center",
dataIndex: 'modelName_dictText'
}, {
title: '识别类型',
align: "center",
dataIndex: 'spaceOne_dictText'
},
{
title: '图片|视频地址',
align: "center",
dataIndex: 'sendUrl'
},
{
title: '图片显示',
align: "center",
dataIndex: 'saveUrl',
scopedSlots: {
customRender: 'imgSlot'
}
},{
title:'是否推送',
align:"center",
dataIndex: 'isPush'
}, {
title:'是否播报',
align:"center",
dataIndex: 'isAudio'
}, {
title:'推送地址',
align:"center",
dataIndex: 'pushId_dictText'
}, {
title:'播报地址',
align:"center",
dataIndex: 'audioId_dictText'
},
{
title: '备注',
align: "center",
dataIndex: 'remake'
},
{
title: '操作',
dataIndex: 'action',
align: "center",
fixed: "right",
width: 200,
scopedSlots: {
customRender: 'action'
}
}
],
url: {
list: "/tab/tabAiModelBund/list",
delete: "/tab/tabAiModelBund/delete",
deleteBatch: "/tab/tabAiModelBund/deleteBatch",
exportXlsUrl: "/tab/tabAiModelBund/exportXls",
importExcelUrl: "tab/tabAiModelBund/importExcel",
identifyUrl: "/tab/tabAiHistory/addIdentify",
identifyCloseUrl: "/tab/tabAiHistory/addIdentifyClose"
},
dictOptions: {
},
superFieldList: [],
}
},
created() {
this.getSuperFieldList();
},
computed: {
importExcelUrl: function() {
return `${window._CONFIG['domianURL']
}/${this.url.importExcelUrl
}`;
},
},
methods: {
initDictConfig() {
},
getSuperFieldList() {
let fieldList = [];
fieldList.push({
type: 'string',
value: 'modelName',
text: '模型名称',
dictCode: ''
})
fieldList.push({
type: 'string',
value: 'sendUrl',
text: '输入图片地址',
dictCode: ''
})
fieldList.push({
type: 'string',
value: 'saveUrl',
text: '保存图片地址',
dictCode: ''
})
fieldList.push({
type: 'string',
value: 'remake',
text: '备注',
dictCode: ''
})
this.superFieldList = fieldList
},
handleIdentify(info) {
console.log("info", this.url);
let that = this;
this.$confirm({
title: "确认识别吗",
content: "手动触发图片识别只会生成一次结果! 但视频会识别到结束",
onOk: function() {
let httpurl = '';
let method = '';
// debugger;
httpurl += that.url.identifyUrl;
method = 'post';
httpAction(httpurl, info, method).then((res) =>
{
if (res.success) {
that.$message.success(res.message);
that.$emit('ok');
} else {
that.$message.warning(res.message);
}
}).finally(() =>
{
that.confirmLoading = false;
})
}
});
},
handleIdentifyClose(info) {
let that = this;
this.$confirm({
title: "确认结束视频结束吗",
content: "结束视频识别结果输出!",
onOk: function() {
let httpurl = '';
let method = '';
// debugger;
httpurl += that.url.identifyCloseUrl;
method = 'post';
httpAction(httpurl, info, method).then((res) =>
{
if (res.success) {
that.$message.success(res.message);
that.$emit('ok');
} else {
that.$message.warning(res.message);
}
}).finally(() =>
{
that.confirmLoading = false;
})
}
});
},
handleOpenVideo(info){
this.$router.push('livecanvas/AddressList');
}
}
}
<
/script>
<style src="@assets/zwyStyle/css/main.css">
<
/style>
<style scoped>
@import '~@assets/less/common.less';
/deep/ .ant-table-scroll{height: calc(100vh - 337px);
}
<
/style>前端就一个点击事件，主要是查看后端识别图片信息的功能。
@Override
public Result<
String> startAi(TabAiModelBund tabAiModelBund, String path, String userId) {
TabAiModel aiModel=modelMapper.selectById(tabAiModelBund.getModelName());
if(aiModel!=null){
switch (aiModel.getSpareOne()){
case "1": //v3
{
log.info("【进入V3开始识别内容】{}",tabAiModelBund.getSpaceTwo());
if(tabAiModelBund.getSpaceOne().equals("0")){
//当前为图片
int a=this.saveIdentify(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
}else{
// 输出视频
// tabAiHistoryService.saveIdentifyVideo(tabAiModelBund,uploadpath);
// 输出坐标 延迟3-5s
//tabAiHistoryService.saveIdentifyLocalVideo(tabAiModelBund,uploadpath,sysUser.getId());
//多线程输出坐标
this.saveIdentifyLocalVideoThread(tabAiModelBund,path,userId);
return Result.OK("视频识别开始");
}
break;
}
case "2":
{
log.info("【进入V5开始识别内容】{}",tabAiModelBund.getSpaceTwo());
if(tabAiModelBund.getSpaceOne().equals("0")){
//当前为图片
if(tabAiModelBund.getSpaceTwo().indexOf("车牌")>
-1||tabAiModelBund.getSpaceTwo().indexOf("区域")>
-1){
log.info("【进入车牌识别内容】{}",tabAiModelBund.getSpaceTwo());
int a=this.saveCarIdentifyV5(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
}else{
int a=this.saveIdentifyYolov5(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
}
}else{
// 输出视频
// tabAiHistoryService.saveIdentifyVideo(tabAiModelBund,uploadpath);
// 输出坐标 延迟3-5s
//tabAiHistoryService.saveIdentifyLocalVideo(tabAiModelBund,uploadpath,sysUser.getId());
//多线程输出坐标
this.saveIdentifyLocalVideoThreadV5(tabAiModelBund,path,userId);
return Result.OK("视频识别开始");
}
break;
}//v5
case "3":
{ log.info("【进入V8开始识别内容】{}",tabAiModelBund.getSpaceTwo());
log.info("【进入V8开始识别内容】{}",tabAiModelBund.getSpaceTwo());
if(tabAiModelBund.getSpaceOne().equals("0")){
//当前为图片
int a=this.saveIdentifyYolov8(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
}else{
// 输出视频
// tabAiHistoryService.saveIdentifyVideo(tabAiModelBund,uploadpath);
// 输出坐标 延迟3-5s
//tabAiHistoryService.saveIdentifyLocalVideo(tabAiModelBund,uploadpath,sysUser.getId());
//多线程输出坐标
this.saveIdentifyLocalVideoThread(tabAiModelBund,path,userId);
return Result.OK("视频识别开始");
}
}//v8
case "4": {
break;
}//json
case "5": {
break;
}//other
case "6": {
//cv
log.info("【进入cv开始识别内容】{}",tabAiModelBund.getSpaceTwo());
if(tabAiModelBund.getSpaceTwo().equals("车牌识别")){
int a=this.saveCarIdentify(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
}
break;
}//json
case "7":{
log.info("【进入OCR文字识别内容】{}",tabAiModelBund.getSpaceTwo());
int a=this.saveStr(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
break;
}
case "9":{
log.info("【进入音频内容】{}",tabAiModelBund.getSpaceTwo());
int a=this.saveAudioStr(tabAiModelBund,path);
if(a==0){
return Result.OK("识别图片成功！");
}
break;
}
}
}
return Result.error("识别失败未发现识别内容");
}图像识别的方法：
/***
* AI模型嵌套模型
* 需要绝对路径
* 输入图片
*/
public static String SendPicYoloV3(String weight, String cfg, String names, String picUrl, String saveName, String uploadpath) throws Exception {
log.info(uploadpath);
Long a=System.currentTimeMillis();
// 加载类别名称
List<
String> classNames = Files.readAllLines(Paths.get(uploadpath+ File.separator +names));
// 加载YOLOv3模型
log.info("cfg地址{}",uploadpath+ File.separator +cfg);
log.info("weight地址{}",uploadpath+ File.separator +weight);
Net net = Dnn.readNetFromDarknet(uploadpath+ File.separator +cfg, uploadpath+ File.separator +weight);
// net.setPreferableBackend(Dnn.DNN_BACKEND_CUDA);
// net.setPreferableBackend(Dnn.DNN_TARGET_CUDA);
net.setPreferableBackend(Dnn.DNN_BACKEND_OPENCV);
net.setPreferableTarget(Dnn.DNN_TARGET_CPU);
// 读取输入图像
log.info("图片地址{}",uploadpath+ File.separator +picUrl);
Mat image = Imgcodecs.imread(uploadpath+ File.separator +picUrl);
// 将图像传递给模型进行目标检测
Mat blob = Dnn.blobFromImage(image, 1.0 / 255, new Size(416, 416), new Scalar(0), true, false);
net.setInput(blob);
// 将图像传递给模型进行目标检测
List<
Mat> result = new ArrayList<
>();
List<
String> outBlobNames = net.getUnconnectedOutLayersNames();
net.forward(result, outBlobNames);
// 处理检测结果
float confThreshold = 0.5f;
List<
Rect2d> boundingBoxes = new ArrayList<
>();
List<
Float> confidences = new ArrayList<
>();
List<
Integer> classIds = new ArrayList<
>();
for (Mat level : result) {
for (int i = 0; i < level.rows();
++i) {
Mat row = level.row(i);
Mat scores = level.row(i).colRange(5, level.cols());
Core.MinMaxLocResult minMaxLocResult = Core.minMaxLoc(scores);
Point classIdPoint = minMaxLocResult.maxLoc;
double confidence = row.get(0, 4)[0];
if (confidence > confThreshold) {
// log.info("classIdPoint"+ classIdPoint);
// log.info("classIdPointx"+ classIdPoint.x);
classIds.add((int) classIdPoint.x);
//记录标签下标
double centerX = row.get(0, 0)[0] * image.cols();
double centerY = row.get(0, 1)[0] * image.rows();
double width = row.get(0, 2)[0] * image.cols();
double height = row.get(0, 3)[0] * image.rows();
double left = centerX - width / 2;
double top = centerY - height / 2;
// 绘制边界框
Rect2d rect = new Rect2d(left, top, width, height);
boundingBoxes.add(rect);
confidences.add((float)confidence);
}
}
}
// 执行非最大抑制，消除重复的边界框
MatOfRect2d boxes = new MatOfRect2d(boundingBoxes.toArray(new Rect2d[0]));
MatOfFloat confidencesMat = new MatOfFloat();
confidencesMat.fromList(confidences);
MatOfInt indices = new MatOfInt();
Dnn.NMSBoxes(boxes, confidencesMat, confThreshold, 0.4f, indices);
if(indices.empty()){
log.info("类别下标啊"+"未识别到内容");
return "error";
}
int[] indicesArray= indices.toArray();
// 获取保留的边界框
log.info(confidences.size()+"类别下标啊"+indicesArray.length);
// 在图像上绘制保留的边界框
int c=0;
for (int idx : indicesArray) {
Rect2d box = boundingBoxes.get(idx);
System.out.println("绘制111111"+"x:"+box.x+"y:"+ box.y+"");
System.out.println("绘制11111111"+"width:"+box.width+"y:"+ box.height+"");
Imgproc.rectangle(image, new Point(box.x, box.y), new Point(box.x + box.width, box.y + box.height),CommonColors(c), 2);
// 添加类别标签
log.info("当前有多少"+confidences.get(idx));
Integer ab=classIds.get(idx);
log.info("类别下标"+ab);
// AIModelYolo3.addChineseText(image, caption,new Point(box.x, box.y - 5));
Imgproc.putText(image, classNames.get(ab), new Point(box.x, box.y - 5), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, CommonColors(c), 1);
c++;
}
String savepath=uploadpath + File.separator + "temp" + File.separator;
if(StringUtils.isNotBlank(saveName)){
savepath+=saveName+".jpg";
}else{
saveName=System.currentTimeMillis()+"";
savepath+=saveName+".jpg";
}
log.info(savepath);
Imgcodecs.imwrite(savepath, image);
Long b=System.currentTimeMillis();
log.info("消耗时间："+(b-a));
return saveName+".jpg";
}
主要调用opencv中的图像方法来识别图片的内容结果。
2、视频识别
待完善。。。。。。
浙公网安备 33010602011771号