python根据用户所选id上传图片视频过对应算法(python实现大文件的分块上传)
1. 项目背景
由于项目中需要用户随时上传图片与视频，并且要求上传的图片或视频需要根据用户的意愿去过对应的算法，写了如下接口
2. 代码实现
2.1 通过django项目调用flask项目接口
@api_view(['POST'])
def deal_video(request):
response = {'state': 1, 'des': 'normal'}
if request.method == 'POST':
ips = []
ports = []
obj = request.FILES.get('obj')
p = dict(request.POST)
print(p)
ids = p['algo']
print(ids)
person = request.POST.get("person", '')
print(person)
time = request.POST.get("time")
f = open(obj.name, 'wb')
for line in obj.chunks():
f.write(line)
f.close()
for id in ids:
query = NodeAlgoManage.objects.filter(algo_id=id).values()
print(query[0]["ip"])
print(query[0]["port"])
ips.append(query[0]["ip"])
ports.append(query[0]["port"])
addr = []
para_data = dict()
for i in range(len(ips)):
addr.append(str(ips[i]) + ":" + str(ports[i]))
# addr.append('192.168.110.200:5002')
print(addr)
para_data = addr
obj_split = obj.name.split(".")
node_query = NodeManagement.objects.filter(node_type="upload").values()
host = node_query[0]["ip"]
port = int(node_query[0]["port"])
username = node_query[0]["username"]
password = node_query[0]["password"]
target_path = node_query[0]["relative_path"]
print(target_path)
transport = paramiko.Transport(host, port)
random_str = ''.join(random.sample(string.ascii_letters + string.digits, 32))
# 本地文件路径
local_file = obj.name
if obj_split[-1] in ["jpg", "png", "webp", "bmp", "tif", "gif"]:
f = open(obj.name, 'rb')
files = {"files": f}
a = dev_upload_image(files, para_data)
print(a)
f.close()
# 上传服务器文件路径
target_file = target_path + 'image/' + random_str + '.' + obj_split[-1]
if a == "True":
UploadAlgoResults.objects.create(person=person, type="image", path=target_file, time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=person, type="image", path=target_file, time=time, algoid=ids,
results="运行失败")
else:
obj_split = obj.name.split(".")
results = True
# cap = cv2.VideoCapture(local_file) # 读取视频
container = av.open(local_file)
stream = container.streams.video[0]
stream.codec_context.skip_frame = 'NONKEY'
for frame in container.decode(stream):
data = pickle.dumps(frame.to_image())
b = dev_upload_video(data, para_data)
if b != "True" or b != "[]":
results = not results
# else:
# cap.release()
# 上传服务器文件路径
target_file = target_path + 'video/' + random_str + '.' + obj_split[-1]
if results:
UploadAlgoResults.objects.create(person=person, type="video", path=target_file, time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=person, type="video", path=target_file, time=time, algoid=ids,
results="运行失败")
# # 建立连接
transport.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(transport)
# # 上传文件
print('===================================')
sftp.put(local_file, target_file)
os.remove(obj.name)
# 关闭连接
transport.close()
return HttpResponse("success")
值得注意的是，上述代码是通过处理视频中的关键帧所实现过算法处理，之前是逐帧过算法的，逐帧处理如下：
@api_view(['POST'])
def deal_video(request):
response = {'state': 1, 'des': 'normal'}
if request.method == 'POST':
ips=[]
ports=[]
obj = request.FILES.get('obj')
p=dict(request.POST)
print(p)
ids=p['algo']
print(ids)
person = request.POST.get("person", '')
print(person)
time = request.POST.get("time")
# ids = request.POST.get("ids",'')
f = open(obj.name, 'wb')
for line in obj.chunks():
f.write(line)
f.close()
for id in ids:
query = NodeAlgoManage.objects.filter(algo_id=id).values()
print(query[0]["ip"])
print(query[0]["port"])
ips.append(query[0]["ip"])
ports.append(query[0]["port"])
addr=[]
para_data=dict()
for i in range(len(ips)):
addr.append(str(ips[i])+":"+str(ports[i]))
# addr.append('192.168.110.200:5002')
print(addr)
# para_data=json.dumps(addr)
para_data=addr
# print(para_data)
# para_data=serializers.serialize(json, d)
obj_split = obj.name.split(".")
node_query = NodeManagement.objects.filter(node_type="upload").values()
host = node_query[0]["ip"]
port = int(node_query[0]["port"])
username = node_query[0]["username"]
password = node_query[0]["password"]
target_path = node_query[0]["relative_path"]
print(target_path)
transport = paramiko.Transport(host, port)
random_str = ''.join(random.sample(string.ascii_letters + string.digits, 32))
# 本地文件路径
local_file = obj.name
if obj_split[-1] in ["jpg","png","webp","bmp","tif","gif"]:
f = open(obj.name, 'rb')
files = {"files": f}
a=dev_upload_image(files,para_data)
print(a)
f.close()
# 上传服务器文件路径
target_file = target_path + 'image/' + random_str + '.' + obj_split[-1]
if a=="True":
UploadAlgoResults.objects.create(person=person,type="image",path=target_file,time=time,algoid=ids,results="运行成功")
else:
UploadAlgoResults.objects.create(person=person, type="image", path=target_file, time=time, algoid=ids,
results="运行失败")
else:
obj_split = obj.name.split(".")
results = True
cap = cv2.VideoCapture(local_file) # 读取视频
while cap.isOpened(): # 当视频被打开时：
ret, frame = cap.read() # 读取视频，读取到的某一帧存储到frame，若是读取成功，ret为True，反之为False
if ret: # 若是读取成功
data = pickle.dumps(frame)
b=dev_upload_video(data,para_data)
if b!="True"or b!="[]":
results = not results
else:
cap.release()
# os.remove(obj.name)
# 上传服务器文件路径
target_file = target_path + 'video/' + random_str + '.' + obj_split[-1]
if results:
UploadAlgoResults.objects.create(person=person, type="video", path=target_file, time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=person, type="video", path=target_file, time=time, algoid=ids,
results="运行失败")
# 建立连接
transport.connect(username=username, password=password)
sftp = paramiko.SFTPClient.from_transport(transport)
# 上传文件
print('===================================')
sftp.put(local_file, target_file)
os.remove(obj.name)
# 关闭连接
transport.close()
return HttpResponse("success")
其中涉及到的方法如下：
def dev_upload_video(data, json):
url = "http://192.168.110.200:5000/upload_video"
# print(data)
# print(json)
params=dict()
for i in range(len(json)):
params[i]=json[i]
# print(params)
respone = requests.post(url, data=data, params=params)
return respone.text
def dev_upload_image(files, json):
url = "http://192.168.110.200:5000/upload_image"
print(files)
print(json)
params=dict()
for i in range(len(json)):
params[i]=json[i]
print(params)
response = requests.post(url, files=files, params=params)
print(response.text)
print('======================')
return response.text
上述方法为调用flask项目中接口的方法这里主要是通过data与params实现传参，之前也有用过json，不过flask是接收不到的，至于为什么，我也不知道，有兴趣的可以研究一下，不过现在想着如果封装一下都放进data里也是可以的
2.2 通过flask项目调用docker内部项目接口
@app.route('/upload_image', methods=['POST'])
def upload_image():
obj = request.files.get('files')
print(obj)
print(request.args)
# print(request.get_json())
para_data = request.args
print(para_data["0"])
print('==========================')
obj.save("upload_image.jpg")
im = Image.open("upload_image.jpg")
results=True
for i in range(len(para_data)):
print(i)
url = "http://"+para_data[str(i)]
print(url)
data = pickle.dumps(im)
# print(data)
response = requests.post(url, data=data)
print(response.reason)
if response.reason!="OK":
results = not results
os.remove("upload_image.jpg")
# if response.text==200
print(results)
return str(results)
@app.route('/upload_video', methods=['POST'])
def upload_video():
print(request.args)
# print(request.get_json())
para_data = request.args
print(para_data["0"])
frame = request.get_data()
results = True
for i in range(len(para_data)):
url = "http://"+para_data[str(i)]
response = requests.post(url, data=frame)
print(response.reason)
if response.reason != "OK":
results = not results
return str(results)
这是flask项目中的方法，实现的主要功能就是调用flask中的算法，至于docker内部怎样封装算法，可以参考本人上一篇文章，最后将整体框架描述一下：django（大脑，只有一个）——flask（胳膊，可以有多个）——封装在docker内部的算法（可以有多个，一个算法封装在一个容器内）
3. 升级版2.0
由于需要，要求上传的文件至少最大可以达到1g，而上述的sftp显然已经满足不了需求，好像是因为sftp传输的文件一旦过大，那么是会开启多线程的，开启完又无法合并，所以需要另找他法，总之sftp报错了，错误只有EOFERROR。。。。。。。所以目前考虑的主要有scp与openssh，下面我将用openssh代替上述sftp，从而实现大文件的传输过算法
#定义ssh远程传输函数
def upload_part(num, offset, data):
#获取数据
print(f"Running thread {num}")
sftp_server = data["sftp_server"]
port = data["port"]
username = data["username"]
password = data["password"]
local_path = data["local_path"]
remote_path = data["remote_path"]
# print(data)
# print(remote_path)
size = os.path.getsize(local_path)
part_size = int(size / threads_count)
#开始连接跨服务器传输大文件
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(sftp_server, port=port, username=username, password=password)
sftp = ssh.open_sftp()
with open(local_path, "rb") as fl:
fl.seek(offset)
with lock:
global created
m = "r+" if created else "w"
created = True
fr = sftp.open(remote_path, m)
with fr:
fr.seek(offset)
fr.set_pipelined(True)
size = 0
while size < part_size:
s = 32768
if size + s > part_size:
s = part_size - size
data = fl.read(s)
fr.write(data)
size += len(data)
if len(data) == 0:
break
except (paramiko.ssh_exception.SSHException) as x:
print(f"Thread {num} failed: {x}")
print(f"Thread {num} done")
# 上传图片视频过算法的接口
# @ratelimit用于限制固定时间内接口调用次数
@api_view(['POST'])
@ratelimit(key='ip', rate='1000/m', block=True)
def deal_video(request):
response = {'state': 1, 'des': 'normal'}
#获取数据
if request.method == 'POST':
ips = []
ports = []
obj = request.FILES.get('obj')
p = dict(request.POST)
print(p)
ids = p['algo']
print(ids)
person = request.POST.get("person", '')
print(person)
time = request.POST.get("time")
log_ip = request.POST.get("log_ip")
#读取文件到本地
f = open(obj.name, 'wb')
for line in obj.chunks():
f.write(line)
f.close()
#开始拼接算法访问地址
for id in ids:
query = NodeAlgoManage.objects.filter(algo_id=id).values()
print(query[0]["ip"])
print(query[0]["port"])
ips.append(query[0]["ip"])
ports.append(query[0]["port"])
addr = []
for i in range(len(ips)):
addr.append(str(ips[i]) + ":" + str(ports[i]))
# addr.append('192.168.110.200:5002')
print(addr)
para_data = addr
#将文件名用.分隔，从而获取后缀与文件名
obj_split = obj.name.split(".")
#获取存储服务器地址与存储位置
node_query = NodeManagement.objects.filter(node_type="upload").values()
#准备与存储服务器建立连接的相关参数
data_upload = {}
print(data_upload)
data_upload["sftp_server"] = node_query[0]["ip"]
data_upload["port"] = int(node_query[0]["port"])
data_upload["username"] = node_query[0]["username"]
data_upload["password"] = node_query[0]["password"]
targete_path = node_query[0]["relative_path"]
print("0000000000000" + targete_path)
print(type(data_upload))
print(type(targete_path))
#生成存储到存储服务器的随机文件名
random_str = ''.join(random.sample(string.ascii_letters + string.digits, 32))
# 本地文件路径
local_file = obj.name
#判断上传文件类型 if图片 else视频
if obj_split[-1] in ["jpg", "png", "webp", "bmp", "tif", "gif"]:
#读取本地图片
f = open(obj.name, 'rb')
files = {"files": f}
#图片过算法
a = dev_upload_image(files, para_data)
print(a)
f.close()
# 上传服务器文件路径
data_upload["remote_path"] = targete_path + 'image/' + random_str + '.' + obj_split[-1]
#根据返回结果，判断算法的运行结果，并且存入数据库
if a == "True":
UploadAlgoResults.objects.create(person=person, type="image", path=data_upload["remote_path"], time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=person, type="image", path=data_upload["remote_path"], time=time, algoid=ids,
results="运行失败")
else:
#读取视频，并获取视频的关键帧
obj_split = obj.name.split(".")
results = True
# cap = cv2.VideoCapture(local_file) # 读取视频
container = av.open(local_file)
stream = container.streams.video[0]
stream.codec_context.skip_frame = 'NONKEY'
#将视频中的关键帧传入算法中
for frame in container.decode(stream):
data = pickle.dumps(frame.to_image())
b = dev_upload_video(data, para_data)
print(b)
if b != "True":
results = not results
# 上传服务器文件路径
data_upload["remote_path"] = targete_path + 'video/' + random_str + '.' + obj_split[-1]
print(results)
#根据返回结果，判断算法的运行结果，并且存入数据库
if results:
UploadAlgoResults.objects.create(person=person, type="video", path=data_upload["remote_path"], time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=person, type="video", path=data_upload["remote_path"], time=time, algoid=ids,
results="运行失败")
# 调用ssh传输方法进行传输，同时开启多线程传输
offset = 0
threads = []
size = os.path.getsize(local_file)
part_size = int(size / threads_count)
data_upload["local_path"] = local_file
for num in range(threads_count):
if num == threads_count - 1:
part_size = size - offset
args = (num, offset, data_upload)
print(f"Starting thread {num} offset {offset} size {part_size}")
thread = threading.Thread(target=upload_part, args=args)
threads.append(thread)
thread.start()
print(f"Started thread {num}")
offset += part_size
for num in range(len(threads)):
print(f"Waiting for thread {num}")
threads[num].join()
print("All thread done")
# 删除本地文件
os.remove(obj.name)
# 返回运行结果
# transport.close()
response['state'] = 0
response['des'] = '运行成功'
#在日志表中插入对应的日志信息
try:
username = Login.objects.filter(ip=log_ip).values("username")[0]["username"]
except Exception as e:
print(e)
username = '未知'
response['state'] = 5
response['des'] = "ip地址不正确"
data_log = {"username": username, "ope": "upload", "ip": log_ip, "des": "离线检测上传图片视频过算法",
"ope_time": datetime.datetime.now()}
add_log(data_log)
#记录运行结束时间，并返回结果
print("=================================================================")
print(datetime.datetime.now())
print("==================================================================")
return JsonResponse(response,safe=False)
注：上述仅是上传图片视频过算法的接口，期间用到的方法在上面，包括在master节点管理中调用flask中接口的方法、flask项目中flask调用docker上面算法的方法，至于docker里的算法怎么封装可以看我其他的文章。。。。。。。。。。。
升级版3.0（将flask项目中代码和并进django）
import datetime
import json
import os
import pickle
import random
import string
import threading
import av
import cv2
import paramiko
import requests as re
from django.contrib.sites import requests
from django.core import serializers
from django.http import HttpResponse, JsonResponse
from django.shortcuts import render
from django_ratelimit.decorators import ratelimit
from moviepy.video.io.VideoFileClip import VideoFileClip
from numpy.ma import clip
from paramiko import sftp_server
from rest_framework.decorators import api_view
from scp import SCPClient
from PIL import Image
from algoresults.models import UploadAlgoResults
from log.views import add_log
from login.models import Login
from master.models import NodeManagement, Node
from master.views import dev_upload_video
from risk.models import Risk
from django.core.paginator import Paginator
from util.crypto import EncDecAES
threads_count = 8
lock = threading.Lock()
created = False
def upload_part(num, offset, data):
print(f"Running thread {num}")
sftp_server = data["sftp_server"]
port = data["port"]
username = data["username"]
password = data["password"]
local_path = data["local_path"]
remote_path = data["remote_path"]
print(data)
print(remote_path)
size = os.path.getsize(local_path)
part_size = int(size / threads_count)
try:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(sftp_server, port=port, username=username, password=password)
sftp = ssh.open_sftp()
with open(local_path, "rb") as fl:
fl.seek(offset)
with lock:
global created
m = "r+" if created else "w"
created = True
fr = sftp.open(remote_path, m)
with fr:
fr.seek(offset)
fr.set_pipelined(True)
size = 0
while size < part_size:
s = 32768
if size + s > part_size:
s = part_size - size
data = fl.read(s)
fr.write(data)
size += len(data)
if len(data) == 0:
break
except (paramiko.ssh_exception.SSHException) as x:
print(f"Thread {num} failed: {x}")
print(f"Thread {num} done")
@api_view(['POST'])
@ratelimit(key='ip', rate='1000/m', block=True)
def deal_video(request):
response = {'state': 1, 'des': 'normal'}
if request.method == 'POST':
ips = []
ports = []
obj = request.FILES.get('obj')
token = request.POST.get("token")
time = request.POST.get("time")
log_ip = request.POST.get("log_ip")
user = Login.objects.filter(token_value=token).values()
p = dict(request.POST)
# print(p)
ids = p['algo']
f = open(obj.name, 'wb')
for line in obj.chunks():
f.write(line)
f.close()
for id in ids:
query = Node.objects.filter(algo_id=id).values()
print(query[0]["ip"])
print(query[0]["port"])
ips.append(query[0]["ip"])
ports.append(query[0]["port"])
addr = []
for i in range(len(ips)):
addr.append(str(ips[i]) + ":" + str(ports[i]))
# addr.append('192.168.110.200:5002')
obj_split = obj.name.split(".")
node_query = NodeManagement.objects.filter(node_type="upload").values()
data_upload = {}
print(data_upload)
data_upload["sftp_server"] = node_query[0]["ip"]
data_upload["port"] = int(node_query[0]["port"])
data_upload["username"] = node_query[0]["username"]
data_upload["password"] = node_query[0]["password"]
targete_path = node_query[0]["relative_path"]
random_str = ''.join(random.sample(string.ascii_letters + string.digits, 32))
params = dict()
for i in range(len(addr)):
params[i] = addr[i]
# 本地文件路径
local_file = obj.name
if obj_split[-1] in ["jpg", "png", "webp", "bmp", "tif", "gif"]:
f = open(obj.name, 'rb')
obj = local_file
im = Image.open(obj)
results = True
for i in range(len(params)):
print(i)
url = "http://" + params[i]
print(url)
data = pickle.dumps(im)
# print(data)
response = re.post(url, data=data)
print(response.reason)
if response.reason != "OK":
results = not results
os.remove("upload_image.jpg")
f.close()
print(results)
# 上传服务器文件路径
data_upload["remote_path"] = targete_path + 'image/' + random_str + '.' + obj_split[-1]
if results:
UploadAlgoResults.objects.create(person=user[0]['username'], type="image",
path=data_upload["remote_path"],
time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=user[0]['username'], type="image",
path=data_upload["remote_path"],
time=time, algoid=ids,
results="运行失败")
else:
obj_split = obj.name.split(".")
results = True
# cap = cv2.VideoCapture(local_file) # 读取视频
container = av.open(local_file)
stream = container.streams.video[0]
stream.codec_context.skip_frame = 'NONKEY'
for frame in container.decode(stream):
data = pickle.dumps(frame.to_image())
# b = dev_upload_video(data, params)
# para_data = params
frame = data
results = True
for i in range(len(params)):
url = "http://" + params[i]
response = re.post(url, data=frame)
print(response.reason)
if response.reason != "OK":
results = not results
# return str(results)
print(results)
# if not results:
# results = not results
# 上传服务器文件路径
data_upload["remote_path"] = targete_path + 'video/' + random_str + '.' + obj_split[-1]
print(results)
if results:
UploadAlgoResults.objects.create(person=user[0]['username'], type="video", path=data_upload["remote_path"],
time=time, algoid=ids,
results="运行成功")
else:
UploadAlgoResults.objects.create(person=user[0]['username'], type="video", path=data_upload["remote_path"],
time=time, algoid=ids,
results="运行失败")
# 建立连接
offset = 0
threads = []
size = os.path.getsize(local_file)
part_size = int(size / threads_count)
data_upload["local_path"] = local_file
for num in range(threads_count):
if num == threads_count - 1:
part_size = size - offset
args = (num, offset, data_upload)
print(f"Starting thread {num} offset {offset} size {part_size}")
thread = threading.Thread(target=upload_part, args=args)
threads.append(thread)
thread.start()
print(f"Started thread {num}")
offset += part_size
for num in range(len(threads)):
print(f"Waiting for thread {num}")
threads[num].join()
print("All thread done")
# 上传文件
os.remove(obj.name)
# 关闭连接
response['state'] = 0
response['des'] = '运行成功'
try:
username = Login.objects.filter(ip=log_ip).values("username")[0]["username"]
except Exception as e:
print(e)
username = '未知'
response['state'] = 5
response['des'] = "ip地址不正确"
data_log = {"username": username, "ope": "upload", "ip": log_ip, "des": "离线检测上传图片视频过算法",
"ope_time": datetime.datetime.now()}
add_log(data_log)
print("=================================================================")
print(datetime.datetime.now())
print("==================================================================")
return JsonResponse(response, safe=False)
参考文章：https://www.saoniuhuo.com/question/detail-2520316.html 用的是下面的一个回答，应该是将参数一改就可以用的
浙公网安备 33010602011771号