mongodb总用方法总结
mongodb总用方法总结
pymongo存在更新不存在插入
{'$set': data, '$setOnInsert': {'create_time': datetime.datetime.now()}}
$setOnInsert里面使用的字段是数据不存在的时候才插入的，存在就不动了，只插入$set里面指定的。
另外$setOnInsert里面使用的字段不能在$set里面再次出现
upsert=True代表的是不存在就插入。
res_content = collection_v2_pongo_keyword.find(filter_, no_cursor_timeout=True, batch_size=5000).allow_disk_use(True)
mongodb中如何删除字段
删除mongo表中字段的方法：
|
1
|
db.collection.update({},{$unset:{"需要删除的字段":""}},false,true) |
update命令格式：
|
1
|
db.collection.update(criteria,objNew,upsert,multi) |
参数说明：
criteria：查询条件
objNew：update对象和一些更新操作符
upsert：如果不存在update的记录，是否插入objNew这个新的文档，true为插入，默认为false，不插入。
multi：默认是false，只更新找到的第一条记录。如果为true，把按条件查询出来的记录全部更新。
collection_v2_pongo_keyword.find(filter_, no_cursor_timeout=True, batch_size=5000).allow_disk_use(True)
# 删除发布list中指定id 的数据. 删除数组对象中指定的数据
"$pull": {"publishlist": {"id": publishlist_id}}
// aggregate db.lily_k_infomation.aggregate([ {$project: { // 字符串拼接 _id:{"$toString":"$_id"}, infoid: "$id", keyword: 1, path: { "$concat" : ["/detail", "$urlpath"]}, // 修改字段名称, 必须引号包裹 // aid: "$allianceid",
// 计算数字长度
aids : {"$size": "$aids"}
} }, // {$match: {}}, ])
db.pongo_wordgroup.aggregate([ { '$match': {'owner': 'rshy'} }, { '$project': { // 不能出现为0的 _id可以 其余字段需要手动写出 //'state': 0, "_id": 0, "name": 1, "id": {"$toString": "$_id"}, 'itemnum': {'$size': '$items'}, // "name": 1, } }, ])
db.v2_pongo_keyword.aggregate([
{"$match": {"industryid2": "10421082"}},
{
"$project": {
"_id": 0,
"id": {"$toObjectId": "$wordgroupid"},
"orgid": 1,
"keyword": 1,
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"wordgroupid": 1,
"baiduindex": 1,
// "createinfo": 1,
}
},
{
"$lookup": {
"from": "v2_pongo_wordgroup",
"localField": "id",
"foreignField": "_id",
"as": "inventory_docs"
},
},
{
"$project": {
"_id": 0,
"orgid": 1,
"keyword": 1,
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"wordgroupid": 1,
"baiduindex": 1,
"createinfo": "$inventory_docs.createinfo",
}
},
], { allowDiskUse: true})
// 查询 db.lily_infomation.find({"links": null}) // 修改字段名称 db.getCollection("lily_infomation").update({}, {$rename: {"specialtopics": "topic"}}, {multi: true})
在MongoDB操作的时候，我们可能会遇到查询【字段对应的数组中包含某个值】的数据，查询语句如下，假设表名为user。
示例1
- 数据
{
id: 1,
state_arr: [
"123",
"456",
"789"
],
date: "2019-09-02"
}
- 查询语句
db.user.find({state_arr:{$elemMatch:{$eq:"123"}}})
示例2
- 数据
{
id: 1,
state_arr: [
{
id: 1,
name: "hhh"
},
{
id: 2,
name: "www"
}
],
date: "2019-09-02"
}
db.lily_infomation.aggregate([
{ '$match': { "tags": {"$elemMatch" : {"id": {"$in" : [11]} } } } },
{ '$sort': { datecreated: -1 } },
{ '$limit': 20 },
])
db.lily_infomation.aggregate([
{ '$match': { "tags": {"$all" : [{"id":1, "name": 3}, {"id":2, "name": 2}] } } },
{ '$sort': { datecreated: -1 } },
{ '$limit': 20 },
])
- 查询语句
db.user.find({state_arr:{$elemMatch:{id:1}}})
// forEach db.getCollection("lily_infomation").find({}).forEach(function(item){ var Id = item._id; var infoid = item.infoid; print(Id, infoid) var li = [] db.lily_specialtopicinformation_new.find( { InfoId: infoid }).forEach(function(item_label){ var SpecialTopicId = item_label.SpecialTopicId; //print(SpecialTopicId); db.lily_specialtopicalliance.find( { id: SpecialTopicId }).forEach(function(item_label_){ var li_id = item_label_.id; var li_name = item_label_.words; //print(li_id, li_name) li.push({"id": li_id, "name": li_name, "urlpath": "/peixun/k_" + li_id + ".html" }) }); // print(li); }); db.getCollection("lily_infomation").update( { _id: Id }, { $set: { topic: li } }); }); // 创建索引 db.publish_tp.createIndex({"uuid": 1}) // or db.syd.find({"$or": [{"lable1": "综合"}, {"lable1": "少儿"}]}) // group db.syd.aggregate([{$group : {_id : "$lable1", total : {$sum : 1}}}]) db.fanwen_c.aggregate([ { $group : {_id : "$status", num_tutorial : {$sum : 1}}, }, { $sort: {_id: 1} } ]) // regex db.publish.find({"title": {"$regex":"_"}, "status": 22}) // $lt $gt db.publish_tp.find({"status": 1, "dateCreated": {"$lt": "2021-06-01"}}).sort({"dateCreated": -1}).count() // count db.asx.find().count() // $rename 修改字段名称 db.getCollection("lily_infomation").update({}, {$rename: {"specialtopics(旧字段名)": "topic(新字段名)"}}, {multi: true}) // // 数组去重 function function unique (arr) { return Array.from(new Set(arr)) } db.getCollection("lily_teacher").update( { _id: item._id }, { $set: { labels: unique(new_labelid) } });
将一个对象数组挨个塞入数组中
params = {
"$set": temp,
"$addToSet": {"labellist": {"$each": labellist}}
}
// $lookup
var db_name = await getDBName(allianceId); var dbo = db.db(db_name); var filter = { labelid1:labelid1, labelid2:labelid2, labelid3:labelid3 }; var optional = [ { $match: filter }, { $lookup: { from: "lily_areacode", localField: "areacode", foreignField: "areacode", as: "docs" } }, { $unwind: "$docs" }, { $project: { _id: 0, province: "$docs.province", city: "$docs.city", longitude: "$docs.longitude", latitude: "$docs.latitude" } } ]; var allow = { allowDiskUse: true}; console.log(optional); var result = await dbo.collection("lily_tdk_list").aggregate(optional,allow).toArray();
var optional = [ { $match: filter }, { $sort: {datecreated: -1} }, { $limit: size }, { $project: { 'allianceid':1, "infoid":1, "title": 1, "datecreated": 1, "dateupdated": 1, "content": { $function: { body: 'function(name) { \ let d = name[0] ; \ let content = d.content; \ content = content.replace(/<[^>]+>/g,""); \ return content.substr(0, 150) ; \ }', args: [ "$contents" ], lang: "js" } }, // 新增回复了量字段 "reply": { "$size": "$contents"}, "urlpath": 1, "imagepath":1, "labelid1": 1, "labelid2": 1, "labelid3": 1, "labelname1": 1, "labelname2": 1, "labelname3": 1, "teachershowid":1, "teachernickname": 1, "teacherheadimage":1, "infotype": 1, "show": "$view.show" } }];
var optional = [ { $limit: page*size },{ $skip: (page-1)*size }, { $project: { '_id':0, 'infoid':1, "title": 1, "datecreated": 1, "urlpath": 1 } }];
var optional = [ { $match: filter }, {"$unwind":"$info"}, { $limit: 100 }, { $match: filter2 }, { $project: { kemu: 1, period: 1, knowledge:{ $function:{ body:'function(maps){\ for(var i in maps){\ if(maps[i].indexOf("知识点")!=-1){\ return maps[i];\ } \ }\ }', args: [ "$info.zsd" ], lang: "js" } } } }];
var optional = [ { $match: filter }, { $project: { '_id':1, "keyword":1, "urlpath": 1, "labelid1": 1, "labelid2": 1, } }, { $sample: { size: size } } ];
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"name": 1,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"itemnum": {f"$size": f"$items"},
"ii": {
"$function": {
"body": 'function(name) { \
return name + "------"; \
}',
"args": ["$name"],
"lang": "js"
}
}
}
},
]
print(pipeline)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
// 对数组中全部元素进行修改
db.getCollection("pongo_content").update({"templateid": "simian-vpc-96232940694139658ae408deec27ae51", "titles": {"$elemMatch": {"id": "simian_vpt-527d8d3cad874dc598de0d91e1fa99fe"}}}, { "$set": { "titles.$.status": 1, } } )
// 对数组中指定元素进行修改 db.getCollection("pongo_content").update({ "templateid": "simian-vpc-96232940694139658ae408deec27ae51", }, { "$set": { "titles.$[].status": 1, } }, // { "multi": true } )
// 对数组中满足条件元素进行修改 db.getCollection("pongo_content").update({ "templateid": "simian-vpc-96232940694139658ae408deec27ae51", }, { "$set": { "titles.$[elem].status": 2, } }, { "multi": true, arrayFilters: [ { "elem.status": 222222 } ] } )
参考：https://docs.mongodb.com/manual/reference/operator/update/positional-filtered/
【MongoDB】查询字段对应的数组中包含某个值 在MongoDB操作的时候，我们可能会遇到查询【字段对应的数组中包含某个值】的数据，查询语句如下，假设表名为user。 示例1 数据 { id: 1, state_arr: [ "123", "456", "789" ], date: "2019-09-02" } 查询语句 db.user.find({state_arr:{$elemMatch:{$eq:"123"}}}) 示例2 数据 { id: 1, state_arr: [ { id: 1, name: "hhh" }, { id: 2, name: "www" } ], date: "2019-09-02" } 查询语句 db.user.find({state_arr:{$elemMatch:{id:1}}})
if aid: filter_.update({"publishlist": {"$elemMatch": {"aid": str(aid)}}}) pipeline = [ {"$match": filter_}, {"$sort": {"keyword": 1}}, {"$limit": page * size}, {"$skip": (page - 1) * size}, {"$unwind": {"path": "$publishlist", "preserveNullAndEmptyArrays": True}}, # { # "$group": { # "_id": "$publishlist.aid", # }, # }, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, if aid: filter_.update({"publishlist": {"$elemMatch": {"aid": str(aid)}}}) pipeline = [ {"$match": filter_}, {"$sort": {"keyword": 1}}, {"$limit": page * size}, {"$skip": (page - 1) * size}, {"$unwind": {"path": "$publishlist", "preserveNullAndEmptyArrays": True}}, # { # "$group": { # "_id": "$publishlist.aid", # }, # }, { "$project": { # _id 可以设置0 别的参数不可以 "_id": 0, "owner": 1, "wordgroup": 1, "wordgroupid": 1, "ownerid": 1, "id": {f"$toString": f"$_id"}, "industry1": 1, "industryid1": 1, "industry2": 1, "industryid2": 1, "keyword": 1, "area": 1, "orgid": 1, "orgname": 1, "baiduindex": 1, "paid": "$publishlist.id", "aid": "$publishlist.aid", "aname": "$publishlist.aname", "labelname1": "$publishlist.labelname1", "labelname2": "$publishlist.labelname2", "labelname3": "$publishlist.labelname3", } }, ] if aid: pipeline.insert(-1, {"$match": {"publishlist.aid": str(aid)}},) "paid": "$publishlist.id", "aid": "$publishlist.aid", "aname": "$publishlist.aname", "labelname1": "$publishlist.labelname1", "labelname2": "$publishlist.labelname2", "labelname3": "$publishlist.labelname3", } }, ] if aid: pipeline.insert(-1, {"$match": {"publishlist.aid": str(aid)}},)
# _*_ coding: utf-8 _*_
from django.shortcuts import render, HttpResponse
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt, csrf_protect
# Create your views here.
import os
import oss2
import uuid
import sys
import ast
import time
import json
import redis
import hashlib
import requests
import random
from tools.mysql_client import LogMysqlApe, LogMysql, LogMysqlApeT, LogMysqlAlliance
from tools.excel_client import read_excel
from upload_pro.settings import logger
from urllib.parse import urlparse, unquote
from django.conf import settings
from tools.mongodb_client import MongoDBClient
from bson import ObjectId
from bson import json_util
from bson.json_util import dumps
import pymongo
from pymongo.errors import DuplicateKeyError
from tools.format_time import dateRange
def get_uuid():
res = str(uuid.uuid4())
UUID = ''.join(res.split('-'))
return UUID
def md5_me(key):
md5 = hashlib.md5()
md5.update(str(key).encode('utf-8'))
value = md5.hexdigest()
return value
# 把格式化时间转换成时间戳
def str_to_timestamp(str_time=None, format='%Y-%m-%d %H:%M:%S'):
if str_time:
time_tuple = time.strptime(str_time, format) # 把格式化好的时间转换成元祖
result = time.mktime(time_tuple) # 把时间元祖转换成时间戳
return int(result)
return int(time.time())
# 把时间戳转换成格式化
def timestamp_to_str(timestamp=None, format='%Y-%m-%d %H:%M:%S'):
if timestamp:
time_tuple = time.localtime(timestamp) # 把时间戳转换成时间元祖
result = time.strftime(format, time_tuple) # 把时间元祖转换成格式化好的时间
return result
else:
return time.strftime(format)
# redis主库
class Redis(object):
conn = None
def __init__(self):
# poll = redis.ConnectionPool(host='47.94.218.171', port=6379, db=11, password='root1234@A')
# 生产 2 为测试 3为线上
poll = redis.ConnectionPool(host='192.168.5.219', port=6379, db=settings.APE_REDIS_DB, password='root1234@A', decode_responses=True)
# 本地测试
# poll = redis.ConnectionPool(host='192.168.10.10', port=7000, db=11)
self.conn = redis.Redis(connection_pool=poll)
ape_db = settings.APE_DB
db = settings.PONGO_DB
pongo_wordgroup = "pongo_wordgroup"
pongo_content = "pongo_content"
pongo_category = "pongo_category"
pongo_person = "pongo_person"
pongo_industry = "pongo_industry"
pongo_keyword = "pongo_keyword"
pongo_title = "pongo_title"
pongo_industry_ids = "pongo_industry_ids"
pongo_tdk_title = "pongo_tdk_title"
pongo_alliance = "pongo_alliance"
pongo_publish_task = "pongo_publish_task"
simian_viewpoint_table = "simian_viewpoint_table"
simian_wordgroup_table = "simian_wordgroup_table"
simian_vptitle_table = "simian_vptitle_table"
simian_wordgroup_viewpoint_table = "simian_wordgroup_viewpoint_table"
simian_vpcontent_table = "simian_vpcontent_table"
pongo_produce_vpc_vpt = "pongo.list.produce_vpc_vpt"
CODE_DICT = {
"0000": "成功",
"1000": "不存在标题id",
"1010": "操作无权限",
"1011": "此一级行业已经分配二级行业，不能删除!",
"1012": "行业已分配角色，不能删除!",
"1013": "无此机构数据!",
"1014": "titles字段数据有误!",
"1015": "该词分类名称已存在!",
"1016": "没有机构，故没有数据!",
"1017": "没有人物，故没有数据!",
"1018": "此关键词已插入!",
"1019": "需要带行业进行筛选!",
"1020": "传入数据为空!",
"1021": "请选择行业!",
"1022": "标题重复!",
"1023": "不满足生成替换文的条件!",
"1024": "不满足删除条件!",
"1025": "没有权限!",
"1026": "数据有误!",
}
# 词组列表
@csrf_exempt
def getWordgroupList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
filter_ = {}
if ownerid:
filter_ = {"ownerid": ownerid}
if owner:
filter_ = {"owner": owner}
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"name": 1,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"itemnum": {f"$size": f"$items"},
"ii": {
"$function": {
"body": 'function(name) { \
return name + "------"; \
}',
"args": ["$name"],
"lang": "js"
}
}
}
},
]
print(pipeline)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词组详情
@csrf_exempt
def getWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
wordgroupid = request.GET.get('wordgroupid', '')
name = request.GET.get('wordgroupname', '')
items = request.GET.get('items', [])
owner = request.GET.get('owner', '')
ownerid = str(request.GET.get('ownerid', ''))
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industry2 = request.GET.get('industry2', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
filter_ = {"orgid": orgid, "name": name}
params = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"items": 1,
"owner": 1,
"ownerid": 1,
}
print(filter_)
# res = collection.find_one(filter_, no_cursor_timeout=True)
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 新增词组
@csrf_exempt
def newWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
name = get_data.get('name', '')
items = get_data.get('items', [])
owner = get_data.get('owner', '')
ownerid = str(get_data.get('ownerid', ''))
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
industry1 = get_data.get('industry1', '')
industryid1 = get_data.get('industryid1', '')
industry2 = get_data.get('industry2', '')
industryid2 = get_data.get('industryid2', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
filter_ = {"orgid": orgid, "name": name}
res_filter = collection.find_one(filter_)
if res_filter:
for item in items:
params = {
"$addToSet": {"items": item},
"$set": {
"dateupdated": TIME,
},
}
collection.update_one(filter_, params)
else:
data = {
"name": name,
"industry1": industry1,
"industryid1": industryid1,
"industry2": industry2,
"industryid2": industryid2,
"items": items,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 10,
"state": 0
}
collection.insert_one(data)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": "",
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改词组
@csrf_exempt
def updateWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
name = get_data.get('name', '')
items = get_data.get('items', [])
wordgroupid = get_data.get('wordgroupid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
params = {
"$set": {
"name": name,
"items": items,
"dateupdated": TIME,
},
}
filter_ = {"_id": ObjectId(wordgroupid)}
res = collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": wordgroupid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 删除词组
@csrf_exempt
def deleteWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
name = get_data.get('name', '')
items = get_data.get('items', [])
wordgroupid = get_data.get('wordgroupid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
collection_del = db_[pongo_wordgroup+'_del']
filter_ = {"_id": ObjectId(wordgroupid)}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
res = collection.delete_one(filter_)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": wordgroupid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 文章模板列表
@csrf_exempt
def getContentList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
categoryid = request.GET.get('categoryid', '')
category = request.GET.get('category', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = filter_2 = {}
if ownerid:
filter_ = {"ownerid": ownerid}
if owner:
filter_ = {"owner": owner}
if categoryid:
filter_2 = {"categoryid": categoryid}
if category:
filter_2 = {"category": category}
filter_.update(filter_2)
pipeline = [
{"$match": filter_},
{"$sort": {"datecreated": -1}},
{
"$project": {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"description": 1,
"owner": 1,
"ownerid": 1,
"categoryid": 1,
"category": 1,
"content": 1,
"contenttemplate": 1,
"titlelist": 1,
"titles": 1,
"state": 1,
}
},
]
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 文章模板详情
@csrf_exempt
def getContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
industry1 = request.GET.get('industry1', '')
industry2 = request.GET.get('industry2', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
viewpointid = request.GET.get('viewpointid', '')
viewpointname = request.GET.get('viewpointname', '')
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
templateid = request.GET.get('templateid', '')
templatename = request.GET.get('templatename', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = {"templateid": templateid}
params = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"description": 1,
"owner": 1,
"ownerid": 1,
"categoryid": 1,
"category": 1,
"content": 1,
"contenttemplate": 1,
"wordgroups": 1,
"property": 1,
"orgid": 1,
"orgname": 1,
"titlelist": 1,
"titles": 1,
"industry2": 1,
"industryid2": 1,
"industry1": 1,
"industryid1": 1,
"viewpointname": 1,
"viewpointid": 1,
"templatename": 1,
"templatid": 1,
"datecreated": 1,
"dateupdated": 1,
"statistics": 1,
"state": 1,
}
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 新增模板
@csrf_exempt
def newContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
name = get_data.get('name', '')
description = get_data.get('description', '')
owner = get_data.get('owner', '')
ownerid = str(get_data.get('ownerid', ''))
categoryid = get_data.get('categoryid', '')
category = get_data.get('category', '')
content = get_data.get('content', '')
wordgroups = get_data.get('wordgroups', [])
property = get_data.get('property', {"distinct": 3})
titles = get_data.get('titles', [])
contenttemplate = content.replace("【", '').replace("】", '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
data = {
"name": name,
"titles": titles,
"description": description,
"owner": owner,
"ownerid": ownerid,
"categoryid": categoryid,
"category": category,
"content": content,
"contenttemplate": contenttemplate,
"wordgroups": wordgroups,
"property": property,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
res = collection.insert_one(data)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res.inserted_id,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改模板
@csrf_exempt
def updateContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
titles = get_data.get('titles', [])
content = get_data.get('content', '')
wordgroups = get_data.get('wordgroups', [])
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
owner = get_data.get('owner', '')
ownerid = str(get_data.get('ownerid', ''))
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
categoryid = get_data.get('categoryid', '')
category = get_data.get('category', '')
property = get_data.get('property', {"distinct": 3})
TIME = timestamp_to_str()
if type(titles) != type([]):
return JsonResponse({"code": "1014", "message": CODE_DICT["1014"]})
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
params = {
"$set": {
"titles": titles,
"content": content,
"wordgroups": wordgroups,
"property": property,
"dateupdated": TIME,
}
}
filter_ = {"templateid": templateid}
res = collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": templateid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改新增模板内容
@csrf_exempt
def updateContentTemplateInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = get_data.get('industry1', '')
industry2 = get_data.get('industry2', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
contenttemplate = get_data.get('contenttemplate', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
try:
params = {
"$set": {
"contenttemplate": contenttemplate,
"dateupdated": TIME,
}
}
filter_ = {"templateid": templateid}
res = collection.update_one(filter_, params)
except DuplicateKeyError:
return JsonResponse({"code": "10003", "message": f"{contenttemplate}-已存在"})
return JsonResponse({
"code": "0000",
"message": "成功",
"data": templateid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 删除模板
@csrf_exempt
def deleteContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
contentid = get_data.get('contentid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
collection_del = db_[pongo_content + '_del']
filter_ = {"_id": ObjectId(contentid)}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
res = collection.delete_one(filter_)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": contentid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 分类列表
@csrf_exempt
def getCategoryList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_category]
filter_ = {}
if ownerid:
filter_ = {"ownerid": ownerid}
if owner:
filter_ = {"owner": owner}
pipeline = [
{"$match": filter_},
{
"$project": {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"owner": 1,
"ownerid": 1,
}
},
]
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 人物行业列表
@csrf_exempt
def getPersonList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
level = request.GET.get('level', [])
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
# logger.info(f"request.GET: {str(request.GET)}")
logger.info(f"level: {str(level)} --- {str(type(level))}")
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_person]
filter_ = {}
if orgid:
filter_ = {"orgid": orgid}
if ownerid:
filter_.update({"ownerid": ownerid})
if "ORG_ADMIN" in level:
filter_.pop("ownerid", '')
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"name": 1,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"industryid": 1,
"pid": 1,
"iid": 1,
"orgid": 1,
"orgname": 1,
}
},
]
logger.info(pipeline)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
temp = {}
# if res:
# for item in res:
# temp.setdefault(item.get('industry1'), []).append({
# "value": item.get('industry2'),
# "label": item.get('industry2'),
# "industry": item.get('industry'),
# "industryid": item.get('industryid'),
# })
# for k, v in temp.items():
# t = {
# "value": k,
# "label": k,
# "children": v
# }
# data.append(t)
# res_data = {
# "industrylist": data
# }
t = []
if res:
for item in res:
# print(item)
p_id = item.get('pid')
_id = item.get('id')
iid = item.get('iid')
industryid = item.get('industryid')
industry = item.get('industry')
if p_id == '0':
t.append({"label": industry, "id": _id, "value": industryid, "pid": p_id, "iid": iid})
else:
temp.setdefault(p_id, []).append({
"label": industry,
"pid": item.get('pid'),
"id": _id,
"value": industryid,
})
for v in t:
P_id = v.get('iid')
v.update({"children": temp.get(P_id, [])})
data.append(v)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": {"industrylist": data},
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 人物行业详情
@csrf_exempt
def getPersonInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
personid = request.GET.get('personid', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_person]
filter_ = {"_id": ObjectId(personid)}
params = {
"_id": 0,
"name": 1,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industrylist": 1,
"orgid": 1,
"orgname": 1,
}
print(filter_)
# res = collection.find_one(filter_, no_cursor_timeout=True)
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 新增人物行业关系
@csrf_exempt
def newPersonInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
level = request.GET.get('level', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', [])
industry2 = request.GET.get('industry2', [])
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_person]
data = {
"level": level,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"industry1": industry1,
"industry2": industry2,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
res = collection.insert_one(data)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res.inserted_id,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改人物行业关系
@csrf_exempt
def updatePersonInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
level = get_data.get('level', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
children = get_data.get('children', [])
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
collection_p = db_[pongo_person]
filter_person = {"orgid": orgid, "ownerid": ownerid}
pipeline_person = [
{"$match": filter_person},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"industryid": 1,
"pid": 1,
"iid": 1,
"level": 1,
"orgid": 1,
"orgname": 1,
"datecreated": 1,
"dateupdated": 1,
}
},
]
res_person = collection_p.aggregate(pipeline=pipeline_person, allowDiskUse=True)
industry_data = {}
old_industry_list = []
old_pid_list = []
for item in res_person:
_id = item.pop('id')
industryid = item.get('industryid')
iid = item.get('iid')
pid = item.get('pid')
old_industry_list.append(iid)
old_pid_list.append(pid)
industry_data.update(item)
collection_p.delete_one({"_id": ObjectId(_id)})
for item in old_industry_list:
params = {
"$set": {
"disabled": False,
"dateupdated": TIME,
},
}
filter_ = {"_id": ObjectId(item)}
collection.update_one(filter_, params)
for item in children:
industry_data_t = {}
_id = item
item_t = collection.find_one({"_id": ObjectId(_id)})
industry = item_t.get('industry')
industryid = item_t.get('industryid')
pid = item_t.get('pid')
industry_data_t.update(industry_data)
industry_data_t.update({
"iid": _id,
"ownerid": ownerid,
"owner": owner,
"level": level,
"orgid": orgid,
"orgname": orgname,
"industry": industry,
"industryid": industryid,
"pid": pid,
"datecreated": TIME,
"dateupdated": TIME,
})
industry_data_t.pop('_id', '')
collection_p.insert_one(industry_data_t)
params = {
"$set": {
"disabled": True,
"dateupdated": TIME,
},
}
filter_ = {"_id": ObjectId(_id)}
collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": 0,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 删除人物行业关系
@csrf_exempt
def deletePersonInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
orgid = get_data.get('orgid', '')
ownerid = str(get_data.get('ownerid', ''))
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection_p = db_[pongo_person]
collection_industry = db_[pongo_industry]
collection_del = db_[pongo_person+'_del']
filter_person = {"orgid": orgid, "ownerid": ownerid}
pipeline_person = [
{"$match": filter_person},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"industryid": 1,
"pid": 1,
"iid": 1,
"level": 1,
"orgid": 1,
"orgname": 1,
"datecreated": 1,
"dateupdated": 1,
}
},
]
res_person = collection_p.aggregate(pipeline=pipeline_person, allowDiskUse=True)
t = []
if res_person:
for item in res_person:
# print(item)
p_id = item.get('pid')
iid = item.get('iid')
if p_id != '0':
t.append(iid)
item.update({"dateupdated": _datecreated})
collection_del.insert_one(item)
print(t)
# 将行业属性disabled变为false getIndustryList2
for r_id in t:
collection_industry.update_one({"_id": ObjectId(r_id)}, {"$set": {"dateupdated": _datecreated, "disabled": False}})
collection_p.delete_many(filter_person)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": "ok",
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 行业列表
@csrf_exempt
def getIndustryList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
level = request.GET.get('level', [])
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
filter_ = {}
if orgid:
logger.info(str(request.GET))
if "ORG_ADMIN" in level:
filter_.update({"orgid": orgid,})
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"industryid": 1,
"pid": 1,
"disabled": 1,
"orgid": 1,
"orgname": 1,
}
},
]
print(pipeline)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
temp = {}
t = []
if res:
for item in res:
# print(item)
p_id = item.get('pid')
_id = item.get('id')
industryid = item.get('industryid')
industry = item.get('industry')
if p_id == '0':
t.append({"title": industry, "id": _id, "industryid": industryid, "pid": p_id})
else:
temp.setdefault(p_id, []).append({
"title": industry,
"pid": item.get('pid'),
"id": _id,
"industryid": industryid,
})
for v in t:
P_id = v.get('id')
v.update({"children": temp.get(P_id, [])})
data.append(v)
else:
return JsonResponse({"code": "1025", "message": CODE_DICT['1025']})
else:
return JsonResponse({"code": "1013", "message": CODE_DICT['1013']})
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 行业列表--与任务表关联关系
@csrf_exempt
def getIndustryList2(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', '0'))
owner = request.GET.get('owner', '')
level = request.GET.get('level', [])
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
collection_p = db_[pongo_person]
filter_ = {}
filter_p = {}
if "ORG_ADMIN" in level:
filter_.update({"orgid": orgid})
if ownerid:
filter_p.update({"ownerid": ownerid, "orgid": orgid})
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"industryid": 1,
"pid": 1,
"disabled": 1,
"orgid": 1,
"orgname": 1,
}
},
]
pipeline_p = [
{"$match": filter_p},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"industryid": 1,
"disabled": 1,
"orgid": 1,
"orgname": 1,
"pid": 1,
"iid": 1,
}
},
]
print(pipeline)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
res2 = collection_p.aggregate(pipeline=pipeline_p, allowDiskUse=True)
if res2:
temp_2 = {}
t_2 = []
for item in res2:
# print(item)
p_id = item.get('pid')
i_id = item.get('iid')
_id = item.get('id')
industryid = item.get('industryid')
industry = item.get('industry')
if p_id == '0':
t_2.append({"title": industry, "id": _id, "industryid": industryid, "pid": p_id})
else:
temp_2.setdefault(i_id, []).append({
"title": industry,
"pid": item.get('pid'),
"id": _id,
"industryid": industryid,
})
if res:
data = []
temp = {}
t = []
for item in res:
# print(item)
p_id = item.get('pid')
_id = item.get('id')
industryid = item.get('industryid')
industry = item.get('industry')
disabled = item.get('disabled')
if p_id == '0':
t.append({"title": industry, "id": _id, "industryid": industryid, "pid": p_id, "expand": True, "disabled": True,})
else:
temp.setdefault(p_id, []).append({
"title": industry,
"pid": item.get('pid'),
"id": _id,
"industryid": industryid,
"disabled": disabled,
})
print("pipeline_person", temp_2)
for v in t:
P_id = v.get('id')
children = temp.get(P_id, [])
children_new = []
for i in children:
if i.get('id') in temp_2:
i.update({"checked": True})
i.pop("disabled")
else:
i.update({"checked": False})
children_new.append(i)
v.update({"children": children_new})
data.append(v)
# #####################
# p_industry = {}
# for item in res2:
# p_industry.setdefault(item.get('industry1'), []).append(item.get('industry2'))
# data = []
# temp = {}
# d_temp = {}
# if res:
# for item in res:
# temp.setdefault(item.get('industry1'), []).append({"title": item.get('industry2'), "disabled": item.get('disabled'), "id": item.get("id")})
# d_temp.setdefault(item.get('industry1'), []).append(item.get('industry2'))
# # "checked ": True
# print(temp)
# for k, v in temp.items():
# vv_new = []
# for vv in v:
# if vv.get('title', '') in p_industry.get(k, []):
# vv.update({"checked": True, "title": vv.get('title', ''),})
# vv.pop("disabled")
# else:
# vv.update({"checked": False, "title": vv.get('title', ''), "disabled": vv.get('disabled', '')})
# vv_new.append(vv)
# t = {
# "title": k,
# "expand": True,
# "disabled": True,
# "children": vv_new
# }
# data.append(t)
# #####################
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 行业详情
@csrf_exempt
def getIndustryInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
industryid = request.GET.get('industryid', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
filter_ = {"_id": ObjectId(industryid)}
params = {
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"orgid": 1,
"orgname": 1,
}
print(filter_)
# res = collection.find_one(filter_, no_cursor_timeout=True)
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 新增行业
@csrf_exempt
def newIndustryInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
level = get_data.get('level', [])
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
industry = get_data.get('title', '')
pid = get_data.get('pid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
collection_industry_ids = db_[pongo_industry_ids]
data = {}
ids_dict = collection_industry_ids.find_one({"id": 1})
industry1id = int(ids_dict.get('industry1id'))
industry2id = int(ids_dict.get('industry2id'))
if 'ORG_ADMIN' in level:
if pid == '0':
# 新增一级
data_indsert = {
"level": level,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"industry": industry,
"industryid": str(industry1id),
"disabled": False,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0,
"pid": "0"
}
res = collection.insert_one(data_indsert)
pipeline = [
{"$match": {"_id": ObjectId(res.inserted_id)}},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"level": 1,
"orgid": 1,
"orgname": 1,
"datecreated": 1,
"dateupdated": 1,
"pid": 1,
}
},
]
data = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
collection_industry_ids.update_one({"id": 1}, {"$set": {"industry1id": str(industry1id + 1), "dateupdated": TIME}})
else:
ids_dict = collection.find_one({"_id": ObjectId(pid)})
# 新增二级
data_indsert = {
"level": level,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"industry": industry,
"industryid": ids_dict.get("industryid") + str(industry2id),
"disabled": False,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0,
"pid": pid,
}
res = collection.insert_one(data_indsert)
pipeline = [
{"$match": {"_id": res.inserted_id}},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"level": 1,
"orgid": 1,
"orgname": 1,
"industryid": 1,
"datecreated": 1,
"dateupdated": 1,
"pid": 1,
}
},
]
data = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
collection_industry_ids.update_one({"id": 1}, {
"$set": {"industry2id": str(industry2id + 1), "dateupdated": TIME}})
else:
return JsonResponse({"code": "1010", "message": CODE_DICT['1010']})
return JsonResponse({
"code": "0000",
"message": "成功",
"data": list(data),
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改行业
@csrf_exempt
def updateIndustryInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
level = get_data.get('level', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
indusrty = get_data.get('title', '')
indusrtyid = get_data.get('indusrtyid', '')
industry_id = get_data.get('id')
TIME = timestamp_to_str()
logger.info(str(get_data))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
collection_p = db_[pongo_person]
filter_industry = {"_id": ObjectId(industry_id)}
logger.info(f"filter_industry：" + str(filter_industry))
params_i = {
"$set": {
"industry": indusrty,
"dateupdated": TIME,
},
}
r_i = collection.update_one(filter_industry, params_i)
# ret.matched_count, ret.modified_count
# 前者是匹配到文档的数目，而后者是修改了文档的个数。
logger.info(f"匹配：" + str(r_i.matched_count))
logger.info(f"修改：" + str(r_i.modified_count))
params_p = {
"$set": {
"industry": indusrty,
"dateupdated": TIME,
},
}
filter_p = {"iid": industry_id}
r_p = collection_p.update_many(filter_p, params_p)
logger.info(f"匹配：" + str(r_p.matched_count))
logger.info(f"修改：" + str(r_p.modified_count))
return JsonResponse({
"code": "0000",
"message": "成功",
"data": 0,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 删除行业
@csrf_exempt
def deleteIndustryInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industryid = get_data.get('id', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_industry]
collection_del = db_[pongo_industry+'_del']
filter_ = {"_id": ObjectId(industryid)}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
pid = data.get('pid')
disabled = data.get('disabled')
if pid == '0':
filter_ = {"pid": industryid}
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry": 1,
"level": 1,
"orgid": 1,
"orgname": 1,
"industryid": 1,
"datecreated": 1,
"dateupdated": 1,
"pid": 1,
"disabled": 1,
}
},
]
data = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
flag = 0
data_t = []
for item in data:
disabled = item.get('disabled', True)
if disabled == True:
flag = 1
break
data_t.append(item)
if flag:
code = "1011"
message = "此一级行业已经分配二级行业，不能删除"
else:
for item in data_t:
item.update({"dateupdated": _datecreated})
collection_del.insert_one(item)
collection.delete_many(filter_)
filter_ = {"_id": ObjectId(industryid)}
collection.delete_one(filter_)
code = "0000"
message = "删除成功"
else:
if disabled == False:
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
collection.delete_one(filter_)
code = "0000"
message = "删除成功"
else:
code = "1012"
message = "行业已分配角色，不能删除"
return JsonResponse({
"code": code,
"message": CODE_DICT[code],
"data": "ok",
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# ##################################
# 关键词
# ##################################
# 发版词上传
@csrf_exempt
def baseKeywordUpload(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
try:
with MongoDBClient() as mg_client, LogMysqlApeT(ape_db) as m_client:
db_ = mg_client[db]
# 获取collection
collection = db_[pongo_keyword]
path_info = request.path_info
jgname = path_info.split('/')[-1]
file_obj_list = request.FILES.getlist('file')
orgid = request.POST.get('orgid', '')
orgname = request.POST.get('orgname', '')
sql = f"select id,wordgroupname from {simian_wordgroup_table} where orgid='{orgid}' and status=0 "
res_w = m_client.search_db(sql)
wordgroupname_dict = {}
for item_w in res_w:
wordgroupname_dict.update({item_w.get("wordgroupname"): item_w.get("id")})
file_path_list = []
filePath = []
flag = 0
for file_obj in file_obj_list:
file_path = r'uploadFiles' + os.sep + file_obj.name
file_path_list.append(file_path)
filePath.append(file_obj.name)
with open(file_path, 'wb') as f:
for i in file_obj.chunks():
f.write(i)
logger.info('文件路径：%s' % str(file_path))
res = read_excel(file_path)
error_list = [["提示信息", "关键词名称", "词分组名称", "一级行业", "二级行业"]]
collection_industry = db_[pongo_industry]
industry_dict = {}
res_all = collection_industry.find({})
for item_i in res_all:
pid = item_i.get('pid')
industry = item_i.get('industry')
industryid = item_i.get('industryid')
if pid == '0':
industry = industry + '_1'
else:
industry = industry + '_2'
industry_dict.update({industry: industryid})
pass
logger.info(str(industry_dict))
count = 0
for item in res:
# logger.info(str(item))
# print(len(item), item)
TIME = timestamp_to_str(time.time() + count)
if item[1]:
keyword = item[3]
industry1 = item[1]
industry2 = item[2]
area = item[4]
baiduindex = item[5]
wordgroup = item[6]
industryid2 = industry_dict.get(industry2 + '_2', '')
industryid1 = industry_dict.get(industry1 + '_1', '')
if not industryid1 or not industryid2:
k_t = ["该关键词对应行业不存在", keyword, wordgroup, industry1, industry2]
error_list.append(k_t)
count += 1
continue
if wordgroup not in wordgroupname_dict:
k_t = ["该分组不存在", keyword, wordgroup, industry1, industry2]
error_list.append(k_t)
count += 1
continue
data = {
"keyword": keyword,
"industry1": industry1,
"industryid1": industryid1,
"industry2": industry2,
"industryid2": industryid2,
"baiduindex": baiduindex,
"area": area,
"orgid": orgid,
"orgname": orgname,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0,
"publishlist": [],
"wordgroup": wordgroup,
"wordgroupid": wordgroupname_dict.get(wordgroup, ''),
}
try:
collection.insert_one(data)
except DuplicateKeyError:
k_t = ["该关键词已存在", keyword, wordgroup, industry1, industry2]
error_list.append(k_t)
count += 1
# 上传成功之后删除文件
for path in file_path_list:
try:
os.remove(path)
except Exception as e:
pass
if len(error_list) != 1:
code = "1026"
return JsonResponse(
{"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated, })
else:
code = "0000"
error_list = []
return JsonResponse(
{"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated, })
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated,})
return JsonResponse({"code": "00", "message": "no this method", "datecreated": _datecreated,})
# 发布任务对应词和站点关系上传上传
@csrf_exempt
def publishKeywordUpload(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
try:
logger.info('# 开始上传文件 #')
with MongoDBClient() as mg_client, LogMysqlApeT(ape_db) as m_client:
db_ = mg_client[db]
# 获取collection
collection = db_[pongo_keyword]
path_info = request.path_info
jgname = path_info.split('/')[-1]
file_obj_list = request.FILES.getlist('file')
aid = request.POST.get('aid', '')
aname = request.POST.get('aname', '')
orgid = request.POST.get('orgid', '')
orgname = request.POST.get('orgname', '')
# sql = f"select id,wordgroupname from {simian_wordgroup_table} where orgid='{orgid}' and status=0 "
# res_w = m_client.search_db(sql)
# wordgroupname_dict = {}
# for item_w in res_w:
# wordgroupname_dict.update({item_w.get("wordgroupname"): item_w.get("id")})
file_path_list = []
filePath = []
for file_obj in file_obj_list:
file_path = r'uploadFiles' + os.sep + file_obj.name
file_path_list.append(file_path)
filePath.append(file_obj.name)
with open(file_path, 'wb') as f:
for i in file_obj.chunks():
f.write(i)
logger.info('文件路径：%s' % str(file_path))
res = read_excel(file_path)
TIME = timestamp_to_str(time.time())
error_list = [["提示信息", "词组名称", "关键词名称"]]
for item in res:
if item[0]:
_id = item[0]
keyword = item[1]
industry1 = item[2]
industry2 = item[3]
area = item[4]
baiduindex = item[5]
wordgroup = item[6]
# if wordgroup not in wordgroupname_dict:
# k_t = ["该分组不存在", wordgroup, keyword]
# error_list.append(k_t)
# continue
labelname1 = item[7]
labelname2 = item[8]
labelname3 = item[9]
# 0 待编辑
# 1 已编辑
# 2 已发布
params = {
"$push": {"publishlist": {
"id": get_uuid(),
"aid": str(aid),
"aname": aname,
"labelname1": labelname1,
"labelname2": labelname2,
"labelname3": labelname3,
"title": "",
# 看不见 计划中
"status": 10,
"onlinedate": "",
}},
"$set": {
"dateupdated": TIME,
# "wordgroup": wordgroup,
# "wordgroupid": wordgroupname_dict.get(wordgroup, ''),
},
}
filter_ = {"_id": ObjectId(_id)}
res = collection.update_one(filter_, params)
yes = res.modified_count
if not yes:
k_t = ["该词对应id不存在", wordgroup, keyword]
error_list.append(k_t)
continue
# 上传成功之后删除文件
for path in file_path_list:
try:
os.remove(path)
except Exception as e:
pass
# 增删改查
if len(error_list) != 1:
code = "1026"
return JsonResponse({"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated,})
else:
code = "0000"
error_list = []
return JsonResponse({"code": code, "data": error_list, "message": CODE_DICT[code], "datecreated": _datecreated,})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated,})
return JsonResponse({"code": "00", "message": "no this method", "datecreated": _datecreated,})
# 关键词列表 --- 基础词任务
@csrf_exempt
def getKeywordList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
keyword = request.GET.get('keyword', '')
area = request.GET.get('area', '')
wordgroup = request.GET.get('wordgroup', '')
wordgroupid = request.GET.get('wordgroupid', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industryid1 = request.GET.get('industryid1', '')
industry2 = request.GET.get('industry2', '')
industryid2 = request.GET.get('industryid2', '')
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
filter_ = {}
if orgid:
filter_.update({"orgid": orgid})
if industryid1:
filter_.update({"industryid1": industryid1})
if industryid2:
filter_.update({"industryid2": industryid2})
if area:
filter_.update({"area": area})
if wordgroup:
filter_.update({"wordgroup": wordgroup})
if keyword:
filter_.update({"keyword": {"$regex": keyword}})
pipeline = [
{"$match": filter_},
{"$sort": {"datecreated": -1}},
{"$limit": page * size},
{"$skip": (page - 1) * size},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroup": 1,
"wordgroupid": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
"baiduindex": 1,
}
},
]
print(pipeline)
total = collection.count_documents(filter=filter_)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
return JsonResponse({
"code": "0000",
"message": "成功",
"total": total,
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词列表 --- 基础词任务 ---- 下载
@csrf_exempt
def getKeywordListDownload(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
keyword = request.GET.get('keyword', '')
area = request.GET.get('area', '')
wordgroup = request.GET.get('wordgroup', '')
wordgroupid = request.GET.get('wordgroupid', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industryid1 = request.GET.get('industryid1', '')
industry2 = request.GET.get('industry2', '')
industryid2 = request.GET.get('industryid2', '')
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
filter_ = {}
if orgid:
filter_.update({"orgid": orgid})
if industryid1:
filter_.update({"industryid1": industryid1})
if industryid2:
filter_.update({"industryid2": industryid2})
if area:
filter_.update({"area": area})
if wordgroup:
filter_.update({"wordgroup": wordgroup})
if keyword:
filter_.update({"keyword": {"$regex": keyword}})
pipeline = [
{"$match": filter_},
{"$sort": {"datecreated": -1}},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroup": 1,
"wordgroupid": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
"baiduindex": 1,
}
},
]
print(pipeline)
total = collection.count_documents(filter=filter_)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
return JsonResponse({
"code": "0000",
"message": "成功",
"total": total,
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词列表 --- 发布任务 词
@csrf_exempt
def getTaskKeywordList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
keyword = request.GET.get('keyword', '')
area = request.GET.get('area', '')
aid = int(request.GET.get('aid', 0))
wordgroup = request.GET.get('wordgroup', '')
wordgroupid = request.GET.get('wordgroupid', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industryid1 = request.GET.get('industryid1', '')
industry2 = request.GET.get('industry2', '')
industryid2 = request.GET.get('industryid2', '')
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
filter_ = {}
if orgid:
filter_.update({"orgid": orgid})
if industryid1:
filter_.update({"industryid1": industryid1})
if industryid2:
filter_.update({"industryid2": industryid2})
if area:
filter_.update({"area": area})
if wordgroup:
filter_.update({"wordgroup": wordgroup})
if keyword:
filter_.update({"keyword": {"$regex": keyword}})
if aid:
filter_.update({"publishlist": {"$elemMatch": {"aid": str(aid)}}})
pipeline = [
{"$match": filter_},
{"$sort": {"keyword": 1}},
{"$limit": page * size},
{"$skip": (page - 1) * size},
{"$unwind": {"path": "$publishlist", "preserveNullAndEmptyArrays": True}},
# {
# "$group": {
# "_id": "$publishlist.aid",
# },
# },
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroup": 1,
"wordgroupid": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
"baiduindex": 1,
"paid": "$publishlist.id",
"aid": "$publishlist.aid",
"aname": "$publishlist.aname",
"labelname1": "$publishlist.labelname1",
"labelname2": "$publishlist.labelname2",
"labelname3": "$publishlist.labelname3",
}
},
]
if aid:
pipeline.insert(-1, {"$match": {"publishlist.aid": str(aid)}},)
logger.info(pipeline)
total = collection.count_documents(filter=filter_)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
return JsonResponse({
"code": "0000",
"message": "成功",
"total": total,
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词详情
@csrf_exempt
def getKeywordInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
keywordid = request.GET.get('keywordid', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
filter_ = {"_id": ObjectId(keywordid)}
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroupid": 1,
"wordgroup": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
}
print(filter_)
# res = collection.find_one(filter_, no_cursor_timeout=True)
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词新增
@csrf_exempt
def newKeywordInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
keyword = get_data.get('keyword', '')
wordgroup = get_data.get('wordgroup', '')
wordgroupid = get_data.get('wordgroupid', '')
area = get_data.get('area', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
# level = get_data.get('level', [])
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
industry1 = get_data.get('industry1', '')
industryid1 = get_data.get('industryid1', '')
industry2 = get_data.get('industry2', '')
industryid2 = get_data.get('industryid2', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
data = {
"orgid": orgid,
"keyword": keyword,
"orgname": orgname,
"wordgroup": wordgroup,
"wordgroupid": wordgroupid,
"area": area,
"owner": owner,
"ownerid": ownerid,
"industry1": industry1,
"industryid1": industryid1,
"industry2": industry2,
"industryid2": industryid2,
"datecreated": TIME,
"dateupdated": TIME,
"publishlist": [],
"publishsum": 5,
"statevalue": "待编辑",
"sort": 100,
"state": 0
}
try:
res = collection.insert_one(data)
except DuplicateKeyError:
code = "1018"
return JsonResponse({"code": code, "message": f"（{keyword}）-{CODE_DICT[code]}"})
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res.inserted_id,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词编辑
@csrf_exempt
def updateKeywordInfo(request):
logger.info("updateKeywordInfo")
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
keywordid = get_data.get('keywordid', '')
keywordlist = get_data.get('keywordlist', [])
# keywordlist = json.loads(keywordlist)
wordgroup = get_data.get('wordgroup', '')
wordgroupid = get_data.get('wordgroupid', '')
keyword = get_data.get('keyword', '')
area = get_data.get('area', '')
logger.info(f'{str(get_data)}')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
xgz = {}
if keyword:
xgz.update({"keyword": keyword})
if keyword:
xgz.update({"area": area})
if wordgroupid:
xgz.update({"wordgroup": wordgroup, "wordgroupid": wordgroupid})
xgz.update({"dateupdated": TIME})
params = {
"$set": xgz,
}
code = "0000"
if keywordid:
try:
filter_ = {"_id": ObjectId(keywordid)}
res = collection.update_one(filter_, params)
except DuplicateKeyError:
code = "1018"
return JsonResponse({"code": code, "message": f"（{keyword}）-{CODE_DICT[code]}"})
count_all = len(keywordlist)
count = 0
if keywordlist:
flag = 0
error_list = []
for item in keywordlist:
try:
keywordid_t = item.get('id')
keyword_t = item.get('keyword')
filter_t = {"_id": ObjectId(keywordid_t)}
res = collection.update_one(filter_t, params)
count += int(res.modified_count)
except DuplicateKeyError:
flag = 1
error_list.append(keyword_t)
code = "1018"
if flag:
return JsonResponse({"code": code, "message": f"（{','.join(error_list)}）-{CODE_DICT[code]}"})
return JsonResponse({
"code": code,
"message": CODE_DICT[code],
"data": f"一共修改{count_all}个，成功{count}个！",
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词删除
@csrf_exempt
def deleteKeywordInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
keywordid = get_data.get('keywordid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
collection_del = db_[pongo_keyword+'_del']
filter_ = {"_id": ObjectId(keywordid)}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
res = collection.delete_one(filter_)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": keywordid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类列表- 有分页
@csrf_exempt
def getWordCategoryList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
wordgroup = request.GET.get('wordgroup', '')
wordgroupid = request.GET.get('wordgroupid', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industryid1 = request.GET.get('industryid1', '')
industry2 = request.GET.get('industry2', '')
industryid2 = request.GET.get('industryid2', '')
description = request.GET.get("description", '') # 描述
data = []
if orgid:
sql = f" select * from {simian_wordgroup_table} where (status=0) "
total_sql = f" select count(*) as n from {simian_wordgroup_table} where (status=0) "
sql += f" and orgid='{orgid}' "
total_sql += f" and orgid='{orgid}' "
# if ownerid:
# sql += f" and ownerid='{ownerid}' "
# total_sql += f" and ownerid='{ownerid}' "
if industryid1:
sql += f" and industryid1='{industryid1}' "
total_sql += f" and industryid1='{industryid1}' "
if industryid2:
sql += f" and industryid2='{industryid2}' "
total_sql += f" and industryid2='{industryid2}' "
if wordgroup:
sql += f" and wordgroupname like '%%{wordgroup}%%' "
total_sql += f" and wordgroupname like '%%{wordgroup}%%' "
sql += f" order by createtime DESC limit {(page-1)* size}, {size*page} "
with LogMysqlApeT(ape_db) as m_client:
data = m_client.search_db(sql)
total_ = m_client.search_db(total_sql)
total = 0
if total_:
total = total_[0].get('n')
return JsonResponse({
"code": "0000",
"message": "成功",
"total": total,
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
# else:
# code = "1017"
# return JsonResponse({"code": code, "message": CODE_DICT[code]})
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类列表- 没有分页
@csrf_exempt
def getWordCategoryListAll(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
wordgroup = request.GET.get('wordgroup', '')
wordgroupid = request.GET.get('wordgroupid', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industryid1 = request.GET.get('industryid1', '')
industry2 = request.GET.get('industry2', '')
industryid2 = request.GET.get('industryid2', '')
description = request.GET.get("description", '') # 描述
data = []
if orgid:
sql = f" select wordgroupname as wordgroup, id from {simian_wordgroup_table} where (status=0) "
total_sql = f" select count(*) as n from {simian_wordgroup_table} where (status=0) "
sql += f" and orgid='{orgid}' "
total_sql += f" and orgid='{orgid}' "
# if ownerid:
# sql += f" and ownerid='{ownerid}' "
# total_sql += f" and ownerid='{ownerid}' "
if industryid1:
sql += f" and industryid1='{industryid1}' "
total_sql += f" and industry1id='{industryid1}' "
if industryid2:
sql += f" and industryid2='{industryid2}' "
total_sql += f" and industryid2='{industryid2}' "
sql += f" order by createtime DESC "
# sql += f" limit {(page-1)* size}, {size*page} "
with LogMysqlApeT(ape_db) as m_client:
data = m_client.search_db(sql)
# total_ = m_client.search_db(total_sql)
# total = 0
# if total_:
# total = total_[0].get('n')
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
else:
code = "1019"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类详情
@csrf_exempt
def getWordCategoryInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
wordgroupid = request.GET.get('wordgroupid', '')
with LogMysqlApeT(ape_db) as m_client:
sql = f" select * from {simian_wordgroup_table} where id='{wordgroupid}' "
res = m_client.search_db(sql)
data = {}
if res:
data = res[0]
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类添加
@csrf_exempt
def newWordCategoryInfo(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
industry1 = get_data.get("industry1", '') # 行业
industryid1 = get_data.get("industryid1", '') # 行业
industry2 = get_data.get("industry2", '') # 行业
industryid2 = get_data.get("industryid2", '') # 行业
wgname = get_data.get("wordgroup", '') # 词分类名称
owner = get_data.get("owner", '') # 所属人
ownerid = get_data.get("ownerid", '') # 所属人id
orgname = get_data.get("orgname", '') # 机构名称
orgid = get_data.get("orgid", '') # 机构id
description = get_data.get("description", '') # 描述
# 查询词是否存在
condition = f"""
select count(id) as count from {simian_wordgroup_table}
where wordgroupname = '{wgname}' and orgid='{orgid}' and industryid2='{industryid2}' and status=0
"""
logger.info(condition)
wgname_count = m_client.search_db(condition)
if wgname_count[0]["count"] > 0:
return JsonResponse({"code": "1015", "data": wgname_count[0]["count"], "message": CODE_DICT['1015']})
else:
TIME = timestamp_to_str(time.time())
data = {
"id": f"wg-{ownerid}-" + get_uuid(),
"industry": industry1,
"industryid1": industryid1,
"industryid2": industryid2,
"industry2": industry2,
"wordgroupname": wgname,
"owner": owner,
"ownerid": ownerid,
"orgname": orgname,
"orgid": orgid,
"description": description,
"createtime": TIME,
"updatetime": TIME,
"status": 0
}
result = m_client.updata_db(data, simian_wordgroup_table)
return JsonResponse({"code": "0000", "data": result, "message": 'success'})
except Exception as e:
return JsonResponse({"code": "1002", "msg": str(e)})
return JsonResponse({"code": "1001", "msg": "no this method"})
# 词分类编辑
@csrf_exempt
def updateWordCategoryInfo(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
industry1 = get_data.get("industry1", '') # 行业
industryid1 = get_data.get("industryid1", '') # 行业
industry2 = get_data.get("industry2", '') # 行业
industryid2 = get_data.get("industryid2", '') # 行业
wordgroupid = get_data.get("wordgroupid", '') # 词分类名称
wgname = get_data.get("wordgroup", '') # 词分类名称
owner = get_data.get("owner", '') # 所属人
ownerid = get_data.get("ownerid", '') # 所属人id
orgname = get_data.get("orgname", '') # 机构名称
orgid = get_data.get("orgid", '') # 机构id
description = get_data.get("description", '') # 描述
TIME = timestamp_to_str(time.time())
# 查询词是否存在
condition = f"""
select count(id) as count from {simian_wordgroup_table}
where wordgroupname = '{wgname}' and orgid='{orgid}' and industryid2='{industryid2}' and status=0
"""
logger.info(condition)
wgname_count = m_client.search_db(condition)
if wgname_count[0]["count"] > 0:
return JsonResponse(
{"code": "1015", "data": wgname_count[0]["count"], "message": CODE_DICT['1015']})
data = {
"id": wordgroupid,
"wordgroupname": wgname,
"description": "修改了词分组",
"createtime": TIME,
"updatetime": TIME
}
result = m_client.updata_db(data, simian_wordgroup_table)
return JsonResponse({"code": "0000", "data": result, "message": 'success'})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类删除
@csrf_exempt
def deleteWordCategoryInfo(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
wordgroupid = get_data.get("wordgroupid", '') # 词分类id
TIME = timestamp_to_str(time.time())
res = m_client.search_db(
f"""select wordgroupname from {simian_wordgroup_table} where id= '{wordgroupid}' """)
v = res[0].get('wordgroupname', '') if res else wordgroupid
data = {
"id": wordgroupid,
"wordgroupname": v+'___',
"status": 1,
"updatetime": TIME
}
result = m_client.updata_db(data, simian_wordgroup_table)
return JsonResponse({"code": "0000", "data": result, "message": 'success'})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点列表
@csrf_exempt
def getViewPointList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
viewpointid = request.GET.get('viewpointid', '')
viewpointname = request.GET.get('viewpointname', '')
description = request.GET.get('description', '')
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
orgid = request.GET.get('orgid', '')
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industry2 = request.GET.get('industry2', '')
industryid1 = request.GET.get('industryid1', '')
industryid2 = request.GET.get('industryid2', '')
data = []
sql = f" select * from {simian_viewpoint_table} where (status=0) "
total_sql = f" select count(*) as n from {simian_viewpoint_table} where (status=0) "
if orgid:
sql += f" and orgid='{orgid}' "
total_sql += f" and orgid='{orgid}' "
# if ownerid:
# sql += f" and ownerid='{ownerid}' "
# total_sql += f" and ownerid='{ownerid}' "
if industryid1:
sql += f" and industryid1='{industryid1}' "
total_sql += f" and industryid1='{industryid1}' "
if industryid2:
sql += f" and industryid2='{industryid2}' "
total_sql += f" and industryid2='{industryid2}' "
if viewpointname:
sql += f" and viewpoint like '%%{viewpointname}%%' "
total_sql += f" and viewpoint like '%%{viewpointname}%%' "
# if description:
# sql += f" and description like '%%{description}%%' "
# total_sql += f" and description like '%%{description}%%' "
sql += f" order by createtime DESC limit {(page - 1) * size}, {size * page} "
with LogMysqlApeT(ape_db) as m_client:
data = m_client.search_db(sql)
total_ = m_client.search_db(total_sql)
total = 0
if total_:
total = total_[0].get('n')
return JsonResponse({
"code": "0000",
"message": "成功",
"total": total,
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
else:
code = "1021"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点详情
@csrf_exempt
def getViewPointInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
viewpointid = request.GET.get('viewpointid', '')
with LogMysqlApeT(ape_db) as m_client:
sql = f" select * from {simian_viewpoint_table} where id='{viewpointid}' "
res = m_client.search_db(sql)
data = {}
if res:
data = res[0]
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点添加
@csrf_exempt
def newViewPointInfo(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
industry = get_data.get("industry1", '') # 行业
industry2 = get_data.get("industry2", '') # 行业
viewpointname = get_data.get("viewpointname", '') # 观点名称
owner = get_data.get("owner", '') # 所属人
ownerid = get_data.get("ownerid", '') # 所属人id
orgname = get_data.get("orgname", '') # 机构名称
orgid = get_data.get("orgid", '') # 机构id
description = get_data.get("description", '') # 写作方向
industryid1 = get_data.get('industryid1', '')
industryid2 = get_data.get('industryid2', '')
# 查询词是否存在
condition = f"""
select count(id) as count from {simian_viewpoint_table}
where viewpoint = '{viewpointname}' and orgid='{orgid}' and industryid2='{industryid2}'
"""
logger.info(condition)
viewpointgnamec = m_client.search_db(condition)
if viewpointgnamec[0]["count"] > 0:
return JsonResponse({"code": "0000", "data": viewpointgnamec[0]["count"], "message": '该观点名称已存在'})
TIME = timestamp_to_str(time.time())
data = {
"id": f"vp-{ownerid}-" + get_uuid(),
"industry": industry,
"industry2": industry2,
"industryid1": industryid1,
"industryid2": industryid2,
"viewpoint": viewpointname,
"description": description,
"owner": owner,
"ownerid": ownerid,
"orgname": orgname,
"orgid": orgid,
"createtime": TIME,
"updatetime": TIME,
"status": 0
}
result = m_client.updata_db(data, simian_viewpoint_table)
return JsonResponse({"code": "0000", "data": result, "message": 'success'})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点编辑
@csrf_exempt
def updateViewPointInfo(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
viewpointid = get_data.get("viewpointid", '') # 观点id
viewpointname = get_data.get("viewpointname", '') # 观点id
description = get_data.get("description", '') # 观点id
TIME = timestamp_to_str(time.time())
data = {
"id": viewpointid,
"viewpoint": viewpointname,
"description": description,
"updatetime": TIME
}
result = m_client.updata_db(data, simian_viewpoint_table)
return JsonResponse({"code": "0000", "data": result, "message": 'success'})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点删除
@csrf_exempt
def deleteViewPointInfo(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
viewpointid = get_data.get("viewpointid", '') # 观点id
res = m_client.search_db(f"""select viewpoint from {simian_viewpoint_table} where id= '{viewpointid}' """)
v = res[0].get('viewpoint', '') if res else viewpointid
TIME = timestamp_to_str(time.time())
data = {
"id": viewpointid,
"viewpoint": v + '__',
"status": 1,
"updatetime": TIME
}
result = m_client.updata_db(data, simian_viewpoint_table)
return JsonResponse({"code": "0000", "data": result, "message": 'success'})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点模板列表
@csrf_exempt
def getTemplateList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
viewpointid = request.GET.get('viewpointid', '')
viewpointname = request.GET.get('viewpointname', '')
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
industryid1 = request.GET.get('industryid1', '')
industryid2 = request.GET.get('industryid2', '')
state = int(request.GET.get('state', 10000))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = filter_2 = {}
if orgid:
filter_.update({"orgid": orgid})
if industryid1:
filter_.update({"industryid1": industryid1})
if industryid2:
filter_.update({"industryid2": industryid2})
if viewpointid:
filter_.update({"viewpointid": viewpointid})
if state != 10000:
filter_.update({"state": state})
pipeline = [
{"$match": filter_},
{"$sort": {"datecreated": -1}},
{
"$project": {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"industry1": 1,
"industry2": 1,
"industryid1": 1,
"industryid2": 1,
"templateid": 1,
"templatename": 1,
"state": 1,
}
},
]
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点模板详情
@csrf_exempt
def getTemplateInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
industry1 = request.GET.get("industry1", '') # 行业
industry2 = request.GET.get("industry2", '') # 行业
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
viewpointid = request.GET.get('viewpointid', '')
viewpointname = request.GET.get('viewpointname', '')
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
templateid = request.GET.get('templateid', '')
templatename = request.GET.get('templatename', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = {"templateid": templateid}
params = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"templateid": 1,
"templatename": 1,
"industry1": 1,
"industry2": 1,
"state": 1
}
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 新增观点模板
@csrf_exempt
def newTemplateInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = get_data.get("industry1", '') # 行业
industry2 = get_data.get("industry2", '') # 行业
industryid1 = get_data.get("industryid1", '')
industryid2 = get_data.get("industryid2", '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templatename = get_data.get('templatename', '')
description = get_data.get('description', '')
TIME = timestamp_to_str()
templateid = f"{viewpointid}-t-" + get_uuid()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
data = {
"industry1": industry1,
"industry2": industry2,
"industryid1": industryid1,
"industryid2": industryid2,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"viewpointid": viewpointid,
"viewpointname": viewpointname,
"description": description,
"templateid": templateid,
"templatename": templatename,
"titles": [],
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
res = collection.insert_one(data)
# try:
# res = collection.insert_one(data)
# except DuplicateKeyError:
# return JsonResponse({"code": "10003", "message": f"（{templatename}）-此模板名称已存在"})
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res.inserted_id,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改观点模板
@csrf_exempt
def updateTemplateInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = request.GET.get("industry1", '') # 行业
industry2 = request.GET.get("industry2", '') # 行业
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
description = get_data.get('description', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
params = {
"$set": {
"templatename": templatename,
"dateupdated": TIME,
}
}
filter_ = {"templateid": templateid}
res = collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": templateid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 删除观点模板
@csrf_exempt
def deleteTemplateInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
collection_del = db_[pongo_content + '_del']
filter_ = {"templateid": templateid}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
res = collection.delete_one(filter_)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": templateid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点标题列表
@csrf_exempt
def getTitleList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
industry1 = request.GET.get("industry1", '') # 行业
industryid1 = request.GET.get("industryid1", '') # 行业
industry2 = request.GET.get("industry2", '') # 行业
industryid2 = request.GET.get("industryid2", '') # 行业
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
viewpointid = request.GET.get('viewpointid', '')
viewpointname = request.GET.get('viewpointname', '')
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = filter_2 = {}
if orgid:
filter_.update({"orgid": orgid})
if ownerid:
filter_.update({"ownerid": ownerid})
pipeline = [
{"$match": filter_},
{
"$project": {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"templateid": 1,
"templatename": 1,
"titlelist": 1,
"contenttemplate": 1
}
},
]
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = {}
if res:
data = list(res)[0]
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 观点标题详情 -- 无用
@csrf_exempt
def getTitleInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
industry1 = request.GET.get("industry1", '') # 行业
industry2 = request.GET.get("industry2", '') # 行业
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
viewpointid = request.GET.get('viewpointid', '')
viewpointname = request.GET.get('viewpointname', '')
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
templateid = request.GET.get('templateid', '')
templatename = request.GET.get('templatename', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = {"templateid": templateid}
params = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"templateid": 1,
"templatename": 1,
"titlelist": 1,
}
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 新增观点标题
@csrf_exempt
def newTitleInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = get_data.get('industry1', '')
industryid1 = get_data.get('industryid1', '')
industry2 = get_data.get('industry2', '')
industryid2 = get_data.get('industryid2', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
titleid = get_data.get('titleid', '')
title = get_data.get('title', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
collection_t = db_[pongo_title]
params_r = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"templateid": 1,
"templatename": 1,
"titles": 1,
}
if not titleid:
titleid_insert = templateid + '-' + get_uuid()
title_insert = title.replace('【', '').replace('】', '')
data = {
"titleid": titleid_insert,
"title": title_insert,
"industry1": industry1,
"industryid1": industryid1,
"industryid2": industryid2,
"industry2": industry2,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"viewpointid": viewpointid,
"viewpointname": viewpointname,
"templateid": templateid,
"templatename": templatename,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
try:
collection_t.insert_one(data)
params = {
"$push": {"titles": {"id": titleid_insert, "templatetitlename": title_insert,
"title": title_insert, "status": 0}},
"$set": {
"dateupdated": TIME,
}
}
filter_ = {"templateid": templateid}
collection.update_one(filter_, params)
res = collection.find_one(filter_, params_r, no_cursor_timeout=True)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except DuplicateKeyError:
return JsonResponse({"code": "10003", "message": f"{title}-已存在"})
else:
code = "1000"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 修改观点标题
@csrf_exempt
def updateTitleInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = get_data.get('industry1', '')
industryid1 = get_data.get('industryid1', '')
industry2 = get_data.get('industry2', '')
industryid2 = get_data.get('industryid2', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
titleid = get_data.get('titleid', '')
title = get_data.get('title', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
collection_t = db_[pongo_title]
params_r = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"templateid": 1,
"templatename": 1,
"titles": 1,
}
if not titleid:
titleid_insert = templateid+'-'+get_uuid()
title_insert = title.replace('【', '').replace('】', '')
data = {
"titleid": titleid_insert,
"title": title_insert,
"industry1": industry1,
"industryid1": industryid1,
"industryid2": industryid2,
"industry2": industry2,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"viewpointid": viewpointid,
"viewpointname": viewpointname,
"templateid": templateid,
"templatename": templatename,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
try:
collection_t.insert_one(data)
params = {
"$push": {"titles": {"id": titleid_insert, "templatetitlename": title_insert, "title": title_insert, "status": 0}},
"$set": {
"dateupdated": TIME,
}
}
filter_ = {"templateid": templateid}
collection.update_one(filter_, params)
res = collection.find_one(filter_, params_r, no_cursor_timeout=True)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except DuplicateKeyError:
return JsonResponse({"code": "10003", "message": f"{title}-已存在"})
else:
filter_ = {"templateid": templateid}
titleid_insert = templateid + '-' + get_uuid()
title_insert = title.replace('【', '').replace('】', '')
data = {
"title": title_insert,
"titleid": titleid_insert,
"industry1": industry1,
"industryid1": industryid1,
"industryid2": industryid2,
"industry2": industry2,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"viewpointid": viewpointid,
"viewpointname": viewpointname,
"templateid": templateid,
"templatename": templatename,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
try:
collection_t.insert_one(data)
res = collection.find_one(filter_, params_r, no_cursor_timeout=True)
titles = res.get('titles', [])
titlelist_new = []
for item in titles:
titleid_old = item.get('id', '')
title_old = item.get('title', '')
if titleid_old == titleid:
item.update({"templatetitlename": title_insert, "title": title_insert})
titlelist_new.append(item)
filter_t = {"title": title_old}
collection_t.delete_one(filter_t)
else:
titlelist_new.append(item)
params = {
"$set": {
"titles": titlelist_new,
"dateupdated": TIME,
}
}
collection.update_one(filter_, params)
# res = collection.find_one_and_update(filter_, params, params_r)
res = collection.find_one(filter_, params_r, no_cursor_timeout=True)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except DuplicateKeyError:
return JsonResponse({"code": "10003", "message": f"{title}-已存在"})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 删除观点标题
@csrf_exempt
def deleteTitleInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = get_data.get('industry1', '')
industry2 = get_data.get('industry2', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
title = get_data.get('title', '')
titleid = get_data.get('titleid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
collection_t = db_[pongo_title]
filter_ = {"templateid": templateid}
params_r = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"orgid": 1,
"orgname": 1,
"owner": 1,
"ownerid": 1,
"viewpointid": 1,
"viewpointname": 1,
"templateid": 1,
"templatename": 1,
"titles": 1,
}
res_data = collection.find_one(filter_, params_r, no_cursor_timeout=True)
titles = res_data.get('titles', [])
titlelist_new = []
for item in titles:
titleid_old = item.get('id', '')
title_old = item.get('templatetitlename', '')
title_name = item.get('title', '')
if titleid_old != titleid:
titlelist_new.append(item)
else:
filter_t = {"title": title_old}
collection_t.delete_one(filter_t)
collection_t_del = db_[pongo_title+'_del']
collection_t_del.insert_one({"id": titleid_old, "templatetitlename": title_old, "title": title_name})
params = {
"$set": {
"dateupdated": TIME,
"titles": titlelist_new
}
}
filter_ = {"templateid": templateid}
collection.update_one(filter_, params)
res = collection.find_one(filter_, params_r, no_cursor_timeout=True)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 判断观点标题是否重复
@csrf_exempt
def judgeTitleInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
industry1 = get_data.get('industry1', '')
industry2 = get_data.get('industry2', '')
orgid = get_data.get('orgid', '')
orgname = get_data.get('orgname', '')
viewpointid = get_data.get('viewpointid', '')
viewpointname = get_data.get('viewpointname', '')
ownerid = str(get_data.get('ownerid', ''))
owner = get_data.get('owner', '')
templateid = get_data.get('templateid', '')
templatename = get_data.get('templatename', '')
title = get_data.get('title', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_title]
data = {
"title": title,
"orgid": orgid,
"orgname": orgname,
"owner": owner,
"ownerid": ownerid,
"viewpointid": viewpointid,
"viewpointname": viewpointname,
"templateid": templateid,
"templatename": templatename,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
try:
collection.insert_one(data)
except DuplicateKeyError:
return JsonResponse({"code": "10003", "message": f"{title}-已存在"})
return JsonResponse({
"code": "0000",
"message": "成功",
"data": templateid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# ##########################################################################
# 词分类管理
# #########################################################################
# 词分类列表
@csrf_exempt
def simian_wordgroup_findall(request):
table1 = f"{simian_wordgroup_table} wt"
table2 = f"{simian_wordgroup_viewpoint_table} wv"
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
industry1 = get_data.get("industry1", '') # 行业
owner = get_data.get("owner", '') # 所属人
ownerid = get_data.get("ownerid", '') # 所属人id
orgname = get_data.get("orgname", '') # 所属人机构
orgid = get_data.get("orgid", '') # 所属人机构id
industry2 = get_data.get("industry2", '') # 二级行业
industryid1 = get_data.get("industryid1", '') # 一级行业id
industryid2 = get_data.get("industryid2", '') # 二级行业id
wordgroupname = get_data.get("wordgroupname", '') # 词名称
page = get_data.get("page", 1) # 页数
size = get_data.get("size", 20) # 条数
condition = f"""select wt.id, wt.wordgroupname ,wt.industry,wt.industry2,wt.industryid1,wt.industryid2,
wt.orgid,wt.orgname,wt.ownerid,wt.owner,
wv.tagmust, wv.owner, wv.viewpoint,wv.tagcity,wv.tagbrand, wt.status,
wv.tagscience, wv.taglocation, wt.description from {table1} left join {table2} ON wt.id = wv.wgid where wt.status=0 """
count = f"""select count(id) as total from {table1} where 1=1 and status=0 """
if orgid:
condition += " and wt.orgid='{}' ".format(orgid)
count += " and orgid='{}' ".format(orgid)
# if ownerid:
# condition += " and wt.ownerid='{}' ".format(ownerid)
# count += " and ownerid='{}' ".format(ownerid)
if industryid1:
condition += " and wt.industryid1='{}' ".format(industryid1)
count += " and industryid1='{}' ".format(industryid1)
if industryid2:
condition += " and wt.industryid2='{}' ".format(industryid2)
count += " and industryid2='{}' ".format(industryid2)
if wordgroupname:
condition += " and wt.wordgroupname='{}' ".format(wordgroupname)
count += " and wordgroupname='{}' ".format(wordgroupname)
condition += " order by wt.industry,wt.wordgroupname limit {},{}".format((page - 1) * size, size)
logger.info(condition)
datalist = m_client.search_db(condition)
total = m_client.search_db(count)
data = {"datalist": datalist, "total": total[0]}
logger.info('# 获取词分类数据 #')
code = "0000"
return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code]})
# else:
# code = "1019"
# return JsonResponse({"code": code, "message": CODE_DICT[code]})
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类配置查询
@csrf_exempt
def simian_wordgroup_viewpoint_deatils(request):
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
get_data = json.loads(request.body)
industry1 = get_data.get("industry1", '') # 行业
owner = get_data.get("owner", '') # 所属人
ownerid = get_data.get("ownerid", '') # 所属人id
orgname = get_data.get("orgname", '') # 所属人机构
orgid = get_data.get("orgid", '') # 所属人机构id
industry2 = get_data.get("industry2", '') # 二级行业
wordgroupname = get_data.get("wordgroupname", '') # 词名称
wgid = get_data.get("wgid", '') # 词Id
industryid1 = get_data.get("industryid1", '') # 一级行业id
industryid2 = get_data.get("industryid2", '') # 二级行业id
condition = f"""select a.*,b.* FROM
(
select svt.id as viewpointid, svt.viewpoint,svt.industry2,svt.industryid2,svt.industryid1, svt.orgid,svt.orgname,svt.ownerid,svt.owner
FROM simian_viewpoint_table svt , (SELECT industry2,industryid2,orgid,ownerid FROM simian_wordgroup_table WHERE id = '{wgid}') swt
WHERE svt.industryid2 = swt.industryid2 and svt.orgid = swt.orgid
) a
LEFT JOIN
(SELECT wvt.id, wt.id as wgid, wt.wordgroupname, wt.industry, wt.owner, wvt.tagmust, wvt.taglocation, wt.description,wvt.viewpointuuid,
wvt.tagcity,wvt.tagbrand,wvt.tagscience
FROM simian_wordgroup_viewpoint_table wvt,
simian_wordgroup_table wt WHERE wvt.wgid = wt.id AND wvt.wgid = '{wgid}') b
ON a.viewpointid = b.viewpointuuid ORDER BY b.id DESC,a.viewpointid"""
data = m_client.search_db(condition)
code = "0000"
return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 词分类观点关系保存
@csrf_exempt
def simian_wordgroup_viewpoint_add(request):
table = "simian_wordgroup_viewpoint_table"
if request.method == 'POST':
get_data = {}
try:
with LogMysqlApeT(ape_db) as m_client:
data = json.loads(request.body)
if data:
wgid = data[0].get("wgid", '')
delsql = "delete from {} where wgid = '{}'".format(table, wgid)
m_client.del_db(delsql)
TIME = timestamp_to_str(time.time())
for item in range(len(data)):
condition = {
"id": "simian-wg-vp-" + get_uuid(),
"wgid": data[item]['wgid'],
"tagmust": data[item]['tagmust'],
"tagcity": data[item]['tagcity'],
"tagbrand": data[item]['tagbrand'],
"tagscience": data[item]['tagscience'],
"wordgroupname": data[item]['wordgroupname'],
"owner": data[item]['owner'],
"ownerid": data[item]['ownerid'],
"orgid": data[item]['orgid'],
"orgname": data[item]['orgname'],
"industry": data[item]['industry'],
"industry2": data[item]['industry2'],
"industryid1": data[item]['industryid1'],
"industryid2": data[item]['industryid2'],
"viewpoint": data[item]['viewpoint'],
"viewpointuuid": data[item]['viewpointid'],
"industryviewpointmd5": md5_me(data[item]['industry']) + '-' + md5_me(data[item]['viewpoint']),
"createtime": TIME,
"updatetime": TIME,
"status": 0,
"taglocation": data[item]['taglocation']
}
logger.info(str(condition))
flag = m_client.updata_db(condition, table)
else:
code = "1020"
return JsonResponse({"code": code, "data": "", "message": CODE_DICT[code]})
code = "0000"
return JsonResponse({"code": code, "data": flag, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# ####################################################################
# 请求脚本
# ####################################################################
# 将观点模板同步
@csrf_exempt
def oldTemplageInsertNew(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
with LogMysqlApeT(ape_db) as m_client:
sql = f""" SELECT * FROM simian_vpcontent_table WHERE property='母模板' AND deleted='0' """
data_list = m_client.search_db(sql)
with MongoDBClient() as mg_client:
db_ = mg_client[db]
# 获取collection
collection = db_[pongo_content]
for item in data_list:
industry1 = item.get('industry', '')
industry2 = item.get('industry2', '')
industryid1 = ''
industryid2 = ''
if industry2 == '成人英语':
industryid1 = '1001'
industryid2 = '10011002'
if industry2 == '留学':
industryid1 = '1002'
industryid2 = '10021011'
if industry2 == '雅思':
industryid1 = '1000'
industryid2 = '10001001'
if industry2 == '托福':
industryid1 = '1000'
industryid2 = '10001000'
templateid = item.get('id')
content = item.get('viewpointcontent', '')
templatename = item.get('parentname', '')
viewpointid = item.get('viewpointuuid', '')
viewpointname = item.get('viewpoint', '')
data = {
"wordgroups": [],
"content": content,
"industry1": industry1,
"industry2": industry2,
"industryid1": industryid1,
"industryid2": industryid2,
"orgid": "bae479060ab74020a085ad665921e8f5",
"orgname": "环球雅思",
"owner": "",
"ownerid": "",
"viewpointid": viewpointid,
"viewpointname": viewpointname,
"description": "观点的写作方向",
"templateid": templateid,
"templatename": templatename,
"titles": [],
"datecreated": _datecreated,
"dateupdated": _datecreated,
"sort": 100,
"state": 0
}
collection.insert_one(data)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": templateid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 将观点标题同步
@csrf_exempt
def oldTitleTemplageInsertNew(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
with LogMysqlApeT(ape_db) as m_client:
sql = f""" SELECT * FROM simian_vptitle_table WHERE property='母标题' AND deleted='0' """
data_list_res = m_client.search_db(sql)
data_dict = {}
sql1 = f"""
SELECT contentid, viewpointuuid FROM simian_vptitle_table WHERE property='子标题' AND deleted='0' GROUP BY viewpointuuid ORDER BY contentid DESC
"""
data_list_res_1 = m_client.search_db(sql1)
c_v_dict = v_c_dict = {}
for j in data_list_res_1:
c_v_dict.update({j.get('viewpointuuid'): j.get('contentid')})
v_c_dict.update({j.get('contentid'): j.get('viewpointuuid')})
for i in data_list_res:
contentid_templateid = i.get('contentid')
viewpointuuid = i.get('viewpointuuid')
if not contentid_templateid:
contentid_templateid = c_v_dict.get(viewpointuuid, '0')
title = i.get('viewpointtitle')
tid = i.get('id')
if viewpointuuid == v_c_dict.get(contentid_templateid, ''):
data_dict.setdefault(contentid_templateid, []).append({
"id": tid,
"templatetitlename": title,
"title": title,
})
with MongoDBClient() as mg_client:
db_ = mg_client[db]
# 获取collection
collection = db_[pongo_content]
for k, v in data_dict.items():
params = {
"$set": {
# "dateupdated": TIME,
"titles": v
}
}
filter_ = {"templateid": k}
collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data_dict.get("0", ''),
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# ####################################################################
# 内容生成
# ####################################################################
# 内容生成接口
# 调取此接口，期间不能够进行修改，直接生成内容
@csrf_exempt
def produceContentTitle(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
templateid = get_data.get("templateid", '') # templateid
logger.info('进行内容和标题的替换生成')
logger.info('接收到数据：%s' % str(get_data))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"state": 1
}
filter_ = {"templateid": templateid}
print(filter_)
data_res = collection.find_one(filter_, params, no_cursor_timeout=True)
state = data_res.get('state', '')
if state == 0:
logger.info(f'可以生成替换文')
# 将任务塞入redis队列
r_client = Redis()
r_client.conn.lpush(pongo_produce_vpc_vpt, templateid)
params = {
"$set": {
"state": 1,
"dateupdated": _datecreated,
}
}
filter_ = {"templateid": templateid}
collection.update_one(filter_, params)
code = "0000"
else:
code = "1023"
return JsonResponse({"code": code, "data": "ok", "message": CODE_DICT[code], "datecreated": _datecreated})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# 删除生成的子内容
@csrf_exempt
def deleteReContentTitle(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
templateid = get_data.get("templateid", '') # templateid
logger.info('删除生成的子内容')
logger.info('接收到数据：%s' % str(get_data))
with MongoDBClient() as mg_client, LogMysqlApeT(ape_db) as m_client:
db_ = mg_client[db]
# 获取collection
collection = db_[pongo_content]
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"state": 1
}
filter_ = {"templateid": templateid}
print(filter_)
data_res = collection.find_one(filter_, params, no_cursor_timeout=True)
state = data_res.get('state', '')
if state == 3:
logger.info(f'可以删除')
# 删除子内容
sql_deleted_vptitle = f""" delete from {simian_vptitle_table} where contentid='{templateid}' and used=0 and deleted=0 and property='子标题' """
m_client.del_db(sql_deleted_vptitle)
logger.info(sql_deleted_vptitle)
sql_deleted_vpcontent = f"""delete from {simian_vpcontent_table} where belongtemplate='{templateid}' and deleted=0 and used=0 and property='子模板' """
m_client.del_db(sql_deleted_vpcontent)
logger.info(sql_deleted_vpcontent)
# 删除成功后修改值，可以进行编辑
params = {
"$set": {
"state": 0,
"dateupdated": _datecreated,
}
}
filter_ = {"templateid": templateid}
collection.update_one(filter_, params)
code = "0000"
else:
code = "1024"
return JsonResponse({"code": code, "data": "ok", "message": CODE_DICT[code], "datecreated": _datecreated})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# ####################################################################
# 关键词统计
# ####################################################################
# 关键词统计列表
@csrf_exempt
def keywordStatisticsList(request):
_datecreated = timestamp_to_str()
logger.info("keywordStatisticsList")
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "0000", "message": "success"})
if request.method == "GET":
ownerid = str(request.GET.get('ownerid', ''))
owner = request.GET.get('owner', '')
keyword = request.GET.get('keyword', '')
area = request.GET.get('area', '')
wordgroup = request.GET.get('wordgroup', '')
wordgroupid = request.GET.get('wordgroupid', '')
orgid = request.GET.get('orgid', '')
orgname = request.GET.get('orgname', '')
industry1 = request.GET.get('industry1', '')
industryid1 = request.GET.get('industryid1', '')
industry2 = request.GET.get('industry2', '')
industryid2 = request.GET.get('industryid2', '')
onlinedate = request.GET.get('onlinedate', '')
date = request.GET.get('date', '["", ""]')
date = json.loads(date)
logger.info(str(date))
logger.info(str(type(date)))
status = str(request.GET.get('status', ''))
page = int(request.GET.get('page', 1))
size = int(request.GET.get('size', 10))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
filter_ = {}
if orgid:
filter_.update({"orgid": orgid})
if industryid1:
filter_.update({"industryid1": industryid1})
if industryid2:
filter_.update({"industryid2": industryid2})
if keyword:
filter_.update({"keyword": {"$regex": keyword}})
# 0 待编辑
# 1 已编辑
# 2 已发布
# 10 计划中
if status == "0":
filter_.update({
"publishlist": {"$elemMatch": {"status": 0}},
# "publishlist.status": 0
})
if status == "1":
filter_.update({
"publishlist": {"$elemMatch": {"status": 1}},
# "publishlist.status": 1
})
if status == "2":
filter_.update({
"publishlist": {"$elemMatch": {"status": 2}},
# "publishlist.status": 2
})
if status == "10":
filter_.update({
"publishlist": {"$elemMatch": {"status": 10}},
# "publishlist.status": 10
})
if date[0] != "" and date:
tk = [
{"onlinedate": {"$gte": date[0]}}, {"onlinedate": {"$lte": date[1]}}
]
if status == "0":
tk.append({"status": 0})
if status == "1":
tk.append({"status": 1})
if status == "2":
tk.append({"status": 2})
if status == "10":
tk.append({"status": 10})
tem = {"publishlist": {"$elemMatch": {"$and": tk}}}
filter_.update(tem)
pipeline = [
{"$match": filter_},
{"$sort": {"datecreated": -1, "keyword": 1}},
{"$skip": (page - 1) * size},
{"$limit": size},
# {"$limit": page * size},
# {"$skip": (page - 1) * size},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroup": 1,
"wordgroupid": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
"publishlist": 1,
"publishsum": {f"$size": f"$publishlist"}, # 发布任务总数量
"published": {
"$function": {
"body": """function(name) {
var count = 0;
for(j = 0,len=name.length; j< len; j++) {
if(name[j].status === 2){
count += 1;
}
}
return count ;
} """,
"args": ["$publishlist"],
"lang": "js"
}, # 已发布数量
},
"toeditpublish": {
"$function": {
"body": """function(name) {
var count = 0;
for(j = 0,len=name.length; j< len; j++) {
if(name[j].status === 0){
count += 1;
}
}
return count ;
} """,
"args": ["$publishlist"],
"lang": "js"
}, # 待编辑数量
},
"editpublish": {
"$function": {
"body": """function(name) {
var count = 0;
for(j = 0,len=name.length; j< len; j++) {
if(name[j].status === 1){
count += 1;
}
}
return count ;
} """,
"args": ["$publishlist"],
"lang": "js"
}, # 已编辑数量
},
"prepublish": {
"$function": {
"body": """function(name) {
var count = 0;
for(j = 0,len=name.length; j< len; j++) {
if(name[j].status === 10){
count += 1;
}
}
return count ;
} """,
"args": ["$publishlist"],
"lang": "js"
}, # 计划中数量
},
},
},
]
logger.info(pipeline)
total = collection.count_documents(filter=filter_)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
else:
code = "1016"
return JsonResponse({"code": code, "message": CODE_DICT[code]})
return JsonResponse({
"code": "0000",
"message": "成功",
"total": total,
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
return JsonResponse({"code": "1001", "message": "no this method"})
# 关键词统计标题详情
@csrf_exempt
def getKeywordStatisticsInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
keywordid = request.GET.get('keywordid', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
filter_ = {"_id": ObjectId(keywordid)}
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroupid": 1,
"wordgroup": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
"publishlist": 1,
}
print(filter_)
# res = collection.find_one(filter_, no_cursor_timeout=True)
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = {}
if res:
data = res
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# 关键词统计编辑
@csrf_exempt
def updateKeywordStatisticsInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e)})
keywordid = get_data.get('keywordid', '')
tid = get_data.get('tid', '')
aid = int(get_data.get('aid', 0))
title = get_data.get('title', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
collection_t = db_[pongo_tdk_title]
params_r = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"wordgroupid": 1,
"wordgroup": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
"area": 1,
"orgid": 1,
"orgname": 1,
"publishlist": 1,
}
# filter_ = {"_id": ObjectId(keywordid)}
# print(filter_)
# data_res = collection.find_one(filter_, params, no_cursor_timeout=True)
# publishlist = data_res.get("publishlist", [])
# publishlist_new = []
data_title = {
"tid": tid,
"aid": aid,
"title": title,
"dateupdated": TIME,
"datecreated": TIME,
}
try:
collection_t.insert_one(data_title)
# for item in publishlist:
# item_aid = int(item.get("aid", 0))
# if item_aid == aid:
# item.update({"title": title, "status": 1})
# publishlist_new.append(item)
xgz = {}
xgz.update({"dateupdated": TIME, "publishlist.$.title": title, "publishlist.$.status": 1, "status": 1})
params = {
"$set": xgz,
}
filter_ = {"_id": ObjectId(keywordid), "publishlist": {"$elemMatch": {"id": tid}}}
collection.update_one(filter_, params)
filter_ = {"_id": ObjectId(keywordid)}
print(filter_)
data_res = collection.find_one(filter_, params_r, no_cursor_timeout=True)
publishlist_new = data_res.get("publishlist", [])
code = "0000"
return JsonResponse({
"code": code,
"message": CODE_DICT[code],
"data": publishlist_new,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except DuplicateKeyError:
code = "1022"
return JsonResponse({"code": code, "message": f"（{title}）-{CODE_DICT[code]}"})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# 关键词统计获取覆盖词函数
def get_cover_word_list(get_data):
title = get_data.get("title", '')
orgid = get_data.get("orgid", '')
industryid1 = get_data.get("industryid1", '')
industryid2 = get_data.get("industryid2", '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
}
filter_ = {"orgid": orgid, "industryid1": industryid1, "industryid2": industryid2}
print(filter_)
data_res = collection.find(filter_, params, no_cursor_timeout=True)
#sql = "select keyword from {}".format(ape_keyword_table)
#res = m_client.search_db(sql)
keyword_list = [item_.get('keyword') for item_ in data_res]
res_list = []
for item in keyword_list:
# keyword = item
index = 0
if item:
for keyword_item in item:
index = title.find(keyword_item, index)
print("index", index)
if index < 0:
break
else:
res_list.append(item)
data = list(set(list(filter(lambda x: len(x) > 0, res_list))))
if not data:
data = ['没有覆盖的关键词']
return data
# 关键词统计获取取当前标题的词覆盖有哪些词
@csrf_exempt
def getKeywordCover(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
title = get_data.get("title", '')
orgid = get_data.get("orgid", '')
industryid1 = get_data.get("industryid1", '')
industryid2 = get_data.get("industryid2", '')
logger.info('获取取当前标题的词覆盖有哪些词')
logger.info('接收到数据：%s' % str(get_data))
data = get_cover_word_list(get_data)
code = "0000"
return JsonResponse({"code": code, "data": data, "datecreated": _datecreated, "message": CODE_DICT[code]})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# 获取当前词所在分组的词有哪些
@csrf_exempt
def getWordgroupKeyword(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
keyword = get_data.get("keyword", '') # keyword
kid = get_data.get("kid", '') # kid
orgid = get_data.get("orgid", '') # kid
wordgroup = get_data.get("wordgroup", '') # kid
wordgroupid = get_data.get("wordgroupid", '') # kid
industry1 = get_data.get("industry1", '') # 行业1
industry2 = get_data.get("industry2", '') # 行业2
industryid1 = get_data.get("industryid1", '') # 行业id1
industryid2 = get_data.get("industryid2", '') # 行业id2
logger.info('获取当前词所在分组的词有哪些')
logger.info('接收到数据：%s' % str(get_data))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_keyword]
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"industry1": 1,
"industryid1": 1,
"industry2": 1,
"industryid2": 1,
"keyword": 1,
}
filter_ = {"orgid": orgid, "industryid1": industryid1, "industryid2": industryid2, "wordgroupid": wordgroupid}
print(filter_)
data_res = collection.find(filter_, params, no_cursor_timeout=True)
keyword_list = [item_.get('keyword') for item_ in data_res]
code = "0000"
return JsonResponse({"code": code, "data": keyword_list, "message": CODE_DICT[code], "datecreated": _datecreated})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# ###########################
# 站点接口
# ###########################
# 获取站点列表
@csrf_exempt
def getAllianceList(request):
_datecreated = timestamp_to_str()
if request.method == 'GET':
try:
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_alliance]
params = {
# _id 可以设置0 别的参数不可以
"_id": 0,
"aid": 1,
"aname": 1,
"id": {f"$toString": f"$_id"},
"labellist": 1,
}
filter_ = {}
print(filter_)
data_res = collection.find(filter_, params, no_cursor_timeout=True)
data = list(data_res)
code = "0000"
return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code], "datecreated": _datecreated})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# ###########################
# 发布接口
# ###########################
# # 发布配置
@csrf_exempt
def publishTaskCreate(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
date = get_data.get("date", [])
aid = int(get_data.get("aid", 0))
aname = get_data.get("aname", '')
tcid = get_data.get("id", '')
distribute = int(get_data.get("distribute", '0')) # 1 为话题 2 为关键词
order = int(get_data.get("order", '0')) # 1 为词覆盖 2 为指数
info = get_data.get("info", [])
# publishnum = get_data.get("publishnum", 0)
# industry1 = get_data.get("industry1", '')
# industryid1 = get_data.get("industryid1", '')
# industry2 = get_data.get("industry2", '')
# industryid2 = get_data.get("industryid2", '')
date_list = dateRange(date[0], date[1])
logger.info('发布配置')
logger.info('接收到数据：%s' % str(get_data))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_publish_task]
if tcid:
filter_ = {"_id": ObjectId(tcid)}
res = collection.find_one(filter_)
else:
res = {}
if res:
data = {
"date": date,
"datelist": date_list,
"distribute": distribute,
"order": order,
"info": info,
}
data.update(
{"dateupdated": _datecreated}
)
params_ = {
"$set": data,
}
collection.update_one(filter_, params_)
else:
data = {
"date": date,
"datelist": date_list,
"aid": aid,
"aname": aname,
"distribute": distribute,
"order": order,
"info": info,
"state": 0,
"datecreated": _datecreated,
"dateupdated": _datecreated,
}
collection.insert_one(data)
code = "0000"
return JsonResponse({"code": code, "data": "ok", "message": CODE_DICT[code], "datecreated": _datecreated})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# # 发布配置详情
@csrf_exempt
def publishTaskInfo(request):
_datecreated = timestamp_to_str()
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
aid = int(get_data.get("aid", 0))
state = int(get_data.get("state", 10000))
page = int(get_data.get("page", 1))
size = int(get_data.get("size", 10))
logger.info('发布配置详情')
logger.info('接收到数据：%s' % str(get_data))
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_publish_task]
filter_ = {"aid": aid}
if state != 10000:
filter_.update({"state": state})
pipeline = [
{"$match": filter_},
{"$sort": {"state": 1, "datecreated": -1}},
{"$skip": (page - 1) * size},
{"$limit": size},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"id": {f"$toString": f"$_id"},
"date": 1,
"aid": 1,
"aname": 1,
"distribute": 1,
"order": 1,
"state": 1, # 如果内容正在生成或者生成，不让修改
"labellist": 1,
"info": 1,
"datelist": 1,
}
},
]
print(pipeline)
total = collection.count_documents(filter=filter_)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
code = "0000"
return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code], "datecreated": _datecreated, "total": total})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# # 发布任务生成给词做标记，给标题做预处理
@csrf_exempt
def publishTaskProduce(request):
_datecreated = timestamp_to_str()
logger.info(f'# 发布任务生成给词做标记，给标题做预处理')
if request.method == 'GET':
try:
date = request.GET.get("date", '')
logger.info('发布任务生成给词做标记，给标题做预处理')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_publish_task]
collection_k = db_[pongo_keyword]
filter_ = {"datelist": {"$elemMatch": {"$eq": date}}, "state": 0}
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"id": {f"$toString": f"$_id"},
"date": 1,
"datelist": 1,
"aid": 1,
"aname": 1,
"distribute": 1,
"order": 1,
"state": 1, # 如果内容正在生成或者生成，不让修改
"info": 1,
}
},
]
logger.info(pipeline)
# total = collection.count_documents(filter=filter_)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
logger.info(f'开始遍历配置的发布任务')
for item in res:
t_id = item.get('id')
info = item.get('info', [])
aid = int(item.get('aid', 0))
aname = item.get('aname', '')
distribute = item.get('distribute', '')
order = item.get('order', '')
datelist = item.get('datelist', [])
logger.info(f"datelist - {str(datelist)}")
keyword_count = {}
for i_date in datelist:
logger.info(i_date)
for i_info in info:
labelid1 = i_info.get('labelid1', '')
labelid2 = i_info.get('labelid2', '')
labelname1 = i_info.get('labelname1', '')
labelname2 = i_info.get('labelname2', '')
count = int(i_info.get('count', 0))
filter_k = {
"state": 0,
"publishlist": {"$elemMatch": {"labelname1": labelname1, "labelname2": labelname2}},
}
logger.info(f"找出条件为 --- {str(filter_k)} --- 的词")
pipeline_k = [
{"$match": filter_k},
# {"$limit": count},
{"$sample": {"size": count}},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"id": {f"$toString": f"$_id"},
"keyword": 1,
"publishlist": 1,
}
},
]
total = collection_k.count_documents(filter=filter_k)
tem = {"total": total, "filter_k": str(filter_k)}
keyword_count.setdefault(i_date, []).append(tem)
logger.info(f'查询数据个数为：{total}')
res_k = collection_k.aggregate(pipeline=pipeline_k, allowDiskUse=True)
for k_item in res_k:
k_id = k_item.get('id')
# 标题
publishlist = k_item.get('publishlist', [])
publishlist_new = []
for p_item in publishlist:
p_aid = int(p_item.get('aid', 0))
if p_aid:
if int(aid) == p_aid:
p_item.update({"title": "", "status": 0, "onlinedate": i_date})
publishlist_new.append(p_item)
# 将状态值变为 state变为 10
filter_k = {"_id": ObjectId(k_id)}
params_k = {
"$set": {
"dateupdated": _datecreated,
"publishlist": publishlist_new,
"state": 10,
"prepstatus": 1,
"pstatus": 0
},
}
collection_k.update_one(filter_k, params_k)
pass
filter_t = {"_id": ObjectId(t_id)}
params_t = {
"$set": {
"dateupdated": _datecreated,
"state": 1,
"keyword_count": keyword_count,
},
}
collection.update_one(filter_t, params_t)
code = "0000"
return JsonResponse({"code": code, "data": data, "message": CODE_DICT[code], "datecreated": _datecreated})
except Exception as e:
return JsonResponse({"code": "1002", "message": str(e), "datecreated": _datecreated})
return JsonResponse({"code": "1001", "message": "no this method", "datecreated": _datecreated})
# _*_ coding: utf-8 _*_
from django.shortcuts import render, HttpResponse
from django.http import JsonResponse
from django.views.decorators.csrf import csrf_exempt, csrf_protect
# Create your views here.
import os
import oss2
import uuid
import sys
import ast
import time
import json
import redis
import hashlib
import requests
import random
from tools.mysql_client import LogMysqlApe, LogMysql, LogMysqlApeT, LogMysqlAlliance
from tools.excel_client import read_excel
from upload_pro.settings import logger
from urllib.parse import urlparse, unquote
from django.conf import settings
from tools.mongodb_client import MongoDBClient
from bson import ObjectId
from bson import json_util
from bson.json_util import dumps
def get_uuid():
res = str(uuid.uuid4())
UUID = ''.join(res.split('-'))
return UUID
def md5_me(key):
md5 = hashlib.md5()
md5.update(str(key).encode('utf-8'))
value = md5.hexdigest()
return value
# 把格式化时间转换成时间戳
def str_to_timestamp(str_time=None, format='%Y-%m-%d %H:%M:%S'):
if str_time:
time_tuple = time.strptime(str_time, format) # 把格式化好的时间转换成元祖
result = time.mktime(time_tuple) # 把时间元祖转换成时间戳
return int(result)
return int(time.time())
# 把时间戳转换成格式化
def timestamp_to_str(timestamp=None, format='%Y-%m-%d %H:%M:%S'):
if timestamp:
time_tuple = time.localtime(timestamp) # 把时间戳转换成时间元祖
result = time.strftime(format, time_tuple) # 把时间元祖转换成格式化好的时间
return result
else:
return time.strftime(format)
db = settings.PONGO_DB
pongo_wordgroup = "pongo_wordgroup"
pongo_content = "pongo_content"
pongo_category = "pongo_category"
# 词组列表
@csrf_exempt
def getWordgroupList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 0, "message": "success"})
if request.method == "GET":
ownerid = request.GET.get('ownerid', '')
owner = request.GET.get('owner', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
filter_ = {}
if ownerid:
filter_ = {"ownerid": ownerid}
if owner:
filter_ = {"owner": owner}
pipeline = [
{"$match": filter_},
{
"$project": {
# _id 可以设置0 别的参数不可以
"_id": 0,
"name": 1,
"owner": 1,
"ownerid": 1,
"id": {f"$toString": f"$_id"},
"itemnum": {f"$size": f"$items"},
}
},
]
print(pipeline)
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 词组详情
@csrf_exempt
def getWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
wordgroupid = request.GET.get('wordgroupid', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
filter_ = {"_id": ObjectId(wordgroupid)}
params = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"items": 1,
"owner": 1,
"ownerid": 1,
}
print(filter_)
# res = collection.find_one(filter_, no_cursor_timeout=True)
res = collection.find_one(filter_, params, no_cursor_timeout=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 新增词组
@csrf_exempt
def newWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
name = get_data.get('name', '')
items = get_data.get('items', [])
owner = get_data.get('owner', '')
ownerid = get_data.get('ownerid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
data = {
"name": name,
"items": items,
"owner": owner,
"ownerid": ownerid,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 10,
"state": 0
}
res = collection.insert_one(data)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res.inserted_id,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 修改词组
@csrf_exempt
def updateWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
name = get_data.get('name', '')
items = get_data.get('items', [])
wordgroupid = get_data.get('wordgroupid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
params = {
"$set": {
"name": name,
"items": items,
"dateupdated": TIME,
},
}
filter_ = {"_id": ObjectId(wordgroupid)}
res = collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": wordgroupid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 删除词组
@csrf_exempt
def deleteWordgroupInfo(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
name = get_data.get('name', '')
items = get_data.get('items', [])
wordgroupid = get_data.get('wordgroupid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_wordgroup]
collection_del = db_[pongo_wordgroup+'_del']
filter_ = {"_id": ObjectId(wordgroupid)}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
res = collection.delete_one(filter_)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": wordgroupid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 文章模板列表
@csrf_exempt
def getContentList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 0, "message": "success"})
if request.method == "GET":
ownerid = request.GET.get('ownerid', '')
owner = request.GET.get('owner', '')
categoryid = request.GET.get('categoryid', '')
category = request.GET.get('category', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = filter_2 = {}
if ownerid:
filter_ = {"ownerid": ownerid}
if owner:
filter_ = {"owner": owner}
if categoryid:
filter_2 = {"categoryid": categoryid}
if category:
filter_2 = {"category": category}
filter_.update(filter_2)
pipeline = [
{"$match": filter_},
{
"$project": {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"description": 1,
"owner": 1,
"ownerid": 1,
"categoryid": 1,
"category": 1,
"content": 1,
}
},
]
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 文章模板详情
@csrf_exempt
def getContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "GET":
contentid = request.GET.get('contentid', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
filter_ = {"_id": ObjectId(contentid)}
params = {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"description": 1,
"owner": 1,
"ownerid": 1,
"categoryid": 1,
"category": 1,
"content": 1,
"wordgroups": 1,
"property": 1,
"datecreated": 1,
"dateupdated": 1,
}
res = collection.find_one(filter_, params, no_cursor_timeout=True)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 新增模板
@csrf_exempt
def newContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
name = get_data.get('name', '')
description = get_data.get('description', '')
owner = get_data.get('owner', '')
ownerid = get_data.get('ownerid', '')
categoryid = get_data.get('categoryid', '')
category = get_data.get('category', '')
content = get_data.get('content', '')
wordgroups = get_data.get('wordgroups', [])
property = get_data.get('property', {"distinct": 3})
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
data = {
"name": name,
"description": description,
"owner": owner,
"ownerid": ownerid,
"categoryid": categoryid,
"category": category,
"content": content,
"wordgroups": wordgroups,
"property": property,
"datecreated": TIME,
"dateupdated": TIME,
"sort": 100,
"state": 0
}
res = collection.insert_one(data)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": res.inserted_id,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 修改模板
@csrf_exempt
def updateContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
name = get_data.get('name', '')
description = get_data.get('description', '')
owner = get_data.get('owner', '')
ownerid = get_data.get('ownerid', '')
categoryid = get_data.get('categoryid', '')
category = get_data.get('category', '')
content = get_data.get('content', '')
wordgroups = get_data.get('wordgroups', [])
property = get_data.get('property', {"distinct": 3})
contentid = get_data.get('id', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
params = {
"$set": {
"name": name,
"description": description,
"owner": owner,
"ownerid": ownerid,
"categoryid": categoryid,
"category": category,
"content": content,
"wordgroups": wordgroups,
"property": property,
"dateupdated": TIME,
}
}
filter_ = {"_id": ObjectId(contentid)}
res = collection.update_one(filter_, params)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": contentid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 删除模板
@csrf_exempt
def deleteContent(request):
_datecreated = timestamp_to_str()
try:
if request.method == "POST":
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
contentid = get_data.get('contentid', '')
TIME = timestamp_to_str()
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_content]
collection_del = db_[pongo_content + '_del']
filter_ = {"_id": ObjectId(contentid)}
data = collection.find_one(filter_)
data.update({"dateupdated": _datecreated})
collection_del.insert_one(data)
res = collection.delete_one(filter_)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": contentid,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
# 分类列表
@csrf_exempt
def getCategoryList(request):
_datecreated = timestamp_to_str()
try:
if request.method == 'POST':
get_data = {}
try:
get_data = json.loads(request.body)
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 0, "message": "success"})
if request.method == "GET":
ownerid = request.GET.get('ownerid', '')
owner = request.GET.get('owner', '')
with MongoDBClient() as m_client:
db_ = m_client[db]
# 获取collection
collection = db_[pongo_category]
filter_ = {}
if ownerid:
filter_ = {"ownerid": ownerid}
if owner:
filter_ = {"owner": owner}
pipeline = [
{"$match": filter_},
{
"$project": {
"_id": 0,
"id": {f"$toString": f"$_id"},
"name": 1,
"owner": 1,
"ownerid": 1,
}
},
]
res = collection.aggregate(pipeline=pipeline, allowDiskUse=True)
data = []
if res:
data = list(res)
return JsonResponse({
"code": "0000",
"message": "成功",
"data": data,
"datecreated": _datecreated,
}, json_dumps_params={'ensure_ascii': False, "default": json_util.default})
except Exception as e:
return JsonResponse({"code": 1002, "message": str(e)})
return JsonResponse({"code": 1001, "message": "no this method"})
python爬虫的任务数据操作的小技巧
好久没写公众号了，最近太忙了，这里就不多说了。直接根据需求上代码，我想这个应该是大家比较喜欢的，
需求
爬取某网站的项目列表页，获取其url，标题等信息，作为后续爬取详情页的任务url。
先上代码
代码
# -*- coding: utf-8 -*-
# @Time : 2019-11-08 14:04
# @Author : cxa
# @File : motor_helper.py
# @Software: PyCharm
import asyncio
import datetime
from loguru import logger
from motor.motor_asyncio import AsyncIOMotorClient
from collections import Iterable
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except ImportError:
pass
db_configs = {
'host': '127.0.0.1',
'port': '27017',
'db_name': 'mafengwo',
'user': ''
}
class MotorOperation:
def __init__(self):
self.__dict__.update(**db_configs)
if self.user:
self.motor_uri = f"mongodb://{self.user}:{self.passwd}@{self.host}:{self.port}/{self.db_name}?authSource={self.db_name}"
else:
self.motor_uri = f"mongodb://{self.host}:{self.port}/{self.db_name}"
self.client = AsyncIOMotorClient(self.motor_uri)
self.mb = self.client[self.db_name]
async def save_data_with_status(self, items, col="seed_data"):
for item in items:
data = dict()
data["update_time"] = datetime.datetime.now()
data["status"] = 0 # 0初始
data.update(item)
print("data", data)
await self.mb[col].update_one({
"url": item.get("url")},
{'$set': data, '$setOnInsert': {'create_time': datetime.datetime.now()}},
upsert=True)
async def add_index(self, col="seed_data"):
# 添加索引
await self.mb[col].create_index('url')
因为我的爬虫是异步网络模块aiohttp写的，所以选择了pymongo的异步版本motor进行操作。
异步代码的基本属性就是async/await成对的出现，如果把上面的await和async去掉，就是类似pymongo的写法了,这里异步不是重点，重点是我们怎么处理每条数据。
这里除了网页的url，标题等信息，我需要附加3个字段。分别是create_time, status,update_time。
这三个字段分别代表，数据插入数据，状态和更新时间。
那么我为什么添加三个字段呢？
首先，我们需要判断每次的任务数据是否存在，我这里的情况是存在就更新不存在就插入，那么我就需要一个查询条件，作为更新的条件，很显然这里可以使用任务的url作为唯一条件(你还可以使用url+标题做个md5然后保存)。好了查询条件确定，
下面说create_time这个比较好理解就是数据插入时间,关键是为什么还要一个update_time，这个的话和status字段有一定的关系。 画重点：这个status作为后续爬虫进行爬取的一个标志用。目前这个status有4个值，0-4，我这是这样定义的，
0:初始状态 1:抓取中的任务 2:抓取成功 3:抓取失败 4:抓取成功但是没有匹配到任务。
后面随着任务的爬取，状态也是不断变化的，同时我们需要更新update_time为最新的时间。这个目前的话是体现不出来什么作用，它的使用场景是，重复任务的抓取，比如今天我抓取了任务列表里的url1,url2，第二天的时候我如果再抓到，为了区分是抓取失败还是抓取成功，我们根据create_time和update_time就可以进行推断了，如果两者相同而且是当前的日期说明刚抓的，如果update_time的日期比create_time新可以说明，抓到了重复的任务。关于字段的设计就啰嗦这么写。
下面是实现，我们可以通过update_one方法，对数据作存在或者插入操作，因为url作为查询条件，后面量大的话就最好添加一个索引。也就是上面的 add_index方法。
好了最好说插入更新的具体代码
需要注意的是
{'$set': data, '$setOnInsert': {'create_time': datetime.datetime.now()}}
$setOnInsert里面使用的字段是数据不存在的时候才插入的，存在就不动了，只插入$set里面指定的。
另外$setOnInsert里面使用的字段不能在$set里面再次出现
upsert=True代表的是不存在就插入。
大概就这么多，不明白的可以给我留言，或者添加微信进群交流。
浙公网安备 33010602011771号
[Ctrl+Enter快捷键提交]
【推荐】MASA Framework 开启全新的.NET应用开发体验
【推荐】下一步，敏捷！云可达科技SpecDD敏捷开发专区