1.run_py.sh shell后台运行文件
#!/bin/bash
#name1=`basename ${0%\.*}` # 获取传入的文件名 去掉后缀 /b/c/a.py-->a
#echo $name1
#name2=$(echo $name1 | sed 's/hbase_id//') # 字符串替换
#echo $name2
#NAME=$name1'.py' # $1 运行时输入参数 为文件名称 # 拼接字符串
#NAME='jmtional_browser_policy.py' # $1 运行时输入参数 为文件名称
NAME=$1 # $1 运行时输入参数 为文件名称
echo $NAME
NAME=`basename ${NAME%%.*}` # 去掉文件后缀
if [ -z "$NAME" ]; then
echo "STRING is empty"
NAME="aa"
fi
echo $NAME
echo "---------------"
ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
echo $ID
echo "---------------"
for id in $ID
do
kill -9 $id
echo "killed $id"
done
echo "---------------"
sleep 1
echo "chrome"
echo "---------------"
ID=`ps -ef | grep "chrome" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
echo $ID
echo "---------------"
for id in $ID
do
kill -9 $id
echo "killed $id"
done
echo "---------------"
sleep 1
cd /home/yscredit/yoyo/sx_zc # 进入目录
sleep 1
current_dir=$(cd $(dirname $0); pwd)
echo $current_dir
if [ ! -d "$current_dir/logs" ]; then
echo "$current_dir/logs does not exist"
`mkdir $current_dir/logs`
fi
echo "---------------"
echo "nohup python3.10 $current_dir/$NAME.py > $current_dir/logs/$NAME.log 2>&1 &"
echo "---------------"
echo "tail -f $current_dir/logs/$NAME.log"
sleep 1
cd /home/yscredit/yoyo/sx_zc
`nohup python3.10 $current_dir/$NAME.py > $current_dir/logs/$NAME.log 2>&1 &`
echo "启动成功"
2.run_py.py python运行目录下的Python文件
# -*- coding: UTF-8 -*-
import concurrent.futures
import os
import platform
import re
import subprocess
Logger.config(level="info", processname=os.path.splitext(os.path.basename(__file__))[0])
logger = Logger
# def run_main():
# system = platform.system()
# path = os.path.abspath('.')
# for spider_name in os.listdir("."):
# if spider_name.endswith(".py") and "ggzyjy" in spider_name:
# if 'indo' in system:
# spider_path = '{}\{}'.format(path, spider_name)
# else:
# spider_path = '{}/{}'.format(path, spider_name)
# logger.info(spider_path)
# res = subprocess.call("python {}".format(spider_path), shell=True)
# logger.info(res)
def run_spider(spider_path):
logger.info(spider_path)
try:
if os.environ.get("yu_localhosts", "") == "localhost":
res = subprocess.call("python {}".format(spider_path), shell=True)
else:
res = subprocess.call("python3.6 {}".format(spider_path), shell=True)
logger.info(res)
except Exception as e:
logger.error("爬虫启动失败:{]".format(e))
if __name__ == '__main__':
# run_main()
for i in range(1):
try:
data_list = []
system = platform.system()
path = os.path.abspath('.')
for spider_name in os.listdir("."):
if spider_name.endswith(".py") and "ggzyjy" in spider_name:
index = re.findall(r'\d+', spider_name)
if index:
index = int(index[0])
else:
index = 0
if index != 14:
if 'indo' in system:
spider_path = '{}\{}'.format(path, spider_name)
else:
spider_path = '{}/{}'.format(path, spider_name)
data_list.append(spider_path)
with concurrent.futures.ThreadPoolExecutor(max_workers=59) as executor:
future_to_url = {executor.submit(run_spider, data): data for data in data_list}
except Exception as e:
print("报错")
3.手动多线程运行
import subprocess
class copyFile(object):
def __init__(self):
self.py_file = "zc_file_html00.py"
self.shell_file = "hbase_id00.sh"
self.start = 13
self.end = 99
# self.copy_file()
self.crontab_shell()
# self.run_shell()
def copy_file(self):
for i in range(self.start, self.end + 1):
for file_name in [self.py_file, self.shell_file]:
with open(file_name, 'r') as source_file:
py_content = source_file.read()
if i < 10:
file_name_ = file_name.replace("00", "0{}".format(str(i)))
else:
file_name_ = file_name.replace("00", "{}".format(str(i)))
with open(file_name_, 'w') as destination_file:
destination_file.write(py_content)
print('文件复制成功:{}'.format(file_name_))
def crontab_shell(self):
for i in range(self.start, self.end + 1):
b = i
if b > 59:
b = b - 60
if i < 10:
print('{} * * * * /bin/sh /home/yscredit/yoyo/zc/hbase_id0{}.sh > /home/yscredit/yoyo/zc/hbase_id.log 2>&1 &'.format(str(b), str(i)))
else:
print('{} * * * * /bin/sh /home/yscredit/yoyo/zc/hbase_id{}.sh > /home/yscredit/yoyo/zc/hbase_id.log 2>&1 &'.format(str(b), str(i)))
def run_shell(self):
for i in range(self.start, self.end + 1):
if i < 10:
print(subprocess.call("bash /home/yscredit/yoyo/zc/hbase_id0{}.sh".format(str(i)), shell=True))
else:
print(subprocess.call("bash /home/yscredit/yoyo/zc/hbase_id{}.sh".format(str(i)), shell=True))
def run_shell(self):
for filename in os.listdir("."):
if ".py" in filename and filename.count('_') == 2:
print(filename)
print(subprocess.call("bash run_py.sh {}".format(str(filename)), shell=True))
if __name__ == "__main__":
copyFile()
4.kills.sh 关闭某个线程
#!/bin/sh
NAME=$1 # $1 运行时输入参数 为文件名称
if [ -z "$NAME" ]; then
echo "STRING is empty"
NAME="aa"
fi
echo $NAME
ID=`ps -ef | grep "$NAME" | grep -v "$0" | grep -v "grep" | awk '{print $2}'`
echo $ID
echo "---------------"
for id in $ID
do
kill -9 $id
echo "killed $id"
done
echo "---------------"
5.定时启动任务
04 18 * * * /usr/bin/bash /home/yscredit/yoyo/sx_zc/jmtional_browser_policy.sh > /home/yscredit/yoyo/sx_zc/logs/jmtional_browser_policy.log 2>&1 &
