MySQL一张表迁移到MongoDB

MySQL:
lan-dev-zt-v2-mysql.bndxqc.com:3316
表：  bonade_operate_center.saas_oper_log

mysql --login-path=root-10.2.21.57   bonade_operate_center  -A


SELECT max(oper_id) FROM saas_oper_log;   -- 1846879


MongoDB
链接信息： 10.2.29.81:20001    

mongo  10.2.29.81:20001  -udevuser -p "devuser.COM2020"   --authenticationDatabase=admin


导出CSV格式：


import pandas as pd
import pymysql

# MySQL连接信息
connection = pymysql.connect(
    host="10.2.21.57",
    user="arch_wt",
    password="2uID@8SNF2OhGecg",
    database="bonade_operate_center",
    port=3316
)

# SQL查询语句
query = "SELECT * FROM saas_oper_log where oper_id <=1846879"

# 执行查询并将结果保存为DataFrame对象
df = pd.read_sql(query, connection)

# 保存DataFrame为CSV文件
df.to_csv("saas_oper_log.csv", index=False)


导入MongoDB：
 
mongoimport -h 10.2.29.81:20001 -u devuser -p "devuser.COM2020" --authenticationDatabase=admin -d test -c saas_oper_log --type csv --headerline /data/tmp_backup/saas_oper_log.csv

 

> db.saas_oper_log.find({}, {oper_id: 1}).sort({oper_id: -1}).limit(1)
{ "_id" : ObjectId("66136d72b8a76f08d202efb3"), "oper_id" : 1846879 }
> db.saas_oper_log.find({}).count();
1745354



增量：

import pandas as pd
import pymysql

# MySQL连接信息
connection = pymysql.connect(
    host="10.2.21.57",
    user="arch_wt",
    password="2uID@8SNF2OhGecg",
    database="bonade_operate_center",
    port=3316
)

# SQL查询语句
query = "SELECT * FROM saas_oper_log where oper_id>1846879"

# 执行查询并将结果保存为DataFrame对象
df = pd.read_sql(query, connection)

# 保存DataFrame为CSV文件
df.to_csv("saas_oper_log-incr.csv", index=False)


导入MongoDB：
 
mongoimport -h 10.2.29.81:20001 -u devuser -p "devuser.COM2020" --authenticationDatabase=admin -d test -c saas_oper_log --type csv --headerline /data/tmp_backup/saas_oper_log-incr.csv