aiomysql,aiomongo,aioredis

pool = await aiomysql.create_pool(host=self.redis_ip, port=3306,
                                          user='root', password='123',
                                          db='laxiao', loop=loop, charset='utf8',
                                          autocommit=True)
        async with pool.acquire() as sqlconn:
            async with sqlconn.cursor() as cur:
                try:
                    insert_sql = 'insert into laxiao(cid,company,captical,valuetype,person,tel,province,city,postcode,business,num,product) values("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}","{}")'.format(
                        item['cid'], item['名称'], item['资本'], item['类型'], item['联系人'], item['电话'], item['省份'], item['城市'],
                        item['邮编'], item['行业'], item['人数'], item['产品'])
                    await  cur.execute(insert_sql)
                    print(item['cid'], 'done')
                except Exception as e:
                    print(e)




mongoconn = AsyncIOMotorClient('mongodb://'+self.redis_ip+':27017')
db = mongoconn.laxiao
        collection = db.company
        await collection.update({'cid': item['cid']}, {'$set': item}, True)
        print(item['cid'], 'mongo-done')




conn = await aioredis.create_redis('redis://'+self.redis_ip, db=self.redis_db, loop=loop)
isdo = await conn.hget("seen_list", seen_list_id)  # 判断是否重复
if (isdo != None):
    continue
await conn.hset("seen_list", seen_list_id, '1')  # 放入set

if await conn.lpop('waitting_urls') == None:
    await asyncio.sleep(5)
wait conn.lpush('waitting_urls', url)

  

posted @ 2019-04-10 15:40  逐梦~前行  阅读(484)  评论(0)    收藏  举报