python3 无限爬虫

import requests
from queue import SimpleQueue
import re

q = SimpleQueue()

crawled = set()

def has_crawled(url):
    url = url.replace('http://', '').replace('https://', '').replace('/', '')
    if url in crawled:
        return True
    crawled.add(url)
    return False

u = 'http://baidu.com'
q.put(u)
has_crawled(u)

while not q.empty():
    url = q.get()
    try:
        r = requests.get(url)
    except BaseException:
        break
    except Exception:
        continue
    print(url)
    ls = re.findall('(https?://[A-Za-z0-9.]*?)[^A-Za-z0-9.]', r.text)
    for e in ls:
        if not has_crawled(e):
            q.put(e)