python入门（2）-目录文件列举和Beautiful Soup简单解析

功能：

　　1.列举一个目录下的文件

　　2.利用BeautifulSoup简单解析正文内容，然后保存

待完善：

　　1.多线程支持

　　2.适配器支持（for雷锋网和36氪两个网站网页）

"""
parser
    for parsing html file from leiphone.com and 36kr.com
    contact xiaoyang
"""

#
# @author:  xiaoyang
# @contact: hityixiaoyang@gmail.com
# @version:
# @describ: parse a html file from leiphone.com
# @log:
#           1.2012-11-22 create
#           2.2012-11-23 add FileCollect and ParseTask class
#

import sys
import urllib2
import codecs
import os
from bs4 import BeautifulSoup

# global def
OUT_FILE_PREFIX = "out"
OUT_CNT = 0

#
FileCollectDBG=False
ParseTaskDbg=True

def errPrint(code, msg=''):
    print >> sys.stderr, __doc__ % globals()
    if msg:
        print >> sys.stderr, msg
        sys.exit(code)

# for LeiPhone.com
def SaveResLP(doc,filename):
    print "!LOCK!"
    fp=None
    try:
        fp=open(filename,"w")
        fp.write(doc)
    except IOError as errStr:
        errPrint(1, errStr)
    finally:
        fp.close()
        print "!UNLOCK!"
    return True

# foe 36kr.com
def SaveRes36K(doc,filename):
   print "!LOCK!"
   print "!UNLOCK!"
   return True

class FileCollect:
	def __init__(self, root):
		self.root = root
		self.dlist = []
		self.flist = []
	def init(self):
		for root, dirs, files in os.walk(self.root):
			self.dlist += dirs
			for afile in files: 
				self.flist.append(root + afile)
		return True

class ParseTask:
    def __init__(self, savedFileName):
		self.soup = None
		self.savedCnt = 0
		self.doneCnt = 0
		self.savedFileName = savedFileName
    def parse(self, readFileName):
        fp = None
        content = None
        try:
            fp = open(readFileName, "r")
            if fp is not None:
                self.soup = BeautifulSoup(fp.read())
            else:
                errPrint(1, "fopen failed!")
            content=self.soup.find_all(id="content_main")
            self.doneCnt=self.doneCnt+1
           
            if self.doneCnt >= self.savedCnt:
                SaveResLP(str(content[0]),self.savedFileName)
                self.doneCnt=0
        except IOError as errStr:
			errPrint(1, errStr)
        finally:
			if fp is not None:
				fp.close()

if FileCollectDBG:		
    fc = FileCollect("/opt/project/")
    fc.init()
    print "dlist:\r\n", fc.dlist
    print "flist:\r\n", fc.flist
elif ParseTaskDbg:
    newTask=ParseTask("out.html")
    newTask.parse("1119-vv-dolby.html")
    print "saved OK!\r\n"