爬取起点中文网某本小说实例：

1 # -*-coding:utf8-*- 2 import requests 3 import urllib 4 import urllib2 5 from bs4 import BeautifulSoup 6 import sys 7 reload(sys) 8 sys.setdefaultencoding('utf-8')       #读写中文内容 9 10 class QDZW:11     def __init__(self, url):12         self.url = url           #网址13         self.user_agent = 'Mozilla/5.0 (Windows NT 6.1; (Windows NT 6.1; WOW64; rv:62.0) Gecko/20100101 Firefox/61.0'      #请求头属性（模拟浏览器访问）14         self.headers = {
   'User-Agent' : self.user_agent}15 16     #获取网址静态HTML（如果未访问成功，则持续访问，直到访问成功为止）17     def gethtml(self):18         19             try:20                 #return requests.get(self.url，headers = self.headers,time_out = 2)21                 request = urllib2.Request(self.url,headers = self.headers)22                 response = urllib2.urlopen(request,time_out = 2)    #设置访问时长限制23                 temp = False24                 return response25             except Exception as e:26                 print e27     #解析HTML内容，获取需要的内容           28     def novel(self):29         while True:30             r = self.gethtml()31             while r == 'None':         #r内容为空表示访问超时，则继续访问32                 r = self.gethtml()     33             #print r.encoding    #输出编码类型34             soup = BeautifulSoup(r ,"html.parser")     #利用BeautifulSoup解析网址内容，class前加 . id前加 #35             #select输出内容为数组，select参数从body中一级结点开始即可，可逐级叠加定位36             #XXX.text为读取标签中的内容  XXX.attrs['xx']为读取 XXX中属性为xx的值 ，XXX为HTML格式的文本37             section_name = soup.select(' .wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .text-head .j_chapterName')[0].text  38             print section_name39             section_text = soup.select('.wrap #j_readMainWrap #j_chapterBox .text-wrap .main-text-wrap .read-content')[0].text40             section_text = self.spliting(section_text)41             #print section_text42             self.writing(section_name + "\r\n" + section_text)43 44             next_url = soup.select('.wrap #j_readMainWrap .chapter-control #j_chapterNext')[0]45             #print next_url.text 46             if next_url.text == "下一章":47                 self.url = "https:" + next_url.attrs['href']48             else:49                 break50     #在去掉
的位置添加换行符号“\r\n”51     def spliting(self, string):52         str_split = string.split()  #以split()中参数为标准，切割字符串53         string = ""54         for text in str_split:55             if text != "":56                 string = string + "    " + text + "\r\n"57         return string58     #输出读取的novel内容59     def writing(self, novel):60         outputs = open(unicode('至尊剑皇.txt','utf8'),'a')      #a为追加型写入61         outputs.write(novel)62         outputs.close()63         64 qdzw = QDZW('https://read.qidian.com/chapter/JDViC81SWM41/pskSqRrpbXDgn4SMoDUcDQ2')     #参数为初始网址  65 qdzw.novel()