Python多線程爬蟲實戰_爬取糗事百科段子的實例_python

'''
#此處代碼為普通爬蟲
import urllib.request
import urllib.error
import re
headers = ("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36")
opener = urllib.request.build_opener()
opener.addheaders = [headers]
urllib.request.install_opener(opener)
for i in range(1,2):
 url = "https://www.qiushibaike.com/8hr/page/"+str(i)+"/"
 pagedata = urllib.request.urlopen(url).read().decode("utf-8","ignore")
 pattern = '<p class="content">.*?<span>(.*?)</span>(.*?)</p>'
 datalist = re.compile(pattern,re.S).findall(pagedata)
 for j in range(0,len(datalist)):
 print("第"+str(i)+"頁第"+str(j)+"個段子內容是：")
 print(datalist[j])
'''
'''
#此處為多線程介紹代碼
import threading #導入多線程包
class A(threading.Thread): #創建一個多線程A
 def init(self): #必須包含的兩個方法之一:初始化線程
 threading.Thread.init(self)
 def run(self): #必須包含的兩個方法之一：線程運行方法
 for i in range(0,11):
 print("我是線程A")
class B(threading.Thread): #創建一個多線程A
 def init(self): #必須包含的兩個方法之一:初始化線程
 threading.Thread.init(self)
 def run(self): #必須包含的兩個方法之一：線程運行方法
 for i in range(0,11):
 print("我是線程B")
t1 = A() #線程實例化
t1.start() #線程運行
t2 = B()
t2.start()
'''

#此處為修改后的多線程爬蟲
#使用多線程進行奇偶頁的爬取
import urllib.request
import urllib.error
import re
import threading
headers = ("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36")
opener = urllib.request.build_opener()
opener.addheaders = [headers]
urllib.request.install_opener(opener)
class one(threading.Thread): #爬取奇數頁內容
 def init(self):
 threading.Thread.init(self)
 def run(self):
 for i in range(1,12,2):
 url = "https://www.qiushibaike.com/8hr/page/"+str(i)+"/"
 pagedata = urllib.request.urlopen(url).read().decode("utf-8","ignore")
 pattern = '<p class="content">.*?<span>(.*?)</span>(.*?)</p>'
 datalist = re.compile(pattern,re.S).findall(pagedata)
 for j in range(0,len(datalist)):
 print("第"+str(i)+"頁第"+str(j)+"段子內容為：")
 print(datalist[j])

class two(threading.Thread): #爬取奇數頁內容
 def init(self):
 threading.Thread.init(self)
 def run(self):
 for i in range(2,12,2):
 url = "https://www.qiushibaike.com/8hr/page/"+str(i)+"/"
 pagedata = urllib.request.urlopen(url).read().decode("utf-8","ignore")
 pattern = '<p class="content">.*?<span>(.*?)</span>(.*?)</p>'
 datalist = re.compile(pattern,re.S).findall(pagedata)
 for j in range(0,len(datalist)):
 print("第"+str(i)+"頁第"+str(j)+"段子內容為：")
 print(datalist[j])
t1 = one()
t2 = two()
t1.start()
t2.start()