Python獲取趕集網(wǎng)招聘信息的方法

import re
import urllib
import urllib.request
#獲取趕集網(wǎng)數(shù)據(jù)
def begin(url):
 #要偽裝成的瀏覽器(我這個是用的chrome)
 headers = ('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36')
 opener = urllib.request.build_opener()
 #將要偽裝成的瀏覽器添加到對應(yīng)的http頭部
 opener.addheaders=[headers]
 #讀取相應(yīng)的url
 data = opener.open(url).read()
 #將獲得的html解碼為utf-8
 data=data.decode('utf-8')
 return data
#處理數(shù)據(jù)，返回字典城市對應(yīng)城市的url
def get_cityinfo(data):
 city_info1=re.findall(r'dl>(.*?)</dl>',data,re.S)
 city_info2=re.findall(r'(<a.*?</a>)',city_info1[0],re.S)
 city_dict={}
 for each in city_info2:
 key=re.findall('>(.*?)</',each,re.S)
 city_url=re.findall('href="(.*?)"',each,re.S)
 city_dict[key[0]]=city_url[0]
 return city_dict
#獲取所有的a標(biāo)簽，并從a標(biāo)簽中獲取信息
def a_info(data):
 a_info=re.findall(r'(<a.*?</a>)',data[0],re.S)
 a_dict={}
 for each in a_info:
 key=re.findall('>(.*?)</',each,re.S)
 a_url=re.findall('"(.*?)"',each,re.S)
 a_dict[key[0]]=a_url[0]
 return a_dict
#獲取對應(yīng)城市信息的所有分類
def get_cityinfoclass():
 #目前先獲取招聘信息，后面有時間再繼續(xù)補充完善這個模塊
 info= 'zhaopin/'
 return info
#獲取對應(yīng)城市和其城市分類的信息
def getzhaopin(city_info,infoclass):
 #先搞成都的信息招聘
 city_url=city_info['成都']
 cdzp_url=city_url+infoclass
 cdzp_info=begin(cdzp_url)
 return city_url,cdzp_info
#獲取成都市招聘信息
def get_zhaopin_info(city_url,cdzp_info):
 allzp_info=re.findall('class="f-all-news"(.*?)</p>',cdzp_info,re.S)
 a_dict={}
 class_info=re.findall('<dd>(.*?)</dd>',allzp_info[0],re.S)
 for each in class_info:
 a_info=re.findall(r'(<a.*?</a>)',each,re.S)
 for each1 in a_info:
 key=re.findall('>(.*?)</',each1,re.S)
 a_url=re.findall('href="/(.*?)"',each1,re.S)
 a_dict[key[0].strip()]=city_url+a_url[0]
 return a_dict
#獲取招聘信息的具體內(nèi)容
def get_city_zpinfo_detail(url):
 #先獲取軟件工程師
 sorft_engineer=(zp_class_info['軟件工程師'])
 job_url_info=begin(sorft_engineer)
 get_detail_info(job_url_info)
#處理詳情頁的信息
def get_detail_info(list_info):
 job_info=re.findall('<dl class="list-noimg job-list clearfix"(.*?)</dl',list_info,re.S)
 print(job_info[0])
if __name__=='__main__':
 url='http://www.ganji.com/index.htm';
 data=begin(url);
 #所有城市信息
 city_info=get_cityinfo(data)
 #對應(yīng)的分類
 infoclass=get_cityinfoclass()
 cdzp_url,xiaoshou=getzhaopin(city_info,infoclass)
 #獲取招聘的分類信息
 zp_class_info=get_zhaopin_info(cdzp_url,xiaoshou)
 get_city_zpinfo_detail(zp_class_info)