这篇文章主要介绍了三个python爬虫项目实例代码，使用了urllib2库，文中示例代码非常详细，对大家的学习或者工作具有一定的参考学习价值，需要的朋友可以参考下。
python爬虫代码示例分享
一、爬取故事段子：推荐学习：python视频教程
注：部分代码无法正常运行，但仍有一定的参考价值。
#encoding=utf-8import urllib2 import re class neihanba(): def spider(self): ''' 爬虫的主调度器 ''' isflow=true#判断是否进行下一页 page=1 while isflow: url="http://www.neihanpa.com/article/list_5_"+str(page)+".html" html=self.load(url) self.deal(html,page) panduan=raw_input("是否继续(y/n)!") if panduan=="y": isflow=true page+=1 else: isflow=false def load(self,url): ''' 针对url地址进行全部爬去 :param url: url地址 :return: 返回爬去的内容 ''' header = { "user-agent": " mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.36 (khtml, like gecko) chrome/67.0.3396.79 safari/537.36" } request = urllib2.request(url, headers=header) response = urllib2.urlopen(request) html = response.read() return html def deal(self,html,page): ''' 对之前爬去的内容进行正则匹配，匹配出标题和正文内容 :param html:之前爬去的内容 :param page: 正在爬去的页码 ''' parrten=re.compile('<li class="piclist\d+">(.*?)</li>',re.s) titlelist=parrten.findall(html) for title in titlelist: parrten1=re.compile('<a href="/article/\d+.html" rel="external nofollow" >(.*)</a>') ti1=parrten1.findall(title) parrten2=re.compile('<div class="f18 mb20">(.*?)</div>',re.s) til2=parrten2.findall(title) for t in ti1: tr=t.replace("<b>","").replace("</b>","") self.writedata(tr,page) for t in til2: tr=t.replace("<p>","").replace("</p>","").replace("<br>","").replace("<br />","").replace("&ldquo","\"").replace("&rdquo","\"") self.writedata(tr,page) def writedata(self,context,page): ''' 将最终爬去的内容写入文件中 :param context: 匹配好的内容 :param page: 当前爬去的页码数 ''' filename = "di" + str(page) + "yehtml.txt" with open(filename, "a") as file: file.writelines(context + "\n")if __name__ == '__main__': n=neihanba() n.spider()
二、爬取智联：
#encoding=utf-8import urllibimport urllib2 import re class zhilian(): def spider(self,position,workplace): ''' 爬虫的主调度器 :param position: 职位 :param workplace: 工作地点 ''' url="http://sou.zhaopin.com/jobs/searchresult.ashx?" url+=urllib.urlencode({"jl":workplace}) url+="&" url+=urllib.urlencode({"kw":position}) isflow=true#是否进行下一页的爬去 page=1 while isflow: url+="&"+str(page) html=self.load(url) self.deal1(html,page) panduan = raw_input("是否继续爬虫下一页(y/n)!") if panduan == "y": isflow = true page += 1 else: isflow = false def load(self,url): ''' 针对url地址进行全部爬去 :param url: url地址 :return: 返回爬去的内容 ''' header = { "user-agent": " mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.36 (khtml, like gecko) chrome/67.0.3396.79 safari/537.36" } request = urllib2.request(url, headers=header) response = urllib2.urlopen(request) html = response.read() return html def deal1(self,html,page): ''' 对之前爬去的内容进行正则匹配，匹配职位所对应的链接 :param html:之前爬去的内容 :param page: 正在爬去的页码 ''' parrten=re.compile('<a\s+style="font-weight:\s+bold"\s+par="ssidkey=y&ss=\d+&ff=\d+&sg=\w+&so=\d+"\s+href="(.*?)" rel="external nofollow" target="_blank">.*?</a>',re.s) til=parrten.findall(html)#爬去链接 for t in til: self.deal2(t,page) def deal2(self,t,page): ''' 进行二次爬虫，然后在新的页面中对公司、薪资、工作经验进行匹配 :param t: url地址 :param page: 当前匹配的页数 ''' html=self.load(t)#返回二次爬虫的内容 parrten1=re.compile('<a\s+onclick=".*?"\s+href=".*?" rel="external nofollow" \s+target="_blank">(.*?)\s+.*?<img\s+class=".*?"\s+src=".*?"\s+border="\d+"\s+vinfo=".*?"></a>',re.s) parrten2=re.compile('<li><span>职位月薪：</span><strong>(.*?) <a.*?>.*?</a></strong></li>',re.s) parrent3=re.compile('<li><span>工作经验：</span><strong>(.*?)</strong></li>',re.s) til1=parrten1.findall(html) til2=parrten2.findall(html) til3=parrent3.findall(html) str="" for t in til1: t=t.replace('<img title="专属页面" src="//img03.zhaopin.cn/2012/img/jobs/icon.png" border="0" />',"") str+=t str+="\t" for t in til2: str+=t str += "\t" for t in til3: str+=t self.writedata(str,page) def writedata(self,context,page): ''' 将最终爬去的内容写入文件中 :param context: 匹配好的内容 :param page: 当前爬去的页码数 ''' filename = "di" + str(page) + "yehtml.txt" with open(filename, "a") as file: file.writelines(context + "\n")if __name__ == '__main__': position=raw_input("请输入职位：") workplace=raw_input("请输入工作地点：") z=zhilian() z.spider(position,workplace)
三、爬取贴吧：
#encoding=utf-8import urllibimport urllib2 import re class teiba(): def spider(self,name,startpage,endpage): url="http://tieba.baidu.com/f?ie=utf-8&" url+=urllib.urlencode({"kw":name}) for page in range(startpage,endpage+1): pn=50*(page-1) urlfull=url+"&"+urllib.urlencode({"pn":pn}) html=self.loadpage(url) self.dealpage(html,page) def loadpage(self,url): header={ "user-agent":" mozilla/5.0 (windows nt 6.1; wow64) applewebkit/537.36 (khtml, like gecko) chrome/67.0.3396.79 safari/537.36" } request=urllib2.request(url,headers=header) response=urllib2.urlopen(request) html=response.read() return html def dealpage(self,html,page): partten=re.compile(r'<a\s+rel="noreferrer"\s+href="/p/\d+" rel="external nofollow" \s+title=".*?"\s+target="_blank" class="j_th_tit\s+">(.*?)</a>',re.s) titlelist=partten.findall(html) rstr=r'<span\s+class="topic-tag"\s+data-name=".*?">#(.*?)#</span>' for title in titlelist: title=re.sub(rstr,"",title) self.writepage(title,page) def writepage(self,context,page): filename="di"+str(page)+"yehtml.txt" with open(filename,"a") as file: file.writelines(context+"\n")if __name__ == '__main__': name=raw_input("请输入贴吧名：") startpage=raw_input("请输入起始页：") endpage=raw_input("请输入终止页：") t=teiba() t.spider(name,int(startpage),int(endpage))
更多相关教程，请关注python教程栏目。
以上就是python爬虫代码示例分享的详细内容。