最基本的抓取网页内容的代码实现：
#!/usr/bin/env python from urllib import urlretrieve def firstnonblank(lines): for eachline in lines: if not eachline.strip(): continue else: return eachline def firstlast(webpage): f = open(webpage) lines = f.readlines() f.close() print firstnonblank(lines), lines.reverse() print firstnonblank(lines), def download(url='http://www',process=firstlast): try: retval = urlretrieve(url)[0] except ioerror: retval = none if retval: process(retval) if __name__ == '__main__': download()
利用urllib模块，来实现一个网页中针对图片的抓取功能：
import urllib.request import socket import re import sys import os targetdir = rc:\users\elqstux\desktop\pic def destfile(path): if not os.path.isdir(targetdir): os.mkdir(targetdir) pos = path.rindex('/') t = os.path.join(targetdir, path[pos+1:]) return t if __name__ == __main__: hostname = http://www.douban.com req = urllib.request.request(hostname) webpage = urllib.request.urlopen(req) contentbytes = webpage.read() for link, t in set(re.findall(r'(http:[^\s]*?(jpg|png|gif))', str(contentbytes))): print(link) urllib.request.urlretrieve(link, destfile(link))
import urllib.request import socket import re import sys import os targetdir = rh:\pic def destfile(path): if not os.path.isdir(targetdir): os.mkdir(targetdir) pos = path.rindex('/') t = os.path.join(targetdir, path[pos+1:]) #会以/作为分隔 return t if __name__ == __main__: hostname = http://www.douban.com/ req = urllib.request.request(hostname) webpage = urllib.request.urlopen(req) contentbytes = webpage.read() match = re.findall(r'(http:[^\s]*?(jpg|png|gif))', str(contentbytes) )#r'(http:[^\s]*?(jpg|png|gif))'中包含两层圆括号，故有两个分组， #上面会返回列表，括号中匹配的内容才会出现在列表中 for picname, pictype in match: print(picname) print(pictype) ''''' 输出： http://img3.douban.com/pics/blank.gif gif http://img3.douban.com/icon/g111328-1.jpg jpg http://img3.douban.com/pics/blank.gif gif http://img3.douban.com/icon/g197523-19.jpg jpg http://img3.douban.com/pics/blank.gif gif ... '''