[Python網路編程]gevent httpclient以及網頁編碼

07-17

http://blog.csdn.net/yueguanghaidao/article/details/27688047

2014

之前看到geventhttpclient這個項目，https://github.com/gwik/geventhttpclient，官方文檔說非常快，由於響應使用了C的解析，所以我一直想把這玩意用到項目中，

這兩天一直在糾結這玩意，說實在一句話，比較難用，封裝的不給力，最大缺陷如下：

1.不支持重定向，重定向需要自己來寫，很費事

2.新建的httpclient對象只能發送同域名的請求

這相當的蛋疼，我花了一點時間封裝了一下，解決了上面的兩個問題，還增加了自動編解碼問題，代碼如下：

[python] view plain copy

#!/usr/bin/envpython
#-*-encoding:UTF-8-*-
importre
fromgeventhttpclient.urlimportURL
fromgeventhttpclient.clientimportHTTPClient,HTTPClientPool
fromurlparseimporturljoin
#fromcore.commonimporturljoin
HEADERS={
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"User-Agent":"Mozilla/5.0(WindowsNT6.1;WOW64;rv:29.0)Gecko/20100101Firefox/29.0"
}
DEFAULF_METHOD="GET"
MAX_REDIRECT_TIME=10
DEFAULT_PAGE_ENCODING="utf8"
classDifferDomainException(Exception):
"""
ifrequestdifferentdomainurl,geventhttpclientwillthrowit,
seegevent.client"raiseValueError("InvalidhostinURL")"
"""
def__init__(self,uri):
self.args=uri
self.uri=uri
classMaxRedirectException(Exception):
def__init__(self,response):
self.args=response
self.response=response
classHTTP(HTTPClient):
defrequest(self,request_uri,method=DEFAULF_METHOD,body=b"",headers={},follow_redirect=True,redirects=MAX_REDIRECT_TIME):
ifbodyandmethod==DEFAULF_METHOD:
method="POST"
h=[k.title()forkinheaders.iterkeys()]
headers.update(dict([(k,v)fork,vinHEADERS.iteritems()ifknotinh]))
response=super(HTTP,self).request(method,request_uri,body,headers)
iffollow_redirectandresponse.status_codein(301,302,303,307)andresponse.methodin("GET","POST"):
ifredirects:
location=response.get("location")orresponse.get("content-location")orresponse.get("uri")
iflocation:
location=urljoin(request_uri,location)
ifnotlocation.startswith(self._base_url_string):
raiseDifferDomainException(location)
returnself.request(location,method,body,headers,follow_redirect,redirects-1)
else:
raiseMaxRedirectException(response)
returnresponse
classHTTPPool(HTTPClientPool):
defget_client(self,url):
ifnotisinstance(url,URL):
url=URL(url)
client_key=url.host,url.port
try:
returnself.clients[client_key]
exceptKeyError:
client=HTTP.from_url(url,**self.client_args)
self.clients[client_key]=client
returnclient
_POLL=HTTPPool(network_timeout=100,connection_timeout=100)
META_CHARSET_REGEX=re.compile(r"(?si)<head>.*<metahttp-equiv="?content-type"?[^>]+charset=(?P<result>[^">]+).*</head>")
defdecodePage(content,content_type):
httpCharset,metaCharset=None,None
ifcontent_typeandcontent_type.find("charset=")!=-1:
httpCharset=content_type.split("charset=")[-1]
match=META_CHARSET_REGEX.search(content)
ifmatch:
metaCharset=match.group("result")
printhttpCharset,metaCharset
charset=httpCharsetormetaCharsetorDEFAULT_PAGE_ENCODING
returncontent.decode(charset).encode(DEFAULT_PAGE_ENCODING)
defrequest(request_uri,method=DEFAULF_METHOD,body=b"",headers={},follow_redirect=True,auto_read=True):
client=_POLL.get_client(request_uri)
response=None
try:
response=client.request(request_uri,method,body,headers,follow_redirect)
exceptDifferDomainException,e:
print"DifferDomainException:"+e.uri
response=request(e.uri,method,body,headers,follow_redirect)
exceptMaxRedirectException,e:
print"maxredirect"
response=e.response#willreturnpreviousresponse,ofcourseredirectresponse
exceptException,e:
printstr(e)
ifauto_readandresponse:
withresponse:
response.content=decodePage(response.read(),response.get("content-type"))
returnresponse
deftest():
#printrequest("http://127.0.0.1/re.php",follow_redirect=False)
#printrequest("http://127.0.0.1/re.php",follow_redirect=True).content
r=request("http://www.baidu.com/",follow_redirect=False)
#baiduutf8utf8
printr.content[:10]
r=request("http://www.163.com/",follow_redirect=False)
#163gbkgb2312
printr.content[:10]
test()

在測試網頁編碼問題遇到了一些問題，看下面：

由於頭部的請求先到，所以我們一般認為返回的內容編碼是先根據頭部的，如果沒有再看頁面編碼。

我們看網易的編碼，頭部為gbk，網頁為gb2312，但用gb2312解碼竟然有問題，？？？我很不解，各位大大們為啥呢？

但用頭部gbk解碼是正常的，這也證明了頭部編碼優先。按理說網頁編碼是告訴瀏覽器以gb2312顯示，但明顯有問題，瀏覽器怎麼做到的？

我們再看新浪的，這更讓我鬱悶了，誰來拯救我啊？