- should have a method to save/load cookies
"""
-#from __future__ import with_statement
-
import os
-#import md5
import urllib
import urllib2
-#import mimetypes
import cookielib
txheaders = {
'Accept':'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png',
'Accept-Language':'en,en-us;q=0.5',
-# 'Accept-Encoding': 'gzip, deflate',
'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
-# 'Keep-Alive': '300',
-# 'Connection': 'keep-alive',
-# 'Cache-Control': 'max-age=0',
}
for key,value in extraheaders.iteritems():
txheaders[key] = value
self.cookies.extract_cookies(openerdirector,req)
if only_head:
return openerdirector
- #if openerdirector.headers.has_key('content-length'):
- # length = long(openerdirector.headers['content-length'])
- #else:
- # length = 0
- #dlength = 0
- #if fd:
- # while True:
- # data = openerdirector.read(1024)
- # dlength += len(data)
- # fd.write(data)
- # if onprogress:
- # onprogress(length,dlength)
- # if not data:
- # break
- #else:
- # data = ''
- # while True:
- # newdata = openerdirector.read(1024)
- # dlength += len(newdata)
- # data += newdata
- # if onprogress:
- # onprogress(length,dlength)
- # if not newdata:
- # break
- # #data = openerdirector.read()
- # if not (self.cacher is None):
- # self.cacher[key] = data
- #try:
- # d2= GzipFile(fileobj=cStringIO.StringIO(data)).read()
- # data = d2
- #except IOError:
- # pass
return openerdirector.read()
except urllib2.URLError:
cnt += 1
if self.debug:
print "MozillaEmulator: urllib2.URLError, retryting ",cnt
-# def post_multipart(self,url,fields, files, forbid_redirect=True):
-# """Post fields and files to an http host as multipart/form-data.
-# fields is a sequence of (name, value) elements for regular form fields.
-# files is a sequence of (name, filename, value) elements for data to be uploaded as files
-# Return the server's response page.
-# """
-# content_type, post_data = encode_multipart_formdata(fields, files)
-# result = self.download(url,post_data, {
-# 'Content-Type': content_type,
-# 'Content-Length': str(len(post_data))
-# },
-# forbid_redirect=forbid_redirect
-# )
-# return result
-
class HTTPNoRedirector(urllib2.HTTPRedirectHandler):
"""This is a custom http redirect handler that FORBIDS redirection."""
newurl = headers.getheaders('uri')[0]
e.newurl = newurl
raise e
-
-
-#def encode_multipart_formdata(fields, files):
-# """
-# fields is a sequence of (name, value) elements for regular form fields.
-# files is a sequence of (name, filename, value) elements for data to be uploaded as files
-# Return (content_type, body) ready for httplib.HTTP instance
-# """
-# BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'
-# CRLF = '\r\n'
-# L = []
-# for (key, value) in fields:
-# L.append('--' + BOUNDARY)
-# L.append('Content-Disposition: form-data; name="%s"' % key)
-# L.append('')
-# L.append(value)
-# for (key, filename, value) in files:
-# L.append('--' + BOUNDARY)
-# L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
-# L.append('Content-Type: %s' % get_content_type(filename))
-# L.append('')
-# L.append(value)
-# L.append('--' + BOUNDARY + '--')
-# L.append('')
-# body = CRLF.join(L)
-# content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
-# return content_type, body
-#
-#
-#def get_content_type(filename):
-# return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
-#