--- a/thirdparty/google_appengine/google/appengine/api/urlfetch.py Tue May 12 13:02:10 2009 +0200
+++ b/thirdparty/google_appengine/google/appengine/api/urlfetch.py Tue May 12 15:39:52 2009 +0200
@@ -30,6 +30,7 @@
import urllib2
import urlparse
+from google.appengine.api import apiproxy_rpc
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import urlfetch_service_pb
from google.appengine.api.urlfetch_errors import *
@@ -186,13 +187,29 @@
return False
+def __create_rpc(deadline=None, callback=None):
+ """DO NOT USE. WILL CHANGE AND BREAK YOUR CODE.
+
+ Creates an RPC object for use with the urlfetch API.
+
+ Args:
+ deadline: deadline in seconds for the operation.
+ callback: callable to invoke on completion.
+
+ Returns:
+ A _URLFetchRPC object.
+ """
+ return _URLFetchRPC(deadline, callback)
+
+
def fetch(url, payload=None, method=GET, headers={}, allow_truncated=False,
- follow_redirects=True):
+ follow_redirects=True, deadline=None):
"""Fetches the given HTTP URL, blocking until the result is returned.
Other optional parameters are:
method: GET, POST, HEAD, PUT, or DELETE
- payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE)
+ payload: POST or PUT payload (implies method is not GET, HEAD, or DELETE).
+ this is ignored if the method is not POST or PUT.
headers: dictionary of HTTP headers to send with the request
allow_truncated: if true, truncate large responses and return them without
error. otherwise, ResponseTooLargeError will be thrown when a response is
@@ -204,6 +221,7 @@
information. If false, you see the HTTP response yourself,
including the 'Location' header, and redirects are not
followed.
+ deadline: deadline in seconds for the operation.
We use a HTTP/1.1 compliant proxy to fetch the result.
@@ -218,73 +236,173 @@
of the returned structure, so HTTP errors like 404 do not result in an
exception.
"""
- if isinstance(method, basestring):
- method = method.upper()
- method = _URL_STRING_MAP.get(method, method)
- if method not in _VALID_METHODS:
- raise InvalidMethodError('Invalid method %s.' % str(method))
+ rpc = __create_rpc(deadline=deadline)
+ rpc.make_call(url, payload, method, headers, follow_redirects)
+ return rpc.get_result(allow_truncated)
+
+
+class _URLFetchRPC(object):
+ """A RPC object that manages the urlfetch RPC.
+
+ Its primary functions are the following:
+ 1. Convert error codes to the URLFetchServiceError namespace and raise them
+ when get_result is called.
+ 2. Wrap the urlfetch response with a _URLFetchResult object.
+ """
- if _is_fetching_self(url, method):
- raise InvalidURLError("App cannot fetch the same URL as the one used for "
- "the request.")
+ def __init__(self, deadline=None, callback=None):
+ """Construct a new url fetch RPC.
- request = urlfetch_service_pb.URLFetchRequest()
- response = urlfetch_service_pb.URLFetchResponse()
- request.set_url(url)
+ Args:
+ deadline: deadline in seconds for the operation.
+ callback: callable to invoke on completion.
+ """
+ self.__rpc = apiproxy_stub_map.CreateRPC('urlfetch')
+ self.__rpc.deadline = deadline
+ self.__rpc.callback = callback
+ self.__called_hooks = False
+
+ def make_call(self, url, payload=None, method=GET, headers={},
+ follow_redirects=True):
+ """Executes the RPC call to fetch a given HTTP URL.
- if method == GET:
- request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
- elif method == POST:
- request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
- elif method == HEAD:
- request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
- elif method == PUT:
- request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
- elif method == DELETE:
- request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
+ See urlfetch.fetch for a thorough description of arguments.
+ """
+ assert self.__rpc.state is apiproxy_rpc.RPC.IDLE
+ if isinstance(method, basestring):
+ method = method.upper()
+ method = _URL_STRING_MAP.get(method, method)
+ if method not in _VALID_METHODS:
+ raise InvalidMethodError('Invalid method %s.' % str(method))
+
+ if _is_fetching_self(url, method):
+ raise InvalidURLError("App cannot fetch the same URL as the one used for "
+ "the request.")
+
+ self.__request = urlfetch_service_pb.URLFetchRequest()
+ self.__response = urlfetch_service_pb.URLFetchResponse()
+ self.__result = None
+ self.__request.set_url(url)
- if payload and (method == POST or method == PUT):
- request.set_payload(payload)
+ if method == GET:
+ self.__request.set_method(urlfetch_service_pb.URLFetchRequest.GET)
+ elif method == POST:
+ self.__request.set_method(urlfetch_service_pb.URLFetchRequest.POST)
+ elif method == HEAD:
+ self.__request.set_method(urlfetch_service_pb.URLFetchRequest.HEAD)
+ elif method == PUT:
+ self.__request.set_method(urlfetch_service_pb.URLFetchRequest.PUT)
+ elif method == DELETE:
+ self.__request.set_method(urlfetch_service_pb.URLFetchRequest.DELETE)
+
+ if payload and (method == POST or method == PUT):
+ self.__request.set_payload(payload)
+
+ for key, value in headers.iteritems():
+ header_proto = self.__request.add_header()
+ header_proto.set_key(key)
+ header_proto.set_value(str(value))
+
+ self.__request.set_followredirects(follow_redirects)
+ if self.__rpc.deadline:
+ self.__request.set_deadline(self.__rpc.deadline)
+
+ apiproxy_stub_map.apiproxy.GetPreCallHooks().Call(
+ 'urlfetch', 'Fetch', self.__request, self.__response)
+ self.__rpc.MakeCall('urlfetch', 'Fetch', self.__request, self.__response)
- for key, value in headers.iteritems():
- header_proto = request.add_header()
- header_proto.set_key(key)
- header_proto.set_value(str(value))
+ def wait(self):
+ """Waits for the urlfetch RPC to finish. Idempotent.
+ """
+ assert self.__rpc.state is not apiproxy_rpc.RPC.IDLE
+ if self.__rpc.state is apiproxy_rpc.RPC.RUNNING:
+ self.__rpc.Wait()
+
+ def check_success(self, allow_truncated=False):
+ """Check success and convert RPC exceptions to urlfetch exceptions.
+
+ This method waits for the RPC if it has not yet finished, and calls the
+ post-call hooks on the first invocation.
- request.set_followredirects(follow_redirects)
+ Args:
+ allow_truncated: if False, an error is raised if the response was
+ truncated.
+
+ Raises:
+ InvalidURLError if the url was invalid.
+ DownloadError if there was a problem fetching the url.
+ ResponseTooLargeError if the response was either truncated (and
+ allow_truncated is false) or if it was too big for us to download.
+ """
+ assert self.__rpc.state is not apiproxy_rpc.RPC.IDLE
+ if self.__rpc.state is apiproxy_rpc.RPC.RUNNING:
+ self.wait()
- try:
- apiproxy_stub_map.MakeSyncCall('urlfetch', 'Fetch', request, response)
- except apiproxy_errors.ApplicationError, e:
- if (e.application_error ==
- urlfetch_service_pb.URLFetchServiceError.INVALID_URL):
- raise InvalidURLError(str(e))
- if (e.application_error ==
- urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR):
- raise DownloadError(str(e))
- if (e.application_error ==
- urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR):
- raise DownloadError(str(e))
- if (e.application_error ==
- urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE):
- raise ResponseTooLargeError(None)
- if (e.application_error ==
- urlfetch_service_pb.URLFetchServiceError.DEADLINE_EXCEEDED):
- raise DownloadError(str(e))
- raise e
- result = _URLFetchResult(response)
+ try:
+ self.__rpc.CheckSuccess()
+ if not self.__called_hooks:
+ self.__called_hooks = True
+ apiproxy_stub_map.apiproxy.GetPostCallHooks().Call(
+ 'urlfetch', 'Fetch', self.__request, self.__response)
+ except apiproxy_errors.ApplicationError, e:
+ if (e.application_error ==
+ urlfetch_service_pb.URLFetchServiceError.INVALID_URL):
+ raise InvalidURLError(str(e))
+ if (e.application_error ==
+ urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR):
+ raise DownloadError(str(e))
+ if (e.application_error ==
+ urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR):
+ raise DownloadError(str(e))
+ if (e.application_error ==
+ urlfetch_service_pb.URLFetchServiceError.RESPONSE_TOO_LARGE):
+ raise ResponseTooLargeError(None)
+ if (e.application_error ==
+ urlfetch_service_pb.URLFetchServiceError.DEADLINE_EXCEEDED):
+ raise DownloadError(str(e))
+ raise e
- if not allow_truncated and response.contentwastruncated():
- raise ResponseTooLargeError(result)
+ if self.__response.contentwastruncated() and not allow_truncated:
+ raise ResponseTooLargeError(_URLFetchResult(self.__response))
+
+ def get_result(self, allow_truncated=False):
+ """Returns the RPC result or raises an exception if the rpc failed.
+
+ This method waits for the RPC if not completed, and checks success.
+
+ Args:
+ allow_truncated: if False, an error is raised if the response was
+ truncated.
- return result
+ Returns:
+ The urlfetch result.
+
+ Raises:
+ Error if the rpc has not yet finished.
+ InvalidURLError if the url was invalid.
+ DownloadError if there was a problem fetching the url.
+ ResponseTooLargeError if the response was either truncated (and
+ allow_truncated is false) or if it was too big for us to download.
+ """
+ if self.__result is None:
+ self.check_success(allow_truncated)
+ self.__result = _URLFetchResult(self.__response)
+ return self.__result
+
Fetch = fetch
class _URLFetchResult(object):
- """A Pythonic representation of our fetch response protocol buffer."""
+ """A Pythonic representation of our fetch response protocol buffer.
+ """
+
def __init__(self, response_proto):
+ """Constructor.
+
+ Args:
+ response_proto: the URLFetchResponse proto buffer to wrap.
+ """
self.__pb = response_proto
self.content = response_proto.content()
self.status_code = response_proto.statuscode()