1 #!/usr/bin/env python

     2 #

     3 # Copyright 2007 Google Inc.

     4 #

     5 # Licensed under the Apache License, Version 2.0 (the "License");

     6 # you may not use this file except in compliance with the License.

     7 # You may obtain a copy of the License at

     8 #

     9 #     http://www.apache.org/licenses/LICENSE-2.0

    10 #

    11 # Unless required by applicable law or agreed to in writing, software

    12 # distributed under the License is distributed on an "AS IS" BASIS,

    13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    14 # See the License for the specific language governing permissions and

    15 # limitations under the License.

    16 #

    17 

    18 """Stub version of the urlfetch API, based on httplib."""

    19 

    20 

    21 

    22 import httplib

    23 import logging

    24 import socket

    25 import urlparse

    26 

    27 from google.appengine.api import urlfetch_errors

    28 from google.appengine.api import urlfetch_service_pb

    29 from google.appengine.runtime import apiproxy_errors

    30 

    31 

    32 MAX_RESPONSE_SIZE = 2 ** 24

    33 

    34 MAX_REDIRECTS = 5

    35 

    36 REDIRECT_STATUSES = frozenset([

    37   httplib.MOVED_PERMANENTLY,

    38   httplib.FOUND,

    39   httplib.SEE_OTHER,

    40   httplib.TEMPORARY_REDIRECT,

    41 ])

    42 

    43 

    44 class URLFetchServiceStub(object):

    45   """Stub version of the urlfetch API to be used with apiproxy_stub_map."""

    46 

    47   def MakeSyncCall(self, service, call, request, response):

    48     """The main RPC entry point.

    49 

    50     Arg:

    51       service: Must be 'urlfetch'.

    52       call: A string representing the rpc to make.  Must be part of

    53         URLFetchService.

    54       request: A protocol buffer of the type corresponding to 'call'.

    55       response: A protocol buffer of the type corresponding to 'call'.

    56     """

    57     assert service == 'urlfetch'

    58     assert request.IsInitialized()

    59 

    60     attr = getattr(self, '_Dynamic_' + call)

    61     attr(request, response)

    62 

    63   def _Dynamic_Fetch(self, request, response):

    64     """Trivial implementation of URLFetchService::Fetch().

    65 

    66     Args:

    67       request: the fetch to perform, a URLFetchRequest

    68       response: the fetch response, a URLFetchResponse

    69     """

    70     (protocol, host, path, parameters, query, fragment) = urlparse.urlparse(request.url())

    71 

    72     payload = ''

    73     if request.method() == urlfetch_service_pb.URLFetchRequest.GET:

    74       method = 'GET'

    75     elif request.method() == urlfetch_service_pb.URLFetchRequest.POST:

    76       method = 'POST'

    77       payload = request.payload()

    78     elif request.method() == urlfetch_service_pb.URLFetchRequest.HEAD:

    79       method = 'HEAD'

    80     elif request.method() == urlfetch_service_pb.URLFetchRequest.PUT:

    81       method = 'PUT'

    82       payload = request.payload()

    83     elif request.method() == urlfetch_service_pb.URLFetchRequest.DELETE:

    84       method = 'DELETE'

    85     else:

    86       logging.error('Invalid method: %s', request.method())

    87       raise apiproxy_errors.ApplicationError(

    88         urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR)

    89 

    90     if not (protocol == 'http' or protocol == 'https'):

    91       logging.error('Invalid protocol: %s', protocol)

    92       raise apiproxy_errors.ApplicationError(

    93         urlfetch_service_pb.URLFetchServiceError.INVALID_URL)

    94 

    95     self._RetrieveURL(request.url(), payload, method,

    96                       request.header_list(), response)

    97 

    98   def _RetrieveURL(self, url, payload, method, headers, response):

    99     """Retrieves a URL.

   100 

   101     Args:

   102       url: String containing the URL to access.

   103       payload: Request payload to send, if any.

   104       method: HTTP method to use (e.g., 'GET')

   105       headers: List of additional header objects to use for the request.

   106       response: Response object

   107 

   108     Raises:

   109       Raises an apiproxy_errors.ApplicationError exception with FETCH_ERROR

   110       in cases where:

   111         - MAX_REDIRECTS is exceeded

   112         - The protocol of the redirected URL is bad or missing.

   113     """

   114     last_protocol = ''

   115     last_host = ''

   116 

   117     for redirect_number in xrange(MAX_REDIRECTS + 1):

   118       (protocol, host, path, parameters, query, fragment) = urlparse.urlparse(url)

   119 

   120       if host == '' and protocol == '':

   121         host = last_host

   122         protocol = last_protocol

   123 

   124       adjusted_headers = {

   125         'Content-Length': len(payload),

   126         'Host': host,

   127         'Accept': '*/*',

   128       }

   129       if method == 'POST' and payload:

   130         adjusted_headers['Content-Type'] = 'application/x-www-form-urlencoded'

   131 

   132       for header in headers:

   133         adjusted_headers[header.key().title()] = header.value()

   134 

   135       logging.debug('Making HTTP request: host = %s, '

   136                     'url = %s, payload = %s, headers = %s',

   137                     host, url, payload, adjusted_headers)

   138       try:

   139         if protocol == 'http':

   140           connection = httplib.HTTPConnection(host)

   141         elif protocol == 'https':

   142           connection = httplib.HTTPSConnection(host)

   143         else:

   144           error_msg = 'Redirect specified invalid protocol: "%s"' % protocol

   145           logging.error(error_msg)

   146           raise apiproxy_errors.ApplicationError(

   147               urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)

   148 

   149         last_protocol = protocol

   150         last_host = host

   151 

   152         if query != '':

   153           full_path = path + '?' + query

   154         else:

   155           full_path = path

   156 

   157         try:

   158           connection.request(method, full_path, payload, adjusted_headers)

   159           http_response = connection.getresponse()

   160           http_response_data = http_response.read()

   161         finally:

   162           connection.close()

   163       except (httplib.error, socket.error, IOError), e:

   164         raise apiproxy_errors.ApplicationError(

   165           urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, str(e))

   166 

   167       if http_response.status in REDIRECT_STATUSES:

   168         url = http_response.getheader('Location', None)

   169         if url is None:

   170           error_msg = 'Redirecting response was missing "Location" header'

   171           logging.error(error_msg)

   172           raise apiproxy_errors.ApplicationError(

   173               urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)

   174         else:

   175           method = 'GET'

   176       else:

   177         response.set_statuscode(http_response.status)

   178         response.set_content(http_response_data[:MAX_RESPONSE_SIZE])

   179         for header_key, header_value in http_response.getheaders():

   180           header_proto = response.add_header()

   181           header_proto.set_key(header_key)

   182           header_proto.set_value(header_value)

   183 

   184         if len(http_response_data) > MAX_RESPONSE_SIZE:

   185           response.set_contentwastruncated(True)

   186 

   187         break

   188     else:

   189       error_msg = 'Too many repeated redirects'

   190       logging.error(error_msg)

   191       raise apiproxy_errors.ApplicationError(

   192           urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)