thirdparty/google_appengine/google/appengine/api/urlfetch_stub.py
author Sverre Rabbelier <srabbelier@gmail.com>
Fri, 23 Oct 2009 13:54:11 -0500
changeset 3031 7678f72140e6
parent 2864 2e0b0af889be
permissions -rwxr-xr-x
Update Google App Engine from 1.2.5 to 1.2.7 in thirdparty folder.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     1
#!/usr/bin/env python
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     2
#
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     3
# Copyright 2007 Google Inc.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     4
#
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     5
# Licensed under the Apache License, Version 2.0 (the "License");
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     6
# you may not use this file except in compliance with the License.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     7
# You may obtain a copy of the License at
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     8
#
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
     9
#     http://www.apache.org/licenses/LICENSE-2.0
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    10
#
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    11
# Unless required by applicable law or agreed to in writing, software
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    12
# distributed under the License is distributed on an "AS IS" BASIS,
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    13
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    14
# See the License for the specific language governing permissions and
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    15
# limitations under the License.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    16
#
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    17
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    18
"""Stub version of the urlfetch API, based on httplib."""
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    19
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    20
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    21
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
    22
import gzip
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    23
import httplib
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    24
import logging
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    25
import socket
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
    26
import StringIO
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    27
import urllib
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    28
import urlparse
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    29
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    30
from google.appengine.api import apiproxy_stub
149
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
    31
from google.appengine.api import urlfetch
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    32
from google.appengine.api import urlfetch_errors
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    33
from google.appengine.api import urlfetch_service_pb
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    34
from google.appengine.runtime import apiproxy_errors
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    35
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    36
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    37
MAX_RESPONSE_SIZE = 2 ** 24
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    38
149
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
    39
MAX_REDIRECTS = urlfetch.MAX_REDIRECTS
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    40
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    41
REDIRECT_STATUSES = frozenset([
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    42
  httplib.MOVED_PERMANENTLY,
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    43
  httplib.FOUND,
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    44
  httplib.SEE_OTHER,
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    45
  httplib.TEMPORARY_REDIRECT,
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    46
])
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    47
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    48
PORTS_ALLOWED_IN_PRODUCTION = (
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    49
    None, '80', '443', '4443', '8080', '8081', '8082', '8083', '8084', '8085',
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    50
    '8086', '8087', '8088', '8089', '8188', '8444', '8990')
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    51
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    52
_API_CALL_DEADLINE = 5.0
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    53
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    54
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    55
_UNTRUSTED_REQUEST_HEADERS = frozenset([
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    56
  'content-length',
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    57
  'host',
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    58
  'vary',
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    59
  'via',
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    60
  'x-forwarded-for',
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    61
])
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    62
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    63
class URLFetchServiceStub(apiproxy_stub.APIProxyStub):
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    64
  """Stub version of the urlfetch API to be used with apiproxy_stub_map."""
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    65
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    66
  def __init__(self, service_name='urlfetch'):
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    67
    """Initializer.
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    68
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    69
    Args:
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    70
      service_name: Service name expected for all calls.
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    71
    """
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
    72
    super(URLFetchServiceStub, self).__init__(service_name)
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    73
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    74
  def _Dynamic_Fetch(self, request, response):
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    75
    """Trivial implementation of URLFetchService::Fetch().
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    76
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    77
    Args:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    78
      request: the fetch to perform, a URLFetchRequest
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    79
      response: the fetch response, a URLFetchResponse
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    80
    """
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    81
    (protocol, host, path, parameters, query, fragment) = urlparse.urlparse(request.url())
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    82
828
f5fd65cc3bf3 Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 686
diff changeset
    83
    payload = None
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    84
    if request.method() == urlfetch_service_pb.URLFetchRequest.GET:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    85
      method = 'GET'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    86
    elif request.method() == urlfetch_service_pb.URLFetchRequest.POST:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    87
      method = 'POST'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    88
      payload = request.payload()
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    89
    elif request.method() == urlfetch_service_pb.URLFetchRequest.HEAD:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    90
      method = 'HEAD'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    91
    elif request.method() == urlfetch_service_pb.URLFetchRequest.PUT:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    92
      method = 'PUT'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    93
      payload = request.payload()
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    94
    elif request.method() == urlfetch_service_pb.URLFetchRequest.DELETE:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    95
      method = 'DELETE'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    96
    else:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    97
      logging.error('Invalid method: %s', request.method())
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    98
      raise apiproxy_errors.ApplicationError(
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
    99
        urlfetch_service_pb.URLFetchServiceError.UNSPECIFIED_ERROR)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   100
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   101
    if not (protocol == 'http' or protocol == 'https'):
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   102
      logging.error('Invalid protocol: %s', protocol)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   103
      raise apiproxy_errors.ApplicationError(
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   104
        urlfetch_service_pb.URLFetchServiceError.INVALID_URL)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   105
2273
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   106
    if not host:
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   107
      logging.error('Missing host.')
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   108
      raise apiproxy_errors.ApplicationError(
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   109
          urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR)
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   110
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   111
    sanitized_headers = self._SanitizeHttpHeaders(_UNTRUSTED_REQUEST_HEADERS,
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   112
                                                  request.header_list())
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   113
    request.clear_header()
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   114
    request.header_list().extend(sanitized_headers)
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   115
    deadline = _API_CALL_DEADLINE
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   116
    if request.has_deadline():
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   117
      deadline = request.deadline()
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   118
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   119
    self._RetrieveURL(request.url(), payload, method,
149
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
   120
                      request.header_list(), response,
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   121
                      follow_redirects=request.followredirects(),
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   122
                      deadline=deadline)
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   123
149
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
   124
  def _RetrieveURL(self, url, payload, method, headers, response,
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   125
                   follow_redirects=True, deadline=_API_CALL_DEADLINE):
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   126
    """Retrieves a URL.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   127
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   128
    Args:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   129
      url: String containing the URL to access.
828
f5fd65cc3bf3 Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 686
diff changeset
   130
      payload: Request payload to send, if any; None if no payload.
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   131
      method: HTTP method to use (e.g., 'GET')
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   132
      headers: List of additional header objects to use for the request.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   133
      response: Response object
149
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
   134
      follow_redirects: optional setting (defaulting to True) for whether or not
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
   135
        we should transparently follow redirects (up to MAX_REDIRECTS)
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   136
      deadline: Number of seconds to wait for the urlfetch to finish.
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   137
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   138
    Raises:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   139
      Raises an apiproxy_errors.ApplicationError exception with FETCH_ERROR
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   140
      in cases where:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   141
        - MAX_REDIRECTS is exceeded
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   142
        - The protocol of the redirected URL is bad or missing.
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   143
    """
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   144
    last_protocol = ''
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   145
    last_host = ''
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   146
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   147
    for redirect_number in xrange(MAX_REDIRECTS + 1):
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   148
      parsed = urlparse.urlparse(url)
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   149
      protocol, host, path, parameters, query, fragment = parsed
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   150
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   151
      port = urllib.splitport(urllib.splituser(host)[1])[1]
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   152
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   153
      if port not in PORTS_ALLOWED_IN_PRODUCTION:
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   154
        logging.warning(
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   155
          'urlfetch received %s ; port %s is not allowed in production!' %
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   156
          (url, port))
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   157
2273
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   158
      if protocol and not host:
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   159
        logging.error('Missing host on redirect; target url is %s' % url)
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   160
        raise apiproxy_errors.ApplicationError(
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   161
          urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR)
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   162
e4cb9c53db3e Load /Users/solydzajs/Desktop/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 828
diff changeset
   163
      if not host and not protocol:
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   164
        host = last_host
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   165
        protocol = last_protocol
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   166
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   167
      adjusted_headers = {
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   168
          'User-Agent':
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   169
          'AppEngine-Google; (+http://code.google.com/appengine)',
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   170
          'Host': host,
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   171
          'Accept-Encoding': 'gzip',
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   172
      }
828
f5fd65cc3bf3 Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 686
diff changeset
   173
      if payload is not None:
f5fd65cc3bf3 Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 686
diff changeset
   174
        adjusted_headers['Content-Length'] = len(payload)
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   175
      if method == 'POST' and payload:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   176
        adjusted_headers['Content-Type'] = 'application/x-www-form-urlencoded'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   177
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   178
      for header in headers:
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   179
        if header.key().title().lower() == 'user-agent':
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   180
          adjusted_headers['User-Agent'] = (
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   181
              '%s %s' %
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   182
              (header.value(), adjusted_headers['User-Agent']))
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   183
        else:
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   184
          adjusted_headers[header.key().title()] = header.value()
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   185
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   186
      logging.debug('Making HTTP request: host = %s, '
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   187
                    'url = %s, payload = %s, headers = %s',
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   188
                    host, url, payload, adjusted_headers)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   189
      try:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   190
        if protocol == 'http':
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   191
          connection = httplib.HTTPConnection(host)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   192
        elif protocol == 'https':
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   193
          connection = httplib.HTTPSConnection(host)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   194
        else:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   195
          error_msg = 'Redirect specified invalid protocol: "%s"' % protocol
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   196
          logging.error(error_msg)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   197
          raise apiproxy_errors.ApplicationError(
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   198
              urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   199
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   200
        last_protocol = protocol
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   201
        last_host = host
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   202
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   203
        if query != '':
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   204
          full_path = path + '?' + query
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   205
        else:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   206
          full_path = path
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   207
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   208
        orig_timeout = socket.getdefaulttimeout()
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   209
        try:
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   210
          socket.setdefaulttimeout(deadline)
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   211
          connection.request(method, full_path, payload, adjusted_headers)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   212
          http_response = connection.getresponse()
2864
2e0b0af889be Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2413
diff changeset
   213
          if method == 'HEAD':
2e0b0af889be Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2413
diff changeset
   214
            http_response_data = ''
2e0b0af889be Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2413
diff changeset
   215
          else:
2e0b0af889be Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2413
diff changeset
   216
            http_response_data = http_response.read()
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   217
        finally:
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   218
          socket.setdefaulttimeout(orig_timeout)
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   219
          connection.close()
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   220
      except (httplib.error, socket.error, IOError), e:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   221
        raise apiproxy_errors.ApplicationError(
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   222
          urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, str(e))
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   223
149
f2e327a7c5de Load ../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents: 109
diff changeset
   224
      if http_response.status in REDIRECT_STATUSES and follow_redirects:
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   225
        url = http_response.getheader('Location', None)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   226
        if url is None:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   227
          error_msg = 'Redirecting response was missing "Location" header'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   228
          logging.error(error_msg)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   229
          raise apiproxy_errors.ApplicationError(
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   230
              urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   231
      else:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   232
        response.set_statuscode(http_response.status)
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   233
        if http_response.getheader('content-encoding') == 'gzip':
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   234
          gzip_stream = StringIO.StringIO(http_response_data)
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   235
          gzip_file = gzip.GzipFile(fileobj=gzip_stream)
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   236
          http_response_data = gzip_file.read()
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   237
        response.set_content(http_response_data[:MAX_RESPONSE_SIZE])
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   238
        for header_key, header_value in http_response.getheaders():
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   239
          if (header_key.lower() == 'content-encoding' and
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   240
              header_value == 'gzip'):
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   241
            continue
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   242
          if header_key.lower() == 'content-length':
2864
2e0b0af889be Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2413
diff changeset
   243
            header_value = str(len(response.content()))
109
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   244
          header_proto = response.add_header()
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   245
          header_proto.set_key(header_key)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   246
          header_proto.set_value(header_value)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   247
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   248
        if len(http_response_data) > MAX_RESPONSE_SIZE:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   249
          response.set_contentwastruncated(True)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   250
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   251
        break
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   252
    else:
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   253
      error_msg = 'Too many repeated redirects'
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   254
      logging.error(error_msg)
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   255
      raise apiproxy_errors.ApplicationError(
620f9b141567 Load ../../google_appengine into trunk/thirdparty/google_appengine.
Todd Larsen <tlarsen@google.com>
parents:
diff changeset
   256
          urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   257
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   258
  def _SanitizeHttpHeaders(self, untrusted_headers, headers):
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   259
    """Cleans "unsafe" headers from the HTTP request/response.
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   260
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   261
    Args:
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   262
      untrusted_headers: set of untrusted headers names
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   263
      headers: list of string pairs, first is header name and the second is header's value
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   264
    """
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   265
    prohibited_headers = [h.key() for h in headers
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   266
                          if h.key().lower() in untrusted_headers]
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   267
    if prohibited_headers:
2413
d0b7dac5325c Update GAE to 1.2.3 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2309
diff changeset
   268
      logging.warn('Stripped prohibited headers from URLFetch request: %s',
2309
be1b94099f2d Update Google App Engine to 1.2.2 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents: 2273
diff changeset
   269
                   prohibited_headers)
686
df109be0567c Load ../../google_appengine_1.1.7/ into
Sverre Rabbelier <srabbelier@gmail.com>
parents: 149
diff changeset
   270
    return (h for h in headers if h.key().lower() not in untrusted_headers)