--- a/thirdparty/google_appengine/google/appengine/api/urlfetch_stub.py Sat Dec 06 14:50:45 2008 +0000
+++ b/thirdparty/google_appengine/google/appengine/api/urlfetch_stub.py Sat Dec 06 16:52:21 2008 +0000
@@ -22,8 +22,10 @@
import httplib
import logging
import socket
+import urllib
import urlparse
+from google.appengine.api import apiproxy_stub
from google.appengine.api import urlfetch
from google.appengine.api import urlfetch_errors
from google.appengine.api import urlfetch_service_pb
@@ -41,25 +43,33 @@
httplib.TEMPORARY_REDIRECT,
])
+PORTS_ALLOWED_IN_PRODUCTION = (
+ None, '80', '443', '4443', '8080', '8081', '8082', '8083', '8084', '8085',
+ '8086', '8087', '8088', '8089', '8188', '8444', '8990')
-class URLFetchServiceStub(object):
+_API_CALL_DEADLINE = 5.0
+
+
+_UNTRUSTED_REQUEST_HEADERS = frozenset([
+ 'content-length',
+ 'host',
+ 'referer',
+ 'user-agent',
+ 'vary',
+ 'via',
+ 'x-forwarded-for',
+])
+
+class URLFetchServiceStub(apiproxy_stub.APIProxyStub):
"""Stub version of the urlfetch API to be used with apiproxy_stub_map."""
- def MakeSyncCall(self, service, call, request, response):
- """The main RPC entry point.
+ def __init__(self, service_name='urlfetch'):
+ """Initializer.
- Arg:
- service: Must be 'urlfetch'.
- call: A string representing the rpc to make. Must be part of
- URLFetchService.
- request: A protocol buffer of the type corresponding to 'call'.
- response: A protocol buffer of the type corresponding to 'call'.
+ Args:
+ service_name: Service name expected for all calls.
"""
- assert service == 'urlfetch'
- assert request.IsInitialized()
-
- attr = getattr(self, '_Dynamic_' + call)
- attr(request, response)
+ super(URLFetchServiceStub, self).__init__(service_name)
def _Dynamic_Fetch(self, request, response):
"""Trivial implementation of URLFetchService::Fetch().
@@ -93,6 +103,11 @@
raise apiproxy_errors.ApplicationError(
urlfetch_service_pb.URLFetchServiceError.INVALID_URL)
+ sanitized_headers = self._SanitizeHttpHeaders(_UNTRUSTED_REQUEST_HEADERS,
+ request.header_list())
+ request.clear_header()
+ request.header_list().extend(sanitized_headers)
+
self._RetrieveURL(request.url(), payload, method,
request.header_list(), response,
follow_redirects=request.followredirects())
@@ -120,7 +135,15 @@
last_host = ''
for redirect_number in xrange(MAX_REDIRECTS + 1):
- (protocol, host, path, parameters, query, fragment) = urlparse.urlparse(url)
+ parsed = urlparse.urlparse(url)
+ protocol, host, path, parameters, query, fragment = parsed
+
+ port = urllib.splitport(urllib.splituser(host)[1])[1]
+
+ if port not in PORTS_ALLOWED_IN_PRODUCTION:
+ logging.warning(
+ 'urlfetch received %s ; port %s is not allowed in production!' %
+ (url, port))
if host == '' and protocol == '':
host = last_host
@@ -159,11 +182,14 @@
else:
full_path = path
+ orig_timeout = socket.getdefaulttimeout()
try:
+ socket.setdefaulttimeout(_API_CALL_DEADLINE)
connection.request(method, full_path, payload, adjusted_headers)
http_response = connection.getresponse()
http_response_data = http_response.read()
finally:
+ socket.setdefaulttimeout(orig_timeout)
connection.close()
except (httplib.error, socket.error, IOError), e:
raise apiproxy_errors.ApplicationError(
@@ -176,8 +202,6 @@
logging.error(error_msg)
raise apiproxy_errors.ApplicationError(
urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)
- else:
- method = 'GET'
else:
response.set_statuscode(http_response.status)
response.set_content(http_response_data[:MAX_RESPONSE_SIZE])
@@ -195,3 +219,12 @@
logging.error(error_msg)
raise apiproxy_errors.ApplicationError(
urlfetch_service_pb.URLFetchServiceError.FETCH_ERROR, error_msg)
+
+ def _SanitizeHttpHeaders(self, untrusted_headers, headers):
+ """Cleans "unsafe" headers from the HTTP request/response.
+
+ Args:
+ untrusted_headers: set of untrusted headers names
+ headers: list of string pairs, first is header name and the second is header's value
+ """
+ return (h for h in headers if h.key().lower() not in untrusted_headers)