author | Lennard de Rijk <ljvderijk@gmail.com> |
Tue, 29 Sep 2009 20:10:08 +0200 | |
changeset 2992 | c860b26481be |
parent 2864 | 2e0b0af889be |
permissions | -rwxr-xr-x |
2864
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
1 |
#!/usr/bin/env python |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
2 |
# |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
3 |
# Copyright 2007 Google Inc. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
4 |
# |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
5 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
6 |
# you may not use this file except in compliance with the License. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
7 |
# You may obtain a copy of the License at |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
8 |
# |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
9 |
# http://www.apache.org/licenses/LICENSE-2.0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
10 |
# |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
11 |
# Unless required by applicable law or agreed to in writing, software |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
12 |
# distributed under the License is distributed on an "AS IS" BASIS, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
13 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
14 |
# See the License for the specific language governing permissions and |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
15 |
# limitations under the License. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
16 |
# |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
17 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
18 |
"""Client-side transfer throttling for use with remote_api_stub. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
19 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
20 |
This module is used to configure rate limiting for programs accessing |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
21 |
AppEngine services through remote_api. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
22 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
23 |
See the Throttle class for more information. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
24 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
25 |
An example with throttling: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
26 |
--- |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
27 |
from google.appengine.ext import db |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
28 |
from google.appengine.ext.remote_api import remote_api_stub |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
29 |
from google.appengine.ext.remote_api import throttle |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
30 |
from myapp import models |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
31 |
import getpass |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
32 |
import threading |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
33 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
34 |
def auth_func(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
35 |
return (raw_input('Username:'), getpass.getpass('Password:')) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
36 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
37 |
remote_api_stub.ConfigureRemoteDatastore('my-app', '/remote_api', auth_func) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
38 |
full_throttle = throttle.DefaultThrottle(multiplier=1.0) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
39 |
throttle.ThrottleRemoteDatastore(full_throttle) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
40 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
41 |
# Register any threads that will be using the datastore with the throttler |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
42 |
full_throttle.Register(threading.currentThread()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
43 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
44 |
# Now you can access the remote datastore just as if your code was running on |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
45 |
# App Engine, and you don't need to worry about exceeding quota limits! |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
46 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
47 |
houses = models.House.all().fetch(100) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
48 |
for a_house in houses: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
49 |
a_house.doors += 1 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
50 |
db.put(houses) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
51 |
--- |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
52 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
53 |
This example limits usage to the default free quota levels. The multiplier |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
54 |
kwarg to throttle.DefaultThrottle can be used to scale the throttle levels |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
55 |
higher or lower. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
56 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
57 |
Throttles can also be constructed directly for more control over the limits |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
58 |
for different operations. See the Throttle class and the constants following |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
59 |
it for details. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
60 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
61 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
62 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
63 |
import logging |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
64 |
import threading |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
65 |
import time |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
66 |
import urllib2 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
67 |
import urlparse |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
68 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
69 |
from google.appengine.api import apiproxy_stub_map |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
70 |
from google.appengine.ext.remote_api import remote_api_stub |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
71 |
from google.appengine.tools import appengine_rpc |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
72 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
73 |
logger = logging.getLogger('google.appengine.ext.remote_api.throttle') |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
74 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
75 |
MINIMUM_THROTTLE_SLEEP_DURATION = 0.001 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
76 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
77 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
78 |
class Error(Exception): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
79 |
"""Base class for errors in this module.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
80 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
81 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
82 |
class ThreadNotRegisteredError(Error): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
83 |
"""An unregistered thread has accessed the throttled datastore stub.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
84 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
85 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
86 |
class UnknownThrottleNameError(Error): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
87 |
"""A transfer was added for an unknown throttle name.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
88 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
89 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
90 |
def InterruptibleSleep(sleep_time): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
91 |
"""Puts thread to sleep, checking this threads exit_flag four times a second. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
92 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
93 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
94 |
sleep_time: Time to sleep. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
95 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
96 |
slept = 0.0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
97 |
epsilon = .0001 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
98 |
thread = threading.currentThread() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
99 |
while slept < sleep_time - epsilon: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
100 |
remaining = sleep_time - slept |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
101 |
this_sleep_time = min(remaining, 0.25) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
102 |
time.sleep(this_sleep_time) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
103 |
slept += this_sleep_time |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
104 |
if hasattr(thread, 'exit_flag') and thread.exit_flag: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
105 |
return |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
106 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
107 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
108 |
class Throttle(object): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
109 |
"""A base class for upload rate throttling. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
110 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
111 |
Transferring large number of entities, too quickly, could trigger |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
112 |
quota limits and cause the transfer process to halt. In order to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
113 |
stay within the application's quota, we throttle the data transfer |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
114 |
to a specified limit (across all transfer threads). |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
115 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
116 |
This class tracks a moving average of some aspect of the transfer |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
117 |
rate (bandwidth, records per second, http connections per |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
118 |
second). It keeps two windows of counts of bytes transferred, on a |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
119 |
per-thread basis. One block is the "current" block, and the other is |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
120 |
the "prior" block. It will rotate the counts from current to prior |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
121 |
when ROTATE_PERIOD has passed. Thus, the current block will |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
122 |
represent from 0 seconds to ROTATE_PERIOD seconds of activity |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
123 |
(determined by: time.time() - self.last_rotate). The prior block |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
124 |
will always represent a full ROTATE_PERIOD. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
125 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
126 |
Sleeping is performed just before a transfer of another block, and is |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
127 |
based on the counts transferred *before* the next transfer. It really |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
128 |
does not matter how much will be transferred, but only that for all the |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
129 |
data transferred SO FAR that we have interspersed enough pauses to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
130 |
ensure the aggregate transfer rate is within the specified limit. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
131 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
132 |
These counts are maintained on a per-thread basis, so we do not require |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
133 |
any interlocks around incrementing the counts. There IS an interlock on |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
134 |
the rotation of the counts because we do not want multiple threads to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
135 |
multiply-rotate the counts. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
136 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
137 |
There are various race conditions in the computation and collection |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
138 |
of these counts. We do not require precise values, but simply to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
139 |
keep the overall transfer within the bandwidth limits. If a given |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
140 |
pause is a little short, or a little long, then the aggregate delays |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
141 |
will be correct. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
142 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
143 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
144 |
ROTATE_PERIOD = 600 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
145 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
146 |
def __init__(self, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
147 |
get_time=time.time, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
148 |
thread_sleep=InterruptibleSleep, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
149 |
layout=None): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
150 |
self.get_time = get_time |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
151 |
self.thread_sleep = thread_sleep |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
152 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
153 |
self.start_time = get_time() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
154 |
self.transferred = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
155 |
self.prior_block = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
156 |
self.totals = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
157 |
self.throttles = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
158 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
159 |
self.last_rotate = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
160 |
self.rotate_mutex = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
161 |
if layout: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
162 |
self.AddThrottles(layout) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
163 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
164 |
def AddThrottle(self, name, limit): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
165 |
self.throttles[name] = limit |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
166 |
self.transferred[name] = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
167 |
self.prior_block[name] = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
168 |
self.totals[name] = {} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
169 |
self.last_rotate[name] = self.get_time() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
170 |
self.rotate_mutex[name] = threading.Lock() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
171 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
172 |
def AddThrottles(self, layout): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
173 |
for key, value in layout.iteritems(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
174 |
self.AddThrottle(key, value) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
175 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
176 |
def Register(self, thread): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
177 |
"""Register this thread with the throttler.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
178 |
thread_id = id(thread) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
179 |
for throttle_name in self.throttles.iterkeys(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
180 |
self.transferred[throttle_name][thread_id] = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
181 |
self.prior_block[throttle_name][thread_id] = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
182 |
self.totals[throttle_name][thread_id] = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
183 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
184 |
def VerifyThrottleName(self, throttle_name): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
185 |
if throttle_name not in self.throttles: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
186 |
raise UnknownThrottleNameError('%s is not a registered throttle' % |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
187 |
throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
188 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
189 |
def AddTransfer(self, throttle_name, token_count): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
190 |
"""Add a count to the amount this thread has transferred. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
191 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
192 |
Each time a thread transfers some data, it should call this method to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
193 |
note the amount sent. The counts may be rotated if sufficient time |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
194 |
has passed since the last rotation. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
195 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
196 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
197 |
throttle_name: The name of the throttle to add to. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
198 |
token_count: The number to add to the throttle counter. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
199 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
200 |
self.VerifyThrottleName(throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
201 |
transferred = self.transferred[throttle_name] |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
202 |
try: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
203 |
transferred[id(threading.currentThread())] += token_count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
204 |
except KeyError: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
205 |
thread = threading.currentThread() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
206 |
raise ThreadNotRegisteredError( |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
207 |
'Unregistered thread accessing throttled datastore stub: id = %s\n' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
208 |
'name = %s' % (id(thread), thread.getName())) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
209 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
210 |
if self.last_rotate[throttle_name] + self.ROTATE_PERIOD < self.get_time(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
211 |
self._RotateCounts(throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
212 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
213 |
def Sleep(self, throttle_name=None): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
214 |
"""Possibly sleep in order to limit the transfer rate. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
215 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
216 |
Note that we sleep based on *prior* transfers rather than what we |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
217 |
may be about to transfer. The next transfer could put us under/over |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
218 |
and that will be rectified *after* that transfer. Net result is that |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
219 |
the average transfer rate will remain within bounds. Spiky behavior |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
220 |
or uneven rates among the threads could possibly bring the transfer |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
221 |
rate above the requested limit for short durations. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
222 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
223 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
224 |
throttle_name: The name of the throttle to sleep on. If None or |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
225 |
omitted, then sleep on all throttles. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
226 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
227 |
if throttle_name is None: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
228 |
for throttle_name in self.throttles: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
229 |
self.Sleep(throttle_name=throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
230 |
return |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
231 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
232 |
self.VerifyThrottleName(throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
233 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
234 |
thread = threading.currentThread() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
235 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
236 |
while True: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
237 |
duration = self.get_time() - self.last_rotate[throttle_name] |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
238 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
239 |
total = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
240 |
for count in self.prior_block[throttle_name].values(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
241 |
total += count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
242 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
243 |
if total: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
244 |
duration += self.ROTATE_PERIOD |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
245 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
246 |
for count in self.transferred[throttle_name].values(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
247 |
total += count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
248 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
249 |
sleep_time = self._SleepTime(total, self.throttles[throttle_name], |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
250 |
duration) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
251 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
252 |
if sleep_time < MINIMUM_THROTTLE_SLEEP_DURATION: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
253 |
break |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
254 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
255 |
logger.debug('[%s] Throttling on %s. Sleeping for %.1f ms ' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
256 |
'(duration=%.1f ms, total=%d)', |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
257 |
thread.getName(), throttle_name, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
258 |
sleep_time * 1000, duration * 1000, total) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
259 |
self.thread_sleep(sleep_time) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
260 |
if thread.exit_flag: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
261 |
break |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
262 |
self._RotateCounts(throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
263 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
264 |
def _SleepTime(self, total, limit, duration): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
265 |
"""Calculate the time to sleep on a throttle. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
266 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
267 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
268 |
total: The total amount transferred. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
269 |
limit: The amount per second that is allowed to be sent. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
270 |
duration: The amount of time taken to send the total. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
271 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
272 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
273 |
A float for the amount of time to sleep. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
274 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
275 |
if not limit: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
276 |
return 0.0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
277 |
return max(0.0, (total / limit) - duration) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
278 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
279 |
def _RotateCounts(self, throttle_name): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
280 |
"""Rotate the transfer counters. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
281 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
282 |
If sufficient time has passed, then rotate the counters from active to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
283 |
the prior-block of counts. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
284 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
285 |
This rotation is interlocked to ensure that multiple threads do not |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
286 |
over-rotate the counts. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
287 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
288 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
289 |
throttle_name: The name of the throttle to rotate. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
290 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
291 |
self.VerifyThrottleName(throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
292 |
self.rotate_mutex[throttle_name].acquire() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
293 |
try: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
294 |
next_rotate_time = self.last_rotate[throttle_name] + self.ROTATE_PERIOD |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
295 |
if next_rotate_time >= self.get_time(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
296 |
return |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
297 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
298 |
for name, count in self.transferred[throttle_name].items(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
299 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
300 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
301 |
self.prior_block[throttle_name][name] = count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
302 |
self.transferred[throttle_name][name] = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
303 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
304 |
self.totals[throttle_name][name] += count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
305 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
306 |
self.last_rotate[throttle_name] = self.get_time() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
307 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
308 |
finally: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
309 |
self.rotate_mutex[throttle_name].release() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
310 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
311 |
def TotalTransferred(self, throttle_name): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
312 |
"""Return the total transferred, and over what period. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
313 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
314 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
315 |
throttle_name: The name of the throttle to total. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
316 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
317 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
318 |
A tuple of the total count and running time for the given throttle name. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
319 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
320 |
total = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
321 |
for count in self.totals[throttle_name].values(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
322 |
total += count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
323 |
for count in self.transferred[throttle_name].values(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
324 |
total += count |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
325 |
return total, self.get_time() - self.start_time |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
326 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
327 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
328 |
BANDWIDTH_UP = 'http-bandwidth-up' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
329 |
BANDWIDTH_DOWN = 'http-bandwidth-down' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
330 |
REQUESTS = 'http-requests' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
331 |
HTTPS_BANDWIDTH_UP = 'https-bandwidth-up' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
332 |
HTTPS_BANDWIDTH_DOWN = 'https-bandwidth-down' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
333 |
HTTPS_REQUESTS = 'https-requests' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
334 |
DATASTORE_CALL_COUNT = 'datastore-call-count' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
335 |
ENTITIES_FETCHED = 'entities-fetched' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
336 |
ENTITIES_MODIFIED = 'entities-modified' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
337 |
INDEX_MODIFICATIONS = 'index-modifications' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
338 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
339 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
340 |
DEFAULT_LIMITS = { |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
341 |
BANDWIDTH_UP: 100000, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
342 |
BANDWIDTH_DOWN: 100000, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
343 |
REQUESTS: 15, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
344 |
HTTPS_BANDWIDTH_UP: 100000, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
345 |
HTTPS_BANDWIDTH_DOWN: 100000, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
346 |
HTTPS_REQUESTS: 15, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
347 |
DATASTORE_CALL_COUNT: 120, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
348 |
ENTITIES_FETCHED: 400, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
349 |
ENTITIES_MODIFIED: 400, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
350 |
INDEX_MODIFICATIONS: 1600, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
351 |
} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
352 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
353 |
NO_LIMITS = { |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
354 |
BANDWIDTH_UP: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
355 |
BANDWIDTH_DOWN: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
356 |
REQUESTS: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
357 |
HTTPS_BANDWIDTH_UP: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
358 |
HTTPS_BANDWIDTH_DOWN: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
359 |
HTTPS_REQUESTS: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
360 |
DATASTORE_CALL_COUNT: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
361 |
ENTITIES_FETCHED: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
362 |
ENTITIES_MODIFIED: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
363 |
INDEX_MODIFICATIONS: None, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
364 |
} |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
365 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
366 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
367 |
def DefaultThrottle(multiplier=1.0): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
368 |
"""Return a Throttle instance with multiplier * the quota limits.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
369 |
layout = dict([(name, multiplier * limit) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
370 |
for (name, limit) in DEFAULT_LIMITS.iteritems()]) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
371 |
return Throttle(layout=layout) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
372 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
373 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
374 |
class ThrottleHandler(urllib2.BaseHandler): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
375 |
"""A urllib2 handler for http and https requests that adds to a throttle.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
376 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
377 |
def __init__(self, throttle): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
378 |
"""Initialize a ThrottleHandler. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
379 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
380 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
381 |
throttle: A Throttle instance to call for bandwidth and http/https request |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
382 |
throttling. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
383 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
384 |
self.throttle = throttle |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
385 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
386 |
def AddRequest(self, throttle_name, req): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
387 |
"""Add to bandwidth throttle for given request. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
388 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
389 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
390 |
throttle_name: The name of the bandwidth throttle to add to. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
391 |
req: The request whose size will be added to the throttle. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
392 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
393 |
size = 0 |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
394 |
for key, value in req.headers.iteritems(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
395 |
size += len('%s: %s\n' % (key, value)) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
396 |
for key, value in req.unredirected_hdrs.iteritems(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
397 |
size += len('%s: %s\n' % (key, value)) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
398 |
(unused_scheme, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
399 |
unused_host_port, url_path, |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
400 |
unused_query, unused_fragment) = urlparse.urlsplit(req.get_full_url()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
401 |
size += len('%s %s HTTP/1.1\n' % (req.get_method(), url_path)) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
402 |
data = req.get_data() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
403 |
if data: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
404 |
size += len(data) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
405 |
self.throttle.AddTransfer(throttle_name, size) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
406 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
407 |
def AddResponse(self, throttle_name, res): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
408 |
"""Add to bandwidth throttle for given response. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
409 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
410 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
411 |
throttle_name: The name of the bandwidth throttle to add to. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
412 |
res: The response whose size will be added to the throttle. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
413 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
414 |
content = res.read() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
415 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
416 |
def ReturnContent(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
417 |
return content |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
418 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
419 |
res.read = ReturnContent |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
420 |
size = len(content) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
421 |
headers = res.info() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
422 |
for key, value in headers.items(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
423 |
size += len('%s: %s\n' % (key, value)) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
424 |
self.throttle.AddTransfer(throttle_name, size) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
425 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
426 |
def http_request(self, req): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
427 |
"""Process an HTTP request. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
428 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
429 |
If the throttle is over quota, sleep first. Then add request size to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
430 |
throttle before returning it to be sent. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
431 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
432 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
433 |
req: A urllib2.Request object. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
434 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
435 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
436 |
The request passed in. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
437 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
438 |
self.throttle.Sleep(BANDWIDTH_UP) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
439 |
self.throttle.Sleep(BANDWIDTH_DOWN) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
440 |
self.AddRequest(BANDWIDTH_UP, req) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
441 |
return req |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
442 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
443 |
def https_request(self, req): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
444 |
"""Process an HTTPS request. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
445 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
446 |
If the throttle is over quota, sleep first. Then add request size to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
447 |
throttle before returning it to be sent. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
448 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
449 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
450 |
req: A urllib2.Request object. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
451 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
452 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
453 |
The request passed in. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
454 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
455 |
self.throttle.Sleep(HTTPS_BANDWIDTH_UP) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
456 |
self.throttle.Sleep(HTTPS_BANDWIDTH_DOWN) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
457 |
self.AddRequest(HTTPS_BANDWIDTH_UP, req) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
458 |
return req |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
459 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
460 |
def http_response(self, unused_req, res): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
461 |
"""Process an HTTP response. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
462 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
463 |
The size of the response is added to the bandwidth throttle and the request |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
464 |
throttle is incremented by one. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
465 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
466 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
467 |
unused_req: The urllib2 request for this response. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
468 |
res: A urllib2 response object. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
469 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
470 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
471 |
The response passed in. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
472 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
473 |
self.AddResponse(BANDWIDTH_DOWN, res) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
474 |
self.throttle.AddTransfer(REQUESTS, 1) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
475 |
return res |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
476 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
477 |
def https_response(self, unused_req, res): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
478 |
"""Process an HTTPS response. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
479 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
480 |
The size of the response is added to the bandwidth throttle and the request |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
481 |
throttle is incremented by one. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
482 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
483 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
484 |
unused_req: The urllib2 request for this response. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
485 |
res: A urllib2 response object. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
486 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
487 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
488 |
The response passed in. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
489 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
490 |
self.AddResponse(HTTPS_BANDWIDTH_DOWN, res) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
491 |
self.throttle.AddTransfer(HTTPS_REQUESTS, 1) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
492 |
return res |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
493 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
494 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
495 |
class ThrottledHttpRpcServer(appengine_rpc.HttpRpcServer): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
496 |
"""Provides a simplified RPC-style interface for HTTP requests. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
497 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
498 |
This RPC server uses a Throttle to prevent exceeding quotas. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
499 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
500 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
501 |
def __init__(self, throttle, *args, **kwargs): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
502 |
"""Initialize a ThrottledHttpRpcServer. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
503 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
504 |
Also sets request_manager.rpc_server to the ThrottledHttpRpcServer instance. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
505 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
506 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
507 |
throttle: A Throttles instance. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
508 |
args: Positional arguments to pass through to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
509 |
appengine_rpc.HttpRpcServer.__init__ |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
510 |
kwargs: Keyword arguments to pass through to |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
511 |
appengine_rpc.HttpRpcServer.__init__ |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
512 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
513 |
self.throttle = throttle |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
514 |
appengine_rpc.HttpRpcServer.__init__(self, *args, **kwargs) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
515 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
516 |
def _GetOpener(self): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
517 |
"""Returns an OpenerDirector that supports cookies and ignores redirects. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
518 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
519 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
520 |
A urllib2.OpenerDirector object. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
521 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
522 |
opener = appengine_rpc.HttpRpcServer._GetOpener(self) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
523 |
opener.add_handler(ThrottleHandler(self.throttle)) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
524 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
525 |
return opener |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
526 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
527 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
528 |
def ThrottledHttpRpcServerFactory(throttle): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
529 |
"""Create a factory to produce ThrottledHttpRpcServer for a given throttle. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
530 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
531 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
532 |
throttle: A Throttle instance to use for the ThrottledHttpRpcServer. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
533 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
534 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
535 |
A factory to produce a ThrottledHttpRpcServer. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
536 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
537 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
538 |
def MakeRpcServer(*args, **kwargs): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
539 |
"""Factory to produce a ThrottledHttpRpcServer. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
540 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
541 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
542 |
args: Positional args to pass to ThrottledHttpRpcServer. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
543 |
kwargs: Keyword args to pass to ThrottledHttpRpcServer. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
544 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
545 |
Returns: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
546 |
A ThrottledHttpRpcServer instance. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
547 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
548 |
kwargs['account_type'] = 'HOSTED_OR_GOOGLE' |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
549 |
kwargs['save_cookies'] = True |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
550 |
rpc_server = ThrottledHttpRpcServer(throttle, *args, **kwargs) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
551 |
return rpc_server |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
552 |
return MakeRpcServer |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
553 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
554 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
555 |
class Throttler(object): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
556 |
def PrehookHandler(self, service, call, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
557 |
handler = getattr(self, '_Prehook_' + call, None) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
558 |
if handler: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
559 |
handler(request, response) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
560 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
561 |
def PosthookHandler(self, service, call, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
562 |
handler = getattr(self, '_Posthook_' + call, None) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
563 |
if handler: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
564 |
handler(request, response) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
565 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
566 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
567 |
def SleepHandler(*throttle_names): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
568 |
def SleepOnThrottles(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
569 |
for throttle_name in throttle_names: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
570 |
self._DatastoreThrottler__throttle.Sleep(throttle_name) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
571 |
return SleepOnThrottles |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
572 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
573 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
574 |
class DatastoreThrottler(Throttler): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
575 |
def __init__(self, throttle): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
576 |
Throttler.__init__(self) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
577 |
self.__throttle = throttle |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
578 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
579 |
def AddCost(self, cost_proto): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
580 |
"""Add costs from the Cost protobuf.""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
581 |
self.__throttle.AddTransfer(INDEX_MODIFICATIONS, cost_proto.index_writes()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
582 |
self.__throttle.AddTransfer(ENTITIES_MODIFIED, cost_proto.entity_writes()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
583 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
584 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
585 |
_Prehook_Put = SleepHandler(ENTITIES_MODIFIED, INDEX_MODIFICATIONS) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
586 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
587 |
def _Posthook_Put(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
588 |
self.AddCost(response.cost()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
589 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
590 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
591 |
_Prehook_Get = SleepHandler(ENTITIES_FETCHED) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
592 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
593 |
def _Posthook_Get(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
594 |
self.__throttle.AddTransfer(ENTITIES_FETCHED, response.entity_size()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
595 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
596 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
597 |
_Prehook_RunQuery = SleepHandler(ENTITIES_FETCHED) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
598 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
599 |
def _Posthook_RunQuery(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
600 |
if not response.keys_only(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
601 |
self.__throttle.AddTransfer(ENTITIES_FETCHED, response.result_size()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
602 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
603 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
604 |
_Prehook_Next = SleepHandler(ENTITIES_FETCHED) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
605 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
606 |
def _Posthook_Next(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
607 |
if not response.keys_only(): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
608 |
self.__throttle.AddTransfer(ENTITIES_FETCHED, response.result_size()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
609 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
610 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
611 |
_Prehook_Delete = SleepHandler(ENTITIES_MODIFIED, INDEX_MODIFICATIONS) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
612 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
613 |
def _Posthook_Delete(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
614 |
self.AddCost(response.cost()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
615 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
616 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
617 |
_Prehook_Commit = SleepHandler() |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
618 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
619 |
def _Posthook_Commit(self, request, response): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
620 |
self.AddCost(response.cost()) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
621 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
622 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
623 |
def ThrottleRemoteDatastore(throttle, remote_datastore_stub=None): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
624 |
"""Install the given throttle for the remote datastore stub. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
625 |
|
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
626 |
Args: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
627 |
throttle: A Throttle instance to limit datastore access rates |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
628 |
remote_datastore_stub: The datstore stub instance to throttle, for |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
629 |
testing purposes. |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
630 |
""" |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
631 |
if not remote_datastore_stub: |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
632 |
remote_datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3') |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
633 |
if not isinstance(remote_datastore_stub, remote_api_stub.RemoteDatastoreStub): |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
634 |
raise remote_api_stub.ConfigurationError('remote_api is not configured.') |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
635 |
throttler = DatastoreThrottler(throttle) |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
636 |
remote_datastore_stub._PreHookHandler = throttler.PrehookHandler |
2e0b0af889be
Update Google App Engine from 1.2.3 to 1.2.5 in thirdparty folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
637 |
remote_datastore_stub._PostHookHandler = throttler.PosthookHandler |