2832
|
1 |
import cProfile
|
|
2 |
import ppstats
|
|
3 |
|
|
4 |
from google.appengine.ext import webapp
|
|
5 |
from google.appengine.api import memcache
|
|
6 |
import google.appengine.ext.webapp.util
|
|
7 |
|
|
8 |
import os.path
|
|
9 |
import logging
|
|
10 |
import re
|
|
11 |
import random
|
|
12 |
import string
|
|
13 |
import zlib
|
|
14 |
|
|
15 |
mc_client = memcache.Client()
|
|
16 |
|
|
17 |
alphanumeric = string.letters + string.digits
|
|
18 |
|
|
19 |
global_profiler = None
|
|
20 |
|
|
21 |
class GAEProfiler(object):
|
|
22 |
_save_every = 10
|
|
23 |
|
|
24 |
def __init__(self):
|
|
25 |
self.is_profiling = False
|
|
26 |
self._profiler = None
|
|
27 |
self.num_requests = 0
|
|
28 |
self.requests_profiled = 0
|
|
29 |
self.request_regex = None
|
|
30 |
self.profile_key = ''.join([random.choice(alphanumeric) for x in range(4)])
|
|
31 |
|
|
32 |
def start_profiling(self, request_regex=None, num_requests=0):
|
|
33 |
"start profiling with this object, setting # of requests and filter"
|
|
34 |
if self.is_profiling:
|
|
35 |
return
|
|
36 |
|
|
37 |
self.is_profiling = True
|
|
38 |
if self._profiler is None:
|
|
39 |
self._profiler = cProfile.Profile()
|
|
40 |
self.num_requests = num_requests
|
|
41 |
if request_regex:
|
|
42 |
self.request_regex = re.compile(request_regex)
|
|
43 |
|
|
44 |
def stop_profiling(self):
|
|
45 |
self.is_profiling = False
|
|
46 |
|
|
47 |
def resume_profiling(self):
|
|
48 |
self.is_profiling = True
|
|
49 |
|
|
50 |
def has_profiler(self):
|
|
51 |
return self._profiler is not None
|
|
52 |
|
|
53 |
def get_pstats(self):
|
|
54 |
"return a ppstats object from current profile data"
|
|
55 |
gae_base_dir = '/'.join(webapp.__file__.split('/')[:-5])
|
|
56 |
sys_base_dir = '/'.join(logging.__file__.split('/')[:-2])
|
|
57 |
|
|
58 |
stats = ppstats.Stats(self._profiler)
|
|
59 |
stats.hide_directory(gae_base_dir, 'GAEHome')
|
|
60 |
stats.hide_directory(sys_base_dir, 'SysHome')
|
|
61 |
stats.strip_dirs()
|
|
62 |
return stats
|
|
63 |
|
|
64 |
def runcall(self, func, *args, **kwargs):
|
|
65 |
"profile one call, incrementing requests_profiled and maybe saving stats"
|
|
66 |
self.requests_profiled += 1
|
|
67 |
if self._profiler:
|
|
68 |
ret = self._profiler.runcall(func, *args, **kwargs)
|
|
69 |
else:
|
|
70 |
ret = func(*args, **kwargs)
|
|
71 |
|
|
72 |
# if (self.requests_profiled % self._save_every) == 0 or \
|
|
73 |
# self.requests_profiled == self.num_requests:
|
|
74 |
# self.save_pstats_to_memcache()
|
|
75 |
self.save_pstats_to_memcache()
|
|
76 |
return ret
|
|
77 |
|
|
78 |
def should_profile_request(self):
|
|
79 |
"check for # of requests profiled and that SCRIPT_NAME matches regex"
|
|
80 |
env = dict(os.environ)
|
|
81 |
script_name = env.get('SCRIPT_NAME', '')
|
|
82 |
logging.info(script_name)
|
|
83 |
|
|
84 |
if self.num_requests and self.requests_profiled >= self.num_requests:
|
|
85 |
return False
|
|
86 |
|
|
87 |
if self.request_regex and not self.request_regex.search(script_name):
|
|
88 |
return False
|
|
89 |
|
|
90 |
return True
|
|
91 |
|
|
92 |
def save_pstats_to_memcache(self):
|
|
93 |
"save stats from profiler object to memcache"
|
|
94 |
ps = self.get_pstats()
|
|
95 |
output = ps.dump_stats_pickle()
|
|
96 |
compressed_data = zlib.compress(output, 3)
|
|
97 |
cache_key = cache_key_for_profile(self.profile_key)
|
|
98 |
mc_client.set(cache_key, compressed_data)
|
|
99 |
logging.info("Saved pstats to memcache with key %s" % cache_key)
|
|
100 |
|
|
101 |
|
|
102 |
|
|
103 |
def get_global_profiler():
|
|
104 |
global global_profiler
|
|
105 |
if not global_profiler:
|
|
106 |
global_profiler = GAEProfiler()
|
|
107 |
|
|
108 |
return global_profiler
|
|
109 |
|
|
110 |
def cache_key_for_profile(profile_key):
|
|
111 |
"generate a memcache key"
|
|
112 |
return "ProfileData.%s" % profile_key
|
|
113 |
|
|
114 |
def load_pstats_from_memcache(profile_key):
|
|
115 |
"retrieve ppstats object"
|
|
116 |
mc_data = mc_client.get(cache_key_for_profile(profile_key))
|
|
117 |
if not mc_data:
|
|
118 |
return None
|
|
119 |
|
|
120 |
return ppstats.from_gz(mc_data)
|
|
121 |
|
|
122 |
def get_stats_from_global_or_request(request_obj):
|
|
123 |
"get pstats for a key, or the global pstats"
|
|
124 |
key = request_obj.get('key', '')
|
|
125 |
if key:
|
|
126 |
return load_pstats_from_memcache(key)
|
|
127 |
else:
|
|
128 |
gp = get_global_profiler()
|
|
129 |
if not gp.has_profiler():
|
|
130 |
return None
|
|
131 |
return gp.get_pstats()
|
|
132 |
|
|
133 |
class show_profile(webapp.RequestHandler):
|
|
134 |
def get(self):
|
|
135 |
ps = get_stats_from_global_or_request(self.request)
|
|
136 |
if not ps:
|
|
137 |
self.response.out.write("<body><html><h3>No profiler.</h3><html></body>")
|
|
138 |
return
|
|
139 |
|
|
140 |
ps.set_output(self.response.out)
|
|
141 |
sort = self.request.get('sort', 'time')
|
|
142 |
ps.sort_stats(sort)
|
|
143 |
self.response.out.write("<body><html><pre>\n")
|
|
144 |
ps.print_stats(30)
|
|
145 |
self.response.out.write("</pre></html></body>")
|
|
146 |
|
|
147 |
class download_profile_data(webapp.RequestHandler):
|
|
148 |
def get(self):
|
|
149 |
ps = get_stats_from_global_or_request(self.request)
|
|
150 |
if not ps:
|
|
151 |
self.response.out.write("<body><html><h3>No profiler.</h3><html></body>")
|
|
152 |
return
|
|
153 |
|
|
154 |
output = ps.dump_stats_pickle()
|
|
155 |
|
|
156 |
self.response.headers['Content-Type'] = 'application/octet-stream'
|
|
157 |
|
|
158 |
self.response.out.write(output)
|
|
159 |
|
|
160 |
class show_profiler_status(webapp.RequestHandler):
|
|
161 |
def get(self):
|
|
162 |
gp = get_global_profiler()
|
|
163 |
if not gp.has_profiler:
|
|
164 |
self.response.out.write("<body><html><h3>No profiler.</h3><html></body>")
|
|
165 |
return
|
|
166 |
|
|
167 |
self.response.out.write("<html><body>")
|
|
168 |
self.response.out.write("<b>Currently profiling:</b> %s<br>" % gp.is_profiling)
|
|
169 |
self.response.out.write("<b>Profile Key</b>: %s<br>" % gp.profile_key)
|
|
170 |
self.response.out.write("<b>Requests profiled so far:</b> %s<br>" % gp.requests_profiled)
|
|
171 |
self.response.out.write("<b>Requests to profile:</b> %s<br>" % gp.num_requests)
|
|
172 |
self.response.out.write("<b>Request regex:</b> %s<br>" % gp.request_regex)
|
|
173 |
self.response.out.write("</body></html>")
|
|
174 |
|
|
175 |
class start_profiler(webapp.RequestHandler):
|
|
176 |
def get(self):
|
|
177 |
gp = get_global_profiler()
|
|
178 |
gp.start_profiling()
|
|
179 |
self.response.headers['Content-Type'] = "text/plain"
|
|
180 |
self.response.out.write("Started profiling (key: %s).\n" % gp.profile_key)
|
|
181 |
self.response.out.write("Retrieve saved results at <a href='/profiler/show?key=%(key)s'>/profiler/show?key=%(key)s).\n" % {'key':gp.profile_key})
|
|
182 |
|
|
183 |
class stop_profiler(webapp.RequestHandler):
|
|
184 |
def get(self):
|
|
185 |
gp = get_global_profiler()
|
|
186 |
gp.stop_profiling()
|
|
187 |
self.request.out.write("Content-Type: text/plain\n\n")
|
|
188 |
self.request.out.write("done.")
|
|
189 |
|
|
190 |
class save_profile_data(webapp.RequestHandler):
|
|
191 |
def get(self):
|
|
192 |
gp = get_global_profiler()
|
|
193 |
|
|
194 |
|
|
195 |
def _add_our_endpoints(application):
|
|
196 |
"insert our URLs into the application map"
|
|
197 |
url_mapping = [(regex.pattern, handler) for (regex, handler) in application._url_mapping]
|
|
198 |
return webapp.WSGIApplication(url_mapping, debug=True)
|
|
199 |
|
|
200 |
#
|
|
201 |
# wrapper to for webapp applications
|
|
202 |
#
|
|
203 |
def run_wsgi_app(application):
|
|
204 |
"proxy webapp.util's call to profile when needed"
|
|
205 |
gp = get_global_profiler()
|
|
206 |
if gp.is_profiling and gp.should_profile_request():
|
|
207 |
return gp.runcall(google.appengine.ext.webapp.util.run_wsgi_app, *(application,))
|
|
208 |
else:
|
|
209 |
return google.appengine.ext.webapp.util.run_wsgi_app(application)
|
|
210 |
|
|
211 |
#
|
|
212 |
# middleware for django applications
|
|
213 |
#
|
|
214 |
|
|
215 |
class ProfileMiddleware(object):
|
|
216 |
def __init__(self):
|
|
217 |
self.profiler = None
|
|
218 |
|
|
219 |
def process_request(self, request):
|
|
220 |
self.profiler = get_global_profiler()
|
|
221 |
|
|
222 |
def process_view(self, request, callback, callback_args, callback_kwargs):
|
|
223 |
if self.profiler.is_profiling:
|
|
224 |
return self.profiler.runcall(callback, request, *callback_args, **callback_kwargs)
|
|
225 |
|
|
226 |
application = webapp.WSGIApplication(
|
|
227 |
[('/profiler/start', start_profiler),
|
|
228 |
('/profiler/stop', stop_profiler),
|
|
229 |
('/profiler/show', show_profile),
|
|
230 |
('/profiler/download', download_profile_data),
|
|
231 |
('/profiler/status', show_profiler_status),
|
|
232 |
],
|
|
233 |
debug=True)
|
|
234 |
|
|
235 |
|
|
236 |
def main():
|
|
237 |
google.appengine.ext.webapp.util.run_wsgi_app(application)
|
|
238 |
|
|
239 |
if __name__ == '__main__':
|
|
240 |
main()
|