author | Sverre Rabbelier <srabbelier@gmail.com> |
Thu, 05 Feb 2009 23:46:39 +0000 | |
changeset 1228 | b24796279fb9 |
parent 323 | ff1a9aa48cfd |
permissions | -rw-r--r-- |
54
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
1 |
""" |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
2 |
Iterator based sre token scanner |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
3 |
""" |
323
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
4 |
import re |
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
5 |
from re import VERBOSE, MULTILINE, DOTALL |
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
6 |
import sre_parse |
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
7 |
import sre_compile |
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
8 |
import sre_constants |
54
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
9 |
from sre_constants import BRANCH, SUBPATTERN |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
10 |
|
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
11 |
__all__ = ['Scanner', 'pattern'] |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
12 |
|
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
13 |
FLAGS = (VERBOSE | MULTILINE | DOTALL) |
323
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
14 |
|
54
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
15 |
class Scanner(object): |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
16 |
def __init__(self, lexicon, flags=FLAGS): |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
17 |
self.actions = [None] |
323
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
18 |
# Combine phrases into a compound pattern |
54
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
19 |
s = sre_parse.Pattern() |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
20 |
s.flags = flags |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
21 |
p = [] |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
22 |
for idx, token in enumerate(lexicon): |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
23 |
phrase = token.pattern |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
24 |
try: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
25 |
subpattern = sre_parse.SubPattern(s, |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
26 |
[(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))]) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
27 |
except sre_constants.error: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
28 |
raise |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
29 |
p.append(subpattern) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
30 |
self.actions.append(token) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
31 |
|
323
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
32 |
s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work |
54
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
33 |
p = sre_parse.SubPattern(s, [(BRANCH, (None, p))]) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
34 |
self.scanner = sre_compile.compile(p) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
35 |
|
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
36 |
def iterscan(self, string, idx=0, context=None): |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
37 |
""" |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
38 |
Yield match, end_idx for each match |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
39 |
""" |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
40 |
match = self.scanner.scanner(string, idx).match |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
41 |
actions = self.actions |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
42 |
lastend = idx |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
43 |
end = len(string) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
44 |
while True: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
45 |
m = match() |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
46 |
if m is None: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
47 |
break |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
48 |
matchbegin, matchend = m.span() |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
49 |
if lastend == matchend: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
50 |
break |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
51 |
action = actions[m.lastindex] |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
52 |
if action is not None: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
53 |
rval, next_pos = action(m, context) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
54 |
if next_pos is not None and next_pos != matchend: |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
55 |
# "fast forward" the scanner |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
56 |
matchend = next_pos |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
57 |
match = self.scanner.scanner(string, matchend).match |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
58 |
yield rval, matchend |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
59 |
lastend = matchend |
323
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
60 |
|
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
61 |
|
54
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
62 |
def pattern(pattern, flags=FLAGS): |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
63 |
def decorator(fn): |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
64 |
fn.pattern = pattern |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
65 |
fn.regex = re.compile(pattern, flags) |
03e267d67478
Major reorganization of the soc svn repo, to merge into a single App Engine
Todd Larsen <tlarsen@google.com>
parents:
diff
changeset
|
66 |
return fn |
323
ff1a9aa48cfd
Load ../vendor/django into trunk/app/django.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
54
diff
changeset
|
67 |
return decorator |