author | Sverre Rabbelier <srabbelier@gmail.com> |
Wed, 04 Mar 2009 23:26:11 +0000 | |
changeset 1673 | 9f67ec81f1ef |
parent 828 | f5fd65cc3bf3 |
permissions | -rwxr-xr-x |
828
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
1 |
"""ANTLR3 runtime package""" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
2 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
3 |
# begin[licence] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
4 |
# |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
5 |
# [The "BSD licence"] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
6 |
# Copyright (c) 2005-2008 Terence Parr |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
7 |
# All rights reserved. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
8 |
# |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
9 |
# Redistribution and use in source and binary forms, with or without |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
10 |
# modification, are permitted provided that the following conditions |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
11 |
# are met: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
12 |
# 1. Redistributions of source code must retain the above copyright |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
13 |
# notice, this list of conditions and the following disclaimer. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
14 |
# 2. Redistributions in binary form must reproduce the above copyright |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
15 |
# notice, this list of conditions and the following disclaimer in the |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
16 |
# documentation and/or other materials provided with the distribution. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
17 |
# 3. The name of the author may not be used to endorse or promote products |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
18 |
# derived from this software without specific prior written permission. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
19 |
# |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
20 |
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
21 |
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
22 |
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
23 |
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
24 |
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
25 |
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
26 |
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
27 |
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
28 |
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
29 |
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
30 |
# |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
31 |
# end[licensc] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
32 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
33 |
from antlr3.constants import EOF |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
34 |
from antlr3.exceptions import NoViableAltException, BacktrackingFailed |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
35 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
36 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
37 |
class DFA(object): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
38 |
"""@brief A DFA implemented as a set of transition tables. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
39 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
40 |
Any state that has a semantic predicate edge is special; those states |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
41 |
are generated with if-then-else structures in a specialStateTransition() |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
42 |
which is generated by cyclicDFA template. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
43 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
44 |
""" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
45 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
46 |
def __init__( |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
47 |
self, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
48 |
recognizer, decisionNumber, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
49 |
eot, eof, min, max, accept, special, transition |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
50 |
): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
51 |
## Which recognizer encloses this DFA? Needed to check backtracking |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
52 |
self.recognizer = recognizer |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
53 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
54 |
self.decisionNumber = decisionNumber |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
55 |
self.eot = eot |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
56 |
self.eof = eof |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
57 |
self.min = min |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
58 |
self.max = max |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
59 |
self.accept = accept |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
60 |
self.special = special |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
61 |
self.transition = transition |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
62 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
63 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
64 |
def predict(self, input): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
65 |
""" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
66 |
From the input stream, predict what alternative will succeed |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
67 |
using this DFA (representing the covering regular approximation |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
68 |
to the underlying CFL). Return an alternative number 1..n. Throw |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
69 |
an exception upon error. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
70 |
""" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
71 |
mark = input.mark() |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
72 |
s = 0 # we always start at s0 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
73 |
try: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
74 |
for _ in xrange(50000): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
75 |
#print "***Current state = %d" % s |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
76 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
77 |
specialState = self.special[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
78 |
if specialState >= 0: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
79 |
#print "is special" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
80 |
s = self.specialStateTransition(specialState, input) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
81 |
if s == -1: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
82 |
self.noViableAlt(s, input) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
83 |
return 0 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
84 |
input.consume() |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
85 |
continue |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
86 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
87 |
if self.accept[s] >= 1: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
88 |
#print "accept state for alt %d" % self.accept[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
89 |
return self.accept[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
90 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
91 |
# look for a normal char transition |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
92 |
c = input.LA(1) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
93 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
94 |
#print "LA = %d (%r)" % (c, unichr(c) if c >= 0 else 'EOF') |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
95 |
#print "range = %d..%d" % (self.min[s], self.max[s]) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
96 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
97 |
if c >= self.min[s] and c <= self.max[s]: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
98 |
# move to next state |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
99 |
snext = self.transition[s][c-self.min[s]] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
100 |
#print "in range, next state = %d" % snext |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
101 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
102 |
if snext < 0: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
103 |
#print "not a normal transition" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
104 |
# was in range but not a normal transition |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
105 |
# must check EOT, which is like the else clause. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
106 |
# eot[s]>=0 indicates that an EOT edge goes to another |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
107 |
# state. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
108 |
if self.eot[s] >= 0: # EOT Transition to accept state? |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
109 |
#print "EOT trans to accept state %d" % self.eot[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
110 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
111 |
s = self.eot[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
112 |
input.consume() |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
113 |
# TODO: I had this as return accept[eot[s]] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
114 |
# which assumed here that the EOT edge always |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
115 |
# went to an accept...faster to do this, but |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
116 |
# what about predicated edges coming from EOT |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
117 |
# target? |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
118 |
continue |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
119 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
120 |
#print "no viable alt" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
121 |
self.noViableAlt(s, input) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
122 |
return 0 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
123 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
124 |
s = snext |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
125 |
input.consume() |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
126 |
continue |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
127 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
128 |
if self.eot[s] >= 0: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
129 |
#print "EOT to %d" % self.eot[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
130 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
131 |
s = self.eot[s] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
132 |
input.consume() |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
133 |
continue |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
134 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
135 |
# EOF Transition to accept state? |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
136 |
if c == EOF and self.eof[s] >= 0: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
137 |
#print "EOF Transition to accept state %d" \ |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
138 |
# % self.accept[self.eof[s]] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
139 |
return self.accept[self.eof[s]] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
140 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
141 |
# not in range and not EOF/EOT, must be invalid symbol |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
142 |
self.noViableAlt(s, input) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
143 |
return 0 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
144 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
145 |
else: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
146 |
raise RuntimeError("DFA bang!") |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
147 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
148 |
finally: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
149 |
input.rewind(mark) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
150 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
151 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
152 |
def noViableAlt(self, s, input): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
153 |
if self.recognizer._state.backtracking > 0: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
154 |
raise BacktrackingFailed |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
155 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
156 |
nvae = NoViableAltException( |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
157 |
self.getDescription(), |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
158 |
self.decisionNumber, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
159 |
s, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
160 |
input |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
161 |
) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
162 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
163 |
self.error(nvae) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
164 |
raise nvae |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
165 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
166 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
167 |
def error(self, nvae): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
168 |
"""A hook for debugging interface""" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
169 |
pass |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
170 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
171 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
172 |
def specialStateTransition(self, s, input): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
173 |
return -1 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
174 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
175 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
176 |
def getDescription(self): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
177 |
return "n/a" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
178 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
179 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
180 |
## def specialTransition(self, state, symbol): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
181 |
## return 0 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
182 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
183 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
184 |
def unpack(cls, string): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
185 |
"""@brief Unpack the runlength encoded table data. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
186 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
187 |
Terence implemented packed table initializers, because Java has a |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
188 |
size restriction on .class files and the lookup tables can grow |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
189 |
pretty large. The generated JavaLexer.java of the Java.g example |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
190 |
would be about 15MB with uncompressed array initializers. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
191 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
192 |
Python does not have any size restrictions, but the compilation of |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
193 |
such large source files seems to be pretty memory hungry. The memory |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
194 |
consumption of the python process grew to >1.5GB when importing a |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
195 |
15MB lexer, eating all my swap space and I was to impacient to see, |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
196 |
if it could finish at all. With packed initializers that are unpacked |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
197 |
at import time of the lexer module, everything works like a charm. |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
198 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
199 |
""" |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
200 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
201 |
ret = [] |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
202 |
for i in range(len(string) / 2): |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
203 |
(n, v) = ord(string[i*2]), ord(string[i*2+1]) |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
204 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
205 |
# Is there a bitwise operation to do this? |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
206 |
if v == 0xFFFF: |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
207 |
v = -1 |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
208 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
209 |
ret += [v] * n |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
210 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
211 |
return ret |
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
212 |
|
f5fd65cc3bf3
Load /Users/solydzajs/Downloads/google_appengine into
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff
changeset
|
213 |
unpack = classmethod(unpack) |