+
−"""
+
−Iterator based sre token scanner
+
−"""
+
−import re
+
−from re import VERBOSE, MULTILINE, DOTALL
+
−import sre_parse
+
−import sre_compile
+
−import sre_constants
+
−from sre_constants import BRANCH, SUBPATTERN
+
−
+
−__all__ = ['Scanner', 'pattern']
+
−
+
−FLAGS = (VERBOSE | MULTILINE | DOTALL)
+
−
+
−class Scanner(object):
+
−    def __init__(self, lexicon, flags=FLAGS):
+
−        self.actions = [None]
+
−        # Combine phrases into a compound pattern
+
−        s = sre_parse.Pattern()
+
−        s.flags = flags
+
−        p = []
+
−        for idx, token in enumerate(lexicon):
+
−            phrase = token.pattern
+
−            try:
+
−                subpattern = sre_parse.SubPattern(s,
+
−                    [(SUBPATTERN, (idx + 1, sre_parse.parse(phrase, flags)))])
+
−            except sre_constants.error:
+
−                raise
+
−            p.append(subpattern)
+
−            self.actions.append(token)
+
−
+
−        s.groups = len(p) + 1 # NOTE(guido): Added to make SRE validation work
+
−        p = sre_parse.SubPattern(s, [(BRANCH, (None, p))])
+
−        self.scanner = sre_compile.compile(p)
+
−
+
−    def iterscan(self, string, idx=0, context=None):
+
−        """
+
−        Yield match, end_idx for each match
+
−        """
+
−        match = self.scanner.scanner(string, idx).match
+
−        actions = self.actions
+
−        lastend = idx
+
−        end = len(string)
+
−        while True:
+
−            m = match()
+
−            if m is None:
+
−                break
+
−            matchbegin, matchend = m.span()
+
−            if lastend == matchend:
+
−                break
+
−            action = actions[m.lastindex]
+
−            if action is not None:
+
−                rval, next_pos = action(m, context)
+
−                if next_pos is not None and next_pos != matchend:
+
−                    # "fast forward" the scanner
+
−                    matchend = next_pos
+
−                    match = self.scanner.scanner(string, matchend).match
+
−                yield rval, matchend
+
−            lastend = matchend
+
−
+
−
+
−def pattern(pattern, flags=FLAGS):
+
−    def decorator(fn):
+
−        fn.pattern = pattern
+
−        fn.regex = re.compile(pattern, flags)
+
−        return fn
+
−    return decorator