app/simplejson/scanner.py
author Pawel Solyga <Pawel.Solyga@gmail.com>
Sun, 25 Jan 2009 11:31:15 +0000
changeset 975 295d67509412
permissions -rw-r--r--
Add simplejson library to app folder. Patch by: Pawel Solyga Reviewed by: to-be-reviewed
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
975
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     1
"""JSON token scanner
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     2
"""
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     3
import re
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     4
try:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     5
    from simplejson._speedups import make_scanner as c_make_scanner
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     6
except ImportError:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     7
    c_make_scanner = None
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     8
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
     9
__all__ = ['make_scanner']
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    10
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    11
NUMBER_RE = re.compile(
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    12
    r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    13
    (re.VERBOSE | re.MULTILINE | re.DOTALL))
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    14
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    15
def py_make_scanner(context):
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    16
    parse_object = context.parse_object
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    17
    parse_array = context.parse_array
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    18
    parse_string = context.parse_string
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    19
    match_number = NUMBER_RE.match
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    20
    encoding = context.encoding
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    21
    strict = context.strict
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    22
    parse_float = context.parse_float
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    23
    parse_int = context.parse_int
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    24
    parse_constant = context.parse_constant
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    25
    object_hook = context.object_hook
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    26
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    27
    def _scan_once(string, idx):
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    28
        try:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    29
            nextchar = string[idx]
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    30
        except IndexError:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    31
            raise StopIteration
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    32
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    33
        if nextchar == '"':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    34
            return parse_string(string, idx + 1, encoding, strict)
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    35
        elif nextchar == '{':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    36
            return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook)
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    37
        elif nextchar == '[':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    38
            return parse_array((string, idx + 1), _scan_once)
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    39
        elif nextchar == 'n' and string[idx:idx + 4] == 'null':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    40
            return None, idx + 4
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    41
        elif nextchar == 't' and string[idx:idx + 4] == 'true':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    42
            return True, idx + 4
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    43
        elif nextchar == 'f' and string[idx:idx + 5] == 'false':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    44
            return False, idx + 5
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    45
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    46
        m = match_number(string, idx)
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    47
        if m is not None:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    48
            integer, frac, exp = m.groups()
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    49
            if frac or exp:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    50
                res = parse_float(integer + (frac or '') + (exp or ''))
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    51
            else:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    52
                res = parse_int(integer)
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    53
            return res, m.end()
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    54
        elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    55
            return parse_constant('NaN'), idx + 3
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    56
        elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    57
            return parse_constant('Infinity'), idx + 8
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    58
        elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    59
            return parse_constant('-Infinity'), idx + 9
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    60
        else:
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    61
            raise StopIteration
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    62
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    63
    return _scan_once
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    64
295d67509412 Add simplejson library to app folder.
Pawel Solyga <Pawel.Solyga@gmail.com>
parents:
diff changeset
    65
make_scanner = c_make_scanner or py_make_scanner