"""Open an arbitrary URL.

See the following document for more info on URLs:

"Names and Addresses, URIs, URLs, URNs, URCs", at

http://www.w3.org/pub/WWW/Addressing/Overview.html

See also the HTTP spec (from which the error codes are derived):

"HTTP - Hypertext Transfer Protocol", at

http://www.w3.org/pub/WWW/Protocols/

Related standards and specs:

- RFC1808: the "relative URL" spec. (authoritative status)

- RFC1738 - the "URL standard". (authoritative status)

- RFC1630 - the "URI spec". (informational status)

All code but that related to URL parsing has been removed (since it is not

compatible with Google App Engine)from this fork of the original file,

obtained from:

http://svn.python.org/view/*checkout*/python/tags/r252/Lib/urllib.py?content-type=text%2Fplain&rev=60915

"""

import string

import sys

from urlparse import urljoin as basejoin

__all__ = ["quote", "quote_plus", "unquote", "unquote_plus",

           "urlencode", "splittag",

           "basejoin", "unwrap",

           "splittype", "splithost", "splituser", "splitpasswd", "splitport",

           "splitnport", "splitquery", "splitattr", "splitvalue",

           "splitgophertype",]

__version__ = '1.17'    # XXX This version is not always updated :-(

# Utilities to parse URLs (most of these return None for missing parts):

# unwrap('<URL:type://host/path>') --> 'type://host/path'

# splittype('type:opaquestring') --> 'type', 'opaquestring'

# splithost('//host[:port]/path') --> 'host[:port]', '/path'

# splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'

# splitpasswd('user:passwd') -> 'user', 'passwd'

# splitport('host:port') --> 'host', 'port'

# splitquery('/path?query') --> '/path', 'query'

# splittag('/path#tag') --> '/path', 'tag'

# splitattr('/path;attr1=value1;attr2=value2;...') ->

#   '/path', ['attr1=value1', 'attr2=value2', ...]

# splitvalue('attr=value') --> 'attr', 'value'

# splitgophertype('/Xselector') --> 'X', 'selector'

# unquote('abc%20def') -> 'abc def'

# quote('abc def') -> 'abc%20def')

try:

    unicode

except NameError:

    def _is_unicode(x):

        return 0

else:

    def _is_unicode(x):

        return isinstance(x, unicode)

def toBytes(url):

    """toBytes(u"URL") --> 'URL'."""

    # Most URL schemes require ASCII. If that changes, the conversion

    # can be relaxed

    if _is_unicode(url):

        try:

            url = url.encode("ASCII")

        except UnicodeError:

            raise UnicodeError("URL " + repr(url) +

                               " contains non-ASCII characters")

    return url

def unwrap(url):

    """unwrap('<URL:type://host/path>') --> 'type://host/path'."""

    url = url.strip()

    if url[:1] == '<' and url[-1:] == '>':

        url = url[1:-1].strip()

    if url[:4] == 'URL:': url = url[4:].strip()

    return url

_typeprog = None

def splittype(url):

    """splittype('type:opaquestring') --> 'type', 'opaquestring'."""

    global _typeprog

    if _typeprog is None:

        import re

        _typeprog = re.compile('^([^/:]+):')

    match = _typeprog.match(url)

    if match:

        scheme = match.group(1)

        return scheme.lower(), url[len(scheme) + 1:]

    return None, url

_hostprog = None

def splithost(url):

    """splithost('//host[:port]/path') --> 'host[:port]', '/path'."""

    global _hostprog

    if _hostprog is None:

        import re

        _hostprog = re.compile('^//([^/?]*)(.*)$')

    match = _hostprog.match(url)

    if match: return match.group(1, 2)

    return None, url

_userprog = None

def splituser(host):

    """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'."""

    global _userprog

    if _userprog is None:

        import re

        _userprog = re.compile('^(.*)@(.*)$')

    match = _userprog.match(host)

    if match: return map(unquote, match.group(1, 2))

    return None, host

_passwdprog = None

def splitpasswd(user):

    """splitpasswd('user:passwd') -> 'user', 'passwd'."""

    global _passwdprog

    if _passwdprog is None:

        import re

        _passwdprog = re.compile('^([^:]*):(.*)$')

    match = _passwdprog.match(user)

    if match: return match.group(1, 2)

    return user, None

# splittag('/path#tag') --> '/path', 'tag'

_portprog = None

def splitport(host):

    """splitport('host:port') --> 'host', 'port'."""

    global _portprog

    if _portprog is None:

        import re

        _portprog = re.compile('^(.*):([0-9]+)$')

    match = _portprog.match(host)

    if match: return match.group(1, 2)

    return host, None

_nportprog = None

def splitnport(host, defport=-1):

    """Split host and port, returning numeric port.

    Return given default port if no ':' found; defaults to -1.

    Return numerical port if a valid number are found after ':'.

    Return None if ':' but not a valid number."""

    global _nportprog

    if _nportprog is None:

        import re

        _nportprog = re.compile('^(.*):(.*)$')

    match = _nportprog.match(host)

    if match:

        host, port = match.group(1, 2)

        try:

            if not port: raise ValueError, "no digits"

            nport = int(port)

        except ValueError:

            nport = None

        return host, nport

    return host, defport

_queryprog = None

def splitquery(url):

    """splitquery('/path?query') --> '/path', 'query'."""

    global _queryprog

    if _queryprog is None:

        import re

        _queryprog = re.compile('^(.*)\?([^?]*)$')

    match = _queryprog.match(url)

    if match: return match.group(1, 2)

    return url, None

_tagprog = None

def splittag(url):

    """splittag('/path#tag') --> '/path', 'tag'."""

    global _tagprog

    if _tagprog is None:

        import re

        _tagprog = re.compile('^(.*)#([^#]*)$')

    match = _tagprog.match(url)

    if match: return match.group(1, 2)

    return url, None

def splitattr(url):

    """splitattr('/path;attr1=value1;attr2=value2;...') ->

        '/path', ['attr1=value1', 'attr2=value2', ...]."""

    words = url.split(';')

    return words[0], words[1:]

_valueprog = None

def splitvalue(attr):

    """splitvalue('attr=value') --> 'attr', 'value'."""

    global _valueprog

    if _valueprog is None:

        import re

        _valueprog = re.compile('^([^=]*)=(.*)$')

    match = _valueprog.match(attr)

    if match: return match.group(1, 2)

    return attr, None

def splitgophertype(selector):

    """splitgophertype('/Xselector') --> 'X', 'selector'."""

    if selector[:1] == '/' and selector[1:2]:

        return selector[1], selector[2:]

    return None, selector

_hextochr = dict(('%02x' % i, chr(i)) for i in range(256))

_hextochr.update(('%02X' % i, chr(i)) for i in range(256))

def unquote(s):

    """unquote('abc%20def') -> 'abc def'."""

    res = s.split('%')

    for i in xrange(1, len(res)):

        item = res[i]

        try:

            res[i] = _hextochr[item[:2]] + item[2:]

        except KeyError:

            res[i] = '%' + item

        except UnicodeDecodeError:

            res[i] = unichr(int(item[:2], 16)) + item[2:]

    return "".join(res)

def unquote_plus(s):

    """unquote('%7e/abc+def') -> '~/abc def'"""

    s = s.replace('+', ' ')

    return unquote(s)

always_safe = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'

               'abcdefghijklmnopqrstuvwxyz'

               '0123456789' '_.-')

_safemaps = {}

def quote(s, safe = '/'):

    """quote('abc def') -> 'abc%20def'

    Each part of a URL, e.g. the path info, the query, etc., has a

    different set of reserved characters that must be quoted.

    RFC 2396 Uniform Resource Identifiers (URI): Generic Syntax lists

    the following reserved characters.

    reserved    = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" |

                  "$" | ","

    Each of these characters is reserved in some component of a URL,

    but not necessarily in all of them.

    By default, the quote function is intended for quoting the path

    section of a URL.  Thus, it will not encode '/'.  This character

    is reserved, but in typical usage the quote function is being

    called on a path where the existing slash characters are used as

    reserved characters.

    """

    cachekey = (safe, always_safe)

    try:

        safe_map = _safemaps[cachekey]

    except KeyError:

        safe += always_safe

        safe_map = {}

        for i in range(256):

            c = chr(i)

            safe_map[c] = (c in safe) and c or ('%%%02X' % i)

        _safemaps[cachekey] = safe_map

    res = map(safe_map.__getitem__, s)

    return ''.join(res)

def quote_plus(s, safe = ''):

    """Quote the query fragment of a URL; replacing ' ' with '+'"""

    if ' ' in s:

        s = quote(s, safe + ' ')

        return s.replace(' ', '+')

    return quote(s, safe)

def urlencode(query,doseq=0):

    """Encode a sequence of two-element tuples or dictionary into a URL query string.

    If any values in the query arg are sequences and doseq is true, each

    sequence element is converted to a separate parameter.

    If the query arg is a sequence of two-element tuples, the order of the

    parameters in the output will match the order of parameters in the

    input.

    """

    if hasattr(query,"items"):

        # mapping objects

        query = query.items()

    else:

        # it's a bother at times that strings and string-like objects are

        # sequences...

        try:

            # non-sequence items should not work with len()

            # non-empty strings will fail this

            if len(query) and not isinstance(query[0], tuple):

                raise TypeError

            # zero-length sequences of all types will get here and succeed,

            # but that's a minor nit - since the original implementation

            # allowed empty dicts that type of behavior probably should be

            # preserved for consistency

        except TypeError:

            ty,va,tb = sys.exc_info()

            raise TypeError, "not a valid non-string sequence or mapping object", tb

    l = []

    if not doseq:

        # preserve old behavior

        for k, v in query:

            k = quote_plus(str(k))

            v = quote_plus(str(v))

            l.append(k + '=' + v)

    else:

        for k, v in query:

            k = quote_plus(str(k))

            if isinstance(v, str):

                v = quote_plus(v)

                l.append(k + '=' + v)

            elif _is_unicode(v):

                # is there a reasonable way to convert to ASCII?

                # encode generates a string, but "replace" or "ignore"

                # lose information and "strict" can raise UnicodeError

                v = quote_plus(v.encode("ASCII","replace"))

                l.append(k + '=' + v)

            else:

                try:

                    # is this a sufficient test for sequence-ness?

                    x = len(v)

                except TypeError:

                    # not a sequence

                    v = quote_plus(str(v))

                    l.append(k + '=' + v)

                else:

                    # loop over the sequence

                    for elt in v:

                        l.append(k + '=' + quote_plus(str(elt)))

    return '&'.join(l)