1 """Translation helper functions."""

     2 

     3 import locale

     4 import os

     5 import re

     6 import sys

     7 import gettext as gettext_module

     8 from cStringIO import StringIO

     9 

    10 from django.utils.safestring import mark_safe, SafeData

    11 

    12 try:

    13     import threading

    14     hasThreads = True

    15 except ImportError:

    16     hasThreads = False

    17 

    18 if hasThreads:

    19     currentThread = threading.currentThread

    20 else:

    21     def currentThread():

    22         return 'no threading'

    23 

    24 # Translations are cached in a dictionary for every language+app tuple.

    25 # The active translations are stored by threadid to make them thread local.

    26 _translations = {}

    27 _active = {}

    28 

    29 # The default translation is based on the settings file.

    30 _default = None

    31 

    32 # This is a cache for normalized accept-header languages to prevent multiple

    33 # file lookups when checking the same locale on repeated requests.

    34 _accepted = {}

    35 

    36 # Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9.

    37 accept_language_re = re.compile(r'''

    38         ([A-Za-z]{1,8}(?:-[A-Za-z]{1,8})*|\*)   # "en", "en-au", "x-y-z", "*"

    39         (?:;q=(0(?:\.\d{,3})?|1(?:.0{,3})?))?   # Optional "q=1.00", "q=0.8"

    40         (?:\s*,\s*|$)                            # Multiple accepts per header.

    41         ''', re.VERBOSE)

    42 

    43 def to_locale(language, to_lower=False):

    44     """

    45     Turns a language name (en-us) into a locale name (en_US). If 'to_lower' is

    46     True, the last component is lower-cased (en_us).

    47     """

    48     p = language.find('-')

    49     if p >= 0:

    50         if to_lower:

    51             return language[:p].lower()+'_'+language[p+1:].lower()

    52         else:

    53             return language[:p].lower()+'_'+language[p+1:].upper()

    54     else:

    55         return language.lower()

    56 

    57 def to_language(locale):

    58     """Turns a locale name (en_US) into a language name (en-us)."""

    59     p = locale.find('_')

    60     if p >= 0:

    61         return locale[:p].lower()+'-'+locale[p+1:].lower()

    62     else:

    63         return locale.lower()

    64 

    65 class DjangoTranslation(gettext_module.GNUTranslations):

    66     """

    67     This class sets up the GNUTranslations context with regard to output

    68     charset. Django uses a defined DEFAULT_CHARSET as the output charset on

    69     Python 2.4. With Python 2.3, use DjangoTranslation23.

    70     """

    71     def __init__(self, *args, **kw):

    72         from django.conf import settings

    73         gettext_module.GNUTranslations.__init__(self, *args, **kw)

    74         # Starting with Python 2.4, there's a function to define

    75         # the output charset. Before 2.4, the output charset is

    76         # identical with the translation file charset.

    77         try:

    78             self.set_output_charset('utf-8')

    79         except AttributeError:

    80             pass

    81         self.django_output_charset = 'utf-8'

    82         self.__language = '??'

    83 

    84     def merge(self, other):

    85         self._catalog.update(other._catalog)

    86 

    87     def set_language(self, language):

    88         self.__language = language

    89 

    90     def language(self):

    91         return self.__language

    92 

    93     def __repr__(self):

    94         return "<DjangoTranslation lang:%s>" % self.__language

    95 

    96 class DjangoTranslation23(DjangoTranslation):

    97     """

    98     Compatibility class that is only used with Python 2.3.

    99     Python 2.3 doesn't support set_output_charset on translation objects and

   100     needs this wrapper class to make sure input charsets from translation files

   101     are correctly translated to output charsets.

   102 

   103     With a full switch to Python 2.4, this can be removed from the source.

   104     """

   105     def gettext(self, msgid):

   106         res = self.ugettext(msgid)

   107         return res.encode(self.django_output_charset)

   108 

   109     def ngettext(self, msgid1, msgid2, n):

   110         res = self.ungettext(msgid1, msgid2, n)

   111         return res.encode(self.django_output_charset)

   112 

   113 def translation(language):

   114     """

   115     Returns a translation object.

   116 

   117     This translation object will be constructed out of multiple GNUTranslations

   118     objects by merging their catalogs. It will construct a object for the

   119     requested language and add a fallback to the default language, if it's

   120     different from the requested language.

   121     """

   122     global _translations

   123 

   124     t = _translations.get(language, None)

   125     if t is not None:

   126         return t

   127 

   128     from django.conf import settings

   129 

   130     # set up the right translation class

   131     klass = DjangoTranslation

   132     if sys.version_info < (2, 4):

   133         klass = DjangoTranslation23

   134 

   135     globalpath = os.path.join(os.path.dirname(sys.modules[settings.__module__].__file__), 'locale')

   136 

   137     if settings.SETTINGS_MODULE is not None:

   138         parts = settings.SETTINGS_MODULE.split('.')

   139         project = __import__(parts[0], {}, {}, [])

   140         projectpath = os.path.join(os.path.dirname(project.__file__), 'locale')

   141     else:

   142         projectpath = None

   143 

   144     def _fetch(lang, fallback=None):

   145 

   146         global _translations

   147 

   148         loc = to_locale(lang)

   149 

   150         res = _translations.get(lang, None)

   151         if res is not None:

   152             return res

   153 

   154         def _translation(path):

   155             try:

   156                 t = gettext_module.translation('django', path, [loc], klass)

   157                 t.set_language(lang)

   158                 return t

   159             except IOError, e:

   160                 return None

   161 

   162         res = _translation(globalpath)

   163 

   164         def _merge(path):

   165             t = _translation(path)

   166             if t is not None:

   167                 if res is None:

   168                     return t

   169                 else:

   170                     res.merge(t)

   171             return res

   172 

   173         for localepath in settings.LOCALE_PATHS:

   174             if os.path.isdir(localepath):

   175                 res = _merge(localepath)

   176 

   177         if projectpath and os.path.isdir(projectpath):

   178             res = _merge(projectpath)

   179 

   180         for appname in settings.INSTALLED_APPS:

   181             p = appname.rfind('.')

   182             if p >= 0:

   183                 app = getattr(__import__(appname[:p], {}, {}, [appname[p+1:]]), appname[p+1:])

   184             else:

   185                 app = __import__(appname, {}, {}, [])

   186 

   187             apppath = os.path.join(os.path.dirname(app.__file__), 'locale')

   188 

   189             if os.path.isdir(apppath):

   190                 res = _merge(apppath)

   191 

   192         if res is None:

   193             if fallback is not None:

   194                 res = fallback

   195             else:

   196                 return gettext_module.NullTranslations()

   197         _translations[lang] = res

   198         return res

   199 

   200     default_translation = _fetch(settings.LANGUAGE_CODE)

   201     current_translation = _fetch(language, fallback=default_translation)

   202 

   203     return current_translation

   204 

   205 def activate(language):

   206     """

   207     Fetches the translation object for a given tuple of application name and

   208     language and installs it as the current translation object for the current

   209     thread.

   210     """

   211     _active[currentThread()] = translation(language)

   212 

   213 def deactivate():

   214     """

   215     Deinstalls the currently active translation object so that further _ calls

   216     will resolve against the default translation object, again.

   217     """

   218     global _active

   219     if currentThread() in _active:

   220         del _active[currentThread()]

   221 

   222 def deactivate_all():

   223     """

   224     Makes the active translation object a NullTranslations() instance. This is

   225     useful when we want delayed translations to appear as the original string

   226     for some reason.

   227     """

   228     _active[currentThread()] = gettext_module.NullTranslations()

   229 

   230 def get_language():

   231     """Returns the currently selected language."""

   232     t = _active.get(currentThread(), None)

   233     if t is not None:

   234         try:

   235             return to_language(t.language())

   236         except AttributeError:

   237             pass

   238     # If we don't have a real translation object, assume it's the default language.

   239     from django.conf import settings

   240     return settings.LANGUAGE_CODE

   241 

   242 def get_language_bidi():

   243     """

   244     Returns selected language's BiDi layout.

   245     False = left-to-right layout

   246     True = right-to-left layout

   247     """

   248     from django.conf import settings

   249     return get_language() in settings.LANGUAGES_BIDI

   250 

   251 def catalog():

   252     """

   253     Returns the current active catalog for further processing.

   254     This can be used if you need to modify the catalog or want to access the

   255     whole message catalog instead of just translating one string.

   256     """

   257     global _default, _active

   258     t = _active.get(currentThread(), None)

   259     if t is not None:

   260         return t

   261     if _default is None:

   262         from django.conf import settings

   263         _default = translation(settings.LANGUAGE_CODE)

   264     return _default

   265 

   266 def do_translate(message, translation_function):

   267     """

   268     Translates 'message' using the given 'translation_function' name -- which

   269     will be either gettext or ugettext. It uses the current thread to find the

   270     translation object to use. If no current translation is activated, the

   271     message will be run through the default translation object.

   272     """

   273     global _default, _active

   274     t = _active.get(currentThread(), None)

   275     if t is not None:

   276         result = getattr(t, translation_function)(message)

   277     else:

   278         if _default is None:

   279             from django.conf import settings

   280             _default = translation(settings.LANGUAGE_CODE)

   281         result = getattr(_default, translation_function)(message)

   282     if isinstance(message, SafeData):

   283         return mark_safe(result)

   284     return result

   285 

   286 def gettext(message):

   287     return do_translate(message, 'gettext')

   288 

   289 def ugettext(message):

   290     return do_translate(message, 'ugettext')

   291 

   292 def gettext_noop(message):

   293     """

   294     Marks strings for translation but doesn't translate them now. This can be

   295     used to store strings in global variables that should stay in the base

   296     language (because they might be used externally) and will be translated

   297     later.

   298     """

   299     return message

   300 

   301 def do_ntranslate(singular, plural, number, translation_function):

   302     global _default, _active

   303 

   304     t = _active.get(currentThread(), None)

   305     if t is not None:

   306         return getattr(t, translation_function)(singular, plural, number)

   307     if _default is None:

   308         from django.conf import settings

   309         _default = translation(settings.LANGUAGE_CODE)

   310     return getattr(_default, translation_function)(singular, plural, number)

   311 

   312 def ngettext(singular, plural, number):

   313     """

   314     Returns a UTF-8 bytestring of the translation of either the singular or

   315     plural, based on the number.

   316     """

   317     return do_ntranslate(singular, plural, number, 'ngettext')

   318 

   319 def ungettext(singular, plural, number):

   320     """

   321     Returns a unicode strings of the translation of either the singular or

   322     plural, based on the number.

   323     """

   324     return do_ntranslate(singular, plural, number, 'ungettext')

   325 

   326 def check_for_language(lang_code):

   327     """

   328     Checks whether there is a global language file for the given language

   329     code. This is used to decide whether a user-provided language is

   330     available. This is only used for language codes from either the cookies or

   331     session.

   332     """

   333     from django.conf import settings

   334     globalpath = os.path.join(os.path.dirname(sys.modules[settings.__module__].__file__), 'locale')

   335     if gettext_module.find('django', globalpath, [to_locale(lang_code)]) is not None:

   336         return True

   337     else:

   338         return False

   339 

   340 def get_language_from_request(request):

   341     """

   342     Analyzes the request to find what language the user wants the system to

   343     show. Only languages listed in settings.LANGUAGES are taken into account.

   344     If the user requests a sublanguage where we have a main language, we send

   345     out the main language.

   346     """

   347     global _accepted

   348     from django.conf import settings

   349     globalpath = os.path.join(os.path.dirname(sys.modules[settings.__module__].__file__), 'locale')

   350     supported = dict(settings.LANGUAGES)

   351 

   352     if hasattr(request, 'session'):

   353         lang_code = request.session.get('django_language', None)

   354         if lang_code in supported and lang_code is not None and check_for_language(lang_code):

   355             return lang_code

   356 

   357     lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)

   358     if lang_code and lang_code in supported and check_for_language(lang_code):

   359         return lang_code

   360 

   361     accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '')

   362     for accept_lang, unused in parse_accept_lang_header(accept):

   363         if accept_lang == '*':

   364             break

   365 

   366         # We have a very restricted form for our language files (no encoding

   367         # specifier, since they all must be UTF-8 and only one possible

   368         # language each time. So we avoid the overhead of gettext.find() and

   369         # work out the MO file manually.

   370 

   371         # 'normalized' is the root name of the locale in POSIX format (which is

   372         # the format used for the directories holding the MO files).

   373         normalized = locale.locale_alias.get(to_locale(accept_lang, True))

   374         if not normalized:

   375             continue

   376         # Remove the default encoding from locale_alias.

   377         normalized = normalized.split('.')[0]

   378 

   379         if normalized in _accepted:

   380             # We've seen this locale before and have an MO file for it, so no

   381             # need to check again.

   382             return _accepted[normalized]

   383 

   384         for lang, dirname in ((accept_lang, normalized),

   385                 (accept_lang.split('-')[0], normalized.split('_')[0])):

   386             if lang not in supported:

   387                 continue

   388             langfile = os.path.join(globalpath, dirname, 'LC_MESSAGES',

   389                     'django.mo')

   390             if os.path.exists(langfile):

   391                 _accepted[normalized] = lang

   392             return lang

   393 

   394     return settings.LANGUAGE_CODE

   395 

   396 def get_date_formats():

   397     """

   398     Checks whether translation files provide a translation for some technical

   399     message ID to store date and time formats. If it doesn't contain one, the

   400     formats provided in the settings will be used.

   401     """

   402     from django.conf import settings

   403     date_format = ugettext('DATE_FORMAT')

   404     datetime_format = ugettext('DATETIME_FORMAT')

   405     time_format = ugettext('TIME_FORMAT')

   406     if date_format == 'DATE_FORMAT':

   407         date_format = settings.DATE_FORMAT

   408     if datetime_format == 'DATETIME_FORMAT':

   409         datetime_format = settings.DATETIME_FORMAT

   410     if time_format == 'TIME_FORMAT':

   411         time_format = settings.TIME_FORMAT

   412     return date_format, datetime_format, time_format

   413 

   414 def get_partial_date_formats():

   415     """

   416     Checks whether translation files provide a translation for some technical

   417     message ID to store partial date formats. If it doesn't contain one, the

   418     formats provided in the settings will be used.

   419     """

   420     from django.conf import settings

   421     year_month_format = ugettext('YEAR_MONTH_FORMAT')

   422     month_day_format = ugettext('MONTH_DAY_FORMAT')

   423     if year_month_format == 'YEAR_MONTH_FORMAT':

   424         year_month_format = settings.YEAR_MONTH_FORMAT

   425     if month_day_format == 'MONTH_DAY_FORMAT':

   426         month_day_format = settings.MONTH_DAY_FORMAT

   427     return year_month_format, month_day_format

   428 

   429 dot_re = re.compile(r'\S')

   430 def blankout(src, char):

   431     """

   432     Changes every non-whitespace character to the given char.

   433     Used in the templatize function.

   434     """

   435     return dot_re.sub(char, src)

   436 

   437 inline_re = re.compile(r"""^\s*trans\s+((?:".*?")|(?:'.*?'))\s*""")

   438 block_re = re.compile(r"""^\s*blocktrans(?:\s+|$)""")

   439 endblock_re = re.compile(r"""^\s*endblocktrans$""")

   440 plural_re = re.compile(r"""^\s*plural$""")

   441 constant_re = re.compile(r"""_\(((?:".*?")|(?:'.*?'))\)""")

   442 

   443 def templatize(src):

   444     """

   445     Turns a Django template into something that is understood by xgettext. It

   446     does so by translating the Django translation tags into standard gettext

   447     function invocations.

   448     """

   449     from django.template import Lexer, TOKEN_TEXT, TOKEN_VAR, TOKEN_BLOCK

   450     out = StringIO()

   451     intrans = False

   452     inplural = False

   453     singular = []

   454     plural = []

   455     for t in Lexer(src, None).tokenize():

   456         if intrans:

   457             if t.token_type == TOKEN_BLOCK:

   458                 endbmatch = endblock_re.match(t.contents)

   459                 pluralmatch = plural_re.match(t.contents)

   460                 if endbmatch:

   461                     if inplural:

   462                         out.write(' ngettext(%r,%r,count) ' % (''.join(singular), ''.join(plural)))

   463                         for part in singular:

   464                             out.write(blankout(part, 'S'))

   465                         for part in plural:

   466                             out.write(blankout(part, 'P'))

   467                     else:

   468                         out.write(' gettext(%r) ' % ''.join(singular))

   469                         for part in singular:

   470                             out.write(blankout(part, 'S'))

   471                     intrans = False

   472                     inplural = False

   473                     singular = []

   474                     plural = []

   475                 elif pluralmatch:

   476                     inplural = True

   477                 else:

   478                     raise SyntaxError("Translation blocks must not include other block tags: %s" % t.contents)

   479             elif t.token_type == TOKEN_VAR:

   480                 if inplural:

   481                     plural.append('%%(%s)s' % t.contents)

   482                 else:

   483                     singular.append('%%(%s)s' % t.contents)

   484             elif t.token_type == TOKEN_TEXT:

   485                 if inplural:

   486                     plural.append(t.contents)

   487                 else:

   488                     singular.append(t.contents)

   489         else:

   490             if t.token_type == TOKEN_BLOCK:

   491                 imatch = inline_re.match(t.contents)

   492                 bmatch = block_re.match(t.contents)

   493                 cmatches = constant_re.findall(t.contents)

   494                 if imatch:

   495                     g = imatch.group(1)

   496                     if g[0] == '"': g = g.strip('"')

   497                     elif g[0] == "'": g = g.strip("'")

   498                     out.write(' gettext(%r) ' % g)

   499                 elif bmatch:

   500                     for fmatch in constant_re.findall(t.contents):

   501                         out.write(' _(%s) ' % fmatch)

   502                     intrans = True

   503                     inplural = False

   504                     singular = []

   505                     plural = []

   506                 elif cmatches:

   507                     for cmatch in cmatches:

   508                         out.write(' _(%s) ' % cmatch)

   509                 else:

   510                     out.write(blankout(t.contents, 'B'))

   511             elif t.token_type == TOKEN_VAR:

   512                 parts = t.contents.split('|')

   513                 cmatch = constant_re.match(parts[0])

   514                 if cmatch:

   515                     out.write(' _(%s) ' % cmatch.group(1))

   516                 for p in parts[1:]:

   517                     if p.find(':_(') >= 0:

   518                         out.write(' %s ' % p.split(':',1)[1])

   519                     else:

   520                         out.write(blankout(p, 'F'))

   521             else:

   522                 out.write(blankout(t.contents, 'X'))

   523     return out.getvalue()

   524 

   525 def parse_accept_lang_header(lang_string):

   526     """

   527     Parses the lang_string, which is the body of an HTTP Accept-Language

   528     header, and returns a list of (lang, q-value), ordered by 'q' values.

   529 

   530     Any format errors in lang_string results in an empty list being returned.

   531     """

   532     result = []

   533     pieces = accept_language_re.split(lang_string)

   534     if pieces[-1]:

   535         return []

   536     for i in range(0, len(pieces) - 1, 3):

   537         first, lang, priority = pieces[i : i + 3]

   538         if first:

   539             return []

   540         priority = priority and float(priority) or 1.0

   541         result.append((lang, priority))

   542     result.sort(lambda x, y: -cmp(x[1], y[1]))

   543     return result