1 "File-based cache backend"

     2 

     3 import md5

     4 import os, time

     5 try:

     6     import cPickle as pickle

     7 except ImportError:

     8     import pickle

     9 from django.core.cache.backends.base import BaseCache

    10 

    11 class CacheClass(BaseCache):

    12     def __init__(self, dir, params):

    13         BaseCache.__init__(self, params)

    14         

    15         max_entries = params.get('max_entries', 300)

    16         try:

    17             self._max_entries = int(max_entries)

    18         except (ValueError, TypeError):

    19             self._max_entries = 300

    20             

    21         cull_frequency = params.get('cull_frequency', 3)

    22         try:

    23             self._cull_frequency = int(cull_frequency)

    24         except (ValueError, TypeError):

    25             self._cull_frequency = 3

    26             

    27         self._dir = dir

    28         if not os.path.exists(self._dir):

    29             self._createdir()

    30 

    31     def add(self, key, value, timeout=None):

    32         if self.has_key(key):

    33             return None

    34         

    35         self.set(key, value, timeout)

    36 

    37     def get(self, key, default=None):

    38         fname = self._key_to_file(key)

    39         try:

    40             f = open(fname, 'rb')

    41             exp = pickle.load(f)

    42             now = time.time()

    43             if exp < now:

    44                 f.close()

    45                 self._delete(fname)

    46             else:

    47                 return pickle.load(f)

    48         except (IOError, OSError, EOFError, pickle.PickleError):

    49             pass

    50         return default

    51 

    52     def set(self, key, value, timeout=None):

    53         fname = self._key_to_file(key)

    54         dirname = os.path.dirname(fname)

    55         

    56         if timeout is None:

    57             timeout = self.default_timeout

    58             

    59         self._cull()

    60         

    61         try:

    62             if not os.path.exists(dirname):

    63                 os.makedirs(dirname)

    64 

    65             f = open(fname, 'wb')

    66             now = time.time()

    67             pickle.dump(now + timeout, f, pickle.HIGHEST_PROTOCOL)

    68             pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)

    69         except (IOError, OSError):

    70             pass

    71 

    72     def delete(self, key):

    73         try:

    74             self._delete(self._key_to_file(key))

    75         except (IOError, OSError):

    76             pass

    77 

    78     def _delete(self, fname):

    79         os.remove(fname)

    80         try:

    81             # Remove the 2 subdirs if they're empty

    82             dirname = os.path.dirname(fname)

    83             os.rmdir(dirname)

    84             os.rmdir(os.path.dirname(dirname))

    85         except (IOError, OSError):

    86             pass

    87 

    88     def has_key(self, key):

    89         fname = self._key_to_file(key)

    90         try:

    91             f = open(fname, 'rb')

    92             exp = pickle.load(f)

    93             now = time.time()

    94             if exp < now:

    95                 f.close()

    96                 self._delete(fname)

    97                 return False

    98             else:

    99                 return True

   100         except (IOError, OSError, EOFError, pickle.PickleError):

   101             return False

   102 

   103     def _cull(self):

   104         if int(self._num_entries) < self._max_entries:

   105             return

   106         

   107         try:

   108             filelist = os.listdir(self._dir)

   109         except (IOError, OSError):

   110             return

   111         

   112         if self._cull_frequency == 0:

   113             doomed = filelist

   114         else:

   115             doomed = [os.path.join(self._dir, k) for (i, k) in enumerate(filelist) if i % self._cull_frequency == 0]

   116 

   117         for topdir in doomed:

   118             try:

   119                 for root, _, files in os.walk(topdir):

   120                     for f in files:

   121                         self._delete(os.path.join(root, f))

   122             except (IOError, OSError):

   123                 pass

   124 

   125     def _createdir(self):

   126         try:

   127             os.makedirs(self._dir)

   128         except OSError:

   129             raise EnvironmentError, "Cache directory '%s' does not exist and could not be created'" % self._dir

   130 

   131     def _key_to_file(self, key):

   132         """

   133         Convert the filename into an md5 string. We'll turn the first couple

   134         bits of the path into directory prefixes to be nice to filesystems

   135         that have problems with large numbers of files in a directory.

   136         

   137         Thus, a cache key of "foo" gets turnned into a file named

   138         ``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.

   139         """

   140         path = md5.new(key.encode('utf-8')).hexdigest()

   141         path = os.path.join(path[:2], path[2:4], path[4:])

   142         return os.path.join(self._dir, path)

   143 

   144     def _get_num_entries(self):

   145         count = 0

   146         for _,_,files in os.walk(self._dir):

   147             count += len(files)

   148         return count

   149     _num_entries = property(_get_num_entries)

   150