diff -r 6641e941ef1e -r ff1a9aa48cfd app/django/db/models/query.py --- a/app/django/db/models/query.py Tue Oct 14 12:36:55 2008 +0000 +++ b/app/django/db/models/query.py Tue Oct 14 16:00:59 2008 +0000 @@ -1,28 +1,132 @@ -import warnings +try: + set +except NameError: + from sets import Set as set # Python 2.3 fallback -from django.conf import settings from django.db import connection, transaction, IntegrityError -from django.db.models.fields import DateField, FieldDoesNotExist -from django.db.models.query_utils import Q +from django.db.models.fields import DateField +from django.db.models.query_utils import Q, select_related_descend from django.db.models import signals, sql -from django.dispatch import dispatcher from django.utils.datastructures import SortedDict + # Used to control how many objects are worked with at once in some cases (e.g. # when deleting objects). CHUNK_SIZE = 100 ITER_CHUNK_SIZE = CHUNK_SIZE -# Pull into this namespace for backwards compatibility +# The maximum number of items to display in a QuerySet.__repr__ +REPR_OUTPUT_SIZE = 20 + +# Pull into this namespace for backwards compatibility. EmptyResultSet = sql.EmptyResultSet + +class CyclicDependency(Exception): + """ + An error when dealing with a collection of objects that have a cyclic + dependency, i.e. when deleting multiple objects. + """ + pass + + +class CollectedObjects(object): + """ + A container that stores keys and lists of values along with remembering the + parent objects for all the keys. + + This is used for the database object deletion routines so that we can + calculate the 'leaf' objects which should be deleted first. + """ + + def __init__(self): + self.data = {} + self.children = {} + + def add(self, model, pk, obj, parent_model, nullable=False): + """ + Adds an item to the container. + + Arguments: + * model - the class of the object being added. + * pk - the primary key. + * obj - the object itself. + * parent_model - the model of the parent object that this object was + reached through. + * nullable - should be True if this relation is nullable. + + Returns True if the item already existed in the structure and + False otherwise. + """ + d = self.data.setdefault(model, SortedDict()) + retval = pk in d + d[pk] = obj + # Nullable relationships can be ignored -- they are nulled out before + # deleting, and therefore do not affect the order in which objects + # have to be deleted. + if parent_model is not None and not nullable: + self.children.setdefault(parent_model, []).append(model) + return retval + + def __contains__(self, key): + return self.data.__contains__(key) + + def __getitem__(self, key): + return self.data[key] + + def __nonzero__(self): + return bool(self.data) + + def iteritems(self): + for k in self.ordered_keys(): + yield k, self[k] + + def items(self): + return list(self.iteritems()) + + def keys(self): + return self.ordered_keys() + + def ordered_keys(self): + """ + Returns the models in the order that they should be dealt with (i.e. + models with no dependencies first). + """ + dealt_with = SortedDict() + # Start with items that have no children + models = self.data.keys() + while len(dealt_with) < len(models): + found = False + for model in models: + if model in dealt_with: + continue + children = self.children.setdefault(model, []) + if len([c for c in children if c not in dealt_with]) == 0: + dealt_with[model] = None + found = True + if not found: + raise CyclicDependency( + "There is a cyclic dependency of items to be processed.") + + return dealt_with.keys() + + def unordered_keys(self): + """ + Fallback for the case where is a cyclic dependency but we don't care. + """ + return self.data.keys() + + class QuerySet(object): - "Represents a lazy database lookup for a set of objects" + """ + Represents a lazy database lookup for a set of objects. + """ def __init__(self, model=None, query=None): self.model = model self.query = query or sql.Query(self.model, connection) self._result_cache = None self._iter = None + self._sticky_filter = False ######################## # PYTHON MAGIC METHODS # @@ -30,7 +134,7 @@ def __getstate__(self): """ - Allows the Queryset to be pickled. + Allows the QuerySet to be pickled. """ # Force the cache to be fully populated. len(self) @@ -40,12 +144,15 @@ return obj_dict def __repr__(self): - return repr(list(self)) + data = list(self[:REPR_OUTPUT_SIZE + 1]) + if len(data) > REPR_OUTPUT_SIZE: + data[-1] = "...(remaining elements truncated)..." + return repr(data) def __len__(self): # Since __len__ is called quite frequently (for example, as part of # list(qs), we make some effort here to be as efficient as possible - # whilst not messing up any existing iterators against the queryset. + # whilst not messing up any existing iterators against the QuerySet. if self._result_cache is None: if self._iter: self._result_cache = list(self._iter) @@ -87,7 +194,9 @@ return True def __getitem__(self, k): - "Retrieve an item or slice from the set of results." + """ + Retrieves an item or slice from the set of results. + """ if not isinstance(k, (slice, int, long)): raise TypeError assert ((not isinstance(k, slice) and (k >= 0)) @@ -132,6 +241,8 @@ def __and__(self, other): self._merge_sanity_check(other) + if isinstance(other, EmptyQuerySet): + return other._clone() combined = self._clone() combined.query.combine(other.query, sql.AND) return combined @@ -139,6 +250,8 @@ def __or__(self, other): self._merge_sanity_check(other) combined = self._clone() + if isinstance(other, EmptyQuerySet): + return combined combined.query.combine(other.query, sql.OR) return combined @@ -174,11 +287,10 @@ Performs a SELECT COUNT() and returns the number of records as an integer. - If the queryset is already cached (i.e. self._result_cache is set) this - simply returns the length of the cached results set to avoid multiple - SELECT COUNT(*) calls. + If the QuerySet is already fully cached this simply returns the length + of the cached results set to avoid multiple SELECT COUNT(*) calls. """ - if self._result_cache is not None: + if self._result_cache is not None and not self._iter: return len(self._result_cache) return self.query.get_count() @@ -200,11 +312,11 @@ def create(self, **kwargs): """ - Create a new object with the given kwargs, saving it to the database + Creates a new object with the given kwargs, saving it to the database and returning the created object. """ obj = self.model(**kwargs) - obj.save() + obj.save(force_insert=True) return obj def get_or_create(self, **kwargs): @@ -223,10 +335,16 @@ params = dict([(k, v) for k, v in kwargs.items() if '__' not in k]) params.update(defaults) obj = self.model(**params) - obj.save() + sid = transaction.savepoint() + obj.save(force_insert=True) + transaction.savepoint_commit(sid) return obj, True except IntegrityError, e: - return self.get(**kwargs), False + transaction.savepoint_rollback(sid) + try: + return self.get(**kwargs), False + except self.model.DoesNotExist: + raise e def latest(self, field_name=None): """ @@ -275,7 +393,7 @@ while 1: # Collect all the objects to be deleted in this chunk, and all the # objects that are related to the objects that are to be deleted. - seen_objs = SortedDict() + seen_objs = CollectedObjects() for object in del_query[:CHUNK_SIZE]: object._collect_sub_objects(seen_objs) @@ -292,11 +410,14 @@ Updates all elements in the current QuerySet, setting all the given fields to the appropriate values. """ + assert self.query.can_filter(), \ + "Cannot update a query once a slice has been taken." query = self.query.clone(sql.UpdateQuery) query.add_update_values(kwargs) - query.execute_sql(None) + rows = query.execute_sql(None) transaction.commit_unless_managed() self._result_cache = None + return rows update.alters_data = True def _update(self, values): @@ -306,10 +427,12 @@ code (it requires too much poking around at model internals to be useful at that level). """ + assert self.query.can_filter(), \ + "Cannot update a query once a slice has been taken." query = self.query.clone(sql.UpdateQuery) query.add_update_fields(values) - query.execute_sql(None) self._result_cache = None + return query.execute_sql(None) _update.alters_data = True ################################################## @@ -331,23 +454,19 @@ def dates(self, field_name, kind, order='ASC'): """ - Returns a list of datetime objects representing all available dates - for the given field_name, scoped to 'kind'. + Returns a list of datetime objects representing all available dates for + the given field_name, scoped to 'kind'. """ assert kind in ("month", "year", "day"), \ "'kind' must be one of 'year', 'month' or 'day'." assert order in ('ASC', 'DESC'), \ "'order' must be either 'ASC' or 'DESC'." - # Let the FieldDoesNotExist exception propagate. - field = self.model._meta.get_field(field_name, many_to_many=False) - assert isinstance(field, DateField), "%r isn't a DateField." \ - % field_name - return self._clone(klass=DateQuerySet, setup=True, _field=field, - _kind=kind, _order=order) + return self._clone(klass=DateQuerySet, setup=True, + _field_name=field_name, _kind=kind, _order=order) def none(self): """ - Returns an empty queryset. + Returns an empty QuerySet. """ return self._clone(klass=EmptyQuerySet) @@ -391,6 +510,7 @@ def complex_filter(self, filter_obj): """ Returns a new QuerySet instance with filter_obj added to the filters. + filter_obj can be a Q object (or anything with an add_to_query() method) or a dictionary of keyword lookup arguments. @@ -398,14 +518,17 @@ and usually it will be more natural to use other methods. """ if isinstance(filter_obj, Q) or hasattr(filter_obj, 'add_to_query'): - return self._filter_or_exclude(None, filter_obj) + clone = self._clone() + clone.query.add_q(filter_obj) + return clone else: return self._filter_or_exclude(None, **filter_obj) def select_related(self, *fields, **kwargs): """ - Returns a new QuerySet instance that will select related objects. If - fields are specified, they must be ForeignKey fields and only those + Returns a new QuerySet instance that will select related objects. + + If fields are specified, they must be ForeignKey fields and only those related objects are included in the selection. """ depth = kwargs.pop('depth', 0) @@ -425,13 +548,15 @@ def dup_select_related(self, other): """ - Copies the related selection status from the queryset 'other' to the - current queryset. + Copies the related selection status from the QuerySet 'other' to the + current QuerySet. """ self.query.select_related = other.query.select_related def order_by(self, *field_names): - """Returns a new QuerySet instance with the ordering changed.""" + """ + Returns a new QuerySet instance with the ordering changed. + """ assert self.query.can_filter(), \ "Cannot reorder a query once a slice has been taken." obj = self._clone() @@ -448,9 +573,9 @@ return obj def extra(self, select=None, where=None, params=None, tables=None, - order_by=None, select_params=None): + order_by=None, select_params=None): """ - Add extra SQL fragments to the query. + Adds extra SQL fragments to the query. """ assert self.query.can_filter(), \ "Cannot change a query once a slice has been taken" @@ -460,7 +585,7 @@ def reverse(self): """ - Reverses the ordering of the queryset. + Reverses the ordering of the QuerySet. """ clone = self._clone() clone.query.standard_ordering = not clone.query.standard_ordering @@ -473,7 +598,10 @@ def _clone(self, klass=None, setup=False, **kwargs): if klass is None: klass = self.__class__ - c = klass(model=self.model, query=self.query.clone()) + query = self.query.clone() + if self._sticky_filter: + query.filter_is_sticky = True + c = klass(model=self.model, query=query) c.__dict__.update(kwargs) if setup and hasattr(c, '_setup_query'): c._setup_query() @@ -491,13 +619,28 @@ except StopIteration: self._iter = None + def _next_is_sticky(self): + """ + Indicates that the next filter call and the one following that should + be treated as a single filter. This is only important when it comes to + determining when to reuse tables for many-to-many filters. Required so + that we can filter naturally on the results of related managers. + + This doesn't return a clone of the current QuerySet (it returns + "self"). The method is only used internally and should be immediately + followed by a filter() that does create a clone. + """ + self._sticky_filter = True + return self + def _merge_sanity_check(self, other): """ - Checks that we are merging two comparable queryset classes. + Checks that we are merging two comparable QuerySet classes. By default + this does nothing, but see the ValuesQuerySet for an example of where + it's useful. """ - if self.__class__ is not other.__class__: - raise TypeError("Cannot merge querysets of different types ('%s' and '%s'." - % (self.__class__.__name__, other.__class__.__name__)) + pass + class ValuesQuerySet(QuerySet): def __init__(self, *args, **kwargs): @@ -509,7 +652,9 @@ # names of the model fields to select. def iterator(self): - self.query.trim_extra_select(self.extra_names) + if (not self.extra_names and + len(self.field_names) != len(self.model._meta.fields)): + self.query.trim_extra_select(self.extra_names) names = self.query.extra_select.keys() + self.field_names for row in self.query.results_iter(): yield dict(zip(names, row)) @@ -519,7 +664,7 @@ Constructs the field_names list that the values query will be retrieving. - Called by the _clone() method after initialising the rest of the + Called by the _clone() method after initializing the rest of the instance. """ self.extra_names = [] @@ -560,6 +705,7 @@ raise TypeError("Merging '%s' classes must involve the same values in each case." % self.__class__.__name__) + class ValuesListQuerySet(ValuesQuerySet): def iterator(self): self.query.trim_extra_select(self.extra_names) @@ -568,7 +714,7 @@ yield row[0] elif not self.query.extra_select: for row in self.query.results_iter(): - yield row + yield tuple(row) else: # When extra(select=...) is involved, the extra cols come are # always at the start of the row, so we need to reorder the fields @@ -583,6 +729,7 @@ clone.flat = self.flat return clone + class DateQuerySet(QuerySet): def iterator(self): return self.query.results_iter() @@ -591,28 +738,38 @@ """ Sets up any special features of the query attribute. - Called by the _clone() method after initialising the rest of the + Called by the _clone() method after initializing the rest of the instance. """ self.query = self.query.clone(klass=sql.DateQuery, setup=True) self.query.select = [] - self.query.add_date_select(self._field.column, self._kind, self._order) - if self._field.null: - self.query.add_filter(('%s__isnull' % self._field.name, True)) + field = self.model._meta.get_field(self._field_name, many_to_many=False) + assert isinstance(field, DateField), "%r isn't a DateField." \ + % field.name + self.query.add_date_select(field, self._kind, self._order) + if field.null: + self.query.add_filter(('%s__isnull' % field.name, False)) def _clone(self, klass=None, setup=False, **kwargs): c = super(DateQuerySet, self)._clone(klass, False, **kwargs) - c._field = self._field + c._field_name = self._field_name c._kind = self._kind if setup and hasattr(c, '_setup_query'): c._setup_query() return c + class EmptyQuerySet(QuerySet): def __init__(self, model=None, query=None): super(EmptyQuerySet, self).__init__(model, query) self._result_cache = [] + def __and__(self, other): + return self._clone() + + def __or__(self, other): + return other._clone() + def count(self): return 0 @@ -629,22 +786,9 @@ # (it raises StopIteration immediately). yield iter([]).next() -# QOperator, QNot, QAnd and QOr are temporarily retained for backwards -# compatibility. All the old functionality is now part of the 'Q' class. -class QOperator(Q): - def __init__(self, *args, **kwargs): - warnings.warn('Use Q instead of QOr, QAnd or QOperation.', - DeprecationWarning, stacklevel=2) - super(QOperator, self).__init__(*args, **kwargs) - -QOr = QAnd = QOperator - -def QNot(q): - warnings.warn('Use ~q instead of QNot(q)', DeprecationWarning, stacklevel=2) - return ~q def get_cached_row(klass, row, index_start, max_depth=0, cur_depth=0, - requested=None): + requested=None): """ Helper function that recursively returns an object with the specified related attributes already populated. @@ -655,10 +799,14 @@ restricted = requested is not None index_end = index_start + len(klass._meta.fields) - obj = klass(*row[index_start:index_end]) + fields = row[index_start:index_end] + if not [x for x in fields if x is not None]: + # If we only have a list of Nones, there was not related object. + obj = None + else: + obj = klass(*fields) for f in klass._meta.fields: - if (not f.rel or (not restricted and f.null) or - (restricted and f.name not in requested) or f.rel.parent_link): + if not select_related_descend(f, restricted, requested): continue if restricted: next = requested[f.name] @@ -668,56 +816,73 @@ cur_depth+1, next) if cached_row: rel_obj, index_end = cached_row - setattr(obj, f.get_cache_name(), rel_obj) + if obj is not None: + setattr(obj, f.get_cache_name(), rel_obj) return obj, index_end + def delete_objects(seen_objs): """ Iterate through a list of seen classes, and remove any instances that are referred to. """ - ordered_classes = seen_objs.keys() - ordered_classes.reverse() - - for cls in ordered_classes: - seen_objs[cls] = seen_objs[cls].items() - seen_objs[cls].sort() + try: + ordered_classes = seen_objs.keys() + except CyclicDependency: + # If there is a cyclic dependency, we cannot in general delete the + # objects. However, if an appropriate transaction is set up, or if the + # database is lax enough, it will succeed. So for now, we go ahead and + # try anyway. + ordered_classes = seen_objs.unordered_keys() - # Pre notify all instances to be deleted - for pk_val, instance in seen_objs[cls]: - dispatcher.send(signal=signals.pre_delete, sender=cls, - instance=instance) + obj_pairs = {} + for cls in ordered_classes: + items = seen_objs[cls].items() + items.sort() + obj_pairs[cls] = items - pk_list = [pk for pk,instance in seen_objs[cls]] + # Pre-notify all instances to be deleted. + for pk_val, instance in items: + signals.pre_delete.send(sender=cls, instance=instance) + + pk_list = [pk for pk,instance in items] del_query = sql.DeleteQuery(cls, connection) del_query.delete_batch_related(pk_list) update_query = sql.UpdateQuery(cls, connection) - for field in cls._meta.fields: - if field.rel and field.null and field.rel.to in seen_objs: - update_query.clear_related(field, pk_list) + for field, model in cls._meta.get_fields_with_model(): + if (field.rel and field.null and field.rel.to in seen_objs and + filter(lambda f: f.column == field.column, + field.rel.to._meta.fields)): + if model: + sql.UpdateQuery(model, connection).clear_related(field, + pk_list) + else: + update_query.clear_related(field, pk_list) - # Now delete the actual data + # Now delete the actual data. for cls in ordered_classes: - seen_objs[cls].reverse() - pk_list = [pk for pk,instance in seen_objs[cls]] + items = obj_pairs[cls] + items.reverse() + + pk_list = [pk for pk,instance in items] del_query = sql.DeleteQuery(cls, connection) del_query.delete_batch(pk_list) # Last cleanup; set NULLs where there once was a reference to the # object, NULL the primary key of the found objects, and perform # post-notification. - for pk_val, instance in seen_objs[cls]: + for pk_val, instance in items: for field in cls._meta.fields: if field.rel and field.null and field.rel.to in seen_objs: setattr(instance, field.attname, None) - dispatcher.send(signal=signals.post_delete, sender=cls, - instance=instance) + signals.post_delete.send(sender=cls, instance=instance) setattr(instance, cls._meta.pk.attname, None) transaction.commit_unless_managed() + def insert_query(model, values, return_id=False, raw_values=False): """ Inserts a new record for the given model. This provides an interface to @@ -727,4 +892,3 @@ query = sql.InsertQuery(model, connection) query.insert_values(values, raw_values) return query.execute_sql(return_id) -