|
1 # LayerMapping -- A Django Model/OGR Layer Mapping Utility |
|
2 """ |
|
3 The LayerMapping class provides a way to map the contents of OGR |
|
4 vector files (e.g. SHP files) to Geographic-enabled Django models. |
|
5 |
|
6 This grew out of my personal needs, specifically the code repetition |
|
7 that went into pulling geometries and fields out of an OGR layer, |
|
8 converting to another coordinate system (e.g. WGS84), and then inserting |
|
9 into a GeoDjango model. |
|
10 |
|
11 Please report any bugs encountered using this utility. |
|
12 |
|
13 Requirements: OGR C Library (from GDAL) required. |
|
14 |
|
15 Usage: |
|
16 lm = LayerMapping(model, source_file, mapping) where, |
|
17 |
|
18 model: |
|
19 GeoDjango model (not an instance) |
|
20 |
|
21 data: |
|
22 OGR-supported data source file (e.g. a shapefile) or |
|
23 gdal.DataSource instance |
|
24 |
|
25 mapping: |
|
26 A python dictionary, keys are strings corresponding |
|
27 to the GeoDjango model field, and values correspond to |
|
28 string field names for the OGR feature, or if the model field |
|
29 is a geographic then it should correspond to the OGR |
|
30 geometry type, e.g. 'POINT', 'LINESTRING', 'POLYGON'. |
|
31 |
|
32 Keyword Args: |
|
33 layer: |
|
34 The index of the layer to use from the Data Source (defaults to 0) |
|
35 |
|
36 source_srs: |
|
37 Use this to specify the source SRS manually (for example, |
|
38 some shapefiles don't come with a '.prj' file). An integer SRID, |
|
39 a string WKT, and SpatialReference objects are valid parameters. |
|
40 |
|
41 encoding: |
|
42 Specifies the encoding of the string in the OGR data source. |
|
43 For example, 'latin-1', 'utf-8', and 'cp437' are all valid |
|
44 encoding parameters. |
|
45 |
|
46 transaction_mode: |
|
47 May be 'commit_on_success' (default) or 'autocommit'. |
|
48 |
|
49 transform: |
|
50 Setting this to False will disable all coordinate transformations. |
|
51 |
|
52 unique: |
|
53 Setting this to the name, or a tuple of names, from the given |
|
54 model will create models unique only to the given name(s). |
|
55 Geometries will from each feature will be added into the collection |
|
56 associated with the unique model. Forces transaction mode to |
|
57 be 'autocommit'. |
|
58 |
|
59 Example: |
|
60 |
|
61 1. You need a GDAL-supported data source, like a shapefile. |
|
62 |
|
63 Assume we're using the test_poly SHP file: |
|
64 >>> from django.contrib.gis.gdal import DataSource |
|
65 >>> ds = DataSource('test_poly.shp') |
|
66 >>> layer = ds[0] |
|
67 >>> print layer.fields # Exploring the fields in the layer, we only want the 'str' field. |
|
68 ['float', 'int', 'str'] |
|
69 >>> print len(layer) # getting the number of features in the layer (should be 3) |
|
70 3 |
|
71 >>> print layer.geom_type # Should be 3 (a Polygon) |
|
72 3 |
|
73 >>> print layer.srs # WGS84 |
|
74 GEOGCS["GCS_WGS_1984", |
|
75 DATUM["WGS_1984", |
|
76 SPHEROID["WGS_1984",6378137,298.257223563]], |
|
77 PRIMEM["Greenwich",0], |
|
78 UNIT["Degree",0.017453292519943295]] |
|
79 |
|
80 2. Now we define our corresponding Django model (make sure to use syncdb): |
|
81 |
|
82 from django.contrib.gis.db import models |
|
83 class TestGeo(models.Model, models.GeoMixin): |
|
84 name = models.CharField(maxlength=25) # corresponds to the 'str' field |
|
85 poly = models.PolygonField(srid=4269) # we want our model in a different SRID |
|
86 objects = models.GeoManager() |
|
87 def __str__(self): |
|
88 return 'Name: %s' % self.name |
|
89 |
|
90 3. Use LayerMapping to extract all the features and place them in the database: |
|
91 |
|
92 >>> from django.contrib.gis.utils import LayerMapping |
|
93 >>> from geoapp.models import TestGeo |
|
94 >>> mapping = {'name' : 'str', # The 'name' model field maps to the 'str' layer field. |
|
95 'poly' : 'POLYGON', # For geometry fields use OGC name. |
|
96 } # The mapping is a dictionary |
|
97 >>> lm = LayerMapping(TestGeo, 'test_poly.shp', mapping) |
|
98 >>> lm.save(verbose=True) # Save the layermap, imports the data. |
|
99 Saved: Name: 1 |
|
100 Saved: Name: 2 |
|
101 Saved: Name: 3 |
|
102 |
|
103 LayerMapping just transformed the three geometries from the SHP file from their |
|
104 source spatial reference system (WGS84) to the spatial reference system of |
|
105 the GeoDjango model (NAD83). If no spatial reference system is defined for |
|
106 the layer, use the `source_srs` keyword with a SpatialReference object to |
|
107 specify one. |
|
108 """ |
|
109 import sys |
|
110 from datetime import date, datetime |
|
111 from decimal import Decimal |
|
112 from django.core.exceptions import ObjectDoesNotExist |
|
113 from django.contrib.gis.db.models import GeometryField |
|
114 from django.contrib.gis.db.backend import SpatialBackend |
|
115 from django.contrib.gis.gdal import CoordTransform, DataSource, \ |
|
116 OGRException, OGRGeometry, OGRGeomType, SpatialReference |
|
117 from django.contrib.gis.gdal.field import \ |
|
118 OFTDate, OFTDateTime, OFTInteger, OFTReal, OFTString, OFTTime |
|
119 from django.contrib.gis.models import GeometryColumns, SpatialRefSys |
|
120 from django.db import models, transaction |
|
121 from django.contrib.localflavor.us.models import USStateField |
|
122 |
|
123 # LayerMapping exceptions. |
|
124 class LayerMapError(Exception): pass |
|
125 class InvalidString(LayerMapError): pass |
|
126 class InvalidDecimal(LayerMapError): pass |
|
127 class InvalidInteger(LayerMapError): pass |
|
128 class MissingForeignKey(LayerMapError): pass |
|
129 |
|
130 class LayerMapping(object): |
|
131 "A class that maps OGR Layers to GeoDjango Models." |
|
132 |
|
133 # Acceptable 'base' types for a multi-geometry type. |
|
134 MULTI_TYPES = {1 : OGRGeomType('MultiPoint'), |
|
135 2 : OGRGeomType('MultiLineString'), |
|
136 3 : OGRGeomType('MultiPolygon'), |
|
137 } |
|
138 |
|
139 # Acceptable Django field types and corresponding acceptable OGR |
|
140 # counterparts. |
|
141 FIELD_TYPES = { |
|
142 models.AutoField : OFTInteger, |
|
143 models.IntegerField : (OFTInteger, OFTReal, OFTString), |
|
144 models.FloatField : (OFTInteger, OFTReal), |
|
145 models.DateField : OFTDate, |
|
146 models.DateTimeField : OFTDateTime, |
|
147 models.EmailField : OFTString, |
|
148 models.TimeField : OFTTime, |
|
149 models.DecimalField : (OFTInteger, OFTReal), |
|
150 models.CharField : OFTString, |
|
151 models.SlugField : OFTString, |
|
152 models.TextField : OFTString, |
|
153 models.URLField : OFTString, |
|
154 USStateField : OFTString, |
|
155 models.XMLField : OFTString, |
|
156 models.SmallIntegerField : (OFTInteger, OFTReal, OFTString), |
|
157 models.PositiveSmallIntegerField : (OFTInteger, OFTReal, OFTString), |
|
158 } |
|
159 |
|
160 # The acceptable transaction modes. |
|
161 TRANSACTION_MODES = {'autocommit' : transaction.autocommit, |
|
162 'commit_on_success' : transaction.commit_on_success, |
|
163 } |
|
164 |
|
165 def __init__(self, model, data, mapping, layer=0, |
|
166 source_srs=None, encoding=None, |
|
167 transaction_mode='commit_on_success', |
|
168 transform=True, unique=None): |
|
169 """ |
|
170 A LayerMapping object is initialized using the given Model (not an instance), |
|
171 a DataSource (or string path to an OGR-supported data file), and a mapping |
|
172 dictionary. See the module level docstring for more details and keyword |
|
173 argument usage. |
|
174 """ |
|
175 # Getting the DataSource and the associated Layer. |
|
176 if isinstance(data, basestring): |
|
177 self.ds = DataSource(data) |
|
178 else: |
|
179 self.ds = data |
|
180 self.layer = self.ds[layer] |
|
181 |
|
182 # Setting the mapping & model attributes. |
|
183 self.mapping = mapping |
|
184 self.model = model |
|
185 |
|
186 # Checking the layer -- intitialization of the object will fail if |
|
187 # things don't check out before hand. |
|
188 self.check_layer() |
|
189 |
|
190 # Getting the geometry column associated with the model (an |
|
191 # exception will be raised if there is no geometry column). |
|
192 self.geo_col = self.geometry_column() |
|
193 |
|
194 # Checking the source spatial reference system, and getting |
|
195 # the coordinate transformation object (unless the `transform` |
|
196 # keyword is set to False) |
|
197 if transform: |
|
198 self.source_srs = self.check_srs(source_srs) |
|
199 self.transform = self.coord_transform() |
|
200 else: |
|
201 self.transform = transform |
|
202 |
|
203 # Setting the encoding for OFTString fields, if specified. |
|
204 if encoding: |
|
205 # Making sure the encoding exists, if not a LookupError |
|
206 # exception will be thrown. |
|
207 from codecs import lookup |
|
208 lookup(encoding) |
|
209 self.encoding = encoding |
|
210 else: |
|
211 self.encoding = None |
|
212 |
|
213 if unique: |
|
214 self.check_unique(unique) |
|
215 transaction_mode = 'autocommit' # Has to be set to autocommit. |
|
216 self.unique = unique |
|
217 else: |
|
218 self.unique = None |
|
219 |
|
220 # Setting the transaction decorator with the function in the |
|
221 # transaction modes dictionary. |
|
222 if transaction_mode in self.TRANSACTION_MODES: |
|
223 self.transaction_decorator = self.TRANSACTION_MODES[transaction_mode] |
|
224 self.transaction_mode = transaction_mode |
|
225 else: |
|
226 raise LayerMapError('Unrecognized transaction mode: %s' % transaction_mode) |
|
227 |
|
228 #### Checking routines used during initialization #### |
|
229 def check_fid_range(self, fid_range): |
|
230 "This checks the `fid_range` keyword." |
|
231 if fid_range: |
|
232 if isinstance(fid_range, (tuple, list)): |
|
233 return slice(*fid_range) |
|
234 elif isinstance(fid_range, slice): |
|
235 return fid_range |
|
236 else: |
|
237 raise TypeError |
|
238 else: |
|
239 return None |
|
240 |
|
241 def check_layer(self): |
|
242 """ |
|
243 This checks the Layer metadata, and ensures that it is compatible |
|
244 with the mapping information and model. Unlike previous revisions, |
|
245 there is no need to increment through each feature in the Layer. |
|
246 """ |
|
247 # The geometry field of the model is set here. |
|
248 # TODO: Support more than one geometry field / model. However, this |
|
249 # depends on the GDAL Driver in use. |
|
250 self.geom_field = False |
|
251 self.fields = {} |
|
252 |
|
253 # Getting lists of the field names and the field types available in |
|
254 # the OGR Layer. |
|
255 ogr_fields = self.layer.fields |
|
256 ogr_field_types = self.layer.field_types |
|
257 |
|
258 # Function for determining if the OGR mapping field is in the Layer. |
|
259 def check_ogr_fld(ogr_map_fld): |
|
260 try: |
|
261 idx = ogr_fields.index(ogr_map_fld) |
|
262 except ValueError: |
|
263 raise LayerMapError('Given mapping OGR field "%s" not found in OGR Layer.' % ogr_map_fld) |
|
264 return idx |
|
265 |
|
266 # No need to increment through each feature in the model, simply check |
|
267 # the Layer metadata against what was given in the mapping dictionary. |
|
268 for field_name, ogr_name in self.mapping.items(): |
|
269 # Ensuring that a corresponding field exists in the model |
|
270 # for the given field name in the mapping. |
|
271 try: |
|
272 model_field = self.model._meta.get_field(field_name) |
|
273 except models.fields.FieldDoesNotExist: |
|
274 raise LayerMapError('Given mapping field "%s" not in given Model fields.' % field_name) |
|
275 |
|
276 # Getting the string name for the Django field class (e.g., 'PointField'). |
|
277 fld_name = model_field.__class__.__name__ |
|
278 |
|
279 if isinstance(model_field, GeometryField): |
|
280 if self.geom_field: |
|
281 raise LayerMapError('LayerMapping does not support more than one GeometryField per model.') |
|
282 |
|
283 try: |
|
284 gtype = OGRGeomType(ogr_name) |
|
285 except OGRException: |
|
286 raise LayerMapError('Invalid mapping for GeometryField "%s".' % field_name) |
|
287 |
|
288 # Making sure that the OGR Layer's Geometry is compatible. |
|
289 ltype = self.layer.geom_type |
|
290 if not (gtype == ltype or self.make_multi(ltype, model_field)): |
|
291 raise LayerMapError('Invalid mapping geometry; model has %s, feature has %s.' % (fld_name, gtype)) |
|
292 |
|
293 # Setting the `geom_field` attribute w/the name of the model field |
|
294 # that is a Geometry. |
|
295 self.geom_field = field_name |
|
296 fields_val = model_field |
|
297 elif isinstance(model_field, models.ForeignKey): |
|
298 if isinstance(ogr_name, dict): |
|
299 # Is every given related model mapping field in the Layer? |
|
300 rel_model = model_field.rel.to |
|
301 for rel_name, ogr_field in ogr_name.items(): |
|
302 idx = check_ogr_fld(ogr_field) |
|
303 try: |
|
304 rel_field = rel_model._meta.get_field(rel_name) |
|
305 except models.fields.FieldDoesNotExist: |
|
306 raise LayerMapError('ForeignKey mapping field "%s" not in %s fields.' % |
|
307 (rel_name, rel_model.__class__.__name__)) |
|
308 fields_val = rel_model |
|
309 else: |
|
310 raise TypeError('ForeignKey mapping must be of dictionary type.') |
|
311 else: |
|
312 # Is the model field type supported by LayerMapping? |
|
313 if not model_field.__class__ in self.FIELD_TYPES: |
|
314 raise LayerMapError('Django field type "%s" has no OGR mapping (yet).' % fld_name) |
|
315 |
|
316 # Is the OGR field in the Layer? |
|
317 idx = check_ogr_fld(ogr_name) |
|
318 ogr_field = ogr_field_types[idx] |
|
319 |
|
320 # Can the OGR field type be mapped to the Django field type? |
|
321 if not issubclass(ogr_field, self.FIELD_TYPES[model_field.__class__]): |
|
322 raise LayerMapError('OGR field "%s" (of type %s) cannot be mapped to Django %s.' % |
|
323 (ogr_field, ogr_field.__name__, fld_name)) |
|
324 fields_val = model_field |
|
325 |
|
326 self.fields[field_name] = fields_val |
|
327 |
|
328 def check_srs(self, source_srs): |
|
329 "Checks the compatibility of the given spatial reference object." |
|
330 if isinstance(source_srs, SpatialReference): |
|
331 sr = source_srs |
|
332 elif isinstance(source_srs, SpatialRefSys): |
|
333 sr = source_srs.srs |
|
334 elif isinstance(source_srs, (int, basestring)): |
|
335 sr = SpatialReference(source_srs) |
|
336 else: |
|
337 # Otherwise just pulling the SpatialReference from the layer |
|
338 sr = self.layer.srs |
|
339 |
|
340 if not sr: |
|
341 raise LayerMapError('No source reference system defined.') |
|
342 else: |
|
343 return sr |
|
344 |
|
345 def check_unique(self, unique): |
|
346 "Checks the `unique` keyword parameter -- may be a sequence or string." |
|
347 if isinstance(unique, (list, tuple)): |
|
348 # List of fields to determine uniqueness with |
|
349 for attr in unique: |
|
350 if not attr in self.mapping: raise ValueError |
|
351 elif isinstance(unique, basestring): |
|
352 # Only a single field passed in. |
|
353 if unique not in self.mapping: raise ValueError |
|
354 else: |
|
355 raise TypeError('Unique keyword argument must be set with a tuple, list, or string.') |
|
356 |
|
357 #### Keyword argument retrieval routines #### |
|
358 def feature_kwargs(self, feat): |
|
359 """ |
|
360 Given an OGR Feature, this will return a dictionary of keyword arguments |
|
361 for constructing the mapped model. |
|
362 """ |
|
363 # The keyword arguments for model construction. |
|
364 kwargs = {} |
|
365 |
|
366 # Incrementing through each model field and OGR field in the |
|
367 # dictionary mapping. |
|
368 for field_name, ogr_name in self.mapping.items(): |
|
369 model_field = self.fields[field_name] |
|
370 |
|
371 if isinstance(model_field, GeometryField): |
|
372 # Verify OGR geometry. |
|
373 val = self.verify_geom(feat.geom, model_field) |
|
374 elif isinstance(model_field, models.base.ModelBase): |
|
375 # The related _model_, not a field was passed in -- indicating |
|
376 # another mapping for the related Model. |
|
377 val = self.verify_fk(feat, model_field, ogr_name) |
|
378 else: |
|
379 # Otherwise, verify OGR Field type. |
|
380 val = self.verify_ogr_field(feat[ogr_name], model_field) |
|
381 |
|
382 # Setting the keyword arguments for the field name with the |
|
383 # value obtained above. |
|
384 kwargs[field_name] = val |
|
385 |
|
386 return kwargs |
|
387 |
|
388 def unique_kwargs(self, kwargs): |
|
389 """ |
|
390 Given the feature keyword arguments (from `feature_kwargs`) this routine |
|
391 will construct and return the uniqueness keyword arguments -- a subset |
|
392 of the feature kwargs. |
|
393 """ |
|
394 if isinstance(self.unique, basestring): |
|
395 return {self.unique : kwargs[self.unique]} |
|
396 else: |
|
397 return dict((fld, kwargs[fld]) for fld in self.unique) |
|
398 |
|
399 #### Verification routines used in constructing model keyword arguments. #### |
|
400 def verify_ogr_field(self, ogr_field, model_field): |
|
401 """ |
|
402 Verifies if the OGR Field contents are acceptable to the Django |
|
403 model field. If they are, the verified value is returned, |
|
404 otherwise the proper exception is raised. |
|
405 """ |
|
406 if (isinstance(ogr_field, OFTString) and |
|
407 isinstance(model_field, (models.CharField, models.TextField))): |
|
408 if self.encoding: |
|
409 # The encoding for OGR data sources may be specified here |
|
410 # (e.g., 'cp437' for Census Bureau boundary files). |
|
411 val = unicode(ogr_field.value, self.encoding) |
|
412 else: |
|
413 val = ogr_field.value |
|
414 if len(val) > model_field.max_length: |
|
415 raise InvalidString('%s model field maximum string length is %s, given %s characters.' % |
|
416 (model_field.name, model_field.max_length, len(val))) |
|
417 elif isinstance(ogr_field, OFTReal) and isinstance(model_field, models.DecimalField): |
|
418 try: |
|
419 # Creating an instance of the Decimal value to use. |
|
420 d = Decimal(str(ogr_field.value)) |
|
421 except: |
|
422 raise InvalidDecimal('Could not construct decimal from: %s' % ogr_field.value) |
|
423 |
|
424 # Getting the decimal value as a tuple. |
|
425 dtup = d.as_tuple() |
|
426 digits = dtup[1] |
|
427 d_idx = dtup[2] # index where the decimal is |
|
428 |
|
429 # Maximum amount of precision, or digits to the left of the decimal. |
|
430 max_prec = model_field.max_digits - model_field.decimal_places |
|
431 |
|
432 # Getting the digits to the left of the decimal place for the |
|
433 # given decimal. |
|
434 if d_idx < 0: |
|
435 n_prec = len(digits[:d_idx]) |
|
436 else: |
|
437 n_prec = len(digits) + d_idx |
|
438 |
|
439 # If we have more than the maximum digits allowed, then throw an |
|
440 # InvalidDecimal exception. |
|
441 if n_prec > max_prec: |
|
442 raise InvalidDecimal('A DecimalField with max_digits %d, decimal_places %d must round to an absolute value less than 10^%d.' % |
|
443 (model_field.max_digits, model_field.decimal_places, max_prec)) |
|
444 val = d |
|
445 elif isinstance(ogr_field, (OFTReal, OFTString)) and isinstance(model_field, models.IntegerField): |
|
446 # Attempt to convert any OFTReal and OFTString value to an OFTInteger. |
|
447 try: |
|
448 val = int(ogr_field.value) |
|
449 except: |
|
450 raise InvalidInteger('Could not construct integer from: %s' % ogr_field.value) |
|
451 else: |
|
452 val = ogr_field.value |
|
453 return val |
|
454 |
|
455 def verify_fk(self, feat, rel_model, rel_mapping): |
|
456 """ |
|
457 Given an OGR Feature, the related model and its dictionary mapping, |
|
458 this routine will retrieve the related model for the ForeignKey |
|
459 mapping. |
|
460 """ |
|
461 # TODO: It is expensive to retrieve a model for every record -- |
|
462 # explore if an efficient mechanism exists for caching related |
|
463 # ForeignKey models. |
|
464 |
|
465 # Constructing and verifying the related model keyword arguments. |
|
466 fk_kwargs = {} |
|
467 for field_name, ogr_name in rel_mapping.items(): |
|
468 fk_kwargs[field_name] = self.verify_ogr_field(feat[ogr_name], rel_model._meta.get_field(field_name)) |
|
469 |
|
470 # Attempting to retrieve and return the related model. |
|
471 try: |
|
472 return rel_model.objects.get(**fk_kwargs) |
|
473 except ObjectDoesNotExist: |
|
474 raise MissingForeignKey('No ForeignKey %s model found with keyword arguments: %s' % (rel_model.__name__, fk_kwargs)) |
|
475 |
|
476 def verify_geom(self, geom, model_field): |
|
477 """ |
|
478 Verifies the geometry -- will construct and return a GeometryCollection |
|
479 if necessary (for example if the model field is MultiPolygonField while |
|
480 the mapped shapefile only contains Polygons). |
|
481 """ |
|
482 if self.make_multi(geom.geom_type, model_field): |
|
483 # Constructing a multi-geometry type to contain the single geometry |
|
484 multi_type = self.MULTI_TYPES[geom.geom_type.num] |
|
485 g = OGRGeometry(multi_type) |
|
486 g.add(geom) |
|
487 else: |
|
488 g = geom |
|
489 |
|
490 # Transforming the geometry with our Coordinate Transformation object, |
|
491 # but only if the class variable `transform` is set w/a CoordTransform |
|
492 # object. |
|
493 if self.transform: g.transform(self.transform) |
|
494 |
|
495 # Returning the WKT of the geometry. |
|
496 return g.wkt |
|
497 |
|
498 #### Other model methods #### |
|
499 def coord_transform(self): |
|
500 "Returns the coordinate transformation object." |
|
501 try: |
|
502 # Getting the target spatial reference system |
|
503 target_srs = SpatialRefSys.objects.get(srid=self.geo_col.srid).srs |
|
504 |
|
505 # Creating the CoordTransform object |
|
506 return CoordTransform(self.source_srs, target_srs) |
|
507 except Exception, msg: |
|
508 raise LayerMapError('Could not translate between the data source and model geometry: %s' % msg) |
|
509 |
|
510 def geometry_column(self): |
|
511 "Returns the GeometryColumn model associated with the geographic column." |
|
512 # Getting the GeometryColumn object. |
|
513 try: |
|
514 db_table = self.model._meta.db_table |
|
515 geo_col = self.geom_field |
|
516 if SpatialBackend.name == 'oracle': |
|
517 # Making upper case for Oracle. |
|
518 db_table = db_table.upper() |
|
519 geo_col = geo_col.upper() |
|
520 gc_kwargs = {GeometryColumns.table_name_col() : db_table, |
|
521 GeometryColumns.geom_col_name() : geo_col, |
|
522 } |
|
523 return GeometryColumns.objects.get(**gc_kwargs) |
|
524 except Exception, msg: |
|
525 raise LayerMapError('Geometry column does not exist for model. (did you run syncdb?):\n %s' % msg) |
|
526 |
|
527 def make_multi(self, geom_type, model_field): |
|
528 """ |
|
529 Given the OGRGeomType for a geometry and its associated GeometryField, |
|
530 determine whether the geometry should be turned into a GeometryCollection. |
|
531 """ |
|
532 return (geom_type.num in self.MULTI_TYPES and |
|
533 model_field.__class__.__name__ == 'Multi%s' % geom_type.django) |
|
534 |
|
535 def save(self, verbose=False, fid_range=False, step=False, |
|
536 progress=False, silent=False, stream=sys.stdout, strict=False): |
|
537 """ |
|
538 Saves the contents from the OGR DataSource Layer into the database |
|
539 according to the mapping dictionary given at initialization. |
|
540 |
|
541 Keyword Parameters: |
|
542 verbose: |
|
543 If set, information will be printed subsequent to each model save |
|
544 executed on the database. |
|
545 |
|
546 fid_range: |
|
547 May be set with a slice or tuple of (begin, end) feature ID's to map |
|
548 from the data source. In other words, this keyword enables the user |
|
549 to selectively import a subset range of features in the geographic |
|
550 data source. |
|
551 |
|
552 step: |
|
553 If set with an integer, transactions will occur at every step |
|
554 interval. For example, if step=1000, a commit would occur after |
|
555 the 1,000th feature, the 2,000th feature etc. |
|
556 |
|
557 progress: |
|
558 When this keyword is set, status information will be printed giving |
|
559 the number of features processed and sucessfully saved. By default, |
|
560 progress information will pe printed every 1000 features processed, |
|
561 however, this default may be overridden by setting this keyword with an |
|
562 integer for the desired interval. |
|
563 |
|
564 stream: |
|
565 Status information will be written to this file handle. Defaults to |
|
566 using `sys.stdout`, but any object with a `write` method is supported. |
|
567 |
|
568 silent: |
|
569 By default, non-fatal error notifications are printed to stdout, but |
|
570 this keyword may be set to disable these notifications. |
|
571 |
|
572 strict: |
|
573 Execution of the model mapping will cease upon the first error |
|
574 encountered. The default behavior is to attempt to continue. |
|
575 """ |
|
576 # Getting the default Feature ID range. |
|
577 default_range = self.check_fid_range(fid_range) |
|
578 |
|
579 # Setting the progress interval, if requested. |
|
580 if progress: |
|
581 if progress is True or not isinstance(progress, int): |
|
582 progress_interval = 1000 |
|
583 else: |
|
584 progress_interval = progress |
|
585 |
|
586 # Defining the 'real' save method, utilizing the transaction |
|
587 # decorator created during initialization. |
|
588 @self.transaction_decorator |
|
589 def _save(feat_range=default_range, num_feat=0, num_saved=0): |
|
590 if feat_range: |
|
591 layer_iter = self.layer[feat_range] |
|
592 else: |
|
593 layer_iter = self.layer |
|
594 |
|
595 for feat in layer_iter: |
|
596 num_feat += 1 |
|
597 # Getting the keyword arguments |
|
598 try: |
|
599 kwargs = self.feature_kwargs(feat) |
|
600 except LayerMapError, msg: |
|
601 # Something borked the validation |
|
602 if strict: raise |
|
603 elif not silent: |
|
604 stream.write('Ignoring Feature ID %s because: %s\n' % (feat.fid, msg)) |
|
605 else: |
|
606 # Constructing the model using the keyword args |
|
607 is_update = False |
|
608 if self.unique: |
|
609 # If we want unique models on a particular field, handle the |
|
610 # geometry appropriately. |
|
611 try: |
|
612 # Getting the keyword arguments and retrieving |
|
613 # the unique model. |
|
614 u_kwargs = self.unique_kwargs(kwargs) |
|
615 m = self.model.objects.get(**u_kwargs) |
|
616 is_update = True |
|
617 |
|
618 # Getting the geometry (in OGR form), creating |
|
619 # one from the kwargs WKT, adding in additional |
|
620 # geometries, and update the attribute with the |
|
621 # just-updated geometry WKT. |
|
622 geom = getattr(m, self.geom_field).ogr |
|
623 new = OGRGeometry(kwargs[self.geom_field]) |
|
624 for g in new: geom.add(g) |
|
625 setattr(m, self.geom_field, geom.wkt) |
|
626 except ObjectDoesNotExist: |
|
627 # No unique model exists yet, create. |
|
628 m = self.model(**kwargs) |
|
629 else: |
|
630 m = self.model(**kwargs) |
|
631 |
|
632 try: |
|
633 # Attempting to save. |
|
634 m.save() |
|
635 num_saved += 1 |
|
636 if verbose: stream.write('%s: %s\n' % (is_update and 'Updated' or 'Saved', m)) |
|
637 except SystemExit: |
|
638 raise |
|
639 except Exception, msg: |
|
640 if self.transaction_mode == 'autocommit': |
|
641 # Rolling back the transaction so that other model saves |
|
642 # will work. |
|
643 transaction.rollback_unless_managed() |
|
644 if strict: |
|
645 # Bailing out if the `strict` keyword is set. |
|
646 if not silent: |
|
647 stream.write('Failed to save the feature (id: %s) into the model with the keyword arguments:\n' % feat.fid) |
|
648 stream.write('%s\n' % kwargs) |
|
649 raise |
|
650 elif not silent: |
|
651 stream.write('Failed to save %s:\n %s\nContinuing\n' % (kwargs, msg)) |
|
652 |
|
653 # Printing progress information, if requested. |
|
654 if progress and num_feat % progress_interval == 0: |
|
655 stream.write('Processed %d features, saved %d ...\n' % (num_feat, num_saved)) |
|
656 |
|
657 # Only used for status output purposes -- incremental saving uses the |
|
658 # values returned here. |
|
659 return num_saved, num_feat |
|
660 |
|
661 nfeat = self.layer.num_feat |
|
662 if step and isinstance(step, int) and step < nfeat: |
|
663 # Incremental saving is requested at the given interval (step) |
|
664 if default_range: |
|
665 raise LayerMapError('The `step` keyword may not be used in conjunction with the `fid_range` keyword.') |
|
666 beg, num_feat, num_saved = (0, 0, 0) |
|
667 indices = range(step, nfeat, step) |
|
668 n_i = len(indices) |
|
669 |
|
670 for i, end in enumerate(indices): |
|
671 # Constructing the slice to use for this step; the last slice is |
|
672 # special (e.g, [100:] instead of [90:100]). |
|
673 if i+1 == n_i: step_slice = slice(beg, None) |
|
674 else: step_slice = slice(beg, end) |
|
675 |
|
676 try: |
|
677 num_feat, num_saved = _save(step_slice, num_feat, num_saved) |
|
678 beg = end |
|
679 except: |
|
680 stream.write('%s\nFailed to save slice: %s\n' % ('=-' * 20, step_slice)) |
|
681 raise |
|
682 else: |
|
683 # Otherwise, just calling the previously defined _save() function. |
|
684 _save() |