1 # Subversion 1.4/1.5 Python API backend

     2 #

     3 # Copyright(C) 2007 Daniel Holth et al

     4 

     5 import os

     6 import re

     7 import sys

     8 import cPickle as pickle

     9 import tempfile

    10 import urllib

    11 import urllib2

    12 

    13 from mercurial import strutil, util, encoding

    14 from mercurial.i18n import _

    15 

    16 # Subversion stuff. Works best with very recent Python SVN bindings

    17 # e.g. SVN 1.5 or backports. Thanks to the bzr folks for enhancing

    18 # these bindings.

    19 

    20 from cStringIO import StringIO

    21 

    22 from common import NoRepo, MissingTool, commit, encodeargs, decodeargs

    23 from common import commandline, converter_source, converter_sink, mapfile

    24 

    25 try:

    26     from svn.core import SubversionException, Pool

    27     import svn

    28     import svn.client

    29     import svn.core

    30     import svn.ra

    31     import svn.delta

    32     import transport

    33     import warnings

    34     warnings.filterwarnings('ignore',

    35             module='svn.core',

    36             category=DeprecationWarning)

    37 

    38 except ImportError:

    39     pass

    40 

    41 class SvnPathNotFound(Exception):

    42     pass

    43 

    44 def geturl(path):

    45     try:

    46         return svn.client.url_from_path(svn.core.svn_path_canonicalize(path))

    47     except SubversionException:

    48         pass

    49     if os.path.isdir(path):

    50         path = os.path.normpath(os.path.abspath(path))

    51         if os.name == 'nt':

    52             path = '/' + util.normpath(path)

    53         # Module URL is later compared with the repository URL returned

    54         # by svn API, which is UTF-8.

    55         path = encoding.tolocal(path)

    56         return 'file://%s' % urllib.quote(path)

    57     return path

    58 

    59 def optrev(number):

    60     optrev = svn.core.svn_opt_revision_t()

    61     optrev.kind = svn.core.svn_opt_revision_number

    62     optrev.value.number = number

    63     return optrev

    64 

    65 class changedpath(object):

    66     def __init__(self, p):

    67         self.copyfrom_path = p.copyfrom_path

    68         self.copyfrom_rev = p.copyfrom_rev

    69         self.action = p.action

    70 

    71 def get_log_child(fp, url, paths, start, end, limit=0, discover_changed_paths=True,

    72                     strict_node_history=False):

    73     protocol = -1

    74     def receiver(orig_paths, revnum, author, date, message, pool):

    75         if orig_paths is not None:

    76             for k, v in orig_paths.iteritems():

    77                 orig_paths[k] = changedpath(v)

    78         pickle.dump((orig_paths, revnum, author, date, message),

    79                     fp, protocol)

    80 

    81     try:

    82         # Use an ra of our own so that our parent can consume

    83         # our results without confusing the server.

    84         t = transport.SvnRaTransport(url=url)

    85         svn.ra.get_log(t.ra, paths, start, end, limit,

    86                        discover_changed_paths,

    87                        strict_node_history,

    88                        receiver)

    89     except SubversionException, (inst, num):

    90         pickle.dump(num, fp, protocol)

    91     except IOError:

    92         # Caller may interrupt the iteration

    93         pickle.dump(None, fp, protocol)

    94     else:

    95         pickle.dump(None, fp, protocol)

    96     fp.close()

    97     # With large history, cleanup process goes crazy and suddenly

    98     # consumes *huge* amount of memory. The output file being closed,

    99     # there is no need for clean termination.

   100     os._exit(0)

   101 

   102 def debugsvnlog(ui, **opts):

   103     """Fetch SVN log in a subprocess and channel them back to parent to

   104     avoid memory collection issues.

   105     """

   106     util.set_binary(sys.stdin)

   107     util.set_binary(sys.stdout)

   108     args = decodeargs(sys.stdin.read())

   109     get_log_child(sys.stdout, *args)

   110 

   111 class logstream(object):

   112     """Interruptible revision log iterator."""

   113     def __init__(self, stdout):

   114         self._stdout = stdout

   115 

   116     def __iter__(self):

   117         while True:

   118             try:

   119                 entry = pickle.load(self._stdout)

   120             except EOFError:

   121                 raise util.Abort(_('Mercurial failed to run itself, check'

   122                                    ' hg executable is in PATH'))

   123             try:

   124                 orig_paths, revnum, author, date, message = entry

   125             except:

   126                 if entry is None:

   127                     break

   128                 raise SubversionException("child raised exception", entry)

   129             yield entry

   130 

   131     def close(self):

   132         if self._stdout:

   133             self._stdout.close()

   134             self._stdout = None

   135 

   136 

   137 # Check to see if the given path is a local Subversion repo. Verify this by

   138 # looking for several svn-specific files and directories in the given

   139 # directory.

   140 def filecheck(ui, path, proto):

   141     for x in ('locks', 'hooks', 'format', 'db'):

   142         if not os.path.exists(os.path.join(path, x)):

   143             return False

   144     return True

   145 

   146 # Check to see if a given path is the root of an svn repo over http. We verify

   147 # this by requesting a version-controlled URL we know can't exist and looking

   148 # for the svn-specific "not found" XML.

   149 def httpcheck(ui, path, proto):

   150     try:

   151         opener = urllib2.build_opener()

   152         rsp = opener.open('%s://%s/!svn/ver/0/.svn' % (proto, path))

   153         data = rsp.read()

   154     except urllib2.HTTPError, inst:

   155         if inst.code != 404:

   156             # Except for 404 we cannot know for sure this is not an svn repo

   157             ui.warn(_('svn: cannot probe remote repository, assume it could '

   158                       'be a subversion repository. Use --source-type if you '

   159                       'know better.\n'))

   160             return True

   161         data = inst.fp.read()

   162     except:

   163         # Could be urllib2.URLError if the URL is invalid or anything else.

   164         return False

   165     return '<m:human-readable errcode="160013">' in data

   166 

   167 protomap = {'http': httpcheck,

   168             'https': httpcheck,

   169             'file': filecheck,

   170             }

   171 def issvnurl(ui, url):

   172     try:

   173         proto, path = url.split('://', 1)

   174         if proto == 'file':

   175             path = urllib.url2pathname(path)

   176     except ValueError:

   177         proto = 'file'

   178         path = os.path.abspath(url)

   179     if proto == 'file':

   180         path = path.replace(os.sep, '/')

   181     check = protomap.get(proto, lambda *args: False)

   182     while '/' in path:

   183         if check(ui, path, proto):

   184             return True

   185         path = path.rsplit('/', 1)[0]

   186     return False

   187 

   188 # SVN conversion code stolen from bzr-svn and tailor

   189 #

   190 # Subversion looks like a versioned filesystem, branches structures

   191 # are defined by conventions and not enforced by the tool. First,

   192 # we define the potential branches (modules) as "trunk" and "branches"

   193 # children directories. Revisions are then identified by their

   194 # module and revision number (and a repository identifier).

   195 #

   196 # The revision graph is really a tree (or a forest). By default, a

   197 # revision parent is the previous revision in the same module. If the

   198 # module directory is copied/moved from another module then the

   199 # revision is the module root and its parent the source revision in

   200 # the parent module. A revision has at most one parent.

   201 #

   202 class svn_source(converter_source):

   203     def __init__(self, ui, url, rev=None):

   204         super(svn_source, self).__init__(ui, url, rev=rev)

   205 

   206         if not (url.startswith('svn://') or url.startswith('svn+ssh://') or

   207                 (os.path.exists(url) and

   208                  os.path.exists(os.path.join(url, '.svn'))) or

   209                 issvnurl(ui, url)):

   210             raise NoRepo(_("%s does not look like a Subversion repository")

   211                          % url)

   212 

   213         try:

   214             SubversionException

   215         except NameError:

   216             raise MissingTool(_('Subversion python bindings could not be loaded'))

   217 

   218         try:

   219             version = svn.core.SVN_VER_MAJOR, svn.core.SVN_VER_MINOR

   220             if version < (1, 4):

   221                 raise MissingTool(_('Subversion python bindings %d.%d found, '

   222                                     '1.4 or later required') % version)

   223         except AttributeError:

   224             raise MissingTool(_('Subversion python bindings are too old, 1.4 '

   225                                 'or later required'))

   226 

   227         self.lastrevs = {}

   228 

   229         latest = None

   230         try:

   231             # Support file://path@rev syntax. Useful e.g. to convert

   232             # deleted branches.

   233             at = url.rfind('@')

   234             if at >= 0:

   235                 latest = int(url[at + 1:])

   236                 url = url[:at]

   237         except ValueError:

   238             pass

   239         self.url = geturl(url)

   240         self.encoding = 'UTF-8' # Subversion is always nominal UTF-8

   241         try:

   242             self.transport = transport.SvnRaTransport(url=self.url)

   243             self.ra = self.transport.ra

   244             self.ctx = self.transport.client

   245             self.baseurl = svn.ra.get_repos_root(self.ra)

   246             # Module is either empty or a repository path starting with

   247             # a slash and not ending with a slash.

   248             self.module = urllib.unquote(self.url[len(self.baseurl):])

   249             self.prevmodule = None

   250             self.rootmodule = self.module

   251             self.commits = {}

   252             self.paths = {}

   253             self.uuid = svn.ra.get_uuid(self.ra)

   254         except SubversionException:

   255             ui.traceback()

   256             raise NoRepo(_("%s does not look like a Subversion repository")

   257                          % self.url)

   258 

   259         if rev:

   260             try:

   261                 latest = int(rev)

   262             except ValueError:

   263                 raise util.Abort(_('svn: revision %s is not an integer') % rev)

   264 

   265         self.startrev = self.ui.config('convert', 'svn.startrev', default=0)

   266         try:

   267             self.startrev = int(self.startrev)

   268             if self.startrev < 0:

   269                 self.startrev = 0

   270         except ValueError:

   271             raise util.Abort(_('svn: start revision %s is not an integer')

   272                              % self.startrev)

   273 

   274         self.head = self.latest(self.module, latest)

   275         if not self.head:

   276             raise util.Abort(_('no revision found in module %s')

   277                              % self.module)

   278         self.last_changed = self.revnum(self.head)

   279 

   280         self._changescache = None

   281 

   282         if os.path.exists(os.path.join(url, '.svn/entries')):

   283             self.wc = url

   284         else:

   285             self.wc = None

   286         self.convertfp = None

   287 

   288     def setrevmap(self, revmap):

   289         lastrevs = {}

   290         for revid in revmap.iterkeys():

   291             uuid, module, revnum = self.revsplit(revid)

   292             lastrevnum = lastrevs.setdefault(module, revnum)

   293             if revnum > lastrevnum:

   294                 lastrevs[module] = revnum

   295         self.lastrevs = lastrevs

   296 

   297     def exists(self, path, optrev):

   298         try:

   299             svn.client.ls(self.url.rstrip('/') + '/' + urllib.quote(path),

   300                                  optrev, False, self.ctx)

   301             return True

   302         except SubversionException:

   303             return False

   304 

   305     def getheads(self):

   306 

   307         def isdir(path, revnum):

   308             kind = self._checkpath(path, revnum)

   309             return kind == svn.core.svn_node_dir

   310 

   311         def getcfgpath(name, rev):

   312             cfgpath = self.ui.config('convert', 'svn.' + name)

   313             if cfgpath is not None and cfgpath.strip() == '':

   314                 return None

   315             path = (cfgpath or name).strip('/')

   316             if not self.exists(path, rev):

   317                 if cfgpath:

   318                     raise util.Abort(_('expected %s to be at %r, but not found')

   319                                  % (name, path))

   320                 return None

   321             self.ui.note(_('found %s at %r\n') % (name, path))

   322             return path

   323 

   324         rev = optrev(self.last_changed)

   325         oldmodule = ''

   326         trunk = getcfgpath('trunk', rev)

   327         self.tags = getcfgpath('tags', rev)

   328         branches = getcfgpath('branches', rev)

   329 

   330         # If the project has a trunk or branches, we will extract heads

   331         # from them. We keep the project root otherwise.

   332         if trunk:

   333             oldmodule = self.module or ''

   334             self.module += '/' + trunk

   335             self.head = self.latest(self.module, self.last_changed)

   336             if not self.head:

   337                 raise util.Abort(_('no revision found in module %s')

   338                                  % self.module)

   339 

   340         # First head in the list is the module's head

   341         self.heads = [self.head]

   342         if self.tags is not None:

   343             self.tags = '%s/%s' % (oldmodule , (self.tags or 'tags'))

   344 

   345         # Check if branches bring a few more heads to the list

   346         if branches:

   347             rpath = self.url.strip('/')

   348             branchnames = svn.client.ls(rpath + '/' + urllib.quote(branches),

   349                                         rev, False, self.ctx)

   350             for branch in branchnames.keys():

   351                 module = '%s/%s/%s' % (oldmodule, branches, branch)

   352                 if not isdir(module, self.last_changed):

   353                     continue

   354                 brevid = self.latest(module, self.last_changed)

   355                 if not brevid:

   356                     self.ui.note(_('ignoring empty branch %s\n') % branch)

   357                     continue

   358                 self.ui.note(_('found branch %s at %d\n') %

   359                              (branch, self.revnum(brevid)))

   360                 self.heads.append(brevid)

   361 

   362         if self.startrev and self.heads:

   363             if len(self.heads) > 1:

   364                 raise util.Abort(_('svn: start revision is not supported '

   365                                    'with more than one branch'))

   366             revnum = self.revnum(self.heads[0])

   367             if revnum < self.startrev:

   368                 raise util.Abort(

   369                     _('svn: no revision found after start revision %d')

   370                                  % self.startrev)

   371 

   372         return self.heads

   373 

   374     def getchanges(self, rev):

   375         if self._changescache and self._changescache[0] == rev:

   376             return self._changescache[1]

   377         self._changescache = None

   378         (paths, parents) = self.paths[rev]

   379         if parents:

   380             files, self.removed, copies = self.expandpaths(rev, paths, parents)

   381         else:

   382             # Perform a full checkout on roots

   383             uuid, module, revnum = self.revsplit(rev)

   384             entries = svn.client.ls(self.baseurl + urllib.quote(module),

   385                                     optrev(revnum), True, self.ctx)

   386             files = [n for n, e in entries.iteritems()

   387                      if e.kind == svn.core.svn_node_file]

   388             copies = {}

   389             self.removed = set()

   390 

   391         files.sort()

   392         files = zip(files, [rev] * len(files))

   393 

   394         # caller caches the result, so free it here to release memory

   395         del self.paths[rev]

   396         return (files, copies)

   397 

   398     def getchangedfiles(self, rev, i):

   399         changes = self.getchanges(rev)

   400         self._changescache = (rev, changes)

   401         return [f[0] for f in changes[0]]

   402 

   403     def getcommit(self, rev):

   404         if rev not in self.commits:

   405             uuid, module, revnum = self.revsplit(rev)

   406             self.module = module

   407             self.reparent(module)

   408             # We assume that:

   409             # - requests for revisions after "stop" come from the

   410             # revision graph backward traversal. Cache all of them

   411             # down to stop, they will be used eventually.

   412             # - requests for revisions before "stop" come to get

   413             # isolated branches parents. Just fetch what is needed.

   414             stop = self.lastrevs.get(module, 0)

   415             if revnum < stop:

   416                 stop = revnum + 1

   417             self._fetch_revisions(revnum, stop)

   418         commit = self.commits[rev]

   419         # caller caches the result, so free it here to release memory

   420         del self.commits[rev]

   421         return commit

   422 

   423     def gettags(self):

   424         tags = {}

   425         if self.tags is None:

   426             return tags

   427 

   428         # svn tags are just a convention, project branches left in a

   429         # 'tags' directory. There is no other relationship than

   430         # ancestry, which is expensive to discover and makes them hard

   431         # to update incrementally.  Worse, past revisions may be

   432         # referenced by tags far away in the future, requiring a deep

   433         # history traversal on every calculation.  Current code

   434         # performs a single backward traversal, tracking moves within

   435         # the tags directory (tag renaming) and recording a new tag

   436         # everytime a project is copied from outside the tags

   437         # directory. It also lists deleted tags, this behaviour may

   438         # change in the future.

   439         pendings = []

   440         tagspath = self.tags

   441         start = svn.ra.get_latest_revnum(self.ra)

   442         stream = self._getlog([self.tags], start, self.startrev)

   443         try:

   444             for entry in stream:

   445                 origpaths, revnum, author, date, message = entry

   446                 copies = [(e.copyfrom_path, e.copyfrom_rev, p) for p, e

   447                           in origpaths.iteritems() if e.copyfrom_path]

   448                 # Apply moves/copies from more specific to general

   449                 copies.sort(reverse=True)

   450 

   451                 srctagspath = tagspath

   452                 if copies and copies[-1][2] == tagspath:

   453                     # Track tags directory moves

   454                     srctagspath = copies.pop()[0]

   455 

   456                 for source, sourcerev, dest in copies:

   457                     if not dest.startswith(tagspath + '/'):

   458                         continue

   459                     for tag in pendings:

   460                         if tag[0].startswith(dest):

   461                             tagpath = source + tag[0][len(dest):]

   462                             tag[:2] = [tagpath, sourcerev]

   463                             break

   464                     else:

   465                         pendings.append([source, sourcerev, dest])

   466 

   467                 # Filter out tags with children coming from different

   468                 # parts of the repository like:

   469                 # /tags/tag.1 (from /trunk:10)

   470                 # /tags/tag.1/foo (from /branches/foo:12)

   471                 # Here/tags/tag.1 discarded as well as its children.

   472                 # It happens with tools like cvs2svn. Such tags cannot

   473                 # be represented in mercurial.

   474                 addeds = dict((p, e.copyfrom_path) for p, e

   475                               in origpaths.iteritems()

   476                               if e.action == 'A' and e.copyfrom_path)

   477                 badroots = set()

   478                 for destroot in addeds:

   479                     for source, sourcerev, dest in pendings:

   480                         if (not dest.startswith(destroot + '/')

   481                             or source.startswith(addeds[destroot] + '/')):

   482                             continue

   483                         badroots.add(destroot)

   484                         break

   485 

   486                 for badroot in badroots:

   487                     pendings = [p for p in pendings if p[2] != badroot

   488                                 and not p[2].startswith(badroot + '/')]

   489 

   490                 # Tell tag renamings from tag creations

   491                 remainings = []

   492                 for source, sourcerev, dest in pendings:

   493                     tagname = dest.split('/')[-1]

   494                     if source.startswith(srctagspath):

   495                         remainings.append([source, sourcerev, tagname])

   496                         continue

   497                     if tagname in tags:

   498                         # Keep the latest tag value

   499                         continue

   500                     # From revision may be fake, get one with changes

   501                     try:

   502                         tagid = self.latest(source, sourcerev)

   503                         if tagid and tagname not in tags:

   504                             tags[tagname] = tagid

   505                     except SvnPathNotFound:

   506                         # It happens when we are following directories

   507                         # we assumed were copied with their parents

   508                         # but were really created in the tag

   509                         # directory.

   510                         pass

   511                 pendings = remainings

   512                 tagspath = srctagspath

   513         finally:

   514             stream.close()

   515         return tags

   516 

   517     def converted(self, rev, destrev):

   518         if not self.wc:

   519             return

   520         if self.convertfp is None:

   521             self.convertfp = open(os.path.join(self.wc, '.svn', 'hg-shamap'),

   522                                   'a')

   523         self.convertfp.write('%s %d\n' % (destrev, self.revnum(rev)))

   524         self.convertfp.flush()

   525 

   526     def revid(self, revnum, module=None):

   527         return 'svn:%s%s@%s' % (self.uuid, module or self.module, revnum)

   528 

   529     def revnum(self, rev):

   530         return int(rev.split('@')[-1])

   531 

   532     def revsplit(self, rev):

   533         url, revnum = rev.rsplit('@', 1)

   534         revnum = int(revnum)

   535         parts = url.split('/', 1)

   536         uuid = parts.pop(0)[4:]

   537         mod = ''

   538         if parts:

   539             mod = '/' + parts[0]

   540         return uuid, mod, revnum

   541 

   542     def latest(self, path, stop=0):

   543         """Find the latest revid affecting path, up to stop. It may return

   544         a revision in a different module, since a branch may be moved without

   545         a change being reported. Return None if computed module does not

   546         belong to rootmodule subtree.

   547         """

   548         if not path.startswith(self.rootmodule):

   549             # Requests on foreign branches may be forbidden at server level

   550             self.ui.debug('ignoring foreign branch %r\n' % path)

   551             return None

   552 

   553         if not stop:

   554             stop = svn.ra.get_latest_revnum(self.ra)

   555         try:

   556             prevmodule = self.reparent('')

   557             dirent = svn.ra.stat(self.ra, path.strip('/'), stop)

   558             self.reparent(prevmodule)

   559         except SubversionException:

   560             dirent = None

   561         if not dirent:

   562             raise SvnPathNotFound(_('%s not found up to revision %d')

   563                                   % (path, stop))

   564 

   565         # stat() gives us the previous revision on this line of

   566         # development, but it might be in *another module*. Fetch the

   567         # log and detect renames down to the latest revision.

   568         stream = self._getlog([path], stop, dirent.created_rev)

   569         try:

   570             for entry in stream:

   571                 paths, revnum, author, date, message = entry

   572                 if revnum <= dirent.created_rev:

   573                     break

   574 

   575                 for p in paths:

   576                     if not path.startswith(p) or not paths[p].copyfrom_path:

   577                         continue

   578                     newpath = paths[p].copyfrom_path + path[len(p):]

   579                     self.ui.debug("branch renamed from %s to %s at %d\n" %

   580                                   (path, newpath, revnum))

   581                     path = newpath

   582                     break

   583         finally:

   584             stream.close()

   585 

   586         if not path.startswith(self.rootmodule):

   587             self.ui.debug('ignoring foreign branch %r\n' % path)

   588             return None

   589         return self.revid(dirent.created_rev, path)

   590 

   591     def reparent(self, module):

   592         """Reparent the svn transport and return the previous parent."""

   593         if self.prevmodule == module:

   594             return module

   595         svnurl = self.baseurl + urllib.quote(module)

   596         prevmodule = self.prevmodule

   597         if prevmodule is None:

   598             prevmodule = ''

   599         self.ui.debug("reparent to %s\n" % svnurl)

   600         svn.ra.reparent(self.ra, svnurl)

   601         self.prevmodule = module

   602         return prevmodule

   603 

   604     def expandpaths(self, rev, paths, parents):

   605         changed, removed = set(), set()

   606         copies = {}

   607 

   608         new_module, revnum = self.revsplit(rev)[1:]

   609         if new_module != self.module:

   610             self.module = new_module

   611             self.reparent(self.module)

   612 

   613         for i, (path, ent) in enumerate(paths):

   614             self.ui.progress(_('scanning paths'), i, item=path,

   615                              total=len(paths))

   616             entrypath = self.getrelpath(path)

   617 

   618             kind = self._checkpath(entrypath, revnum)

   619             if kind == svn.core.svn_node_file:

   620                 changed.add(self.recode(entrypath))

   621                 if not ent.copyfrom_path or not parents:

   622                     continue

   623                 # Copy sources not in parent revisions cannot be

   624                 # represented, ignore their origin for now

   625                 pmodule, prevnum = self.revsplit(parents[0])[1:]

   626                 if ent.copyfrom_rev < prevnum:

   627                     continue

   628                 copyfrom_path = self.getrelpath(ent.copyfrom_path, pmodule)

   629                 if not copyfrom_path:

   630                     continue

   631                 self.ui.debug("copied to %s from %s@%s\n" %

   632                               (entrypath, copyfrom_path, ent.copyfrom_rev))

   633                 copies[self.recode(entrypath)] = self.recode(copyfrom_path)

   634             elif kind == 0: # gone, but had better be a deleted *file*

   635                 self.ui.debug("gone from %s\n" % ent.copyfrom_rev)

   636                 pmodule, prevnum = self.revsplit(parents[0])[1:]

   637                 parentpath = pmodule + "/" + entrypath

   638                 fromkind = self._checkpath(entrypath, prevnum, pmodule)

   639 

   640                 if fromkind == svn.core.svn_node_file:

   641                     removed.add(self.recode(entrypath))

   642                 elif fromkind == svn.core.svn_node_dir:

   643                     oroot = parentpath.strip('/')

   644                     nroot = path.strip('/')

   645                     children = self._iterfiles(oroot, prevnum)

   646                     for childpath in children:

   647                         childpath = childpath.replace(oroot, nroot)

   648                         childpath = self.getrelpath("/" + childpath, pmodule)

   649                         if childpath:

   650                             removed.add(self.recode(childpath))

   651                 else:

   652                     self.ui.debug('unknown path in revision %d: %s\n' % \

   653                                   (revnum, path))

   654             elif kind == svn.core.svn_node_dir:

   655                 if ent.action == 'M':

   656                     # If the directory just had a prop change,

   657                     # then we shouldn't need to look for its children.

   658                     continue

   659                 if ent.action == 'R' and parents:

   660                     # If a directory is replacing a file, mark the previous

   661                     # file as deleted

   662                     pmodule, prevnum = self.revsplit(parents[0])[1:]

   663                     pkind = self._checkpath(entrypath, prevnum, pmodule)

   664                     if pkind == svn.core.svn_node_file:

   665                         removed.add(self.recode(entrypath))

   666                     elif pkind == svn.core.svn_node_dir:

   667                         # We do not know what files were kept or removed,

   668                         # mark them all as changed.

   669                         for childpath in self._iterfiles(pmodule, prevnum):

   670                             childpath = self.getrelpath("/" + childpath)

   671                             if childpath:

   672                                 changed.add(self.recode(childpath))

   673 

   674                 for childpath in self._iterfiles(path, revnum):

   675                     childpath = self.getrelpath("/" + childpath)

   676                     if childpath:

   677                         changed.add(self.recode(childpath))

   678 

   679                 # Handle directory copies

   680                 if not ent.copyfrom_path or not parents:

   681                     continue

   682                 # Copy sources not in parent revisions cannot be

   683                 # represented, ignore their origin for now

   684                 pmodule, prevnum = self.revsplit(parents[0])[1:]

   685                 if ent.copyfrom_rev < prevnum:

   686                     continue

   687                 copyfrompath = self.getrelpath(ent.copyfrom_path, pmodule)

   688                 if not copyfrompath:

   689                     continue

   690                 self.ui.debug("mark %s came from %s:%d\n"

   691                               % (path, copyfrompath, ent.copyfrom_rev))

   692                 children = self._iterfiles(ent.copyfrom_path, ent.copyfrom_rev)

   693                 for childpath in children:

   694                     childpath = self.getrelpath("/" + childpath, pmodule)

   695                     if not childpath:

   696                         continue

   697                     copytopath = path + childpath[len(copyfrompath):]

   698                     copytopath = self.getrelpath(copytopath)

   699                     copies[self.recode(copytopath)] = self.recode(childpath)

   700 

   701         self.ui.progress(_('scanning paths'), None)

   702         changed.update(removed)

   703         return (list(changed), removed, copies)

   704 

   705     def _fetch_revisions(self, from_revnum, to_revnum):

   706         if from_revnum < to_revnum:

   707             from_revnum, to_revnum = to_revnum, from_revnum

   708 

   709         self.child_cset = None

   710 

   711         def parselogentry(orig_paths, revnum, author, date, message):

   712             """Return the parsed commit object or None, and True if

   713             the revision is a branch root.

   714             """

   715             self.ui.debug("parsing revision %d (%d changes)\n" %

   716                           (revnum, len(orig_paths)))

   717 

   718             branched = False

   719             rev = self.revid(revnum)

   720             # branch log might return entries for a parent we already have

   721 

   722             if rev in self.commits or revnum < to_revnum:

   723                 return None, branched

   724 

   725             parents = []

   726             # check whether this revision is the start of a branch or part

   727             # of a branch renaming

   728             orig_paths = sorted(orig_paths.iteritems())

   729             root_paths = [(p, e) for p, e in orig_paths

   730                           if self.module.startswith(p)]

   731             if root_paths:

   732                 path, ent = root_paths[-1]

   733                 if ent.copyfrom_path:

   734                     branched = True

   735                     newpath = ent.copyfrom_path + self.module[len(path):]

   736                     # ent.copyfrom_rev may not be the actual last revision

   737                     previd = self.latest(newpath, ent.copyfrom_rev)

   738                     if previd is not None:

   739                         prevmodule, prevnum = self.revsplit(previd)[1:]

   740                         if prevnum >= self.startrev:

   741                             parents = [previd]

   742                             self.ui.note(

   743                                 _('found parent of branch %s at %d: %s\n') %

   744                                 (self.module, prevnum, prevmodule))

   745                 else:

   746                     self.ui.debug("no copyfrom path, don't know what to do.\n")

   747 

   748             paths = []

   749             # filter out unrelated paths

   750             for path, ent in orig_paths:

   751                 if self.getrelpath(path) is None:

   752                     continue

   753                 paths.append((path, ent))

   754 

   755             # Example SVN datetime. Includes microseconds.

   756             # ISO-8601 conformant

   757             # '2007-01-04T17:35:00.902377Z'

   758             date = util.parsedate(date[:19] + " UTC", ["%Y-%m-%dT%H:%M:%S"])

   759 

   760             log = message and self.recode(message) or ''

   761             author = author and self.recode(author) or ''

   762             try:

   763                 branch = self.module.split("/")[-1]

   764                 if branch == 'trunk':

   765                     branch = ''

   766             except IndexError:

   767                 branch = None

   768 

   769             cset = commit(author=author,

   770                           date=util.datestr(date),

   771                           desc=log,

   772                           parents=parents,

   773                           branch=branch,

   774                           rev=rev)

   775 

   776             self.commits[rev] = cset

   777             # The parents list is *shared* among self.paths and the

   778             # commit object. Both will be updated below.

   779             self.paths[rev] = (paths, cset.parents)

   780             if self.child_cset and not self.child_cset.parents:

   781                 self.child_cset.parents[:] = [rev]

   782             self.child_cset = cset

   783             return cset, branched

   784 

   785         self.ui.note(_('fetching revision log for "%s" from %d to %d\n') %

   786                      (self.module, from_revnum, to_revnum))

   787 

   788         try:

   789             firstcset = None

   790             lastonbranch = False

   791             stream = self._getlog([self.module], from_revnum, to_revnum)

   792             try:

   793                 for entry in stream:

   794                     paths, revnum, author, date, message = entry

   795                     if revnum < self.startrev:

   796                         lastonbranch = True

   797                         break

   798                     if not paths:

   799                         self.ui.debug('revision %d has no entries\n' % revnum)

   800                         # If we ever leave the loop on an empty

   801                         # revision, do not try to get a parent branch

   802                         lastonbranch = lastonbranch or revnum == 0

   803                         continue

   804                     cset, lastonbranch = parselogentry(paths, revnum, author,

   805                                                        date, message)

   806                     if cset:

   807                         firstcset = cset

   808                     if lastonbranch:

   809                         break

   810             finally:

   811                 stream.close()

   812 

   813             if not lastonbranch and firstcset and not firstcset.parents:

   814                 # The first revision of the sequence (the last fetched one)

   815                 # has invalid parents if not a branch root. Find the parent

   816                 # revision now, if any.

   817                 try:

   818                     firstrevnum = self.revnum(firstcset.rev)

   819                     if firstrevnum > 1:

   820                         latest = self.latest(self.module, firstrevnum - 1)

   821                         if latest:

   822                             firstcset.parents.append(latest)

   823                 except SvnPathNotFound:

   824                     pass

   825         except SubversionException, (inst, num):

   826             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:

   827                 raise util.Abort(_('svn: branch has no revision %s') % to_revnum)

   828             raise

   829 

   830     def getfile(self, file, rev):

   831         # TODO: ra.get_file transmits the whole file instead of diffs.

   832         if file in self.removed:

   833             raise IOError()

   834         mode = ''

   835         try:

   836             new_module, revnum = self.revsplit(rev)[1:]

   837             if self.module != new_module:

   838                 self.module = new_module

   839                 self.reparent(self.module)

   840             io = StringIO()

   841             info = svn.ra.get_file(self.ra, file, revnum, io)

   842             data = io.getvalue()

   843             # ra.get_files() seems to keep a reference on the input buffer

   844             # preventing collection. Release it explicitely.

   845             io.close()

   846             if isinstance(info, list):

   847                 info = info[-1]

   848             mode = ("svn:executable" in info) and 'x' or ''

   849             mode = ("svn:special" in info) and 'l' or mode

   850         except SubversionException, e:

   851             notfound = (svn.core.SVN_ERR_FS_NOT_FOUND,

   852                 svn.core.SVN_ERR_RA_DAV_PATH_NOT_FOUND)

   853             if e.apr_err in notfound: # File not found

   854                 raise IOError()

   855             raise

   856         if mode == 'l':

   857             link_prefix = "link "

   858             if data.startswith(link_prefix):

   859                 data = data[len(link_prefix):]

   860         return data, mode

   861 

   862     def _iterfiles(self, path, revnum):

   863         """Enumerate all files in path at revnum, recursively."""

   864         path = path.strip('/')

   865         pool = Pool()

   866         rpath = '/'.join([self.baseurl, urllib.quote(path)]).strip('/')

   867         entries = svn.client.ls(rpath, optrev(revnum), True, self.ctx, pool)

   868         return ((path + '/' + p) for p, e in entries.iteritems()

   869                 if e.kind == svn.core.svn_node_file)

   870 

   871     def getrelpath(self, path, module=None):

   872         if module is None:

   873             module = self.module

   874         # Given the repository url of this wc, say

   875         #   "http://server/plone/CMFPlone/branches/Plone-2_0-branch"

   876         # extract the "entry" portion (a relative path) from what

   877         # svn log --xml says, ie

   878         #   "/CMFPlone/branches/Plone-2_0-branch/tests/PloneTestCase.py"

   879         # that is to say "tests/PloneTestCase.py"

   880         if path.startswith(module):

   881             relative = path.rstrip('/')[len(module):]

   882             if relative.startswith('/'):

   883                 return relative[1:]

   884             elif relative == '':

   885                 return relative

   886 

   887         # The path is outside our tracked tree...

   888         self.ui.debug('%r is not under %r, ignoring\n' % (path, module))

   889         return None

   890 

   891     def _checkpath(self, path, revnum, module=None):

   892         if module is not None:

   893             prevmodule = self.reparent('')

   894             path = module + '/' + path

   895         try:

   896             # ra.check_path does not like leading slashes very much, it leads

   897             # to PROPFIND subversion errors

   898             return svn.ra.check_path(self.ra, path.strip('/'), revnum)

   899         finally:

   900             if module is not None:

   901                 self.reparent(prevmodule)

   902 

   903     def _getlog(self, paths, start, end, limit=0, discover_changed_paths=True,

   904                 strict_node_history=False):

   905         # Normalize path names, svn >= 1.5 only wants paths relative to

   906         # supplied URL

   907         relpaths = []

   908         for p in paths:

   909             if not p.startswith('/'):

   910                 p = self.module + '/' + p

   911             relpaths.append(p.strip('/'))

   912         args = [self.baseurl, relpaths, start, end, limit, discover_changed_paths,

   913                 strict_node_history]

   914         arg = encodeargs(args)

   915         hgexe = util.hgexecutable()

   916         cmd = '%s debugsvnlog' % util.shellquote(hgexe)

   917         stdin, stdout = util.popen2(cmd)

   918         stdin.write(arg)

   919         try:

   920             stdin.close()

   921         except IOError:

   922             raise util.Abort(_('Mercurial failed to run itself, check'

   923                                ' hg executable is in PATH'))

   924         return logstream(stdout)

   925 

   926 pre_revprop_change = '''#!/bin/sh

   927 

   928 REPOS="$1"

   929 REV="$2"

   930 USER="$3"

   931 PROPNAME="$4"

   932 ACTION="$5"

   933 

   934 if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:log" ]; then exit 0; fi

   935 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-branch" ]; then exit 0; fi

   936 if [ "$ACTION" = "A" -a "$PROPNAME" = "hg:convert-rev" ]; then exit 0; fi

   937 

   938 echo "Changing prohibited revision property" >&2

   939 exit 1

   940 '''

   941 

   942 class svn_sink(converter_sink, commandline):

   943     commit_re = re.compile(r'Committed revision (\d+).', re.M)

   944 

   945     def prerun(self):

   946         if self.wc:

   947             os.chdir(self.wc)

   948 

   949     def postrun(self):

   950         if self.wc:

   951             os.chdir(self.cwd)

   952 

   953     def join(self, name):

   954         return os.path.join(self.wc, '.svn', name)

   955 

   956     def revmapfile(self):

   957         return self.join('hg-shamap')

   958 

   959     def authorfile(self):

   960         return self.join('hg-authormap')

   961 

   962     def __init__(self, ui, path):

   963         converter_sink.__init__(self, ui, path)

   964         commandline.__init__(self, ui, 'svn')

   965         self.delete = []

   966         self.setexec = []

   967         self.delexec = []

   968         self.copies = []

   969         self.wc = None

   970         self.cwd = os.getcwd()

   971 

   972         path = os.path.realpath(path)

   973 

   974         created = False

   975         if os.path.isfile(os.path.join(path, '.svn', 'entries')):

   976             self.wc = path

   977             self.run0('update')

   978         else:

   979             wcpath = os.path.join(os.getcwd(), os.path.basename(path) + '-wc')

   980 

   981             if os.path.isdir(os.path.dirname(path)):

   982                 if not os.path.exists(os.path.join(path, 'db', 'fs-type')):

   983                     ui.status(_('initializing svn repository %r\n') %

   984                               os.path.basename(path))

   985                     commandline(ui, 'svnadmin').run0('create', path)

   986                     created = path

   987                 path = util.normpath(path)

   988                 if not path.startswith('/'):

   989                     path = '/' + path

   990                 path = 'file://' + path

   991 

   992             ui.status(_('initializing svn working copy %r\n')

   993                       % os.path.basename(wcpath))

   994             self.run0('checkout', path, wcpath)

   995 

   996             self.wc = wcpath

   997         self.opener = util.opener(self.wc)

   998         self.wopener = util.opener(self.wc)

   999         self.childmap = mapfile(ui, self.join('hg-childmap'))

  1000         self.is_exec = util.checkexec(self.wc) and util.is_exec or None

  1001 

  1002         if created:

  1003             hook = os.path.join(created, 'hooks', 'pre-revprop-change')

  1004             fp = open(hook, 'w')

  1005             fp.write(pre_revprop_change)

  1006             fp.close()

  1007             util.set_flags(hook, False, True)

  1008 

  1009         xport = transport.SvnRaTransport(url=geturl(path))

  1010         self.uuid = svn.ra.get_uuid(xport.ra)

  1011 

  1012     def wjoin(self, *names):

  1013         return os.path.join(self.wc, *names)

  1014 

  1015     def putfile(self, filename, flags, data):

  1016         if 'l' in flags:

  1017             self.wopener.symlink(data, filename)

  1018         else:

  1019             try:

  1020                 if os.path.islink(self.wjoin(filename)):

  1021                     os.unlink(filename)

  1022             except OSError:

  1023                 pass

  1024             self.wopener(filename, 'w').write(data)

  1025 

  1026             if self.is_exec:

  1027                 was_exec = self.is_exec(self.wjoin(filename))

  1028             else:

  1029                 # On filesystems not supporting execute-bit, there is no way

  1030                 # to know if it is set but asking subversion. Setting it

  1031                 # systematically is just as expensive and much simpler.

  1032                 was_exec = 'x' not in flags

  1033 

  1034             util.set_flags(self.wjoin(filename), False, 'x' in flags)

  1035             if was_exec:

  1036                 if 'x' not in flags:

  1037                     self.delexec.append(filename)

  1038             else:

  1039                 if 'x' in flags:

  1040                     self.setexec.append(filename)

  1041 

  1042     def _copyfile(self, source, dest):

  1043         # SVN's copy command pukes if the destination file exists, but

  1044         # our copyfile method expects to record a copy that has

  1045         # already occurred.  Cross the semantic gap.

  1046         wdest = self.wjoin(dest)

  1047         exists = os.path.lexists(wdest)

  1048         if exists:

  1049             fd, tempname = tempfile.mkstemp(

  1050                 prefix='hg-copy-', dir=os.path.dirname(wdest))

  1051             os.close(fd)

  1052             os.unlink(tempname)

  1053             os.rename(wdest, tempname)

  1054         try:

  1055             self.run0('copy', source, dest)

  1056         finally:

  1057             if exists:

  1058                 try:

  1059                     os.unlink(wdest)

  1060                 except OSError:

  1061                     pass

  1062                 os.rename(tempname, wdest)

  1063 

  1064     def dirs_of(self, files):

  1065         dirs = set()

  1066         for f in files:

  1067             if os.path.isdir(self.wjoin(f)):

  1068                 dirs.add(f)

  1069             for i in strutil.rfindall(f, '/'):

  1070                 dirs.add(f[:i])

  1071         return dirs

  1072 

  1073     def add_dirs(self, files):

  1074         add_dirs = [d for d in sorted(self.dirs_of(files))

  1075                     if not os.path.exists(self.wjoin(d, '.svn', 'entries'))]

  1076         if add_dirs:

  1077             self.xargs(add_dirs, 'add', non_recursive=True, quiet=True)

  1078         return add_dirs

  1079 

  1080     def add_files(self, files):

  1081         if files:

  1082             self.xargs(files, 'add', quiet=True)

  1083         return files

  1084 

  1085     def tidy_dirs(self, names):

  1086         deleted = []

  1087         for d in sorted(self.dirs_of(names), reverse=True):

  1088             wd = self.wjoin(d)

  1089             if os.listdir(wd) == '.svn':

  1090                 self.run0('delete', d)

  1091                 deleted.append(d)

  1092         return deleted

  1093 

  1094     def addchild(self, parent, child):

  1095         self.childmap[parent] = child

  1096 

  1097     def revid(self, rev):

  1098         return u"svn:%s@%s" % (self.uuid, rev)

  1099 

  1100     def putcommit(self, files, copies, parents, commit, source, revmap):

  1101         # Apply changes to working copy

  1102         for f, v in files:

  1103             try:

  1104                 data, mode = source.getfile(f, v)

  1105             except IOError:

  1106                 self.delete.append(f)

  1107             else:

  1108                 self.putfile(f, mode, data)

  1109                 if f in copies:

  1110                     self.copies.append([copies[f], f])

  1111         files = [f[0] for f in files]

  1112 

  1113         for parent in parents:

  1114             try:

  1115                 return self.revid(self.childmap[parent])

  1116             except KeyError:

  1117                 pass

  1118         entries = set(self.delete)

  1119         files = frozenset(files)

  1120         entries.update(self.add_dirs(files.difference(entries)))

  1121         if self.copies:

  1122             for s, d in self.copies:

  1123                 self._copyfile(s, d)

  1124             self.copies = []

  1125         if self.delete:

  1126             self.xargs(self.delete, 'delete')

  1127             self.delete = []

  1128         entries.update(self.add_files(files.difference(entries)))

  1129         entries.update(self.tidy_dirs(entries))

  1130         if self.delexec:

  1131             self.xargs(self.delexec, 'propdel', 'svn:executable')

  1132             self.delexec = []

  1133         if self.setexec:

  1134             self.xargs(self.setexec, 'propset', 'svn:executable', '*')

  1135             self.setexec = []

  1136