1 # convcmd - convert extension commands definition

     2 #

     3 # Copyright 2005-2007 Matt Mackall <mpm@selenic.com>

     4 #

     5 # This software may be used and distributed according to the terms of the

     6 # GNU General Public License version 2 or any later version.

     7 

     8 from common import NoRepo, MissingTool, SKIPREV, mapfile

     9 from cvs import convert_cvs

    10 from darcs import darcs_source

    11 from git import convert_git

    12 from hg import mercurial_source, mercurial_sink

    13 from subversion import svn_source, svn_sink

    14 from monotone import monotone_source

    15 from gnuarch import gnuarch_source

    16 from bzr import bzr_source

    17 from p4 import p4_source

    18 import filemap

    19 

    20 import os, shutil

    21 from mercurial import hg, util, encoding

    22 from mercurial.i18n import _

    23 

    24 orig_encoding = 'ascii'

    25 

    26 def recode(s):

    27     if isinstance(s, unicode):

    28         return s.encode(orig_encoding, 'replace')

    29     else:

    30         return s.decode('utf-8').encode(orig_encoding, 'replace')

    31 

    32 source_converters = [

    33     ('cvs', convert_cvs, 'branchsort'),

    34     ('git', convert_git, 'branchsort'),

    35     ('svn', svn_source, 'branchsort'),

    36     ('hg', mercurial_source, 'sourcesort'),

    37     ('darcs', darcs_source, 'branchsort'),

    38     ('mtn', monotone_source, 'branchsort'),

    39     ('gnuarch', gnuarch_source, 'branchsort'),

    40     ('bzr', bzr_source, 'branchsort'),

    41     ('p4', p4_source, 'branchsort'),

    42     ]

    43 

    44 sink_converters = [

    45     ('hg', mercurial_sink),

    46     ('svn', svn_sink),

    47     ]

    48 

    49 def convertsource(ui, path, type, rev):

    50     exceptions = []

    51     if type and type not in [s[0] for s in source_converters]:

    52         raise util.Abort(_('%s: invalid source repository type') % type)

    53     for name, source, sortmode in source_converters:

    54         try:

    55             if not type or name == type:

    56                 return source(ui, path, rev), sortmode

    57         except (NoRepo, MissingTool), inst:

    58             exceptions.append(inst)

    59     if not ui.quiet:

    60         for inst in exceptions:

    61             ui.write("%s\n" % inst)

    62     raise util.Abort(_('%s: missing or unsupported repository') % path)

    63 

    64 def convertsink(ui, path, type):

    65     if type and type not in [s[0] for s in sink_converters]:

    66         raise util.Abort(_('%s: invalid destination repository type') % type)

    67     for name, sink in sink_converters:

    68         try:

    69             if not type or name == type:

    70                 return sink(ui, path)

    71         except NoRepo, inst:

    72             ui.note(_("convert: %s\n") % inst)

    73     raise util.Abort(_('%s: unknown repository type') % path)

    74 

    75 class progresssource(object):

    76     def __init__(self, ui, source, filecount):

    77         self.ui = ui

    78         self.source = source

    79         self.filecount = filecount

    80         self.retrieved = 0

    81 

    82     def getfile(self, file, rev):

    83         self.retrieved += 1

    84         self.ui.progress(_('getting files'), self.retrieved,

    85                          item=file, total=self.filecount)

    86         return self.source.getfile(file, rev)

    87 

    88     def lookuprev(self, rev):

    89         return self.source.lookuprev(rev)

    90 

    91     def close(self):

    92         self.ui.progress(_('getting files'), None)

    93 

    94 class converter(object):

    95     def __init__(self, ui, source, dest, revmapfile, opts):

    96 

    97         self.source = source

    98         self.dest = dest

    99         self.ui = ui

   100         self.opts = opts

   101         self.commitcache = {}

   102         self.authors = {}

   103         self.authorfile = None

   104 

   105         # Record converted revisions persistently: maps source revision

   106         # ID to target revision ID (both strings).  (This is how

   107         # incremental conversions work.)

   108         self.map = mapfile(ui, revmapfile)

   109 

   110         # Read first the dst author map if any

   111         authorfile = self.dest.authorfile()

   112         if authorfile and os.path.exists(authorfile):

   113             self.readauthormap(authorfile)

   114         # Extend/Override with new author map if necessary

   115         if opts.get('authormap'):

   116             self.readauthormap(opts.get('authormap'))

   117             self.authorfile = self.dest.authorfile()

   118 

   119         self.splicemap = mapfile(ui, opts.get('splicemap'))

   120         self.branchmap = mapfile(ui, opts.get('branchmap'))

   121 

   122     def walktree(self, heads):

   123         '''Return a mapping that identifies the uncommitted parents of every

   124         uncommitted changeset.'''

   125         visit = heads

   126         known = set()

   127         parents = {}

   128         while visit:

   129             n = visit.pop(0)

   130             if n in known or n in self.map:

   131                 continue

   132             known.add(n)

   133             self.ui.progress(_('scanning'), len(known), unit=_('revisions'))

   134             commit = self.cachecommit(n)

   135             parents[n] = []

   136             for p in commit.parents:

   137                 parents[n].append(p)

   138                 visit.append(p)

   139         self.ui.progress(_('scanning'), None)

   140 

   141         return parents

   142 

   143     def toposort(self, parents, sortmode):

   144         '''Return an ordering such that every uncommitted changeset is

   145         preceeded by all its uncommitted ancestors.'''

   146 

   147         def mapchildren(parents):

   148             """Return a (children, roots) tuple where 'children' maps parent

   149             revision identifiers to children ones, and 'roots' is the list of

   150             revisions without parents. 'parents' must be a mapping of revision

   151             identifier to its parents ones.

   152             """

   153             visit = parents.keys()

   154             seen = set()

   155             children = {}

   156             roots = []

   157 

   158             while visit:

   159                 n = visit.pop(0)

   160                 if n in seen:

   161                     continue

   162                 seen.add(n)

   163                 # Ensure that nodes without parents are present in the

   164                 # 'children' mapping.

   165                 children.setdefault(n, [])

   166                 hasparent = False

   167                 for p in parents[n]:

   168                     if not p in self.map:

   169                         visit.append(p)

   170                         hasparent = True

   171                     children.setdefault(p, []).append(n)

   172                 if not hasparent:

   173                     roots.append(n)

   174 

   175             return children, roots

   176 

   177         # Sort functions are supposed to take a list of revisions which

   178         # can be converted immediately and pick one

   179 

   180         def makebranchsorter():

   181             """If the previously converted revision has a child in the

   182             eligible revisions list, pick it. Return the list head

   183             otherwise. Branch sort attempts to minimize branch

   184             switching, which is harmful for Mercurial backend

   185             compression.

   186             """

   187             prev = [None]

   188             def picknext(nodes):

   189                 next = nodes[0]

   190                 for n in nodes:

   191                     if prev[0] in parents[n]:

   192                         next = n

   193                         break

   194                 prev[0] = next

   195                 return next

   196             return picknext

   197 

   198         def makesourcesorter():

   199             """Source specific sort."""

   200             keyfn = lambda n: self.commitcache[n].sortkey

   201             def picknext(nodes):

   202                 return sorted(nodes, key=keyfn)[0]

   203             return picknext

   204 

   205         def makedatesorter():

   206             """Sort revisions by date."""

   207             dates = {}

   208             def getdate(n):

   209                 if n not in dates:

   210                     dates[n] = util.parsedate(self.commitcache[n].date)

   211                 return dates[n]

   212 

   213             def picknext(nodes):

   214                 return min([(getdate(n), n) for n in nodes])[1]

   215 

   216             return picknext

   217 

   218         if sortmode == 'branchsort':

   219             picknext = makebranchsorter()

   220         elif sortmode == 'datesort':

   221             picknext = makedatesorter()

   222         elif sortmode == 'sourcesort':

   223             picknext = makesourcesorter()

   224         else:

   225             raise util.Abort(_('unknown sort mode: %s') % sortmode)

   226 

   227         children, actives = mapchildren(parents)

   228 

   229         s = []

   230         pendings = {}

   231         while actives:

   232             n = picknext(actives)

   233             actives.remove(n)

   234             s.append(n)

   235 

   236             # Update dependents list

   237             for c in children.get(n, []):

   238                 if c not in pendings:

   239                     pendings[c] = [p for p in parents[c] if p not in self.map]

   240                 try:

   241                     pendings[c].remove(n)

   242                 except ValueError:

   243                     raise util.Abort(_('cycle detected between %s and %s')

   244                                        % (recode(c), recode(n)))

   245                 if not pendings[c]:

   246                     # Parents are converted, node is eligible

   247                     actives.insert(0, c)

   248                     pendings[c] = None

   249 

   250         if len(s) != len(parents):

   251             raise util.Abort(_("not all revisions were sorted"))

   252 

   253         return s

   254 

   255     def writeauthormap(self):

   256         authorfile = self.authorfile

   257         if authorfile:

   258             self.ui.status(_('Writing author map file %s\n') % authorfile)

   259             ofile = open(authorfile, 'w+')

   260             for author in self.authors:

   261                 ofile.write("%s=%s\n" % (author, self.authors[author]))

   262             ofile.close()

   263 

   264     def readauthormap(self, authorfile):

   265         afile = open(authorfile, 'r')

   266         for line in afile:

   267 

   268             line = line.strip()

   269             if not line or line.startswith('#'):

   270                 continue

   271 

   272             try:

   273                 srcauthor, dstauthor = line.split('=', 1)

   274             except ValueError:

   275                 msg = _('Ignoring bad line in author map file %s: %s\n')

   276                 self.ui.warn(msg % (authorfile, line.rstrip()))

   277                 continue

   278 

   279             srcauthor = srcauthor.strip()

   280             dstauthor = dstauthor.strip()

   281             if self.authors.get(srcauthor) in (None, dstauthor):

   282                 msg = _('mapping author %s to %s\n')

   283                 self.ui.debug(msg % (srcauthor, dstauthor))

   284                 self.authors[srcauthor] = dstauthor

   285                 continue

   286 

   287             m = _('overriding mapping for author %s, was %s, will be %s\n')

   288             self.ui.status(m % (srcauthor, self.authors[srcauthor], dstauthor))

   289 

   290         afile.close()

   291 

   292     def cachecommit(self, rev):

   293         commit = self.source.getcommit(rev)

   294         commit.author = self.authors.get(commit.author, commit.author)

   295         commit.branch = self.branchmap.get(commit.branch, commit.branch)

   296         self.commitcache[rev] = commit

   297         return commit

   298 

   299     def copy(self, rev):

   300         commit = self.commitcache[rev]

   301 

   302         changes = self.source.getchanges(rev)

   303         if isinstance(changes, basestring):

   304             if changes == SKIPREV:

   305                 dest = SKIPREV

   306             else:

   307                 dest = self.map[changes]

   308             self.map[rev] = dest

   309             return

   310         files, copies = changes

   311         pbranches = []

   312         if commit.parents:

   313             for prev in commit.parents:

   314                 if prev not in self.commitcache:

   315                     self.cachecommit(prev)

   316                 pbranches.append((self.map[prev],

   317                                   self.commitcache[prev].branch))

   318         self.dest.setbranch(commit.branch, pbranches)

   319         try:

   320             parents = self.splicemap[rev].replace(',', ' ').split()

   321             self.ui.status(_('spliced in %s as parents of %s\n') %

   322                            (parents, rev))

   323             parents = [self.map.get(p, p) for p in parents]

   324         except KeyError:

   325             parents = [b[0] for b in pbranches]

   326         source = progresssource(self.ui, self.source, len(files))

   327         newnode = self.dest.putcommit(files, copies, parents, commit,

   328                                       source, self.map)

   329         source.close()

   330         self.source.converted(rev, newnode)

   331         self.map[rev] = newnode

   332 

   333     def convert(self, sortmode):

   334         try:

   335             self.source.before()

   336             self.dest.before()

   337             self.source.setrevmap(self.map)

   338             self.ui.status(_("scanning source...\n"))

   339             heads = self.source.getheads()

   340             parents = self.walktree(heads)

   341             self.ui.status(_("sorting...\n"))

   342             t = self.toposort(parents, sortmode)

   343             num = len(t)

   344             c = None

   345 

   346             self.ui.status(_("converting...\n"))

   347             for i, c in enumerate(t):

   348                 num -= 1

   349                 desc = self.commitcache[c].desc

   350                 if "\n" in desc:

   351                     desc = desc.splitlines()[0]

   352                 # convert log message to local encoding without using

   353                 # tolocal() because the encoding.encoding convert()

   354                 # uses is 'utf-8'

   355                 self.ui.status("%d %s\n" % (num, recode(desc)))

   356                 self.ui.note(_("source: %s\n") % recode(c))

   357                 self.ui.progress(_('converting'), i, unit=_('revisions'),

   358                                  total=len(t))

   359                 self.copy(c)

   360             self.ui.progress(_('converting'), None)

   361 

   362             tags = self.source.gettags()

   363             ctags = {}

   364             for k in tags:

   365                 v = tags[k]

   366                 if self.map.get(v, SKIPREV) != SKIPREV:

   367                     ctags[k] = self.map[v]

   368 

   369             if c and ctags:

   370                 nrev, tagsparent = self.dest.puttags(ctags)

   371                 if nrev and tagsparent:

   372                     # write another hash correspondence to override the previous

   373                     # one so we don't end up with extra tag heads

   374                     tagsparents = [e for e in self.map.iteritems()

   375                                    if e[1] == tagsparent]

   376                     if tagsparents:

   377                         self.map[tagsparents[0][0]] = nrev

   378 

   379             self.writeauthormap()

   380         finally:

   381             self.cleanup()

   382 

   383     def cleanup(self):

   384         try:

   385             self.dest.after()

   386         finally:

   387             self.source.after()

   388         self.map.close()

   389 

   390 def convert(ui, src, dest=None, revmapfile=None, **opts):

   391     global orig_encoding

   392     orig_encoding = encoding.encoding

   393     encoding.encoding = 'UTF-8'

   394 

   395     # support --authors as an alias for --authormap

   396     if not opts.get('authormap'):

   397         opts['authormap'] = opts.get('authors')

   398 

   399     if not dest:

   400         dest = hg.defaultdest(src) + "-hg"

   401         ui.status(_("assuming destination %s\n") % dest)

   402 

   403     destc = convertsink(ui, dest, opts.get('dest_type'))

   404 

   405     try:

   406         srcc, defaultsort = convertsource(ui, src, opts.get('source_type'),

   407                                           opts.get('rev'))

   408     except Exception:

   409         for path in destc.created:

   410             shutil.rmtree(path, True)

   411         raise

   412 

   413     sortmodes = ('branchsort', 'datesort', 'sourcesort')

   414     sortmode = [m for m in sortmodes if opts.get(m)]

   415     if len(sortmode) > 1:

   416         raise util.Abort(_('more than one sort mode specified'))

   417     sortmode = sortmode and sortmode[0] or defaultsort

   418     if sortmode == 'sourcesort' and not srcc.hasnativeorder():

   419         raise util.Abort(_('--sourcesort is not supported by this data source'))

   420 

   421     fmap = opts.get('filemap')

   422     if fmap:

   423         srcc = filemap.filemap_source(ui, srcc, fmap)

   424         destc.setfilemapmode(True)

   425 

   426     if not revmapfile:

   427         try:

   428             revmapfile = destc.revmapfile()

   429         except:

   430             revmapfile = os.path.join(destc, "map")

   431 

   432     c = converter(ui, srcc, destc, revmapfile, opts)

   433     c.convert(sortmode)

   434