1   
   2   
   3   
   4   
   5   
   6   
   7   
   8   
   9   
  10   
  11   
  12   
  13   
  14   
  15   
  16   
  17   
  18   
  19   
  20   
  21   
  22   
  23   
  24   
  25   
  26   
  27   
  28   
  29   
  30   
  31   
  32   
  33   
  34   
  35   
  36   
  37   
  38   
  39   
  40   
  41  """ 
  42  Provides general-purpose utilities. 
  43   
  44  @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList, 
  45         RegexList, _Vertex, DirectedGraph, PathResolverSingleton, 
  46         sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine, 
  47         resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice, 
  48         deriveDayOfWeek, isStartOfWeek, buildNormalizedPath, 
  49         ISO_SECTOR_SIZE, BYTES_PER_SECTOR, 
  50         BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE, 
  51         SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY, 
  52         UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS 
  53   
  54  @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes. 
  55  @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector. 
  56  @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB). 
  57  @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB). 
  58  @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB). 
  59  @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB). 
  60  @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB). 
  61  @var SECONDS_PER_MINUTE: Number of seconds per minute. 
  62  @var MINUTES_PER_HOUR: Number of minutes per hour. 
  63  @var HOURS_PER_DAY: Number of hours per day. 
  64  @var SECONDS_PER_DAY: Number of seconds per day. 
  65  @var UNIT_BYTES: Constant representing the byte (B) unit for conversion. 
  66  @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion. 
  67  @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion. 
  68  @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion. 
  69  @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion. 
  70   
  71  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  72  """ 
  73   
  74   
  75   
  76   
  77   
  78   
  79  import sys 
  80  import math 
  81  import os 
  82  import re 
  83  import time 
  84  import logging 
  85  import string   
  86  from subprocess import Popen, STDOUT, PIPE 
  87   
  88  try: 
  89     import pwd 
  90     import grp 
  91     _UID_GID_AVAILABLE = True 
  92  except ImportError: 
  93     _UID_GID_AVAILABLE = False 
  94   
  95  from CedarBackup2.release import VERSION, DATE 
  96   
  97   
  98   
  99   
 100   
 101   
 102  logger = logging.getLogger("CedarBackup2.log.util") 
 103  outputLogger = logging.getLogger("CedarBackup2.output") 
 104   
 105  ISO_SECTOR_SIZE    = 2048.0    
 106  BYTES_PER_SECTOR   = ISO_SECTOR_SIZE 
 107   
 108  BYTES_PER_KBYTE    = 1024.0 
 109  KBYTES_PER_MBYTE   = 1024.0 
 110  MBYTES_PER_GBYTE   = 1024.0 
 111  BYTES_PER_MBYTE    = BYTES_PER_KBYTE * KBYTES_PER_MBYTE 
 112  BYTES_PER_GBYTE    = BYTES_PER_MBYTE * MBYTES_PER_GBYTE 
 113   
 114  SECONDS_PER_MINUTE = 60.0 
 115  MINUTES_PER_HOUR   = 60.0 
 116  HOURS_PER_DAY      = 24.0 
 117  SECONDS_PER_DAY    = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY 
 118   
 119  UNIT_BYTES         = 0 
 120  UNIT_KBYTES        = 1 
 121  UNIT_MBYTES        = 2 
 122  UNIT_GBYTES        = 4 
 123  UNIT_SECTORS       = 3 
 124   
 125  MTAB_FILE          = "/etc/mtab" 
 126   
 127  MOUNT_COMMAND      = [ "mount", ] 
 128  UMOUNT_COMMAND     = [ "umount", ] 
 129   
 130  DEFAULT_LANGUAGE   = "C" 
 131  LANG_VAR           = "LANG" 
 132  LOCALE_VARS        = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE", 
 133                         "LC_CTYPE", "LC_IDENTIFICATION", 
 134                         "LC_MEASUREMENT", "LC_MESSAGES", 
 135                         "LC_MONETARY", "LC_NAME", "LC_NUMERIC", 
 136                         "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ] 
 144   
 145     """ 
 146     Class representing an "unordered list". 
 147   
 148     An "unordered list" is a list in which only the contents matter, not the 
 149     order in which the contents appear in the list. 
 150   
 151     For instance, we might be keeping track of set of paths in a list, because 
 152     it's convenient to have them in that form.  However, for comparison 
 153     purposes, we would only care that the lists contain exactly the same 
 154     contents, regardless of order. 
 155   
 156     I have come up with two reasonable ways of doing this, plus a couple more 
 157     that would work but would be a pain to implement.  My first method is to 
 158     copy and sort each list, comparing the sorted versions.  This will only work 
 159     if two lists with exactly the same members are guaranteed to sort in exactly 
 160     the same order.  The second way would be to create two Sets and then compare 
 161     the sets.  However, this would lose information about any duplicates in 
 162     either list.  I've decided to go with option #1 for now.  I'll modify this 
 163     code if I run into problems in the future. 
 164   
 165     We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__}, 
 166     C{__le__} and C{__lt__} list methods to change the definition of the various 
 167     comparison operators.  In all cases, the comparison is changed to return the 
 168     result of the original operation I{but instead comparing sorted lists}. 
 169     This is going to be quite a bit slower than a normal list, so you probably 
 170     only want to use it on small lists. 
 171     """ 
 172   
 174        """ 
 175        Definition of C{==} operator for this class. 
 176        @param other: Other object to compare to. 
 177        @return: True/false depending on whether C{self == other}. 
 178        """ 
 179        if other is None: 
 180           return False 
 181        selfSorted = self[:] 
 182        otherSorted = other[:] 
 183        selfSorted.sort() 
 184        otherSorted.sort() 
 185        return selfSorted.__eq__(otherSorted) 
  186   
 188        """ 
 189        Definition of C{!=} operator for this class. 
 190        @param other: Other object to compare to. 
 191        @return: True/false depending on whether C{self != other}. 
 192        """ 
 193        if other is None: 
 194           return True 
 195        selfSorted = self[:] 
 196        otherSorted = other[:] 
 197        selfSorted.sort() 
 198        otherSorted.sort() 
 199        return selfSorted.__ne__(otherSorted) 
  200   
 202        """ 
 203        Definition of S{>=} operator for this class. 
 204        @param other: Other object to compare to. 
 205        @return: True/false depending on whether C{self >= other}. 
 206        """ 
 207        if other is None: 
 208           return True 
 209        selfSorted = self[:] 
 210        otherSorted = other[:] 
 211        selfSorted.sort() 
 212        otherSorted.sort() 
 213        return selfSorted.__ge__(otherSorted) 
  214   
 216        """ 
 217        Definition of C{>} operator for this class. 
 218        @param other: Other object to compare to. 
 219        @return: True/false depending on whether C{self > other}. 
 220        """ 
 221        if other is None: 
 222           return True 
 223        selfSorted = self[:] 
 224        otherSorted = other[:] 
 225        selfSorted.sort() 
 226        otherSorted.sort() 
 227        return selfSorted.__gt__(otherSorted) 
  228   
 230        """ 
 231        Definition of S{<=} operator for this class. 
 232        @param other: Other object to compare to. 
 233        @return: True/false depending on whether C{self <= other}. 
 234        """ 
 235        if other is None: 
 236           return False 
 237        selfSorted = self[:] 
 238        otherSorted = other[:] 
 239        selfSorted.sort() 
 240        otherSorted.sort() 
 241        return selfSorted.__le__(otherSorted) 
  242   
 244        """ 
 245        Definition of C{<} operator for this class. 
 246        @param other: Other object to compare to. 
 247        @return: True/false depending on whether C{self < other}. 
 248        """ 
 249        if other is None: 
 250           return False 
 251        selfSorted = self[:] 
 252        otherSorted = other[:] 
 253        selfSorted.sort() 
 254        otherSorted.sort() 
 255        return selfSorted.__lt__(otherSorted) 
   256   
 263   
 264     """ 
 265     Class representing a list of absolute paths. 
 266   
 267     This is an unordered list. 
 268   
 269     We override the C{append}, C{insert} and C{extend} methods to ensure that 
 270     any item added to the list is an absolute path. 
 271   
 272     Each item added to the list is encoded using L{encodePath}.  If we don't do 
 273     this, we have problems trying certain operations between strings and unicode 
 274     objects, particularly for "odd" filenames that can't be encoded in standard 
 275     ASCII. 
 276     """ 
 277   
 279        """ 
 280        Overrides the standard C{append} method. 
 281        @raise ValueError: If item is not an absolute path. 
 282        """ 
 283        if not os.path.isabs(item): 
 284           raise ValueError("Not an absolute path: [%s]" % item) 
 285        list.append(self, encodePath(item)) 
  286   
 287 -   def insert(self, index, item): 
  288        """ 
 289        Overrides the standard C{insert} method. 
 290        @raise ValueError: If item is not an absolute path. 
 291        """ 
 292        if not os.path.isabs(item): 
 293           raise ValueError("Not an absolute path: [%s]" % item) 
 294        list.insert(self, index, encodePath(item)) 
  295   
 297        """ 
 298        Overrides the standard C{insert} method. 
 299        @raise ValueError: If any item is not an absolute path. 
 300        """ 
 301        for item in seq: 
 302           if not os.path.isabs(item): 
 303              raise ValueError("Not an absolute path: [%s]" % item) 
 304        for item in seq: 
 305           list.append(self, encodePath(item)) 
   306   
 313   
 314     """ 
 315     Class representing a list containing only objects with a certain type. 
 316   
 317     This is an unordered list. 
 318   
 319     We override the C{append}, C{insert} and C{extend} methods to ensure that 
 320     any item added to the list matches the type that is requested.  The 
 321     comparison uses the built-in C{isinstance}, which should allow subclasses of 
 322     of the requested type to be added to the list as well. 
 323   
 324     The C{objectName} value will be used in exceptions, i.e. C{"Item must be a 
 325     CollectDir object."} if C{objectName} is C{"CollectDir"}. 
 326     """ 
 327   
 328 -   def __init__(self, objectType, objectName): 
  329        """ 
 330        Initializes a typed list for a particular type. 
 331        @param objectType: Type that the list elements must match. 
 332        @param objectName: Short string containing the "name" of the type. 
 333        """ 
 334        super(ObjectTypeList, self).__init__() 
 335        self.objectType = objectType 
 336        self.objectName = objectName 
  337   
 339        """ 
 340        Overrides the standard C{append} method. 
 341        @raise ValueError: If item does not match requested type. 
 342        """ 
 343        if not isinstance(item, self.objectType): 
 344           raise ValueError("Item must be a %s object." % self.objectName) 
 345        list.append(self, item) 
  346   
 347 -   def insert(self, index, item): 
  348        """ 
 349        Overrides the standard C{insert} method. 
 350        @raise ValueError: If item does not match requested type. 
 351        """ 
 352        if not isinstance(item, self.objectType): 
 353           raise ValueError("Item must be a %s object." % self.objectName) 
 354        list.insert(self, index, item) 
  355   
 357        """ 
 358        Overrides the standard C{insert} method. 
 359        @raise ValueError: If item does not match requested type. 
 360        """ 
 361        for item in seq: 
 362           if not isinstance(item, self.objectType): 
 363              raise ValueError("All items must be %s objects." % self.objectName) 
 364        list.extend(self, seq) 
   365   
 366   
 367   
 368   
 369   
 370   
 371 -class RestrictedContentList(UnorderedList): 
  372   
 373     """ 
 374     Class representing a list containing only object with certain values. 
 375   
 376     This is an unordered list. 
 377   
 378     We override the C{append}, C{insert} and C{extend} methods to ensure that 
 379     any item added to the list is among the valid values.  We use a standard 
 380     comparison, so pretty much anything can be in the list of valid values. 
 381   
 382     The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be 
 383     one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}. 
 384   
 385     @note:  This class doesn't make any attempt to trap for nonsensical 
 386     arguments.  All of the values in the values list should be of the same type 
 387     (i.e. strings).  Then, all list operations also need to be of that type 
 388     (i.e. you should always insert or append just strings).  If you mix types -- 
 389     for instance lists and strings -- you will likely see AttributeError 
 390     exceptions or other problems. 
 391     """ 
 392   
 393 -   def __init__(self, valuesList, valuesDescr, prefix=None): 
  394        """ 
 395        Initializes a list restricted to containing certain values. 
 396        @param valuesList: List of valid values. 
 397        @param valuesDescr: Short string describing list of values. 
 398        @param prefix: Prefix to use in error messages (None results in prefix "Item") 
 399        """ 
 400        super(RestrictedContentList, self).__init__() 
 401        self.prefix = "Item" 
 402        if prefix is not None: self.prefix = prefix 
 403        self.valuesList = valuesList 
 404        self.valuesDescr = valuesDescr 
  405   
 406 -   def append(self, item): 
  407        """ 
 408        Overrides the standard C{append} method. 
 409        @raise ValueError: If item is not in the values list. 
 410        """ 
 411        if item not in self.valuesList: 
 412           raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 
 413        list.append(self, item) 
  414   
 415 -   def insert(self, index, item): 
  416        """ 
 417        Overrides the standard C{insert} method. 
 418        @raise ValueError: If item is not in the values list. 
 419        """ 
 420        if item not in self.valuesList: 
 421           raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 
 422        list.insert(self, index, item) 
  423   
 424 -   def extend(self, seq): 
  425        """ 
 426        Overrides the standard C{insert} method. 
 427        @raise ValueError: If item is not in the values list. 
 428        """ 
 429        for item in seq: 
 430           if item not in self.valuesList: 
 431              raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 
 432        list.extend(self, seq) 
   433   
 440   
 441     """ 
 442     Class representing a list containing only strings that match a regular expression. 
 443   
 444     If C{emptyAllowed} is passed in as C{False}, then empty strings are 
 445     explicitly disallowed, even if they happen to match the regular expression. 
 446     (C{None} values are always disallowed, since string operations are not 
 447     permitted on C{None}.) 
 448   
 449     This is an unordered list. 
 450   
 451     We override the C{append}, C{insert} and C{extend} methods to ensure that 
 452     any item added to the list matches the indicated regular expression. 
 453   
 454     @note: If you try to put values that are not strings into the list, you will 
 455     likely get either TypeError or AttributeError exceptions as a result. 
 456     """ 
 457   
 458 -   def __init__(self, valuesRegex, emptyAllowed=True, prefix=None): 
  459        """ 
 460        Initializes a list restricted to containing certain values. 
 461        @param valuesRegex: Regular expression that must be matched, as a string 
 462        @param emptyAllowed: Indicates whether empty or None values are allowed. 
 463        @param prefix: Prefix to use in error messages (None results in prefix "Item") 
 464        """ 
 465        super(RegexMatchList, self).__init__() 
 466        self.prefix = "Item" 
 467        if prefix is not None: self.prefix = prefix 
 468        self.valuesRegex = valuesRegex 
 469        self.emptyAllowed = emptyAllowed 
 470        self.pattern = re.compile(self.valuesRegex) 
  471   
 473        """ 
 474        Overrides the standard C{append} method. 
 475        @raise ValueError: If item is None 
 476        @raise ValueError: If item is empty and empty values are not allowed 
 477        @raise ValueError: If item does not match the configured regular expression 
 478        """ 
 479        if item is None or (not self.emptyAllowed and item == ""): 
 480           raise ValueError("%s cannot be empty." % self.prefix) 
 481        if not self.pattern.search(item): 
 482           raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 
 483        list.append(self, item) 
  484   
 485 -   def insert(self, index, item): 
  486        """ 
 487        Overrides the standard C{insert} method. 
 488        @raise ValueError: If item is None 
 489        @raise ValueError: If item is empty and empty values are not allowed 
 490        @raise ValueError: If item does not match the configured regular expression 
 491        """ 
 492        if item is None or (not self.emptyAllowed and item == ""): 
 493           raise ValueError("%s cannot be empty." % self.prefix) 
 494        if not self.pattern.search(item): 
 495           raise ValueError("%s is not valid [%s]" % (self.prefix, item)) 
 496        list.insert(self, index, item) 
  497   
 499        """ 
 500        Overrides the standard C{insert} method. 
 501        @raise ValueError: If any item is None 
 502        @raise ValueError: If any item is empty and empty values are not allowed 
 503        @raise ValueError: If any item does not match the configured regular expression 
 504        """ 
 505        for item in seq: 
 506           if item is None or (not self.emptyAllowed and item == ""): 
 507              raise ValueError("%s cannot be empty." % self.prefix) 
 508           if not self.pattern.search(item): 
 509              raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 
 510        list.extend(self, seq) 
   511   
 512   
 513   
 514   
 515   
 516   
 517 -class RegexList(UnorderedList): 
  518   
 519     """ 
 520     Class representing a list of valid regular expression strings. 
 521   
 522     This is an unordered list. 
 523   
 524     We override the C{append}, C{insert} and C{extend} methods to ensure that 
 525     any item added to the list is a valid regular expression. 
 526     """ 
 527   
 529        """ 
 530        Overrides the standard C{append} method. 
 531        @raise ValueError: If item is not an absolute path. 
 532        """ 
 533        try: 
 534           re.compile(item) 
 535        except re.error: 
 536           raise ValueError("Not a valid regular expression: [%s]" % item) 
 537        list.append(self, item) 
  538   
 539 -   def insert(self, index, item): 
  540        """ 
 541        Overrides the standard C{insert} method. 
 542        @raise ValueError: If item is not an absolute path. 
 543        """ 
 544        try: 
 545           re.compile(item) 
 546        except re.error: 
 547           raise ValueError("Not a valid regular expression: [%s]" % item) 
 548        list.insert(self, index, item) 
  549   
 551        """ 
 552        Overrides the standard C{insert} method. 
 553        @raise ValueError: If any item is not an absolute path. 
 554        """ 
 555        for item in seq: 
 556           try: 
 557              re.compile(item) 
 558           except re.error: 
 559              raise ValueError("Not a valid regular expression: [%s]" % item) 
 560        for item in seq: 
 561           list.append(self, item) 
   562   
 563   
 564   
 565   
 566   
 567   
 568 -class _Vertex(object): 
  569   
 570     """ 
 571     Represents a vertex (or node) in a directed graph. 
 572     """ 
 573   
 575        """ 
 576        Constructor. 
 577        @param name: Name of this graph vertex. 
 578        @type name: String value. 
 579        """ 
 580        self.name = name 
 581        self.endpoints = [] 
 582        self.state = None 
   583   
 585   
 586     """ 
 587     Represents a directed graph. 
 588   
 589     A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set 
 590     B{E} of vertex pairs or edges.  In a directed graph, each edge also has an 
 591     associated direction (from vertext B{v1} to vertex B{v2}).  A C{DirectedGraph} 
 592     object provides a way to construct a directed graph and execute a depth- 
 593     first search. 
 594   
 595     This data structure was designed based on the graphing chapter in 
 596     U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>}, 
 597     by Steven S. Skiena. 
 598   
 599     This class is intended to be used by Cedar Backup for dependency ordering. 
 600     Because of this, it's not quite general-purpose.  Unlike a "general" graph, 
 601     every vertex in this graph has at least one edge pointing to it, from a 
 602     special "start" vertex.  This is so no vertices get "lost" either because 
 603     they have no dependencies or because nothing depends on them. 
 604     """ 
 605   
 606     _UNDISCOVERED = 0 
 607     _DISCOVERED   = 1 
 608     _EXPLORED     = 2 
 609   
 611        """ 
 612        Directed graph constructor. 
 613   
 614        @param name: Name of this graph. 
 615        @type name: String value. 
 616        """ 
 617        if name is None or name == "": 
 618           raise ValueError("Graph name must be non-empty.") 
 619        self._name = name 
 620        self._vertices = {} 
 621        self._startVertex = _Vertex(None)   
  622   
 624        """ 
 625        Official string representation for class instance. 
 626        """ 
 627        return "DirectedGraph(%s)" % self.name 
  628   
 630        """ 
 631        Informal string representation for class instance. 
 632        """ 
 633        return self.__repr__() 
  634   
 636        """ 
 637        Definition of equals operator for this class. 
 638        @param other: Other object to compare to. 
 639        @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 
 640        """ 
 641         
 642        if other is None: 
 643           return 1 
 644        if self.name != other.name: 
 645           if self.name < other.name: 
 646              return -1 
 647           else: 
 648              return 1 
 649        if self._vertices != other._vertices: 
 650           if self._vertices < other._vertices: 
 651              return -1 
 652           else: 
 653              return 1 
 654        return 0 
  655   
 657        """ 
 658        Property target used to get the graph name. 
 659        """ 
 660        return self._name 
  661   
 662     name = property(_getName, None, None, "Name of the graph.") 
 663   
 665        """ 
 666        Creates a named vertex. 
 667        @param name: vertex name 
 668        @raise ValueError: If the vertex name is C{None} or empty. 
 669        """ 
 670        if name is None or name == "": 
 671           raise ValueError("Vertex name must be non-empty.") 
 672        vertex = _Vertex(name) 
 673        self._startVertex.endpoints.append(vertex)   
 674        self._vertices[name] = vertex 
  675   
 677        """ 
 678        Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex. 
 679        @param start: Name of start vertex. 
 680        @param finish: Name of finish vertex. 
 681        @raise ValueError: If one of the named vertices is unknown. 
 682        """ 
 683        try: 
 684           startVertex = self._vertices[start] 
 685           finishVertex = self._vertices[finish] 
 686           startVertex.endpoints.append(finishVertex) 
 687        except KeyError, e: 
 688           raise ValueError("Vertex [%s] could not be found." % e) 
  689   
 691        """ 
 692        Implements a topological sort of the graph. 
 693   
 694        This method also enforces that the graph is a directed acyclic graph, 
 695        which is a requirement of a topological sort. 
 696   
 697        A directed acyclic graph (or "DAG") is a directed graph with no directed 
 698        cycles.  A topological sort of a DAG is an ordering on the vertices such 
 699        that all edges go from left to right.  Only an acyclic graph can have a 
 700        topological sort, but any DAG has at least one topological sort. 
 701   
 702        Since a topological sort only makes sense for an acyclic graph, this 
 703        method throws an exception if a cycle is found. 
 704   
 705        A depth-first search only makes sense if the graph is acyclic.  If the 
 706        graph contains any cycles, it is not possible to determine a consistent 
 707        ordering for the vertices. 
 708   
 709        @note: If a particular vertex has no edges, then its position in the 
 710        final list depends on the order in which the vertices were created in the 
 711        graph.  If you're using this method to determine a dependency order, this 
 712        makes sense: a vertex with no dependencies can go anywhere (and will). 
 713   
 714        @return: Ordering on the vertices so that all edges go from left to right. 
 715   
 716        @raise ValueError: If a cycle is found in the graph. 
 717        """ 
 718        ordering = [] 
 719        for key in self._vertices: 
 720           vertex = self._vertices[key] 
 721           vertex.state = self._UNDISCOVERED 
 722        for key in self._vertices: 
 723           vertex = self._vertices[key] 
 724           if vertex.state == self._UNDISCOVERED: 
 725              self._topologicalSort(self._startVertex, ordering) 
 726        return ordering 
  727   
 729        """ 
 730        Recursive depth first search function implementing topological sort. 
 731        @param vertex: Vertex to search 
 732        @param ordering: List of vertices in proper order 
 733        """ 
 734        vertex.state = self._DISCOVERED 
 735        for endpoint in vertex.endpoints: 
 736           if endpoint.state == self._UNDISCOVERED: 
 737              self._topologicalSort(endpoint, ordering) 
 738           elif endpoint.state != self._EXPLORED: 
 739              raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name)) 
 740        if vertex.name is not None: 
 741           ordering.insert(0, vertex.name) 
 742        vertex.state = self._EXPLORED 
   743   
 750   
 751     """ 
 752     Singleton used for resolving executable paths. 
 753   
 754     Various functions throughout Cedar Backup (including extensions) need a way 
 755     to resolve the path of executables that they use.  For instance, the image 
 756     functionality needs to find the C{mkisofs} executable, and the Subversion 
 757     extension needs to find the C{svnlook} executable.  Cedar Backup's original 
 758     behavior was to assume that the simple name (C{"svnlook"} or whatever) was 
 759     available on the caller's C{$PATH}, and to fail otherwise.   However, this 
 760     turns out to be less than ideal, since for instance the root user might not 
 761     always have executables like C{svnlook} in its path. 
 762   
 763     One solution is to specify a path (either via an absolute path or some sort 
 764     of path insertion or path appending mechanism) that would apply to the 
 765     C{executeCommand()} function.  This is not difficult to implement, but it 
 766     seem like kind of a "big hammer" solution.  Besides that, it might also 
 767     represent a security flaw (for instance, I prefer not to mess with root's 
 768     C{$PATH} on the application level if I don't have to). 
 769   
 770     The alternative is to set up some sort of configuration for the path to 
 771     certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or 
 772     whatever.  This PathResolverSingleton aims to provide a good solution to the 
 773     mapping problem.  Callers of all sorts (extensions or not) can get an 
 774     instance of the singleton.  Then, they call the C{lookup} method to try and 
 775     resolve the executable they are looking for.  Through the C{lookup} method, 
 776     the caller can also specify a default to use if a mapping is not found. 
 777     This way, with no real effort on the part of the caller, behavior can neatly 
 778     degrade to something equivalent to the current behavior if there is no 
 779     special mapping or if the singleton was never initialized in the first 
 780     place. 
 781   
 782     Even better, extensions automagically get access to the same resolver 
 783     functionality, and they don't even need to understand how the mapping 
 784     happens.  All extension authors need to do is document what executables 
 785     their code requires, and the standard resolver configuration section will 
 786     meet their needs. 
 787   
 788     The class should be initialized once through the constructor somewhere in 
 789     the main routine.  Then, the main routine should call the L{fill} method to 
 790     fill in the resolver's internal structures.  Everyone else who needs to 
 791     resolve a path will get an instance of the class using L{getInstance} and 
 792     will then just call the L{lookup} method. 
 793   
 794     @cvar _instance: Holds a reference to the singleton 
 795     @ivar _mapping: Internal mapping from resource name to path. 
 796     """ 
 797   
 798     _instance = None      
 799   
 801        """Helper class to provide a singleton factory method.""" 
  810   
 811     getInstance = _Helper()     
 812   
 819   
 820 -   def lookup(self, name, default=None): 
  821        """ 
 822        Looks up name and returns the resolved path associated with the name. 
 823        @param name: Name of the path resource to resolve. 
 824        @param default: Default to return if resource cannot be resolved. 
 825        @return: Resolved path associated with name, or default if name can't be resolved. 
 826        """ 
 827        value = default 
 828        if name in self._mapping.keys(): 
 829           value = self._mapping[name] 
 830        logger.debug("Resolved command [%s] to [%s].", name, value) 
 831        return value 
  832   
 833 -   def fill(self, mapping): 
  834        """ 
 835        Fills in the singleton's internal mapping from name to resource. 
 836        @param mapping: Mapping from resource name to path. 
 837        @type mapping: Dictionary mapping name to path, both as strings. 
 838        """ 
 839        self._mapping = { } 
 840        for key in mapping.keys(): 
 841           self._mapping[key] = mapping[key] 
   842   
 843   
 844   
 845   
 846   
 847   
 848 -class Pipe(Popen): 
  849     """ 
 850     Specialized pipe class for use by C{executeCommand}. 
 851   
 852     The L{executeCommand} function needs a specialized way of interacting 
 853     with a pipe.  First, C{executeCommand} only reads from the pipe, and 
 854     never writes to it.  Second, C{executeCommand} needs a way to discard all 
 855     output written to C{stderr}, as a means of simulating the shell 
 856     C{2>/dev/null} construct. 
 857     """ 
 858 -   def __init__(self, cmd, bufsize=-1, ignoreStderr=False): 
  859        stderr = STDOUT 
 860        if ignoreStderr: 
 861           devnull = nullDevice() 
 862           stderr = os.open(devnull, os.O_RDWR) 
 863        Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr) 
   864   
 871   
 872     """ 
 873     Class holding runtime diagnostic information. 
 874   
 875     Diagnostic information is information that is useful to get from users for 
 876     debugging purposes.  I'm consolidating it all here into one object. 
 877   
 878     @sort: __init__, __repr__, __str__ 
 879     """ 
 880      
 881   
 883        """ 
 884        Constructor for the C{Diagnostics} class. 
 885        """ 
  886   
 888        """ 
 889        Official string representation for class instance. 
 890        """ 
 891        return "Diagnostics()" 
  892   
 894        """ 
 895        Informal string representation for class instance. 
 896        """ 
 897        return self.__repr__() 
  898   
 900        """ 
 901        Get a map containing all of the diagnostic values. 
 902        @return: Map from diagnostic name to diagnostic value. 
 903        """ 
 904        values = {} 
 905        values['version'] = self.version 
 906        values['interpreter'] = self.interpreter 
 907        values['platform'] = self.platform 
 908        values['encoding'] = self.encoding 
 909        values['locale'] = self.locale 
 910        values['timestamp'] = self.timestamp 
 911        values['supported'] = self.supported 
 912        return values 
  913   
 915        """ 
 916        Pretty-print diagnostic information to a file descriptor. 
 917        @param fd: File descriptor used to print information. 
 918        @param prefix: Prefix string (if any) to place onto printed lines 
 919        @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
 920        """ 
 921        lines = self._buildDiagnosticLines(prefix) 
 922        for line in lines: 
 923           fd.write("%s\n" % line) 
  924   
 926        """ 
 927        Pretty-print diagnostic information using a logger method. 
 928        @param method: Logger method to use for logging (i.e. logger.info) 
 929        @param prefix: Prefix string (if any) to place onto printed lines 
 930        """ 
 931        lines = self._buildDiagnosticLines(prefix) 
 932        for line in lines: 
 933           method("%s" % line) 
  934   
 936        """ 
 937        Build a set of pretty-printed diagnostic lines. 
 938        @param prefix: Prefix string (if any) to place onto printed lines 
 939        @return: List of strings, not terminated by newlines. 
 940        """ 
 941        values = self.getValues() 
 942        keys = values.keys() 
 943        keys.sort() 
 944        tmax = Diagnostics._getMaxLength(keys) + 3   
 945        lines = [] 
 946        for key in keys: 
 947           title = key.title() 
 948           title += (tmax - len(title)) * '.' 
 949           value = values[key] 
 950           line = "%s%s: %s" % (prefix, title, value) 
 951           lines.append(line) 
 952        return lines 
  953   
 954     @staticmethod 
 956        """ 
 957        Get the maximum length from among a list of strings. 
 958        """ 
 959        tmax = 0 
 960        for value in values: 
 961           if len(value) > tmax: 
 962              tmax = len(value) 
 963        return tmax 
  964   
 966        """ 
 967        Property target to get the Cedar Backup version. 
 968        """ 
 969        return "Cedar Backup %s (%s)" % (VERSION, DATE) 
  970   
 972        """ 
 973        Property target to get the Python interpreter version. 
 974        """ 
 975        version = sys.version_info 
 976        return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3]) 
  977   
 979        """ 
 980        Property target to get the filesystem encoding. 
 981        """ 
 982        return sys.getfilesystemencoding() or sys.getdefaultencoding() 
  983   
1004   
1006        """ 
1007        Property target to get the default locale that is in effect. 
1008        """ 
1009        try: 
1010           import locale 
1011           return locale.getdefaultlocale()[0] 
1012        except: 
1013           return "(unknown)" 
 1014   
1016        """ 
1017        Property target to get a current date/time stamp. 
1018        """ 
1019        try: 
1020           import datetime 
1021           return datetime.datetime.utcnow().ctime() + " UTC" 
1022        except: 
1023           return "(unknown)" 
 1024   
1026        """ 
1027        Property target to get the supported value. 
1028        """ 
1029        return "Unsupported as of 11 Nov 2017" 
 1030   
1031     version = property(_getVersion, None, None, "Cedar Backup version.") 
1032     interpreter = property(_getInterpreter, None, None, "Python interpreter version.") 
1033     platform = property(_getPlatform, None, None, "Platform identifying information.") 
1034     encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.") 
1035     locale = property(_getLocale, None, None, "Locale that is in effect.") 
1036     timestamp = property(_getTimestamp, None, None, "Current timestamp.") 
1037     supported = property(_getSupported, None, None, "Whether the version is supported.") 
 1038   
1039   
1040   
1041   
1042   
1043   
1044   
1045   
1046   
1047   
1048 -def sortDict(d): 
 1049     """ 
1050     Returns the keys of the dictionary sorted by value. 
1051   
1052     There are cuter ways to do this in Python 2.4, but we were originally 
1053     attempting to stay compatible with Python 2.3. 
1054   
1055     @param d: Dictionary to operate on 
1056     @return: List of dictionary keys sorted in order by dictionary value. 
1057     """ 
1058     items = d.items() 
1059     items.sort(lambda x, y: cmp(x[1], y[1])) 
1060     return [key for key, value in items] 
 1061   
1062   
1063   
1064   
1065   
1066   
1067 -def removeKeys(d, keys): 
 1068     """ 
1069     Removes all of the keys from the dictionary. 
1070     The dictionary is altered in-place. 
1071     Each key must exist in the dictionary. 
1072     @param d: Dictionary to operate on 
1073     @param keys: List of keys to remove 
1074     @raise KeyError: If one of the keys does not exist 
1075     """ 
1076     for key in keys: 
1077        del d[key] 
 1078   
1079   
1080   
1081   
1082   
1083   
1084 -def convertSize(size, fromUnit, toUnit): 
 1085     """ 
1086     Converts a size in one unit to a size in another unit. 
1087   
1088     This is just a convenience function so that the functionality can be 
1089     implemented in just one place.  Internally, we convert values to bytes and 
1090     then to the final unit. 
1091   
1092     The available units are: 
1093   
1094        - C{UNIT_BYTES} - Bytes 
1095        - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B 
1096        - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB 
1097        - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB 
1098        - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B 
1099   
1100     @param size: Size to convert 
1101     @type size: Integer or float value in units of C{fromUnit} 
1102   
1103     @param fromUnit: Unit to convert from 
1104     @type fromUnit: One of the units listed above 
1105   
1106     @param toUnit: Unit to convert to 
1107     @type toUnit: One of the units listed above 
1108   
1109     @return: Number converted to new unit, as a float. 
1110     @raise ValueError: If one of the units is invalid. 
1111     """ 
1112     if size is None: 
1113        raise ValueError("Cannot convert size of None.") 
1114     if fromUnit == UNIT_BYTES: 
1115        byteSize = float(size) 
1116     elif fromUnit == UNIT_KBYTES: 
1117        byteSize = float(size) * BYTES_PER_KBYTE 
1118     elif fromUnit == UNIT_MBYTES: 
1119        byteSize = float(size) * BYTES_PER_MBYTE 
1120     elif fromUnit == UNIT_GBYTES: 
1121        byteSize = float(size) * BYTES_PER_GBYTE 
1122     elif fromUnit == UNIT_SECTORS: 
1123        byteSize = float(size) * BYTES_PER_SECTOR 
1124     else: 
1125        raise ValueError("Unknown 'from' unit %s." % fromUnit) 
1126     if toUnit == UNIT_BYTES: 
1127        return byteSize 
1128     elif toUnit == UNIT_KBYTES: 
1129        return byteSize / BYTES_PER_KBYTE 
1130     elif toUnit == UNIT_MBYTES: 
1131        return byteSize / BYTES_PER_MBYTE 
1132     elif toUnit == UNIT_GBYTES: 
1133        return byteSize / BYTES_PER_GBYTE 
1134     elif toUnit == UNIT_SECTORS: 
1135        return byteSize / BYTES_PER_SECTOR 
1136     else: 
1137        raise ValueError("Unknown 'to' unit %s." % toUnit) 
 1138   
1139   
1140   
1141   
1142   
1143   
1144 -def displayBytes(bytes, digits=2):  
 1145     """ 
1146     Format a byte quantity so it can be sensibly displayed. 
1147   
1148     It's rather difficult to look at a number like "72372224 bytes" and get any 
1149     meaningful information out of it.  It would be more useful to see something 
1150     like "69.02 MB".  That's what this function does.  Any time you want to display 
1151     a byte value, i.e.:: 
1152   
1153        print "Size: %s bytes" % bytes 
1154   
1155     Call this function instead:: 
1156   
1157        print "Size: %s" % displayBytes(bytes) 
1158   
1159     What comes out will be sensibly formatted.  The indicated number of digits 
1160     will be listed after the decimal point, rounded based on whatever rules are 
1161     used by Python's standard C{%f} string format specifier. (Values less than 1 
1162     kB will be listed in bytes and will not have a decimal point, since the 
1163     concept of a fractional byte is nonsensical.) 
1164   
1165     @param bytes: Byte quantity. 
1166     @type bytes: Integer number of bytes. 
1167   
1168     @param digits: Number of digits to display after the decimal point. 
1169     @type digits: Integer value, typically 2-5. 
1170   
1171     @return: String, formatted for sensible display. 
1172     """ 
1173     if bytes is None: 
1174        raise ValueError("Cannot display byte value of None.") 
1175     bytes = float(bytes) 
1176     if math.fabs(bytes) < BYTES_PER_KBYTE: 
1177        fmt = "%.0f bytes" 
1178        value = bytes 
1179     elif math.fabs(bytes) < BYTES_PER_MBYTE: 
1180        fmt = "%." + "%d" % digits + "f kB" 
1181        value = bytes / BYTES_PER_KBYTE 
1182     elif math.fabs(bytes) < BYTES_PER_GBYTE: 
1183        fmt = "%." + "%d" % digits + "f MB" 
1184        value = bytes / BYTES_PER_MBYTE 
1185     else: 
1186        fmt = "%." + "%d" % digits + "f GB" 
1187        value = bytes / BYTES_PER_GBYTE 
1188     return fmt % value 
1189   
1196     """ 
1197     Gets a reference to a named function. 
1198   
1199     This does some hokey-pokey to get back a reference to a dynamically named 
1200     function.  For instance, say you wanted to get a reference to the 
1201     C{os.path.isdir} function.  You could use:: 
1202   
1203        myfunc = getFunctionReference("os.path", "isdir") 
1204   
1205     Although we won't bomb out directly, behavior is pretty much undefined if 
1206     you pass in C{None} or C{""} for either C{module} or C{function}. 
1207   
1208     The only validation we enforce is that whatever we get back must be 
1209     callable. 
1210   
1211     I derived this code based on the internals of the Python unittest 
1212     implementation.  I don't claim to completely understand how it works. 
1213   
1214     @param module: Name of module associated with function. 
1215     @type module: Something like "os.path" or "CedarBackup2.util" 
1216   
1217     @param function: Name of function 
1218     @type function: Something like "isdir" or "getUidGid" 
1219   
1220     @return: Reference to function associated with name. 
1221   
1222     @raise ImportError: If the function cannot be found. 
1223     @raise ValueError: If the resulting reference is not callable. 
1224   
1225     @copyright: Some of this code, prior to customization, was originally part 
1226     of the Python 2.3 codebase.  Python code is copyright (c) 2001, 2002 Python 
1227     Software Foundation; All Rights Reserved. 
1228     """ 
1229     parts = [] 
1230     if module is not None and module != "": 
1231        parts = module.split(".") 
1232     if function is not None and function != "": 
1233        parts.append(function) 
1234     copy = parts[:] 
1235     while copy: 
1236        try: 
1237           module = __import__(string.join(copy, ".")) 
1238           break 
1239        except ImportError: 
1240           del copy[-1] 
1241           if not copy: raise 
1242        parts = parts[1:] 
1243     obj = module 
1244     for part in parts: 
1245        obj = getattr(obj, part) 
1246     if not callable(obj): 
1247        raise ValueError("Reference to %s.%s is not callable." % (module, function)) 
1248     return obj 
 1249   
1250   
1251   
1252   
1253   
1254   
1255 -def getUidGid(user, group): 
 1256     """ 
1257     Get the uid/gid associated with a user/group pair 
1258   
1259     This is a no-op if user/group functionality is not available on the platform. 
1260   
1261     @param user: User name 
1262     @type user: User name as a string 
1263   
1264     @param group: Group name 
1265     @type group: Group name as a string 
1266   
1267     @return: Tuple C{(uid, gid)} matching passed-in user and group. 
1268     @raise ValueError: If the ownership user/group values are invalid 
1269     """ 
1270     if _UID_GID_AVAILABLE: 
1271        try: 
1272           uid = pwd.getpwnam(user)[2] 
1273           gid = grp.getgrnam(group)[2] 
1274           return (uid, gid) 
1275        except Exception, e: 
1276           logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e) 
1277           raise ValueError("Unable to lookup up uid and gid for passed in user/group.") 
1278     else: 
1279        return (0, 0) 
 1280   
1287     """ 
1288     Changes ownership of path to match the user and group. 
1289   
1290     This is a no-op if user/group functionality is not available on the 
1291     platform, or if the either passed-in user or group is C{None}.  Further, we 
1292     won't even try to do it unless running as root, since it's unlikely to work. 
1293   
1294     @param path: Path whose ownership to change. 
1295     @param user: User which owns file. 
1296     @param group: Group which owns file. 
1297     """ 
1298     if _UID_GID_AVAILABLE: 
1299        if user is None or group is None: 
1300           logger.debug("User or group is None, so not attempting to change owner on [%s].", path) 
1301        elif not isRunningAsRoot(): 
1302           logger.debug("Not root, so not attempting to change owner on [%s].", path) 
1303        else: 
1304           try: 
1305              (uid, gid) = getUidGid(user, group) 
1306              os.chown(path, uid, gid) 
1307           except Exception, e: 
1308              logger.error("Error changing ownership of [%s]: %s", path, e) 
 1309   
1316     """ 
1317     Indicates whether the program is running as the root user. 
1318     """ 
1319     return os.getuid() == 0 
 1320   
1327     """ 
1328     Splits a command line string into a list of arguments. 
1329   
1330     Unfortunately, there is no "standard" way to parse a command line string, 
1331     and it's actually not an easy problem to solve portably (essentially, we 
1332     have to emulate the shell argument-processing logic).  This code only 
1333     respects double quotes (C{"}) for grouping arguments, not single quotes 
1334     (C{'}).  Make sure you take this into account when building your command 
1335     line. 
1336   
1337     Incidentally, I found this particular parsing method while digging around in 
1338     Google Groups, and I tweaked it for my own use. 
1339   
1340     @param commandLine: Command line string 
1341     @type commandLine: String, i.e. "cback --verbose stage store" 
1342   
1343     @return: List of arguments, suitable for passing to C{popen2}. 
1344   
1345     @raise ValueError: If the command line is None. 
1346     """ 
1347     if commandLine is None: 
1348        raise ValueError("Cannot split command line of None.") 
1349     fields = re.findall('[^ "]+|"[^"]+"', commandLine) 
1350     fields = [field.replace('"', '') for field in fields] 
1351     return fields 
 1352   
1359     """ 
1360     Resolves the real path to a command through the path resolver mechanism. 
1361   
1362     Both extensions and standard Cedar Backup functionality need a way to 
1363     resolve the "real" location of various executables.  Normally, they assume 
1364     that these executables are on the system path, but some callers need to 
1365     specify an alternate location. 
1366   
1367     Ideally, we want to handle this configuration in a central location.  The 
1368     Cedar Backup path resolver mechanism (a singleton called 
1369     L{PathResolverSingleton}) provides the central location to store the 
1370     mappings.  This function wraps access to the singleton, and is what all 
1371     functions (extensions or standard functionality) should call if they need to 
1372     find a command. 
1373   
1374     The passed-in command must actually be a list, in the standard form used by 
1375     all existing Cedar Backup code (something like C{["svnlook", ]}).  The 
1376     lookup will actually be done on the first element in the list, and the 
1377     returned command will always be in list form as well. 
1378   
1379     If the passed-in command can't be resolved or no mapping exists, then the 
1380     command itself will be returned unchanged.  This way, we neatly fall back on 
1381     default behavior if we have no sensible alternative. 
1382   
1383     @param command: Command to resolve. 
1384     @type command: List form of command, i.e. C{["svnlook", ]}. 
1385   
1386     @return: Path to command or just command itself if no mapping exists. 
1387     """ 
1388     singleton = PathResolverSingleton.getInstance() 
1389     name = command[0] 
1390     result = command[:] 
1391     result[0] = singleton.lookup(name, name) 
1392     return result 
 1393   
1394   
1395   
1396   
1397   
1398   
1399 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None): 
 1400     """ 
1401     Executes a shell command, hopefully in a safe way. 
1402   
1403     This function exists to replace direct calls to C{os.popen} in the Cedar 
1404     Backup code.  It's not safe to call a function such as C{os.popen()} with 
1405     untrusted arguments, since that can cause problems if the string contains 
1406     non-safe variables or other constructs (imagine that the argument is 
1407     C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/; 
1408     echo"} in the current environment). 
1409   
1410     Instead, it's safer to pass a list of arguments in the style supported bt 
1411     C{popen2} or C{popen4}.  This function actually uses a specialized C{Pipe} 
1412     class implemented using either C{subprocess.Popen} or C{popen2.Popen4}. 
1413   
1414     Under the normal case, this function will return a tuple of C{(status, 
1415     None)} where the status is the wait-encoded return status of the call per 
1416     the C{popen2.Popen4} documentation.  If C{returnOutput} is passed in as 
1417     C{True}, the function will return a tuple of C{(status, output)} where 
1418     C{output} is a list of strings, one entry per line in the output from the 
1419     command.  Output is always logged to the C{outputLogger.info()} target, 
1420     regardless of whether it's returned. 
1421   
1422     By default, C{stdout} and C{stderr} will be intermingled in the output. 
1423     However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be 
1424     included in the output. 
1425   
1426     The C{doNotLog} parameter exists so that callers can force the function to 
1427     not log command output to the debug log.  Normally, you would want to log. 
1428     However, if you're using this function to write huge output files (i.e. 
1429     database backups written to C{stdout}) then you might want to avoid putting 
1430     all that information into the debug log. 
1431   
1432     The C{outputFile} parameter exists to make it easier for a caller to push 
1433     output into a file, i.e. as a substitute for redirection to a file.  If this 
1434     value is passed in, each time a line of output is generated, it will be 
1435     written to the file using C{outputFile.write()}.  At the end, the file 
1436     descriptor will be flushed using C{outputFile.flush()}.  The caller 
1437     maintains responsibility for closing the file object appropriately. 
1438   
1439     @note: I know that it's a bit confusing that the command and the arguments 
1440     are both lists.  I could have just required the caller to pass in one big 
1441     list.  However, I think it makes some sense to keep the command (the 
1442     constant part of what we're executing, i.e. C{"scp -B"}) separate from its 
1443     arguments, even if they both end up looking kind of similar. 
1444   
1445     @note: You cannot redirect output via shell constructs (i.e. C{>file}, 
1446     C{2>/dev/null}, etc.) using this function.  The redirection string would be 
1447     passed to the command just like any other argument.  However, you can 
1448     implement the equivalent to redirection using C{ignoreStderr} and 
1449     C{outputFile}, as discussed above. 
1450   
1451     @note: The operating system environment is partially sanitized before 
1452     the command is invoked.  See L{sanitizeEnvironment} for details. 
1453   
1454     @param command: Shell command to execute 
1455     @type command: List of individual arguments that make up the command 
1456   
1457     @param args: List of arguments to the command 
1458     @type args: List of additional arguments to the command 
1459   
1460     @param returnOutput: Indicates whether to return the output of the command 
1461     @type returnOutput: Boolean C{True} or C{False} 
1462   
1463     @param ignoreStderr: Whether stderr should be discarded 
1464     @type ignoreStderr: Boolean True or False 
1465   
1466     @param doNotLog: Indicates that output should not be logged. 
1467     @type doNotLog: Boolean C{True} or C{False} 
1468   
1469     @param outputFile: File object that all output should be written to. 
1470     @type outputFile: File object as returned from C{open()} or C{file()}. 
1471   
1472     @return: Tuple of C{(result, output)} as described above. 
1473     """ 
1474     logger.debug("Executing command %s with args %s.", command, args) 
1475     outputLogger.info("Executing command %s with args %s.", command, args) 
1476     if doNotLog: 
1477        logger.debug("Note: output will not be logged, per the doNotLog flag.") 
1478        outputLogger.info("Note: output will not be logged, per the doNotLog flag.") 
1479     output = [] 
1480     fields = command[:]         
1481     fields.extend(args) 
1482     try: 
1483        sanitizeEnvironment()    
1484        try: 
1485           pipe = Pipe(fields, ignoreStderr=ignoreStderr) 
1486        except OSError: 
1487            
1488            
1489            
1490           pipe = Pipe(fields, ignoreStderr=ignoreStderr) 
1491        while True: 
1492           line = pipe.stdout.readline() 
1493           if not line: break 
1494           if returnOutput: output.append(line) 
1495           if outputFile is not None: outputFile.write(line) 
1496           if not doNotLog: outputLogger.info(line[:-1])   
1497        if outputFile is not None: 
1498           try:  
1499              outputFile.flush() 
1500           except: pass 
1501        if returnOutput: 
1502           return (pipe.wait(), output) 
1503        else: 
1504           return (pipe.wait(), None) 
1505     except OSError, e: 
1506        try: 
1507           if returnOutput: 
1508              if output != []: 
1509                 return (pipe.wait(), output) 
1510              else: 
1511                 return (pipe.wait(), [ e, ]) 
1512           else: 
1513              return (pipe.wait(), None) 
1514        except UnboundLocalError:   
1515           if returnOutput: 
1516              return (256, []) 
1517           else: 
1518              return (256, None) 
 1519   
1526     """ 
1527     Calculates the age (in days) of a file. 
1528   
1529     The "age" of a file is the amount of time since the file was last used, per 
1530     the most recent of the file's C{st_atime} and C{st_mtime} values. 
1531   
1532     Technically, we only intend this function to work with files, but it will 
1533     probably work with anything on the filesystem. 
1534   
1535     @param path: Path to a file on disk. 
1536   
1537     @return: Age of the file in days (possibly fractional). 
1538     @raise OSError: If the file doesn't exist. 
1539     """ 
1540     currentTime = int(time.time()) 
1541     fileStats = os.stat(path) 
1542     lastUse = max(fileStats.st_atime, fileStats.st_mtime)   
1543     ageInSeconds = currentTime - lastUse 
1544     ageInDays = ageInSeconds / SECONDS_PER_DAY 
1545     return ageInDays 
 1546   
1547   
1548   
1549   
1550   
1551   
1552 -def mount(devicePath, mountPoint, fsType): 
 1553     """ 
1554     Mounts the indicated device at the indicated mount point. 
1555   
1556     For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount 
1557     point C{/media/cdrw} and filesystem type C{iso9660}.  You can safely use any 
1558     filesystem type that is supported by C{mount} on your platform.  If the type 
1559     is C{None}, we'll attempt to let C{mount} auto-detect it.  This may or may 
1560     not work on all systems. 
1561   
1562     @note: This only works on platforms that have a concept of "mounting" a 
1563     filesystem through a command-line C{"mount"} command, like UNIXes.  It 
1564     won't work on Windows. 
1565   
1566     @param devicePath: Path of device to be mounted. 
1567     @param mountPoint: Path that device should be mounted at. 
1568     @param fsType: Type of the filesystem assumed to be available via the device. 
1569   
1570     @raise IOError: If the device cannot be mounted. 
1571     """ 
1572     if fsType is None: 
1573        args = [ devicePath, mountPoint ] 
1574     else: 
1575        args = [ "-t", fsType, devicePath, mountPoint ] 
1576     command = resolveCommand(MOUNT_COMMAND) 
1577     result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0] 
1578     if result != 0: 
1579        raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType)) 
 1580   
1581   
1582   
1583   
1584   
1585   
1586 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0): 
 1587     """ 
1588     Unmounts whatever device is mounted at the indicated mount point. 
1589   
1590     Sometimes, it might not be possible to unmount the mount point immediately, 
1591     if there are still files open there.  Use the C{attempts} and C{waitSeconds} 
1592     arguments to indicate how many unmount attempts to make and how many seconds 
1593     to wait between attempts.  If you pass in zero attempts, no attempts will be 
1594     made (duh). 
1595   
1596     If the indicated mount point is not really a mount point per 
1597     C{os.path.ismount()}, then it will be ignored.  This seems to be a safer 
1598     check then looking through C{/etc/mtab}, since C{ismount()} is already in 
1599     the Python standard library and is documented as working on all POSIX 
1600     systems. 
1601   
1602     If C{removeAfter} is C{True}, then the mount point will be removed using 
1603     C{os.rmdir()} after the unmount action succeeds.  If for some reason the 
1604     mount point is not a directory, then it will not be removed. 
1605   
1606     @note: This only works on platforms that have a concept of "mounting" a 
1607     filesystem through a command-line C{"mount"} command, like UNIXes.  It 
1608     won't work on Windows. 
1609   
1610     @param mountPoint: Mount point to be unmounted. 
1611     @param removeAfter: Remove the mount point after unmounting it. 
1612     @param attempts: Number of times to attempt the unmount. 
1613     @param waitSeconds: Number of seconds to wait between repeated attempts. 
1614   
1615     @raise IOError: If the mount point is still mounted after attempts are exhausted. 
1616     """ 
1617     if os.path.ismount(mountPoint): 
1618        for attempt in range(0, attempts): 
1619           logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint) 
1620           command = resolveCommand(UMOUNT_COMMAND) 
1621           result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0] 
1622           if result != 0: 
1623              logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt) 
1624           elif os.path.ismount(mountPoint): 
1625              logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint) 
1626           else: 
1627              logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt) 
1628              break   
1629           if attempt+1 < attempts:   
1630              if waitSeconds > 0: 
1631                 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds) 
1632                 time.sleep(waitSeconds) 
1633        else: 
1634           if os.path.ismount(mountPoint): 
1635              raise IOError("Unable to unmount [%s] after %d attempts." % (mountPoint, attempts)) 
1636           logger.info("Mount point [%s] seems to have finally gone away.", mountPoint) 
1637        if os.path.isdir(mountPoint) and removeAfter: 
1638           logger.debug("Removing mount point [%s].", mountPoint) 
1639           os.rmdir(mountPoint) 
 1640   
1647     """ 
1648     Indicates whether a specific filesystem device is currently mounted. 
1649   
1650     We determine whether the device is mounted by looking through the system's 
1651     C{mtab} file.  This file shows every currently-mounted filesystem, ordered 
1652     by device.  We only do the check if the C{mtab} file exists and is readable. 
1653     Otherwise, we assume that the device is not mounted. 
1654   
1655     @note: This only works on platforms that have a concept of an mtab file 
1656     to show mounted volumes, like UNIXes.  It won't work on Windows. 
1657   
1658     @param devicePath: Path of device to be checked 
1659   
1660     @return: True if device is mounted, false otherwise. 
1661     """ 
1662     if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK): 
1663        realPath = os.path.realpath(devicePath) 
1664        lines = open(MTAB_FILE).readlines() 
1665        for line in lines: 
1666           (mountDevice, mountPoint, remainder) = line.split(None, 2) 
1667           if mountDevice in [ devicePath, realPath, ]: 
1668              logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint) 
1669              return True 
1670     return False 
 1671   
1678   
1679     r""" 
1680     Safely encodes a filesystem path. 
1681   
1682     Many Python filesystem functions, such as C{os.listdir}, behave differently 
1683     if they are passed unicode arguments versus simple string arguments.  For 
1684     instance, C{os.listdir} generally returns unicode path names if it is passed 
1685     a unicode argument, and string pathnames if it is passed a string argument. 
1686   
1687     However, this behavior often isn't as consistent as we might like.  As an example, 
1688     C{os.listdir} "gives up" if it finds a filename that it can't properly encode 
1689     given the current locale settings.  This means that the returned list is 
1690     a mixed set of unicode and simple string paths.  This has consequences later, 
1691     because other filesystem functions like C{os.path.join} will blow up if they 
1692     are given one string path and one unicode path. 
1693   
1694     On comp.lang.python, Martin v. Löwis explained the C{os.listdir} behavior 
1695     like this:: 
1696   
1697        The operating system (POSIX) does not have the inherent notion that file 
1698        names are character strings. Instead, in POSIX, file names are primarily 
1699        byte strings. There are some bytes which are interpreted as characters 
1700        (e.g. '\x2e', which is '.', or '\x2f', which is '/'), but apart from 
1701        that, most OS layers think these are just bytes. 
1702   
1703        Now, most *people* think that file names are character strings.  To 
1704        interpret a file name as a character string, you need to know what the 
1705        encoding is to interpret the file names (which are byte strings) as 
1706        character strings. 
1707   
1708        There is, unfortunately, no operating system API to carry the notion of a 
1709        file system encoding. By convention, the locale settings should be used 
1710        to establish this encoding, in particular the LC_CTYPE facet of the 
1711        locale. This is defined in the environment variables LC_CTYPE, LC_ALL, 
1712        and LANG (searched in this order). 
1713   
1714        If LANG is not set, the "C" locale is assumed, which uses ASCII as its 
1715        file system encoding. In this locale, '\xe2\x99\xaa\xe2\x99\xac' is not a 
1716        valid file name (at least it cannot be interpreted as characters, and 
1717        hence not be converted to Unicode). 
1718   
1719        Now, your Python script has requested that all file names *should* be 
1720        returned as character (ie. Unicode) strings, but Python cannot comply, 
1721        since there is no way to find out what this byte string means, in terms 
1722        of characters. 
1723   
1724        So we have three options: 
1725   
1726        1. Skip this string, only return the ones that can be converted to Unicode. 
1727           Give the user the impression the file does not exist. 
1728        2. Return the string as a byte string 
1729        3. Refuse to listdir altogether, raising an exception (i.e. return nothing) 
1730   
1731        Python has chosen alternative 2, allowing the application to implement 1 
1732        or 3 on top of that if it wants to (or come up with other strategies, 
1733        such as user feedback). 
1734   
1735     As a solution, he suggests that rather than passing unicode paths into the 
1736     filesystem functions, that I should sensibly encode the path first.  That is 
1737     what this function accomplishes.  Any function which takes a filesystem path 
1738     as an argument should encode it first, before using it for any other purpose. 
1739   
1740     I confess I still don't completely understand how this works.  On a system 
1741     with filesystem encoding "ISO-8859-1", a path C{u"\xe2\x99\xaa\xe2\x99\xac"} 
1742     is converted into the string C{"\xe2\x99\xaa\xe2\x99\xac"}.  However, on a 
1743     system with a "utf-8" encoding, the result is a completely different string: 
1744     C{"\xc3\xa2\xc2\x99\xc2\xaa\xc3\xa2\xc2\x99\xc2\xac"}.  A quick test where I 
1745     write to the first filename and open the second proves that the two strings 
1746     represent the same file on disk, which is all I really care about. 
1747   
1748     @note: As a special case, if C{path} is C{None}, then this function will 
1749     return C{None}. 
1750   
1751     @note: To provide several examples of encoding values, my Debian sarge box 
1752     with an ext3 filesystem has Python filesystem encoding C{ISO-8859-1}.  User 
1753     Anarcat's Debian box with a xfs filesystem has filesystem encoding 
1754     C{ANSI_X3.4-1968}.  Both my iBook G4 running Mac OS X 10.4 and user Dag 
1755     Rende's SuSE 9.3 box both have filesystem encoding C{UTF-8}. 
1756   
1757     @note: Just because a filesystem has C{UTF-8} encoding doesn't mean that it 
1758     will be able to handle all extended-character filenames.  For instance, 
1759     certain extended-character (but not UTF-8) filenames -- like the ones in the 
1760     regression test tar file C{test/data/tree13.tar.gz} -- are not valid under 
1761     Mac OS X, and it's not even possible to extract them from the tarfile on 
1762     that platform. 
1763   
1764     @param path: Path to encode 
1765   
1766     @return: Path, as a string, encoded appropriately 
1767     @raise ValueError: If the path cannot be encoded properly. 
1768     """ 
1769     if path is None: 
1770        return path 
1771     try: 
1772        if isinstance(path, unicode): 
1773           encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 
1774           path = path.encode(encoding) 
1775        return path 
1776     except UnicodeError: 
1777        raise ValueError("Path could not be safely encoded as %s." % encoding) 
 1778   
1785     """ 
1786     Attempts to portably return the null device on this system. 
1787   
1788     The null device is something like C{/dev/null} on a UNIX system.  The name 
1789     varies on other platforms. 
1790     """ 
1791     return os.devnull 
 1792   
1799     """ 
1800     Converts English day name to numeric day of week as from C{time.localtime}. 
1801   
1802     For instance, the day C{monday} would be converted to the number C{0}. 
1803   
1804     @param dayName: Day of week to convert 
1805     @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc. 
1806   
1807     @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible. 
1808     """ 
1809     if dayName.lower() == "monday": 
1810        return 0 
1811     elif dayName.lower() == "tuesday": 
1812        return 1 
1813     elif dayName.lower() == "wednesday": 
1814        return 2 
1815     elif dayName.lower() == "thursday": 
1816        return 3 
1817     elif dayName.lower() == "friday": 
1818        return 4 
1819     elif dayName.lower() == "saturday": 
1820        return 5 
1821     elif dayName.lower() == "sunday": 
1822        return 6 
1823     else: 
1824        return -1   
 1825   
1832     """ 
1833     Indicates whether "today" is the backup starting day per configuration. 
1834   
1835     If the current day's English name matches the indicated starting day, then 
1836     today is a starting day. 
1837   
1838     @param startingDay: Configured starting day. 
1839     @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc. 
1840   
1841     @return: Boolean indicating whether today is the starting day. 
1842     """ 
1843     value = time.localtime().tm_wday == deriveDayOfWeek(startingDay) 
1844     if value: 
1845        logger.debug("Today is the start of the week.") 
1846     else: 
1847        logger.debug("Today is NOT the start of the week.") 
1848     return value 
 1849   
1856     """ 
1857     Returns a "normalized" path based on a path name. 
1858   
1859     A normalized path is a representation of a path that is also a valid file 
1860     name.  To make a valid file name out of a complete path, we have to convert 
1861     or remove some characters that are significant to the filesystem -- in 
1862     particular, the path separator and any leading C{'.'} character (which would 
1863     cause the file to be hidden in a file listing). 
1864   
1865     Note that this is a one-way transformation -- you can't safely derive the 
1866     original path from the normalized path. 
1867   
1868     To normalize a path, we begin by looking at the first character.  If the 
1869     first character is C{'/'} or C{'\\'}, it gets removed.  If the first 
1870     character is C{'.'}, it gets converted to C{'_'}.  Then, we look through the 
1871     rest of the path and convert all remaining C{'/'} or C{'\\'} characters 
1872     C{'-'}, and all remaining whitespace characters to C{'_'}. 
1873   
1874     As a special case, a path consisting only of a single C{'/'} or C{'\\'} 
1875     character will be converted to C{'-'}. 
1876   
1877     @param path: Path to normalize 
1878   
1879     @return: Normalized path as described above. 
1880   
1881     @raise ValueError: If the path is None 
1882     """ 
1883     if path is None: 
1884        raise ValueError("Cannot normalize path None.") 
1885     elif len(path) == 0: 
1886        return path 
1887     elif path == "/" or path == "\\": 
1888        return "-" 
1889     else: 
1890        normalized = path 
1891        normalized = re.sub(r"^\/", "", normalized)   
1892        normalized = re.sub(r"^\\", "", normalized)   
1893        normalized = re.sub(r"^\.", "_", normalized)  
1894        normalized = re.sub(r"\/", "-", normalized)   
1895        normalized = re.sub(r"\\", "-", normalized)   
1896        normalized = re.sub(r"\s", "_", normalized)   
1897        return normalized 
 1898   
1905     """ 
1906     Sanitizes the operating system environment. 
1907   
1908     The operating system environment is contained in C{os.environ}.  This method 
1909     sanitizes the contents of that dictionary. 
1910   
1911     Currently, all it does is reset the locale (removing C{$LC_*}) and set the 
1912     default language (C{$LANG}) to L{DEFAULT_LANGUAGE}.  This way, we can count 
1913     on consistent localization regardless of what the end-user has configured. 
1914     This is important for code that needs to parse program output. 
1915   
1916     The C{os.environ} dictionary is modifed in-place.  If C{$LANG} is already 
1917     set to the proper value, it is not re-set, so we can avoid the memory leaks 
1918     that are documented to occur on BSD-based systems. 
1919   
1920     @return: Copy of the sanitized environment. 
1921     """ 
1922     for var in LOCALE_VARS: 
1923        if os.environ.has_key(var): 
1924           del os.environ[var] 
1925     if os.environ.has_key(LANG_VAR): 
1926        if os.environ[LANG_VAR] != DEFAULT_LANGUAGE:  
1927           os.environ[LANG_VAR] = DEFAULT_LANGUAGE 
1928     return os.environ.copy() 
 1929   
1936     """ 
1937     Deference a soft link, optionally normalizing it to an absolute path. 
1938     @param path: Path of link to dereference 
1939     @param absolute: Whether to normalize the result to an absolute path 
1940     @return: Dereferenced path, or original path if original is not a link. 
1941     """ 
1942     if os.path.islink(path): 
1943        result = os.readlink(path) 
1944        if absolute and not os.path.isabs(result): 
1945           result = os.path.abspath(os.path.join(os.path.dirname(path), result)) 
1946        return result 
1947     return path 
 1948   
1949   
1950   
1951   
1952   
1953   
1954 -def checkUnique(prefix, values): 
 1955     """ 
1956     Checks that all values are unique. 
1957   
1958     The values list is checked for duplicate values.  If there are 
1959     duplicates, an exception is thrown.  All duplicate values are listed in 
1960     the exception. 
1961   
1962     @param prefix: Prefix to use in the thrown exception 
1963     @param values: List of values to check 
1964   
1965     @raise ValueError: If there are duplicates in the list 
1966     """ 
1967     values.sort() 
1968     duplicates = [] 
1969     for i in range(1, len(values)): 
1970        if values[i-1] == values[i]: 
1971           duplicates.append(values[i]) 
1972     if duplicates: 
1973        raise ValueError("%s %s" % (prefix, duplicates)) 
 1974   
1981     """ 
1982     Parses a list of values out of a comma-separated string. 
1983   
1984     The items in the list are split by comma, and then have whitespace 
1985     stripped.  As a special case, if C{commaString} is C{None}, then C{None} 
1986     will be returned. 
1987   
1988     @param commaString: List of values in comma-separated string format. 
1989     @return: Values from commaString split into a list, or C{None}. 
1990     """ 
1991     if commaString is None: 
1992        return None 
1993     else: 
1994        pass1 = commaString.split(",") 
1995        pass2 = [] 
1996        for item in pass1: 
1997           item = item.strip() 
1998           if len(item) > 0: 
1999              pass2.append(item) 
2000        return pass2 
 2001