1   
   2   
   3   
   4   
   5   
   6   
   7   
   8   
   9   
  10   
  11   
  12   
  13   
  14   
  15   
  16   
  17   
  18   
  19   
  20   
  21   
  22   
  23   
  24   
  25   
  26   
  27   
  28   
  29   
  30   
  31   
  32   
  33   
  34   
  35   
  36   
  37   
  38  """ 
  39  Synchonizes a local directory with an Amazon S3 bucket. 
  40   
  41  No configuration is required; all necessary information is taken from the 
  42  command-line.  The only thing configuration would help with is the path 
  43  resolver interface, and it doesn't seem worth it to require configuration just 
  44  to get that. 
  45   
  46  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  47  """ 
  48   
  49   
  50   
  51   
  52   
  53   
  54  import sys 
  55  import os 
  56  import logging 
  57  import getopt 
  58  import json 
  59  import warnings 
  60  import chardet 
  61   
  62   
  63  from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT 
  64  from CedarBackup2.filesystem import FilesystemList 
  65  from CedarBackup2.cli import setupLogging, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE 
  66  from CedarBackup2.util import Diagnostics, splitCommandLine, encodePath 
  67  from CedarBackup2.util import executeCommand 
  68   
  69   
  70   
  71   
  72   
  73   
  74  logger = logging.getLogger("CedarBackup2.log.tools.amazons3") 
  75   
  76  AWS_COMMAND   = [ "aws" ] 
  77   
  78  SHORT_SWITCHES     = "hVbql:o:m:OdsDvw" 
  79  LONG_SWITCHES      = [ 'help', 'version', 'verbose', 'quiet', 
  80                         'logfile=', 'owner=', 'mode=', 
  81                         'output', 'debug', 'stack', 'diagnostics', 
  82                         'verifyOnly', 'ignoreWarnings', ] 
  83   
  84   
  85   
  86   
  87   
  88   
  90   
  91      
  92      
  93      
  94   
  95     """ 
  96     Class representing command-line options for the cback-amazons3-sync script. 
  97   
  98     The C{Options} class is a Python object representation of the command-line 
  99     options of the cback script. 
 100   
 101     The object representation is two-way: a command line string or a list of 
 102     command line arguments can be used to create an C{Options} object, and then 
 103     changes to the object can be propogated back to a list of command-line 
 104     arguments or to a command-line string.  An C{Options} object can even be 
 105     created from scratch programmatically (if you have a need for that). 
 106   
 107     There are two main levels of validation in the C{Options} class.  The first 
 108     is field-level validation.  Field-level validation comes into play when a 
 109     given field in an object is assigned to or updated.  We use Python's 
 110     C{property} functionality to enforce specific validations on field values, 
 111     and in some places we even use customized list classes to enforce 
 112     validations on list members.  You should expect to catch a C{ValueError} 
 113     exception when making assignments to fields if you are programmatically 
 114     filling an object. 
 115   
 116     The second level of validation is post-completion validation.  Certain 
 117     validations don't make sense until an object representation of options is 
 118     fully "complete".  We don't want these validations to apply all of the time, 
 119     because it would make building up a valid object from scratch a real pain. 
 120     For instance, we might have to do things in the right order to keep from 
 121     throwing exceptions, etc. 
 122   
 123     All of these post-completion validations are encapsulated in the 
 124     L{Options.validate} method.  This method can be called at any time by a 
 125     client, and will always be called immediately after creating a C{Options} 
 126     object from a command line and before exporting a C{Options} object back to 
 127     a command line.  This way, we get acceptable ease-of-use but we also don't 
 128     accept or emit invalid command lines. 
 129   
 130     @note: Lists within this class are "unordered" for equality comparisons. 
 131   
 132     @sort: __init__, __repr__, __str__, __cmp__ 
 133     """ 
 134   
 135      
 136      
 137      
 138   
 139 -   def __init__(self, argumentList=None, argumentString=None, validate=True): 
  140        """ 
 141        Initializes an options object. 
 142   
 143        If you initialize the object without passing either C{argumentList} or 
 144        C{argumentString}, the object will be empty and will be invalid until it 
 145        is filled in properly. 
 146   
 147        No reference to the original arguments is saved off by this class.  Once 
 148        the data has been parsed (successfully or not) this original information 
 149        is discarded. 
 150   
 151        The argument list is assumed to be a list of arguments, not including the 
 152        name of the command, something like C{sys.argv[1:]}.  If you pass 
 153        C{sys.argv} instead, things are not going to work. 
 154   
 155        The argument string will be parsed into an argument list by the 
 156        L{util.splitCommandLine} function (see the documentation for that 
 157        function for some important notes about its limitations).  There is an 
 158        assumption that the resulting list will be equivalent to C{sys.argv[1:]}, 
 159        just like C{argumentList}. 
 160   
 161        Unless the C{validate} argument is C{False}, the L{Options.validate} 
 162        method will be called (with its default arguments) after successfully 
 163        parsing any passed-in command line.  This validation ensures that 
 164        appropriate actions, etc. have been specified.  Keep in mind that even if 
 165        C{validate} is C{False}, it might not be possible to parse the passed-in 
 166        command line, so an exception might still be raised. 
 167   
 168        @note: The command line format is specified by the L{_usage} function. 
 169        Call L{_usage} to see a usage statement for the cback script. 
 170   
 171        @note: It is strongly suggested that the C{validate} option always be set 
 172        to C{True} (the default) unless there is a specific need to read in 
 173        invalid command line arguments. 
 174   
 175        @param argumentList: Command line for a program. 
 176        @type argumentList: List of arguments, i.e. C{sys.argv} 
 177   
 178        @param argumentString: Command line for a program. 
 179        @type argumentString: String, i.e. "cback --verbose stage store" 
 180   
 181        @param validate: Validate the command line after parsing it. 
 182        @type validate: Boolean true/false. 
 183   
 184        @raise getopt.GetoptError: If the command-line arguments could not be parsed. 
 185        @raise ValueError: If the command-line arguments are invalid. 
 186        """ 
 187        self._help = False 
 188        self._version = False 
 189        self._verbose = False 
 190        self._quiet = False 
 191        self._logfile = None 
 192        self._owner = None 
 193        self._mode = None 
 194        self._output = False 
 195        self._debug = False 
 196        self._stacktrace = False 
 197        self._diagnostics = False 
 198        self._verifyOnly = False 
 199        self._ignoreWarnings = False 
 200        self._sourceDir = None 
 201        self._s3BucketUrl = None 
 202        if argumentList is not None and argumentString is not None: 
 203           raise ValueError("Use either argumentList or argumentString, but not both.") 
 204        if argumentString is not None: 
 205           argumentList = splitCommandLine(argumentString) 
 206        if argumentList is not None: 
 207           self._parseArgumentList(argumentList) 
 208           if validate: 
 209              self.validate() 
  210   
 211   
 212      
 213      
 214      
 215   
 221   
 223        """ 
 224        Informal string representation for class instance. 
 225        """ 
 226        return self.__repr__() 
  227   
 228   
 229      
 230      
 231      
 232   
 318   
 319   
 320      
 321      
 322      
 323   
 325        """ 
 326        Property target used to set the help flag. 
 327        No validations, but we normalize the value to C{True} or C{False}. 
 328        """ 
 329        if value: 
 330           self._help = True 
 331        else: 
 332           self._help = False 
  333   
 335        """ 
 336        Property target used to get the help flag. 
 337        """ 
 338        return self._help 
  339   
 341        """ 
 342        Property target used to set the version flag. 
 343        No validations, but we normalize the value to C{True} or C{False}. 
 344        """ 
 345        if value: 
 346           self._version = True 
 347        else: 
 348           self._version = False 
  349   
 351        """ 
 352        Property target used to get the version flag. 
 353        """ 
 354        return self._version 
  355   
 357        """ 
 358        Property target used to set the verbose flag. 
 359        No validations, but we normalize the value to C{True} or C{False}. 
 360        """ 
 361        if value: 
 362           self._verbose = True 
 363        else: 
 364           self._verbose = False 
  365   
 367        """ 
 368        Property target used to get the verbose flag. 
 369        """ 
 370        return self._verbose 
  371   
 373        """ 
 374        Property target used to set the quiet flag. 
 375        No validations, but we normalize the value to C{True} or C{False}. 
 376        """ 
 377        if value: 
 378           self._quiet = True 
 379        else: 
 380           self._quiet = False 
  381   
 383        """ 
 384        Property target used to get the quiet flag. 
 385        """ 
 386        return self._quiet 
  387   
 389        """ 
 390        Property target used to set the logfile parameter. 
 391        @raise ValueError: If the value cannot be encoded properly. 
 392        """ 
 393        if value is not None: 
 394           if len(value) < 1: 
 395              raise ValueError("The logfile parameter must be a non-empty string.") 
 396        self._logfile = encodePath(value) 
  397   
 399        """ 
 400        Property target used to get the logfile parameter. 
 401        """ 
 402        return self._logfile 
  403   
 405        """ 
 406        Property target used to set the owner parameter. 
 407        If not C{None}, the owner must be a C{(user,group)} tuple or list. 
 408        Strings (and inherited children of strings) are explicitly disallowed. 
 409        The value will be normalized to a tuple. 
 410        @raise ValueError: If the value is not valid. 
 411        """ 
 412        if value is None: 
 413           self._owner = None 
 414        else: 
 415           if isinstance(value, str): 
 416              raise ValueError("Must specify user and group tuple for owner parameter.") 
 417           if len(value) != 2: 
 418              raise ValueError("Must specify user and group tuple for owner parameter.") 
 419           if len(value[0]) < 1 or len(value[1]) < 1: 
 420              raise ValueError("User and group tuple values must be non-empty strings.") 
 421           self._owner = (value[0], value[1]) 
  422   
 424        """ 
 425        Property target used to get the owner parameter. 
 426        The parameter is a tuple of C{(user, group)}. 
 427        """ 
 428        return self._owner 
  429   
 431        """ 
 432        Property target used to set the mode parameter. 
 433        """ 
 434        if value is None: 
 435           self._mode = None 
 436        else: 
 437           try: 
 438              if isinstance(value, str): 
 439                 value = int(value, 8) 
 440              else: 
 441                 value = int(value) 
 442           except TypeError: 
 443              raise ValueError("Mode must be an octal integer >= 0, i.e. 644.") 
 444           if value < 0: 
 445              raise ValueError("Mode must be an octal integer >= 0. i.e. 644.") 
 446           self._mode = value 
  447   
 449        """ 
 450        Property target used to get the mode parameter. 
 451        """ 
 452        return self._mode 
  453   
 455        """ 
 456        Property target used to set the output flag. 
 457        No validations, but we normalize the value to C{True} or C{False}. 
 458        """ 
 459        if value: 
 460           self._output = True 
 461        else: 
 462           self._output = False 
  463   
 465        """ 
 466        Property target used to get the output flag. 
 467        """ 
 468        return self._output 
  469   
 471        """ 
 472        Property target used to set the debug flag. 
 473        No validations, but we normalize the value to C{True} or C{False}. 
 474        """ 
 475        if value: 
 476           self._debug = True 
 477        else: 
 478           self._debug = False 
  479   
 481        """ 
 482        Property target used to get the debug flag. 
 483        """ 
 484        return self._debug 
  485   
 487        """ 
 488        Property target used to set the stacktrace flag. 
 489        No validations, but we normalize the value to C{True} or C{False}. 
 490        """ 
 491        if value: 
 492           self._stacktrace = True 
 493        else: 
 494           self._stacktrace = False 
  495   
 497        """ 
 498        Property target used to get the stacktrace flag. 
 499        """ 
 500        return self._stacktrace 
  501   
 503        """ 
 504        Property target used to set the diagnostics flag. 
 505        No validations, but we normalize the value to C{True} or C{False}. 
 506        """ 
 507        if value: 
 508           self._diagnostics = True 
 509        else: 
 510           self._diagnostics = False 
  511   
 513        """ 
 514        Property target used to get the diagnostics flag. 
 515        """ 
 516        return self._diagnostics 
  517   
 519        """ 
 520        Property target used to set the verifyOnly flag. 
 521        No validations, but we normalize the value to C{True} or C{False}. 
 522        """ 
 523        if value: 
 524           self._verifyOnly = True 
 525        else: 
 526           self._verifyOnly = False 
  527   
 529        """ 
 530        Property target used to get the verifyOnly flag. 
 531        """ 
 532        return self._verifyOnly 
  533   
 535        """ 
 536        Property target used to set the ignoreWarnings flag. 
 537        No validations, but we normalize the value to C{True} or C{False}. 
 538        """ 
 539        if value: 
 540           self._ignoreWarnings = True 
 541        else: 
 542           self._ignoreWarnings = False 
  543   
 545        """ 
 546        Property target used to get the ignoreWarnings flag. 
 547        """ 
 548        return self._ignoreWarnings 
  549   
 551        """ 
 552        Property target used to set the sourceDir parameter. 
 553        """ 
 554        if value is not None: 
 555           if len(value) < 1: 
 556              raise ValueError("The sourceDir parameter must be a non-empty string.") 
 557        self._sourceDir = value 
  558   
 560        """ 
 561        Property target used to get the sourceDir parameter. 
 562        """ 
 563        return self._sourceDir 
  564   
 566        """ 
 567        Property target used to set the s3BucketUrl parameter. 
 568        """ 
 569        if value is not None: 
 570           if len(value) < 1: 
 571              raise ValueError("The s3BucketUrl parameter must be a non-empty string.") 
 572        self._s3BucketUrl = value 
  573   
 575        """ 
 576        Property target used to get the s3BucketUrl parameter. 
 577        """ 
 578        return self._s3BucketUrl 
  579   
 580     help = property(_getHelp, _setHelp, None, "Command-line help (C{-h,--help}) flag.") 
 581     version = property(_getVersion, _setVersion, None, "Command-line version (C{-V,--version}) flag.") 
 582     verbose = property(_getVerbose, _setVerbose, None, "Command-line verbose (C{-b,--verbose}) flag.") 
 583     quiet = property(_getQuiet, _setQuiet, None, "Command-line quiet (C{-q,--quiet}) flag.") 
 584     logfile = property(_getLogfile, _setLogfile, None, "Command-line logfile (C{-l,--logfile}) parameter.") 
 585     owner = property(_getOwner, _setOwner, None, "Command-line owner (C{-o,--owner}) parameter, as tuple C{(user,group)}.") 
 586     mode = property(_getMode, _setMode, None, "Command-line mode (C{-m,--mode}) parameter.") 
 587     output = property(_getOutput, _setOutput, None, "Command-line output (C{-O,--output}) flag.") 
 588     debug = property(_getDebug, _setDebug, None, "Command-line debug (C{-d,--debug}) flag.") 
 589     stacktrace = property(_getStacktrace, _setStacktrace, None, "Command-line stacktrace (C{-s,--stack}) flag.") 
 590     diagnostics = property(_getDiagnostics, _setDiagnostics, None, "Command-line diagnostics (C{-D,--diagnostics}) flag.") 
 591     verifyOnly = property(_getVerifyOnly, _setVerifyOnly, None, "Command-line verifyOnly (C{-v,--verifyOnly}) flag.") 
 592     ignoreWarnings = property(_getIgnoreWarnings, _setIgnoreWarnings, None, "Command-line ignoreWarnings (C{-w,--ignoreWarnings}) flag.") 
 593     sourceDir = property(_getSourceDir, _setSourceDir, None, "Command-line sourceDir, source of sync.") 
 594     s3BucketUrl = property(_getS3BucketUrl, _setS3BucketUrl, None, "Command-line s3BucketUrl, target of sync.") 
 595   
 596   
 597      
 598      
 599      
 600   
 602        """ 
 603        Validates command-line options represented by the object. 
 604   
 605        Unless C{--help} or C{--version} are supplied, at least one action must 
 606        be specified.  Other validations (as for allowed values for particular 
 607        options) will be taken care of at assignment time by the properties 
 608        functionality. 
 609   
 610        @note: The command line format is specified by the L{_usage} function. 
 611        Call L{_usage} to see a usage statement for the cback script. 
 612   
 613        @raise ValueError: If one of the validations fails. 
 614        """ 
 615        if not self.help and not self.version and not self.diagnostics: 
 616           if self.sourceDir is None or self.s3BucketUrl is None: 
 617              raise ValueError("Source directory and S3 bucket URL are both required.") 
  618   
 620        """ 
 621        Extracts options into a list of command line arguments. 
 622   
 623        The original order of the various arguments (if, indeed, the object was 
 624        initialized with a command-line) is not preserved in this generated 
 625        argument list.   Besides that, the argument list is normalized to use the 
 626        long option names (i.e. --version rather than -V).  The resulting list 
 627        will be suitable for passing back to the constructor in the 
 628        C{argumentList} parameter.  Unlike L{buildArgumentString}, string 
 629        arguments are not quoted here, because there is no need for it. 
 630   
 631        Unless the C{validate} parameter is C{False}, the L{Options.validate} 
 632        method will be called (with its default arguments) against the 
 633        options before extracting the command line.  If the options are not valid, 
 634        then an argument list will not be extracted. 
 635   
 636        @note: It is strongly suggested that the C{validate} option always be set 
 637        to C{True} (the default) unless there is a specific need to extract an 
 638        invalid command line. 
 639   
 640        @param validate: Validate the options before extracting the command line. 
 641        @type validate: Boolean true/false. 
 642   
 643        @return: List representation of command-line arguments. 
 644        @raise ValueError: If options within the object are invalid. 
 645        """ 
 646        if validate: 
 647           self.validate() 
 648        argumentList = [] 
 649        if self._help: 
 650           argumentList.append("--help") 
 651        if self.version: 
 652           argumentList.append("--version") 
 653        if self.verbose: 
 654           argumentList.append("--verbose") 
 655        if self.quiet: 
 656           argumentList.append("--quiet") 
 657        if self.logfile is not None: 
 658           argumentList.append("--logfile") 
 659           argumentList.append(self.logfile) 
 660        if self.owner is not None: 
 661           argumentList.append("--owner") 
 662           argumentList.append("%s:%s" % (self.owner[0], self.owner[1])) 
 663        if self.mode is not None: 
 664           argumentList.append("--mode") 
 665           argumentList.append("%o" % self.mode) 
 666        if self.output: 
 667           argumentList.append("--output") 
 668        if self.debug: 
 669           argumentList.append("--debug") 
 670        if self.stacktrace: 
 671           argumentList.append("--stack") 
 672        if self.diagnostics: 
 673           argumentList.append("--diagnostics") 
 674        if self.verifyOnly: 
 675           argumentList.append("--verifyOnly") 
 676        if self.ignoreWarnings: 
 677           argumentList.append("--ignoreWarnings") 
 678        if self.sourceDir is not None: 
 679           argumentList.append(self.sourceDir) 
 680        if self.s3BucketUrl is not None: 
 681           argumentList.append(self.s3BucketUrl) 
 682        return argumentList 
  683   
 685        """ 
 686        Extracts options into a string of command-line arguments. 
 687   
 688        The original order of the various arguments (if, indeed, the object was 
 689        initialized with a command-line) is not preserved in this generated 
 690        argument string.   Besides that, the argument string is normalized to use 
 691        the long option names (i.e. --version rather than -V) and to quote all 
 692        string arguments with double quotes (C{"}).  The resulting string will be 
 693        suitable for passing back to the constructor in the C{argumentString} 
 694        parameter. 
 695   
 696        Unless the C{validate} parameter is C{False}, the L{Options.validate} 
 697        method will be called (with its default arguments) against the options 
 698        before extracting the command line.  If the options are not valid, then 
 699        an argument string will not be extracted. 
 700   
 701        @note: It is strongly suggested that the C{validate} option always be set 
 702        to C{True} (the default) unless there is a specific need to extract an 
 703        invalid command line. 
 704   
 705        @param validate: Validate the options before extracting the command line. 
 706        @type validate: Boolean true/false. 
 707   
 708        @return: String representation of command-line arguments. 
 709        @raise ValueError: If options within the object are invalid. 
 710        """ 
 711        if validate: 
 712           self.validate() 
 713        argumentString = "" 
 714        if self._help: 
 715           argumentString += "--help " 
 716        if self.version: 
 717           argumentString += "--version " 
 718        if self.verbose: 
 719           argumentString += "--verbose " 
 720        if self.quiet: 
 721           argumentString += "--quiet " 
 722        if self.logfile is not None: 
 723           argumentString += "--logfile \"%s\" " % self.logfile 
 724        if self.owner is not None: 
 725           argumentString += "--owner \"%s:%s\" " % (self.owner[0], self.owner[1]) 
 726        if self.mode is not None: 
 727           argumentString += "--mode %o " % self.mode 
 728        if self.output: 
 729           argumentString += "--output " 
 730        if self.debug: 
 731           argumentString += "--debug " 
 732        if self.stacktrace: 
 733           argumentString += "--stack " 
 734        if self.diagnostics: 
 735           argumentString += "--diagnostics " 
 736        if self.verifyOnly: 
 737           argumentString += "--verifyOnly " 
 738        if self.ignoreWarnings: 
 739           argumentString += "--ignoreWarnings " 
 740        if self.sourceDir is not None: 
 741           argumentString +=  "\"%s\" " % self.sourceDir 
 742        if self.s3BucketUrl is not None: 
 743           argumentString +=  "\"%s\" " % self.s3BucketUrl 
 744        return argumentString 
  745   
 747        """ 
 748        Internal method to parse a list of command-line arguments. 
 749   
 750        Most of the validation we do here has to do with whether the arguments 
 751        can be parsed and whether any values which exist are valid.  We don't do 
 752        any validation as to whether required elements exist or whether elements 
 753        exist in the proper combination (instead, that's the job of the 
 754        L{validate} method). 
 755   
 756        For any of the options which supply parameters, if the option is 
 757        duplicated with long and short switches (i.e. C{-l} and a C{--logfile}) 
 758        then the long switch is used.  If the same option is duplicated with the 
 759        same switch (long or short), then the last entry on the command line is 
 760        used. 
 761   
 762        @param argumentList: List of arguments to a command. 
 763        @type argumentList: List of arguments to a command, i.e. C{sys.argv[1:]} 
 764   
 765        @raise ValueError: If the argument list cannot be successfully parsed. 
 766        """ 
 767        switches = { } 
 768        opts, remaining = getopt.getopt(argumentList, SHORT_SWITCHES, LONG_SWITCHES) 
 769        for o, a in opts:   
 770           switches[o] = a 
 771        if switches.has_key("-h") or switches.has_key("--help"): 
 772           self.help = True 
 773        if switches.has_key("-V") or switches.has_key("--version"): 
 774           self.version = True 
 775        if switches.has_key("-b") or switches.has_key("--verbose"): 
 776           self.verbose = True 
 777        if switches.has_key("-q") or switches.has_key("--quiet"): 
 778           self.quiet = True 
 779        if switches.has_key("-l"): 
 780           self.logfile = switches["-l"] 
 781        if switches.has_key("--logfile"): 
 782           self.logfile = switches["--logfile"] 
 783        if switches.has_key("-o"): 
 784           self.owner = switches["-o"].split(":", 1) 
 785        if switches.has_key("--owner"): 
 786           self.owner = switches["--owner"].split(":", 1) 
 787        if switches.has_key("-m"): 
 788           self.mode = switches["-m"] 
 789        if switches.has_key("--mode"): 
 790           self.mode = switches["--mode"] 
 791        if switches.has_key("-O") or switches.has_key("--output"): 
 792           self.output = True 
 793        if switches.has_key("-d") or switches.has_key("--debug"): 
 794           self.debug = True 
 795        if switches.has_key("-s") or switches.has_key("--stack"): 
 796           self.stacktrace = True 
 797        if switches.has_key("-D") or switches.has_key("--diagnostics"): 
 798           self.diagnostics = True 
 799        if switches.has_key("-v") or switches.has_key("--verifyOnly"): 
 800           self.verifyOnly = True 
 801        if switches.has_key("-w") or switches.has_key("--ignoreWarnings"): 
 802           self.ignoreWarnings = True 
 803        try: 
 804           (self.sourceDir, self.s3BucketUrl) = remaining 
 805        except ValueError: 
 806           pass 
   807   
 808   
 809   
 810   
 811   
 812   
 813   
 814   
 815   
 816   
 818     """ 
 819     Implements the command-line interface for the C{cback-amazons3-sync} script. 
 820   
 821     Essentially, this is the "main routine" for the cback-amazons3-sync script.  It does 
 822     all of the argument processing for the script, and then also implements the 
 823     tool functionality. 
 824   
 825     This function looks pretty similiar to C{CedarBackup2.cli.cli()}.  It's not 
 826     easy to refactor this code to make it reusable and also readable, so I've 
 827     decided to just live with the duplication. 
 828   
 829     A different error code is returned for each type of failure: 
 830   
 831        - C{1}: The Python interpreter version is < 2.7 
 832        - C{2}: Error processing command-line arguments 
 833        - C{3}: Error configuring logging 
 834        - C{5}: Backup was interrupted with a CTRL-C or similar 
 835        - C{6}: Error executing other parts of the script 
 836   
 837     @note: This script uses print rather than logging to the INFO level, because 
 838     it is interactive.  Underlying Cedar Backup functionality uses the logging 
 839     mechanism exclusively. 
 840   
 841     @return: Error code as described above. 
 842     """ 
 843     try: 
 844        if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 7]: 
 845           sys.stderr.write("Python 2 version 2.7 or greater required.\n") 
 846           return 1 
 847     except: 
 848         
 849        sys.stderr.write("Python 2 version 2.7 or greater required.\n") 
 850        return 1 
 851   
 852     try: 
 853        options = Options(argumentList=sys.argv[1:]) 
 854     except Exception, e: 
 855        _usage() 
 856        sys.stderr.write(" *** Error: %s\n" % e) 
 857        return 2 
 858   
 859     if options.help: 
 860        _usage() 
 861        return 0 
 862     if options.version: 
 863        _version() 
 864        return 0 
 865     if options.diagnostics: 
 866        _diagnostics() 
 867        return 0 
 868   
 869     if options.stacktrace: 
 870        logfile = setupLogging(options) 
 871     else: 
 872        try: 
 873           logfile = setupLogging(options) 
 874        except Exception as e: 
 875           sys.stderr.write("Error setting up logging: %s\n" % e) 
 876           return 3 
 877   
 878     logger.info("Cedar Backup Amazon S3 sync run started.") 
 879     logger.info("Options were [%s]", options) 
 880     logger.info("Logfile is [%s]", logfile) 
 881     Diagnostics().logDiagnostics(method=logger.info) 
 882   
 883     if options.stacktrace: 
 884        _executeAction(options) 
 885     else: 
 886        try: 
 887           _executeAction(options) 
 888        except KeyboardInterrupt: 
 889           logger.error("Backup interrupted.") 
 890           logger.info("Cedar Backup Amazon S3 sync run completed with status 5.") 
 891           return 5 
 892        except Exception, e: 
 893           logger.error("Error executing backup: %s", e) 
 894           logger.info("Cedar Backup Amazon S3 sync run completed with status 6.") 
 895           return 6 
 896   
 897     logger.info("Cedar Backup Amazon S3 sync run completed with status 0.") 
 898     return 0 
  899   
 900   
 901   
 902   
 903   
 904   
 905   
 906   
 907   
 908   
 910     """ 
 911     Prints usage information for the cback-amazons3-sync script. 
 912     @param fd: File descriptor used to print information. 
 913     @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
 914     """ 
 915     fd.write("\n") 
 916     fd.write(" Usage: cback-amazons3-sync [switches] sourceDir s3bucketUrl\n") 
 917     fd.write("\n") 
 918     fd.write(" Cedar Backup Amazon S3 sync tool.\n") 
 919     fd.write("\n") 
 920     fd.write(" This Cedar Backup utility synchronizes a local directory to an Amazon S3\n") 
 921     fd.write(" bucket.  After the sync is complete, a validation step is taken.  An\n") 
 922     fd.write(" error is reported if the contents of the bucket do not match the\n") 
 923     fd.write(" source directory, or if the indicated size for any file differs.\n") 
 924     fd.write(" This tool is a wrapper over the AWS CLI command-line tool.\n") 
 925     fd.write("\n") 
 926     fd.write(" The following arguments are required:\n") 
 927     fd.write("\n") 
 928     fd.write("   sourceDir            The local source directory on disk (must exist)\n") 
 929     fd.write("   s3BucketUrl          The URL to the target Amazon S3 bucket\n") 
 930     fd.write("\n") 
 931     fd.write(" The following switches are accepted:\n") 
 932     fd.write("\n") 
 933     fd.write("   -h, --help           Display this usage/help listing\n") 
 934     fd.write("   -V, --version        Display version information\n") 
 935     fd.write("   -b, --verbose        Print verbose output as well as logging to disk\n") 
 936     fd.write("   -q, --quiet          Run quietly (display no output to the screen)\n") 
 937     fd.write("   -l, --logfile        Path to logfile (default: %s)\n" % DEFAULT_LOGFILE) 
 938     fd.write("   -o, --owner          Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1])) 
 939     fd.write("   -m, --mode           Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE) 
 940     fd.write("   -O, --output         Record some sub-command (i.e. aws) output to the log\n") 
 941     fd.write("   -d, --debug          Write debugging information to the log (implies --output)\n") 
 942     fd.write("   -s, --stack          Dump Python stack trace instead of swallowing exceptions\n")  
 943     fd.write("   -D, --diagnostics    Print runtime diagnostics to the screen and exit\n") 
 944     fd.write("   -v, --verifyOnly     Only verify the S3 bucket contents, do not make changes\n") 
 945     fd.write("   -w, --ignoreWarnings Ignore warnings about problematic filename encodings\n") 
 946     fd.write("\n") 
 947     fd.write(" Typical usage would be something like:\n") 
 948     fd.write("\n") 
 949     fd.write("   cback-amazons3-sync /home/myuser s3://example.com-backup/myuser\n") 
 950     fd.write("\n") 
 951     fd.write(" This will sync the contents of /home/myuser into the indicated bucket.\n") 
 952     fd.write("\n") 
  953   
 954   
 955   
 956   
 957   
 958   
 960     """ 
 961     Prints version information for the cback script. 
 962     @param fd: File descriptor used to print information. 
 963     @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
 964     """ 
 965     fd.write("\n") 
 966     fd.write(" Cedar Backup Amazon S3 sync tool.\n") 
 967     fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE)) 
 968     fd.write("\n") 
 969     fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL)) 
 970     fd.write(" See CREDITS for a list of included code and other contributors.\n") 
 971     fd.write(" This is free software; there is NO warranty.  See the\n") 
 972     fd.write(" GNU General Public License version 2 for copying conditions.\n") 
 973     fd.write("\n") 
 974     fd.write(" Use the --help option for usage information.\n") 
 975     fd.write("\n") 
  976   
 977   
 978   
 979   
 980   
 981   
 983     """ 
 984     Prints runtime diagnostics information. 
 985     @param fd: File descriptor used to print information. 
 986     @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
 987     """ 
 988     fd.write("\n") 
 989     fd.write("Diagnostics:\n") 
 990     fd.write("\n") 
 991     Diagnostics().printDiagnostics(fd=fd, prefix="   ") 
 992     fd.write("\n") 
  993   
 994   
 995   
 996   
 997   
 998   
1014   
1015   
1016   
1017   
1018   
1019   
1021     """ 
1022     Build a list of files in a source directory 
1023     @param sourceDir: Local source directory 
1024     @return: FilesystemList with contents of source directory 
1025     """ 
1026     if not os.path.isdir(sourceDir): 
1027        raise ValueError("Source directory does not exist on disk.") 
1028     sourceFiles = FilesystemList() 
1029     sourceFiles.addDirContents(sourceDir) 
1030     return sourceFiles 
 1031   
1032   
1033   
1034   
1035   
1036   
1037   
1039     """ 
1040     Check source files, trying to guess which ones will have encoding problems. 
1041     @param sourceDir: Local source directory 
1042     @param sourceDir: Local source directory 
1043     @raises ValueError: If a problem file is found 
1044     @see U{http://opensourcehacker.com/2011/09/16/fix-linux-filename-encodings-with-python/} 
1045     @see U{http://serverfault.com/questions/82821/how-to-tell-the-language-encoding-of-a-filename-on-linux} 
1046     @see U{http://randysofia.com/2014/06/06/aws-cli-and-your-locale/} 
1047     """ 
1048     with warnings.catch_warnings(): 
1049        warnings.simplefilter("ignore")  
1050   
1051        encoding = Diagnostics().encoding 
1052   
1053        failed = False 
1054        for entry in sourceFiles: 
1055           result = chardet.detect(entry) 
1056           source = entry.decode(result["encoding"]) 
1057           try: 
1058              target = source.encode(encoding) 
1059              if source != target: 
1060                 logger.error("Inconsistent encoding for [%s]: got %s, but need %s", entry, result["encoding"], encoding) 
1061                 failed = True 
1062           except UnicodeEncodeError: 
1063              logger.error("Inconsistent encoding for [%s]: got %s, but need %s", entry, result["encoding"], encoding) 
1064              failed = True 
1065   
1066        if not failed: 
1067           logger.info("Completed checking source filename encoding (no problems found).") 
1068        else: 
1069           logger.error("Some filenames have inconsistent encodings and will likely cause sync problems.") 
1070           logger.error("You may be able to fix this by setting a more sensible locale in your environment.") 
1071           logger.error("Aternately, you can rename the problem files to be valid in the indicated locale.") 
1072           logger.error("To ignore this warning and proceed anyway, use --ignoreWarnings") 
1073           raise ValueError("Some filenames have inconsistent encodings and will likely cause sync problems.") 
 1074   
1075   
1076   
1077   
1078   
1079   
1081     """ 
1082     Synchronize a local directory to an Amazon S3 bucket. 
1083     @param sourceDir: Local source directory 
1084     @param s3BucketUrl: Target S3 bucket URL 
1085     """ 
1086      
1087      
1088      
1089     logger.info("Synchronizing local source directory up to Amazon S3.") 
1090     args = [ "s3", "sync", sourceDir, s3BucketUrl, "--delete" ] 
1091     result = executeCommand(AWS_COMMAND, args, returnOutput=False)[0] 
1092     if result != 0: 
1093        raise IOError("Error [%d] calling AWS CLI synchronize bucket." % result) 
 1094   
1095   
1096   
1097   
1098   
1099   
1100 -def _verifyBucketContents(sourceDir, sourceFiles, s3BucketUrl): 
 1101     """ 
1102     Verify that a source directory is equivalent to an Amazon S3 bucket. 
1103     @param sourceDir: Local source directory 
1104     @param sourceFiles: Filesystem list containing contents of source directory 
1105     @param s3BucketUrl: Target S3 bucket URL 
1106     """ 
1107      
1108      
1109      
1110      
1111      
1112      
1113      
1114      
1115      
1116      
1117      
1118      
1119      
1120      
1121      
1122      
1123      
1124      
1125      
1126      
1127      
1128   
1129     (bucket, prefix) = s3BucketUrl.replace("s3://", "").split("/", 1) 
1130   
1131     query = "Contents[].{Key: Key, Size: Size}" 
1132     args = [ "s3api", "list-objects", "--bucket", bucket, "--prefix", prefix, "--query", query, ] 
1133     (result, data) = executeCommand(AWS_COMMAND, args, returnOutput=True) 
1134     if result != 0: 
1135        raise IOError("Error [%d] calling AWS CLI verify bucket contents." % result) 
1136   
1137     contents = { } 
1138     for entry in json.loads("".join(data)): 
1139        key = entry["Key"].replace(prefix, "") 
1140        size = long(entry["Size"]) 
1141        contents[key] = size 
1142   
1143     failed = False 
1144     for entry in sourceFiles: 
1145        if os.path.isfile(entry): 
1146           key = entry.replace(sourceDir, "") 
1147           size = long(os.stat(entry).st_size) 
1148           if not key in contents: 
1149              logger.error("File was apparently not uploaded: [%s]", entry) 
1150              failed = True 
1151           else: 
1152              if size != contents[key]: 
1153                 logger.error("File size differs [%s]: expected %s bytes but got %s bytes", entry, size, contents[key]) 
1154                 failed = True 
1155   
1156     if not failed: 
1157        logger.info("Completed verifying Amazon S3 bucket contents (no problems found).") 
1158     else: 
1159        logger.error("There were differences between source directory and target S3 bucket.") 
1160        raise ValueError("There were differences between source directory and target S3 bucket.") 
 1161   
1162   
1163   
1164   
1165   
1166   
1167  if __name__ == "__main__": 
1168     sys.exit(cli()) 
1169