1   
  2   
  3   
  4   
  5   
  6   
  7   
  8   
  9   
 10   
 11   
 12   
 13   
 14   
 15   
 16   
 17   
 18   
 19   
 20   
 21   
 22   
 23   
 24   
 25   
 26   
 27   
 28   
 29   
 30   
 31   
 32   
 33   
 34   
 35   
 36   
 37   
 38  """ 
 39  Spans staged data among multiple discs 
 40   
 41  This is the Cedar Backup span tool.  It is intended for use by people who stage 
 42  more data than can fit on a single disc.  It allows a user to split staged data 
 43  among more than one disc.  It can't be an extension because it requires user 
 44  input when switching media. 
 45   
 46  Most configuration is taken from the Cedar Backup configuration file, 
 47  specifically the store section.  A few pieces of configuration are taken 
 48  directly from the user. 
 49   
 50  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 51  """ 
 52   
 53   
 54   
 55   
 56   
 57   
 58  import sys 
 59  import os 
 60  import logging 
 61  import tempfile 
 62   
 63   
 64  from CedarBackup2.release import AUTHOR, EMAIL, VERSION, DATE, COPYRIGHT 
 65  from CedarBackup2.util import displayBytes, convertSize, mount, unmount 
 66  from CedarBackup2.util import UNIT_SECTORS, UNIT_BYTES 
 67  from CedarBackup2.config import Config 
 68  from CedarBackup2.filesystem import BackupFileList, compareDigestMaps, normalizeDir 
 69  from CedarBackup2.cli import Options, setupLogging, setupPathResolver 
 70  from CedarBackup2.cli import DEFAULT_CONFIG, DEFAULT_LOGFILE, DEFAULT_OWNERSHIP, DEFAULT_MODE 
 71  from CedarBackup2.actions.constants import STORE_INDICATOR 
 72  from CedarBackup2.actions.util import createWriter 
 73  from CedarBackup2.actions.store import writeIndicatorFile 
 74  from CedarBackup2.actions.util import findDailyDirs 
 75  from CedarBackup2.util import Diagnostics 
 76   
 77   
 78   
 79   
 80   
 81   
 82  logger = logging.getLogger("CedarBackup2.log.tools.span") 
 83   
 84   
 85   
 86   
 87   
 88   
 90   
 91     """ 
 92     Tool-specific command-line options. 
 93   
 94     Most of the cback command-line options are exactly what we need here -- 
 95     logfile path, permissions, verbosity, etc.  However, we need to make a few 
 96     tweaks since we don't accept any actions. 
 97   
 98     Also, a few extra command line options that we accept are really ignored 
 99     underneath.  I just don't care about that for a tool like this. 
100     """ 
101   
103        """ 
104        Validates command-line options represented by the object. 
105        There are no validations here, because we don't use any actions. 
106        @raise ValueError: If one of the validations fails. 
107        """ 
108        pass 
  109   
110   
111   
112   
113   
114   
115   
116   
117   
118   
120     """ 
121     Implements the command-line interface for the C{cback-span} script. 
122   
123     Essentially, this is the "main routine" for the cback-span script.  It does 
124     all of the argument processing for the script, and then also implements the 
125     tool functionality. 
126   
127     This function looks pretty similiar to C{CedarBackup2.cli.cli()}.  It's not 
128     easy to refactor this code to make it reusable and also readable, so I've 
129     decided to just live with the duplication. 
130   
131     A different error code is returned for each type of failure: 
132   
133        - C{1}: The Python interpreter version is < 2.7 
134        - C{2}: Error processing command-line arguments 
135        - C{3}: Error configuring logging 
136        - C{4}: Error parsing indicated configuration file 
137        - C{5}: Backup was interrupted with a CTRL-C or similar 
138        - C{6}: Error executing other parts of the script 
139   
140     @note: This script uses print rather than logging to the INFO level, because 
141     it is interactive.  Underlying Cedar Backup functionality uses the logging 
142     mechanism exclusively. 
143   
144     @return: Error code as described above. 
145     """ 
146     try: 
147        if map(int, [sys.version_info[0], sys.version_info[1]]) < [2, 7]: 
148           sys.stderr.write("Python 2 version 2.7 or greater required.\n") 
149           return 1 
150     except: 
151         
152        sys.stderr.write("Python 2 version 2.7 or greater required.\n") 
153        return 1 
154   
155     try: 
156        options = SpanOptions(argumentList=sys.argv[1:]) 
157     except Exception, e: 
158        _usage() 
159        sys.stderr.write(" *** Error: %s\n" % e) 
160        return 2 
161   
162     if options.help: 
163        _usage() 
164        return 0 
165     if options.version: 
166        _version() 
167        return 0 
168     if options.diagnostics: 
169        _diagnostics() 
170        return 0 
171   
172     if options.stacktrace: 
173        logfile = setupLogging(options) 
174     else: 
175        try: 
176           logfile = setupLogging(options) 
177        except Exception as e: 
178           sys.stderr.write("Error setting up logging: %s\n" % e) 
179           return 3 
180   
181     logger.info("Cedar Backup 'span' utility run started.") 
182     logger.info("Options were [%s]", options) 
183     logger.info("Logfile is [%s]", logfile) 
184   
185     if options.config is None: 
186        logger.debug("Using default configuration file.") 
187        configPath = DEFAULT_CONFIG 
188     else: 
189        logger.debug("Using user-supplied configuration file.") 
190        configPath = options.config 
191   
192     try: 
193        logger.info("Configuration path is [%s]", configPath) 
194        config = Config(xmlPath=configPath) 
195        setupPathResolver(config) 
196     except Exception, e: 
197        logger.error("Error reading or handling configuration: %s", e) 
198        logger.info("Cedar Backup 'span' utility run completed with status 4.") 
199        return 4 
200   
201     if options.stacktrace: 
202        _executeAction(options, config) 
203     else: 
204        try: 
205           _executeAction(options, config) 
206        except KeyboardInterrupt: 
207           logger.error("Backup interrupted.") 
208           logger.info("Cedar Backup 'span' utility run completed with status 5.") 
209           return 5 
210        except Exception, e: 
211           logger.error("Error executing backup: %s", e) 
212           logger.info("Cedar Backup 'span' utility run completed with status 6.") 
213           return 6 
214   
215     logger.info("Cedar Backup 'span' utility run completed with status 0.") 
216     return 0 
 217   
218   
219   
220   
221   
222   
223   
224   
225   
226   
228     """ 
229     Prints usage information for the cback script. 
230     @param fd: File descriptor used to print information. 
231     @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
232     """ 
233     fd.write("\n") 
234     fd.write(" Usage: cback-span [switches]\n") 
235     fd.write("\n") 
236     fd.write(" Cedar Backup 'span' tool.\n") 
237     fd.write("\n") 
238     fd.write(" This Cedar Backup utility spans staged data between multiple discs.\n") 
239     fd.write(" It is a utility, not an extension, and requires user interaction.\n") 
240     fd.write("\n") 
241     fd.write(" The following switches are accepted, mostly to set up underlying\n") 
242     fd.write(" Cedar Backup functionality:\n") 
243     fd.write("\n") 
244     fd.write("   -h, --help     Display this usage/help listing\n") 
245     fd.write("   -V, --version  Display version information\n") 
246     fd.write("   -b, --verbose  Print verbose output as well as logging to disk\n") 
247     fd.write("   -c, --config   Path to config file (default: %s)\n" % DEFAULT_CONFIG) 
248     fd.write("   -l, --logfile  Path to logfile (default: %s)\n" % DEFAULT_LOGFILE) 
249     fd.write("   -o, --owner    Logfile ownership, user:group (default: %s:%s)\n" % (DEFAULT_OWNERSHIP[0], DEFAULT_OWNERSHIP[1])) 
250     fd.write("   -m, --mode     Octal logfile permissions mode (default: %o)\n" % DEFAULT_MODE) 
251     fd.write("   -O, --output   Record some sub-command (i.e. tar) output to the log\n") 
252     fd.write("   -d, --debug    Write debugging information to the log (implies --output)\n") 
253     fd.write("   -s, --stack    Dump a Python stack trace instead of swallowing exceptions\n") 
254     fd.write("\n") 
 255   
256   
257   
258   
259   
260   
262     """ 
263     Prints version information for the cback script. 
264     @param fd: File descriptor used to print information. 
265     @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
266     """ 
267     fd.write("\n") 
268     fd.write(" Cedar Backup 'span' tool.\n") 
269     fd.write(" Included with Cedar Backup version %s, released %s.\n" % (VERSION, DATE)) 
270     fd.write("\n") 
271     fd.write(" Copyright (c) %s %s <%s>.\n" % (COPYRIGHT, AUTHOR, EMAIL)) 
272     fd.write(" See CREDITS for a list of included code and other contributors.\n") 
273     fd.write(" This is free software; there is NO warranty.  See the\n") 
274     fd.write(" GNU General Public License version 2 for copying conditions.\n") 
275     fd.write("\n") 
276     fd.write(" Use the --help option for usage information.\n") 
277     fd.write("\n") 
 278   
279   
280   
281   
282   
283   
285     """ 
286     Prints runtime diagnostics information. 
287     @param fd: File descriptor used to print information. 
288     @note: The C{fd} is used rather than C{print} to facilitate unit testing. 
289     """ 
290     fd.write("\n") 
291     fd.write("Diagnostics:\n") 
292     fd.write("\n") 
293     Diagnostics().printDiagnostics(fd=fd, prefix="   ") 
294     fd.write("\n") 
 295   
296   
297   
298   
299   
300   
301   
303     """ 
304     Implements the guts of the cback-span tool. 
305   
306     @param options: Program command-line options. 
307     @type options: SpanOptions object. 
308   
309     @param config: Program configuration. 
310     @type config: Config object. 
311   
312     @raise Exception: Under many generic error conditions 
313     """ 
314     print "" 
315     print "================================================" 
316     print "           Cedar Backup 'span' tool" 
317     print "================================================" 
318     print "" 
319     print "This the Cedar Backup span tool.  It is used to split up staging" 
320     print "data when that staging data does not fit onto a single disc." 
321     print "" 
322     print "This utility operates using Cedar Backup configuration.  Configuration" 
323     print "specifies which staging directory to look at and which writer device" 
324     print "and media type to use." 
325     print "" 
326     if not _getYesNoAnswer("Continue?", default="Y"): 
327        return 
328     print "===" 
329   
330     print "" 
331     print "Cedar Backup store configuration looks like this:" 
332     print "" 
333     print "   Source Directory...: %s" % config.store.sourceDir 
334     print "   Media Type.........: %s" % config.store.mediaType 
335     print "   Device Type........: %s" % config.store.deviceType 
336     print "   Device Path........: %s" % config.store.devicePath 
337     print "   Device SCSI ID.....: %s" % config.store.deviceScsiId 
338     print "   Drive Speed........: %s" % config.store.driveSpeed 
339     print "   Check Data Flag....: %s" % config.store.checkData 
340     print "   No Eject Flag......: %s" % config.store.noEject 
341     print "" 
342     if not _getYesNoAnswer("Is this OK?", default="Y"): 
343        return 
344     print "===" 
345   
346     (writer, mediaCapacity) = _getWriter(config) 
347   
348     print "" 
349     print "Please wait, indexing the source directory (this may take a while)..." 
350     (dailyDirs, fileList) = _findDailyDirs(config.store.sourceDir) 
351     print "===" 
352   
353     print "" 
354     print "The following daily staging directories have not yet been written to disc:" 
355     print "" 
356     for dailyDir in dailyDirs: 
357        print "   %s" % dailyDir 
358   
359     totalSize = fileList.totalSize() 
360     print "" 
361     print "The total size of the data in these directories is %s." % displayBytes(totalSize) 
362     print "" 
363     if not _getYesNoAnswer("Continue?", default="Y"): 
364        return 
365     print "===" 
366   
367     print "" 
368     print "Based on configuration, the capacity of your media is %s." % displayBytes(mediaCapacity) 
369   
370     print "" 
371     print "Since estimates are not perfect and there is some uncertainly in" 
372     print "media capacity calculations, it is good to have a \"cushion\"," 
373     print "a percentage of capacity to set aside.  The cushion reduces the" 
374     print "capacity of your media, so a 1.5% cushion leaves 98.5% remaining." 
375     print "" 
376     cushion = _getFloat("What cushion percentage?", default=4.5) 
377     print "===" 
378   
379     realCapacity = ((100.0 - cushion)/100.0) * mediaCapacity 
380     minimumDiscs = (totalSize/realCapacity) + 1 
381     print "" 
382     print "The real capacity, taking into account the %.2f%% cushion, is %s." % (cushion, displayBytes(realCapacity)) 
383     print "It will take at least %d disc(s) to store your %s of data." % (minimumDiscs, displayBytes(totalSize)) 
384     print "" 
385     if not _getYesNoAnswer("Continue?", default="Y"): 
386        return 
387     print "===" 
388   
389     happy = False 
390     while not happy: 
391        print "" 
392        print "Which algorithm do you want to use to span your data across" 
393        print "multiple discs?" 
394        print "" 
395        print "The following algorithms are available:" 
396        print "" 
397        print "   first....: The \"first-fit\" algorithm" 
398        print "   best.....: The \"best-fit\" algorithm" 
399        print "   worst....: The \"worst-fit\" algorithm" 
400        print "   alternate: The \"alternate-fit\" algorithm" 
401        print "" 
402        print "If you don't like the results you will have a chance to try a" 
403        print "different one later." 
404        print "" 
405        algorithm = _getChoiceAnswer("Which algorithm?", "worst", [ "first", "best", "worst", "alternate", ]) 
406        print "===" 
407   
408        print "" 
409        print "Please wait, generating file lists (this may take a while)..." 
410        spanSet = fileList.generateSpan(capacity=realCapacity, algorithm="%s_fit" % algorithm) 
411        print "===" 
412   
413        print "" 
414        print "Using the \"%s-fit\" algorithm, Cedar Backup can split your data" % algorithm 
415        print "into %d discs." % len(spanSet) 
416        print "" 
417        counter = 0 
418        for item in spanSet: 
419           counter += 1 
420           print "Disc %d: %d files, %s, %.2f%% utilization" % (counter, len(item.fileList), 
421                                                                displayBytes(item.size), item.utilization) 
422        print "" 
423        if _getYesNoAnswer("Accept this solution?", default="Y"): 
424           happy = True 
425        print "===" 
426   
427     counter = 0 
428     for spanItem in spanSet: 
429        counter += 1 
430        if counter == 1: 
431           print "" 
432           _getReturn("Please place the first disc in your backup device.\nPress return when ready.") 
433           print "===" 
434        else: 
435           print "" 
436           _getReturn("Please replace the disc in your backup device.\nPress return when ready.") 
437           print "===" 
438        _writeDisc(config, writer, spanItem) 
439   
440     _writeStoreIndicator(config, dailyDirs) 
441   
442     print "" 
443     print "Completed writing all discs." 
 444   
445   
446   
447   
448   
449   
451     """ 
452     Returns a list of all daily staging directories that have not yet been 
453     stored. 
454   
455     The store indicator file C{cback.store} will be written to a daily staging 
456     directory once that directory is written to disc.  So, this function looks 
457     at each daily staging directory within the configured staging directory, and 
458     returns a list of those which do not contain the indicator file. 
459   
460     Returned is a tuple containing two items: a list of daily staging 
461     directories, and a BackupFileList containing all files among those staging 
462     directories. 
463   
464     @param stagingDir: Configured staging directory 
465   
466     @return: Tuple (staging dirs, backup file list) 
467     """ 
468     results = findDailyDirs(stagingDir, STORE_INDICATOR) 
469     fileList = BackupFileList() 
470     for item in results: 
471        fileList.addDirContents(item) 
472     return (results, fileList) 
 473   
474   
475   
476   
477   
478   
490   
491   
492   
493   
494   
495   
506   
507   
508   
509   
510   
511   
525   
527     """ 
528     Initialize an ISO image for a span item. 
529     @param config: Cedar Backup configuration 
530     @param writer: Writer to use 
531     @param spanItem: Span item to write 
532     """ 
533     complete = False 
534     while not complete: 
535        try: 
536           print "Initializing image..." 
537           writer.initializeImage(newDisc=True, tmpdir=config.options.workingDir) 
538           for path in spanItem.fileList: 
539              graftPoint = os.path.dirname(path.replace(config.store.sourceDir, "", 1)) 
540              writer.addImageEntry(path, graftPoint) 
541           complete = True 
542        except KeyboardInterrupt, e: 
543           raise e 
544        except Exception, e: 
545           logger.error("Failed to initialize image: %s", e) 
546           if not _getYesNoAnswer("Retry initialization step?", default="Y"): 
547              raise e 
548           print "Ok, attempting retry." 
549           print "===" 
550     print "Completed initializing image." 
 551   
552   
554     """ 
555     Writes a ISO image for a span item. 
556     @param config: Cedar Backup configuration 
557     @param writer: Writer to use 
558     """ 
559     complete = False 
560     while not complete: 
561        try: 
562           print "Writing image to disc..." 
563           writer.writeImage() 
564           complete = True 
565        except KeyboardInterrupt, e: 
566           raise e 
567        except Exception, e: 
568           logger.error("Failed to write image: %s", e) 
569           if not _getYesNoAnswer("Retry this step?", default="Y"): 
570              raise e 
571           print "Ok, attempting retry." 
572           _getReturn("Please replace media if needed.\nPress return when ready.") 
573           print "===" 
574     print "Completed writing image." 
 575   
577     """ 
578     Run a consistency check on an ISO image for a span item. 
579     @param config: Cedar Backup configuration 
580     @param writer: Writer to use 
581     @param spanItem: Span item to write 
582     """ 
583     if config.store.checkData: 
584        complete = False 
585        while not complete: 
586           try: 
587              print "Running consistency check..." 
588              _consistencyCheck(config, spanItem.fileList) 
589              complete = True 
590           except KeyboardInterrupt, e: 
591              raise e 
592           except Exception, e: 
593              logger.error("Consistency check failed: %s", e) 
594              if not _getYesNoAnswer("Retry the consistency check?", default="Y"): 
595                 raise e 
596              if _getYesNoAnswer("Rewrite the disc first?", default="N"): 
597                 print "Ok, attempting retry." 
598                 _getReturn("Please replace the disc in your backup device.\nPress return when ready.") 
599                 print "===" 
600                 _discWriteImage(config, writer) 
601              else: 
602                 print "Ok, attempting retry." 
603                 print "===" 
604        print "Completed consistency check." 
 605   
606   
607   
608   
609   
610   
612     """ 
613     Runs a consistency check against media in the backup device. 
614   
615     The function mounts the device at a temporary mount point in the working 
616     directory, and then compares the passed-in file list's digest map with the 
617     one generated from the disc.  The two lists should be identical. 
618   
619     If no exceptions are thrown, there were no problems with the consistency 
620     check. 
621   
622     @warning: The implementation of this function is very UNIX-specific. 
623   
624     @param config: Config object. 
625     @param fileList: BackupFileList whose contents to check against 
626   
627     @raise ValueError: If the check fails 
628     @raise IOError: If there is a problem working with the media. 
629     """ 
630     logger.debug("Running consistency check.") 
631     mountPoint = tempfile.mkdtemp(dir=config.options.workingDir) 
632     try: 
633        mount(config.store.devicePath, mountPoint, "iso9660") 
634        discList = BackupFileList() 
635        discList.addDirContents(mountPoint) 
636        sourceList = BackupFileList() 
637        sourceList.extend(fileList) 
638        discListDigest = discList.generateDigestMap(stripPrefix=normalizeDir(mountPoint)) 
639        sourceListDigest = sourceList.generateDigestMap(stripPrefix=normalizeDir(config.store.sourceDir)) 
640        compareDigestMaps(sourceListDigest, discListDigest, verbose=True) 
641        logger.info("Consistency check completed.  No problems found.") 
642     finally: 
643        unmount(mountPoint, True, 5, 1)   
 644   
645   
646   
647   
648   
649   
651     """ 
652     Get a yes/no answer from the user. 
653     The default will be placed at the end of the prompt. 
654     A "Y" or "y" is considered yes, anything else no. 
655     A blank (empty) response results in the default. 
656     @param prompt: Prompt to show. 
657     @param default: Default to set if the result is blank 
658     @return: Boolean true/false corresponding to Y/N 
659     """ 
660     if default == "Y": 
661        prompt = "%s [Y/n]: " % prompt 
662     else: 
663        prompt = "%s [y/N]: " % prompt 
664     answer = raw_input(prompt) 
665     if answer in [ None, "", ]: 
666        answer = default 
667     if answer[0] in [ "Y", "y", ]: 
668        return True 
669     else: 
670        return False 
 671   
673     """ 
674     Get a particular choice from the user. 
675     The default will be placed at the end of the prompt. 
676     The function loops until getting a valid choice. 
677     A blank (empty) response results in the default. 
678     @param prompt: Prompt to show. 
679     @param default: Default to set if the result is None or blank. 
680     @param validChoices: List of valid choices (strings) 
681     @return: Valid choice from user. 
682     """ 
683     prompt = "%s [%s]: " % (prompt, default) 
684     answer = raw_input(prompt) 
685     if answer in [ None, "", ]: 
686        answer = default 
687     while answer not in validChoices: 
688        print "Choice must be one of %s" % validChoices 
689        answer = raw_input(prompt) 
690     return answer 
 691   
693     """ 
694     Get a floating point number from the user. 
695     The default will be placed at the end of the prompt. 
696     The function loops until getting a valid floating point number. 
697     A blank (empty) response results in the default. 
698     @param prompt: Prompt to show. 
699     @param default: Default to set if the result is None or blank. 
700     @return: Floating point number from user 
701     """ 
702     prompt = "%s [%.2f]: " % (prompt, default) 
703     while True: 
704        answer = raw_input(prompt) 
705        if answer in [ None, "" ]: 
706           return default 
707        else: 
708           try: 
709              return float(answer) 
710           except ValueError: 
711              print "Enter a floating point number." 
 712   
714     """ 
715     Get a return key from the user. 
716     @param prompt: Prompt to show. 
717     """ 
718     raw_input(prompt) 
 719   
720   
721   
722   
723   
724   
725  if __name__ == "__main__": 
726     sys.exit(cli()) 
727