1   
  2   
  3   
  4   
  5   
  6   
  7   
  8   
  9   
 10   
 11   
 12   
 13   
 14   
 15   
 16   
 17   
 18   
 19   
 20   
 21   
 22   
 23   
 24   
 25   
 26   
 27   
 28   
 29   
 30   
 31   
 32   
 33   
 34   
 35   
 36   
 37   
 38  """ 
 39  Implements the standard 'store' action. 
 40  @sort: executeStore, writeImage, writeStoreIndicator, consistencyCheck 
 41  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
 42  @author: Dmitry Rutsky <rutsky@inbox.ru> 
 43  """ 
 44   
 45   
 46   
 47   
 48   
 49   
 50   
 51  import sys 
 52  import os 
 53  import logging 
 54  import datetime 
 55  import tempfile 
 56   
 57   
 58  from CedarBackup2.filesystem import compareContents 
 59  from CedarBackup2.util import isStartOfWeek 
 60  from CedarBackup2.util import mount, unmount, displayBytes 
 61  from CedarBackup2.actions.util import createWriter, checkMediaState, buildMediaLabel, writeIndicatorFile 
 62  from CedarBackup2.actions.constants import DIR_TIME_FORMAT, STAGE_INDICATOR, STORE_INDICATOR 
 63   
 64   
 65   
 66   
 67   
 68   
 69  logger = logging.getLogger("CedarBackup2.log.actions.store") 
 70   
 71   
 72   
 73   
 74   
 75   
 76   
 77   
 78   
 79   
 80   
 82     """ 
 83     Executes the store backup action. 
 84   
 85     @note: The rebuild action and the store action are very similar.  The 
 86     main difference is that while store only stores a single day's staging 
 87     directory, the rebuild action operates on multiple staging directories. 
 88   
 89     @note: When the store action is complete, we will write a store indicator to 
 90     the daily staging directory we used, so it's obvious that the store action 
 91     has completed. 
 92   
 93     @param configPath: Path to configuration file on disk. 
 94     @type configPath: String representing a path on disk. 
 95   
 96     @param options: Program command-line options. 
 97     @type options: Options object. 
 98   
 99     @param config: Program configuration. 
100     @type config: Config object. 
101   
102     @raise ValueError: Under many generic error conditions 
103     @raise IOError: If there are problems reading or writing files. 
104     """ 
105     logger.debug("Executing the 'store' action.") 
106     if sys.platform == "darwin": 
107        logger.warn("Warning: the store action is not fully supported on Mac OS X.") 
108        logger.warn("See the Cedar Backup software manual for further information.") 
109     if config.options is None or config.store is None: 
110        raise ValueError("Store configuration is not properly filled in.") 
111     if config.store.checkMedia: 
112        checkMediaState(config.store)   
113     rebuildMedia = options.full 
114     logger.debug("Rebuild media flag [%s]", rebuildMedia) 
115     todayIsStart = isStartOfWeek(config.options.startingDay) 
116     stagingDirs = _findCorrectDailyDir(options, config) 
117     writeImageBlankSafe(config, rebuildMedia, todayIsStart, config.store.blankBehavior, stagingDirs) 
118     if config.store.checkData: 
119        if sys.platform == "darwin": 
120           logger.warn("Warning: consistency check cannot be run successfully on Mac OS X.") 
121           logger.warn("See the Cedar Backup software manual for further information.") 
122        else: 
123           logger.debug("Running consistency check of media.") 
124           consistencyCheck(config, stagingDirs) 
125     writeStoreIndicator(config, stagingDirs) 
126     logger.info("Executed the 'store' action successfully.") 
 127   
128   
129   
130   
131   
132   
134     """ 
135     Builds and writes an ISO image containing the indicated stage directories. 
136   
137     The generated image will contain each of the staging directories listed in 
138     C{stagingDirs}.  The directories will be placed into the image at the root by 
139     date, so staging directory C{/opt/stage/2005/02/10} will be placed into the 
140     disc at C{/2005/02/10}. 
141   
142     @note: This function is implemented in terms of L{writeImageBlankSafe}.  The 
143     C{newDisc} flag is passed in for both C{rebuildMedia} and C{todayIsStart}. 
144   
145     @param config: Config object. 
146     @param newDisc: Indicates whether the disc should be re-initialized 
147     @param stagingDirs: Dictionary mapping directory path to date suffix. 
148   
149     @raise ValueError: Under many generic error conditions 
150     @raise IOError: If there is a problem writing the image to disc. 
151     """ 
152     writeImageBlankSafe(config, newDisc, newDisc, None, stagingDirs) 
 153   
154   
155   
156   
157   
158   
160     """ 
161     Builds and writes an ISO image containing the indicated stage directories. 
162   
163     The generated image will contain each of the staging directories listed in 
164     C{stagingDirs}.  The directories will be placed into the image at the root by 
165     date, so staging directory C{/opt/stage/2005/02/10} will be placed into the 
166     disc at C{/2005/02/10}.  The media will always be written with a media 
167     label specific to Cedar Backup. 
168   
169     This function is similar to L{writeImage}, but tries to implement a smarter 
170     blanking strategy. 
171   
172     First, the media is always blanked if the C{rebuildMedia} flag is true. 
173     Then, if C{rebuildMedia} is false, blanking behavior and C{todayIsStart} 
174     come into effect:: 
175   
176        If no blanking behavior is specified, and it is the start of the week, 
177        the disc will be blanked 
178   
179        If blanking behavior is specified, and either the blank mode is "daily" 
180        or the blank mode is "weekly" and it is the start of the week, then 
181        the disc will be blanked if it looks like the weekly backup will not 
182        fit onto the media. 
183   
184        Otherwise, the disc will not be blanked 
185   
186     How do we decide whether the weekly backup will fit onto the media?  That is 
187     what the blanking factor is used for.  The following formula is used:: 
188   
189        will backup fit? = (bytes available / (1 + bytes required) <= blankFactor 
190   
191     The blanking factor will vary from setup to setup, and will probably 
192     require some experimentation to get it right. 
193   
194     @param config: Config object. 
195     @param rebuildMedia: Indicates whether media should be rebuilt 
196     @param todayIsStart: Indicates whether today is the starting day of the week 
197     @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior 
198     @param stagingDirs: Dictionary mapping directory path to date suffix. 
199   
200     @raise ValueError: Under many generic error conditions 
201     @raise IOError: If there is a problem writing the image to disc. 
202     """ 
203     mediaLabel = buildMediaLabel() 
204     writer = createWriter(config) 
205     writer.initializeImage(True, config.options.workingDir, mediaLabel)   
206     for stageDir in stagingDirs.keys(): 
207        logger.debug("Adding stage directory [%s].", stageDir) 
208        dateSuffix = stagingDirs[stageDir] 
209        writer.addImageEntry(stageDir, dateSuffix) 
210     newDisc = _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior) 
211     writer.setImageNewDisc(newDisc) 
212     writer.writeImage() 
 213   
214 -def _getNewDisc(writer, rebuildMedia, todayIsStart, blankBehavior): 
 215     """ 
216     Gets a value for the newDisc flag based on blanking factor rules. 
217   
218     The blanking factor rules are described above by L{writeImageBlankSafe}. 
219   
220     @param writer: Previously configured image writer containing image entries 
221     @param rebuildMedia: Indicates whether media should be rebuilt 
222     @param todayIsStart: Indicates whether today is the starting day of the week 
223     @param blankBehavior: Blank behavior from configuration, or C{None} to use default behavior 
224   
225     @return: newDisc flag to be set on writer. 
226     """ 
227     newDisc = False 
228     if rebuildMedia: 
229        newDisc = True 
230        logger.debug("Setting new disc flag based on rebuildMedia flag.") 
231     else: 
232        if blankBehavior is None: 
233           logger.debug("Default media blanking behavior is in effect.") 
234           if todayIsStart: 
235              newDisc = True 
236              logger.debug("Setting new disc flag based on todayIsStart.") 
237        else: 
238            
239           logger.debug("Optimized media blanking behavior is in effect based on configuration.") 
240           if blankBehavior.blankMode == "daily" or (blankBehavior.blankMode == "weekly" and todayIsStart): 
241              logger.debug("New disc flag will be set based on blank factor calculation.") 
242              blankFactor = float(blankBehavior.blankFactor) 
243              logger.debug("Configured blanking factor: %.2f", blankFactor) 
244              available = writer.retrieveCapacity().bytesAvailable 
245              logger.debug("Bytes available: %s", displayBytes(available)) 
246              required = writer.getEstimatedImageSize() 
247              logger.debug("Bytes required: %s", displayBytes(required)) 
248              ratio = available / (1.0 + required) 
249              logger.debug("Calculated ratio: %.2f", ratio) 
250              newDisc = (ratio <= blankFactor) 
251              logger.debug("%.2f <= %.2f ? %s", ratio, blankFactor, newDisc) 
252           else: 
253              logger.debug("No blank factor calculation is required based on configuration.") 
254     logger.debug("New disc flag [%s].", newDisc) 
255     return newDisc 
 256   
257   
258   
259   
260   
261   
263     """ 
264     Writes a store indicator file into staging directories. 
265   
266     The store indicator is written into each of the staging directories when 
267     either a store or rebuild action has written the staging directory to disc. 
268   
269     @param config: Config object. 
270     @param stagingDirs: Dictionary mapping directory path to date suffix. 
271     """ 
272     for stagingDir in stagingDirs.keys(): 
273        writeIndicatorFile(stagingDir, STORE_INDICATOR, 
274                           config.options.backupUser, 
275                           config.options.backupGroup) 
 276   
277   
278   
279   
280   
281   
283     """ 
284     Runs a consistency check against media in the backup device. 
285   
286     It seems that sometimes, it's possible to create a corrupted multisession 
287     disc (i.e. one that cannot be read) although no errors were encountered 
288     while writing the disc.  This consistency check makes sure that the data 
289     read from disc matches the data that was used to create the disc. 
290   
291     The function mounts the device at a temporary mount point in the working 
292     directory, and then compares the indicated staging directories in the 
293     staging directory and on the media.  The comparison is done via 
294     functionality in C{filesystem.py}. 
295   
296     If no exceptions are thrown, there were no problems with the consistency 
297     check.  A positive confirmation of "no problems" is also written to the log 
298     with C{info} priority. 
299   
300     @warning: The implementation of this function is very UNIX-specific. 
301   
302     @param config: Config object. 
303     @param stagingDirs: Dictionary mapping directory path to date suffix. 
304   
305     @raise ValueError: If the two directories are not equivalent. 
306     @raise IOError: If there is a problem working with the media. 
307     """ 
308     logger.debug("Running consistency check.") 
309     mountPoint = tempfile.mkdtemp(dir=config.options.workingDir) 
310     try: 
311        mount(config.store.devicePath, mountPoint, "iso9660") 
312        for stagingDir in stagingDirs.keys(): 
313           discDir = os.path.join(mountPoint, stagingDirs[stagingDir]) 
314           logger.debug("Checking [%s] vs. [%s].", stagingDir, discDir) 
315           compareContents(stagingDir, discDir, verbose=True) 
316           logger.info("Consistency check completed for [%s].  No problems found.", stagingDir) 
317     finally: 
318        unmount(mountPoint, True, 5, 1)   
 319   
320   
321   
322   
323   
324   
325   
326   
327   
328   
330     """ 
331     Finds the correct daily staging directory to be written to disk. 
332   
333     In Cedar Backup v1.0, we assumed that the correct staging directory matched 
334     the current date.  However, that has problems.  In particular, it breaks 
335     down if collect is on one side of midnite and stage is on the other, or if 
336     certain processes span midnite. 
337   
338     For v2.0, I'm trying to be smarter.  I'll first check the current day.  If 
339     that directory is found, it's good enough.  If it's not found, I'll look for 
340     a valid directory from the day before or day after I{which has not yet been 
341     staged, according to the stage indicator file}.  The first one I find, I'll 
342     use.  If I use a directory other than for the current day I{and} 
343     C{config.store.warnMidnite} is set, a warning will be put in the log. 
344   
345     There is one exception to this rule.  If the C{options.full} flag is set, 
346     then the special "span midnite" logic will be disabled and any existing 
347     store indicator will be ignored.  I did this because I think that most users 
348     who run C{cback --full store} twice in a row expect the command to generate 
349     two identical discs.  With the other rule in place, running that command 
350     twice in a row could result in an error ("no unstored directory exists") or 
351     could even cause a completely unexpected directory to be written to disc (if 
352     some previous day's contents had not yet been written). 
353   
354     @note: This code is probably longer and more verbose than it needs to be, 
355     but at least it's straightforward. 
356   
357     @param options: Options object. 
358     @param config: Config object. 
359   
360     @return: Correct staging dir, as a dict mapping directory to date suffix. 
361     @raise IOError: If the staging directory cannot be found. 
362     """ 
363     oneDay = datetime.timedelta(days=1) 
364     today = datetime.date.today() 
365     yesterday = today - oneDay 
366     tomorrow = today + oneDay 
367     todayDate = today.strftime(DIR_TIME_FORMAT) 
368     yesterdayDate = yesterday.strftime(DIR_TIME_FORMAT) 
369     tomorrowDate = tomorrow.strftime(DIR_TIME_FORMAT) 
370     todayPath = os.path.join(config.stage.targetDir, todayDate) 
371     yesterdayPath = os.path.join(config.stage.targetDir, yesterdayDate) 
372     tomorrowPath = os.path.join(config.stage.targetDir, tomorrowDate) 
373     todayStageInd = os.path.join(todayPath, STAGE_INDICATOR) 
374     yesterdayStageInd = os.path.join(yesterdayPath, STAGE_INDICATOR) 
375     tomorrowStageInd = os.path.join(tomorrowPath, STAGE_INDICATOR) 
376     todayStoreInd = os.path.join(todayPath, STORE_INDICATOR) 
377     yesterdayStoreInd = os.path.join(yesterdayPath, STORE_INDICATOR) 
378     tomorrowStoreInd = os.path.join(tomorrowPath, STORE_INDICATOR) 
379     if options.full: 
380        if os.path.isdir(todayPath) and os.path.exists(todayStageInd): 
381           logger.info("Store process will use current day's stage directory [%s]", todayPath) 
382           return { todayPath:todayDate } 
383        raise IOError("Unable to find staging directory to store (only tried today due to full option).") 
384     else: 
385        if os.path.isdir(todayPath) and os.path.exists(todayStageInd) and not os.path.exists(todayStoreInd): 
386           logger.info("Store process will use current day's stage directory [%s]", todayPath) 
387           return { todayPath:todayDate } 
388        elif os.path.isdir(yesterdayPath) and os.path.exists(yesterdayStageInd) and not os.path.exists(yesterdayStoreInd): 
389           logger.info("Store process will use previous day's stage directory [%s]", yesterdayPath) 
390           if config.store.warnMidnite: 
391              logger.warn("Warning: store process crossed midnite boundary to find data.") 
392           return { yesterdayPath:yesterdayDate } 
393        elif os.path.isdir(tomorrowPath) and os.path.exists(tomorrowStageInd) and not os.path.exists(tomorrowStoreInd): 
394           logger.info("Store process will use next day's stage directory [%s]", tomorrowPath) 
395           if config.store.warnMidnite: 
396              logger.warn("Warning: store process crossed midnite boundary to find data.") 
397           return { tomorrowPath:tomorrowDate } 
398        raise IOError("Unable to find unused staging directory to store (tried today, yesterday, tomorrow).") 
 399