duplicity.collections

1 # -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*- 2 # 3 # Copyright 2002 Ben Escoto <ben@emerose.org> 4 # Copyright 2007 Kenneth Loafman <kenneth@loafman.com> 5 # 6 # This file is part of duplicity. 7 # 8 # Duplicity is free software; you can redistribute it and/or modify it 9 # under the terms of the GNU General Public License as published by the 10 # Free Software Foundation; either version 2 of the License, or (at your 11 # option) any later version. 12 # 13 # Duplicity is distributed in the hope that it will be useful, but 14 # WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 16 # General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with duplicity; if not, write to the Free Software Foundation, 20 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22 """Classes and functions on collections of backup volumes""" 23 24 import types 25 import gettext 26 27 from duplicity import log 28 from duplicity import file_naming 29 from duplicity import path 30 from duplicity import dup_time 31 from duplicity import globals 32 from duplicity import manifest 33 from duplicity.gpg import GPGError 34

35 -class CollectionsError(Exception):

36 pass

37

38 -class BackupSet:

39 """ 40 Backup set - the backup information produced by one session 41 """

42 - def __init__(self, backend):

43 """ 44 Initialize new backup set, only backend is required at first 45 """ 46 self.backend = backend 47 self.info_set = False # true if fields are set 48 self.volume_name_dict = {} # dict from volume number to filename 49 self.remote_manifest_name = None # full name of remote manifest 50 self.local_manifest_path = None # full path to local manifest 51 self.time = None # will be set if is full backup set 52 self.start_time = None # will be set if inc 53 self.end_time = None # will be set if inc 54 self.partial = False # true if a partial backup 55 self.encrypted = False # true if an encrypted backup

56

57 - def is_complete(self):

58 """ 59 Assume complete if found manifest file 60 """ 61 return self.remote_manifest_name

62

63 - def add_filename(self, filename):

64 """ 65 Add a filename to given set. Return true if it fits. 66 67 The filename will match the given set if it has the right 68 times and is of the right type. The information will be set 69 from the first filename given. 70 71 @param filename: name of file to add 72 @type filename: string 73 """ 74 pr = file_naming.parse(filename) 75 if not pr or not (pr.type == "full" or pr.type == "inc"): 76 return False 77 78 if not self.info_set: 79 self.set_info(pr) 80 else: 81 if pr.type != self.type: 82 return False 83 if pr.time != self.time: 84 return False 85 if (pr.start_time != self.start_time or 86 pr.end_time != self.end_time): 87 return False 88 if bool(pr.encrypted) != bool(self.encrypted): 89 if self.partial and pr.encrypted: 90 self.encrypted = pr.encrypted 91 92 if pr.manifest: 93 self.set_manifest(filename) 94 else: 95 assert pr.volume_number is not None 96 assert not self.volume_name_dict.has_key(pr.volume_number), \ 97 (self.volume_name_dict, filename) 98 self.volume_name_dict[pr.volume_number] = filename 99 100 return True

101

102 - def set_info(self, pr):

103 """ 104 Set BackupSet information from ParseResults object 105 106 @param pr: parse results 107 @type pf: ParseResults 108 """ 109 assert not self.info_set 110 self.type = pr.type 111 self.time = pr.time 112 self.start_time = pr.start_time 113 self.end_time = pr.end_time 114 self.time = pr.time 115 self.partial = pr.partial 116 self.encrypted = bool(pr.encrypted) 117 self.info_set = True

118

119 - def set_manifest(self, remote_filename):

120 """ 121 Add local and remote manifest filenames to backup set 122 """ 123 assert not self.remote_manifest_name, (self.remote_manifest_name, 124 remote_filename) 125 self.remote_manifest_name = remote_filename 126 127 for local_filename in globals.archive_dir.listdir(): 128 pr = file_naming.parse(local_filename) 129 if (pr and pr.manifest 130 and pr.type == self.type 131 and pr.time == self.time 132 and pr.start_time == self.start_time 133 and pr.end_time == self.end_time): 134 self.local_manifest_path = \ 135 globals.archive_dir.append(local_filename) 136 break

137

138 - def delete(self):

139 """ 140 Remove all files in set, both local and remote 141 """ 142 rfn = self.get_filenames() 143 rfn.reverse() 144 try: 145 self.backend.delete(rfn) 146 except Exception: 147 log.Debug("BackupSet.delete: missing %s" % rfn) 148 pass 149 for lfn in globals.archive_dir.listdir(): 150 pr = file_naming.parse(lfn) 151 if (pr 152 and pr.time == self.time 153 and pr.start_time == self.start_time 154 and pr.end_time == self.end_time 155 and pr.type != "new-sig" ): 156 # do not remove new sigs from the cache: 157 # they aren't removed from the remote archive, 158 # and subsequent backups will have to resync 159 # which is bad if running non-interactive with encrypt-key 160 try: 161 globals.archive_dir.append(lfn).delete() 162 except Exception: 163 log.Debug("BackupSet.delete: missing %s" % lfn) 164 pass

165

166 - def __str__(self):

167 """ 168 For now just list files in set 169 """ 170 filelist = [] 171 if self.remote_manifest_name: 172 filelist.append(self.remote_manifest_name) 173 filelist.extend(self.volume_name_dict.values()) 174 return "[%s]" % ", ".join(filelist)

175

176 - def get_timestr(self):

177 """ 178 Return time string suitable for log statements 179 """ 180 return dup_time.timetopretty(self.time or self.end_time)

181

182 - def check_manifests(self):

183 """ 184 Make sure remote manifest is equal to local one 185 """ 186 if not self.remote_manifest_name and not self.local_manifest_path: 187 log.FatalError(_("Fatal Error: No manifests found for most recent backup"), 188 log.ErrorCode.no_manifests) 189 assert self.remote_manifest_name, "if only one, should be remote" 190 191 remote_manifest = self.get_remote_manifest() 192 if self.local_manifest_path: 193 local_manifest = self.get_local_manifest() 194 if remote_manifest and self.local_manifest_path and local_manifest: 195 if remote_manifest != local_manifest: 196 log.FatalError(_("Fatal Error: Remote manifest does not match " 197 "local one. Either the remote backup set or " 198 "the local archive directory has been corrupted."), 199 log.ErrorCode.mismatched_manifests) 200 if not remote_manifest: 201 if self.local_manifest_path: 202 remote_manifest = local_manifest 203 else: 204 log.FatalError(_("Fatal Error: Neither remote nor local " 205 "manifest is readable."), 206 log.ErrorCode.unreadable_manifests) 207 remote_manifest.check_dirinfo()

208

209 - def get_local_manifest(self):

210 """ 211 Return manifest object by reading local manifest file 212 """ 213 assert self.local_manifest_path 214 manifest_buffer = self.local_manifest_path.get_data() 215 return manifest.Manifest().from_string(manifest_buffer)

216

217 - def get_remote_manifest(self):

218 """ 219 Return manifest by reading remote manifest on backend 220 """ 221 assert self.remote_manifest_name 222 # Following by MDR. Should catch if remote encrypted with 223 # public key w/o secret key 224 try: 225 manifest_buffer = self.backend.get_data(self.remote_manifest_name) 226 except GPGError, message: 227 #TODO: We check for gpg v1 and v2 messages, should be an error code. 228 if ("secret key not available" in message.args[0] or 229 "No secret key" in message.args[0]): 230 return None 231 else: 232 raise 233 return manifest.Manifest().from_string(manifest_buffer)

234

235 - def get_manifest(self):

236 """ 237 Return manifest object, showing preference for local copy 238 """ 239 if self.local_manifest_path: 240 return self.get_local_manifest() 241 else: 242 return self.get_remote_manifest()

243

244 - def get_filenames(self):

245 """ 246 Return sorted list of (remote) filenames of files in set 247 """ 248 assert self.info_set 249 volume_num_list = self.volume_name_dict.keys() 250 volume_num_list.sort() 251 volume_filenames = map(lambda x: self.volume_name_dict[x], 252 volume_num_list) 253 if self.remote_manifest_name: 254 # For convenience of implementation for restart support, we treat 255 # local partial manifests as this set's remote manifest. But 256 # when specifically asked for a list of remote filenames, we 257 # should not include it. 258 pr = file_naming.parse(self.remote_manifest_name) 259 if not pr or not pr.partial: 260 volume_filenames.append(self.remote_manifest_name) 261 return volume_filenames

262

263 - def get_time(self):

264 """ 265 Return time if full backup, or end_time if incremental 266 """ 267 if self.time: 268 return self.time 269 if self.end_time: 270 return self.end_time 271 assert 0, "Neither self.time nor self.end_time set"

272

273 - def __len__(self):

274 """ 275 Return the number of volumes in the set 276 """ 277 return len(self.volume_name_dict.keys())

278 279

280 -class BackupChain:

281 """ 282 BackupChain - a number of linked BackupSets 283 284 A BackupChain always starts with a full backup set and continues 285 with incremental ones. 286 """

287 - def __init__(self, backend):

288 """ 289 Initialize new chain, only backend is required at first 290 """ 291 self.backend = backend 292 self.fullset = None 293 self.incset_list = [] # sorted list of BackupSets 294 self.start_time, self.end_time = None, None

295

296 - def set_full(self, fullset):

297 """ 298 Add full backup set 299 """ 300 assert not self.fullset and isinstance(fullset, BackupSet) 301 self.fullset = fullset 302 assert fullset.time 303 self.start_time, self.end_time = fullset.time, fullset.time

304

305 - def add_inc(self, incset):

306 """ 307 Add incset to self. Return False if incset does not match 308 """ 309 if self.end_time == incset.start_time: 310 self.incset_list.append(incset) 311 else: 312 if (self.incset_list 313 and incset.start_time == self.incset_list[-1].start_time 314 and incset.end_time > self.incset_list[-1]): 315 log.Info(_("Preferring Backupset over previous one!")) 316 self.incset_list[-1] = incset 317 else: 318 log.Info(_("Ignoring incremental Backupset (start_time: %s; needed: %s)") % 319 (dup_time.timetopretty(incset.start_time), 320 dup_time.timetopretty(self.end_time))) 321 return False 322 self.end_time = incset.end_time 323 log.Info(_("Added incremental Backupset (start_time: %s / end_time: %s)") % 324 (dup_time.timetopretty(incset.start_time), 325 dup_time.timetopretty(incset.end_time))) 326 assert self.end_time 327 return True

328

329 - def delete(self):

330 """ 331 Delete all sets in chain, in reverse order 332 """ 333 for i in range(len(self.incset_list)-1, -1, -1): 334 self.incset_list[i].delete() 335 if self.fullset: 336 self.fullset.delete()

337

338 - def get_sets_at_time(self, time):

339 """ 340 Return a list of sets in chain earlier or equal to time 341 """ 342 older_incsets = filter(lambda s: s.end_time <= time, self.incset_list) 343 return [self.fullset] + older_incsets

344

345 - def get_last(self):

346 """ 347 Return last BackupSet in chain 348 """ 349 if self.incset_list: 350 return self.incset_list[-1] 351 else: 352 return self.fullset

353

354 - def get_first(self):

355 """ 356 Return first BackupSet in chain (ie the full backup) 357 """ 358 return self.fullset

359

360 - def short_desc(self):

361 """ 362 Return a short one-line description of the chain, 363 suitable for log messages. 364 """ 365 return "[%s]-[%s]" % (dup_time.timetopretty(self.start_time), 366 dup_time.timetopretty(self.end_time))

367

368 - def to_log_info(self, prefix=''):

369 """ 370 Return summary, suitable for printing to log 371 """ 372 l = [] 373 for s in self.get_all_sets(): 374 if s.time: 375 type = "full" 376 time = s.time 377 else: 378 type = "inc" 379 time = s.end_time 380 if s.encrypted: 381 enc = "enc" 382 else: 383 enc = "noenc" 384 l.append("%s%s %s %d %s" % (prefix, type, dup_time.timetostring(time), (len(s)), enc)) 385 return l

386

387 - def __str__(self):

388 """ 389 Return string representation, for testing purposes 390 """ 391 set_schema = "%20s %30s %15s" 392 l = ["-------------------------", 393 _("Chain start time: ") + dup_time.timetopretty(self.start_time), 394 _("Chain end time: ") + dup_time.timetopretty(self.end_time), 395 _("Number of contained backup sets: %d") % 396 (len(self.incset_list)+1,), 397 _("Total number of contained volumes: %d") % 398 (self.get_num_volumes(),), 399 set_schema % (_("Type of backup set:"), _("Time:"), _("Num volumes:"))] 400 401 for s in self.get_all_sets(): 402 if s.time: 403 type = _("Full") 404 time = s.time 405 else: 406 type = _("Incremental") 407 time = s.end_time 408 l.append(set_schema % (type, dup_time.timetopretty(time), len(s))) 409 410 l.append("-------------------------") 411 return "\n".join(l)

412

413 - def get_num_volumes(self):

414 """ 415 Return the total number of volumes in the chain 416 """ 417 n = 0 418 for s in self.get_all_sets(): 419 n += len(s) 420 return n

421

422 - def get_all_sets(self):

423 """ 424 Return list of all backup sets in chain 425 """ 426 if self.fullset: 427 return [self.fullset] + self.incset_list 428 else: 429 return self.incset_list

430 431

432 -class SignatureChain:

433 """ 434 A number of linked SignatureSets 435 436 Analog to BackupChain - start with a full-sig, and continue with 437 new-sigs. 438 """

439 - def __init__(self, local, location):

440 """ 441 Return new SignatureChain. 442 443 local should be true iff the signature chain resides in 444 globals.archive_dir and false if the chain is in 445 globals.backend. 446 447 @param local: True if sig chain in globals.archive_dir 448 @type local: Boolean 449 450 @param location: Where the sig chain is located 451 @type location: globals.archive_dir or globals.backend 452 """ 453 if local: 454 self.archive_dir, self.backend = location, None 455 else: 456 self.archive_dir, self.backend = None, location 457 self.fullsig = None # filename of full signature 458 self.inclist = [] # list of filenames of incremental signatures 459 self.start_time, self.end_time = None, None

460

461 - def __str__(self):

462 """ 463 Local or Remote and List of files in the set 464 """ 465 if self.archive_dir: 466 place = _("local") 467 else: 468 place = _("remote") 469 filelist = [] 470 if self.fullsig: 471 filelist.append(self.fullsig) 472 filelist.extend(self.inclist) 473 return "%s: [%s]" % (place, ", ".join(filelist))

474

475 - def check_times(self, time_list):

476 """ 477 Check to make sure times are in whole seconds 478 """ 479 for time in time_list: 480 if type(time) not in (types.LongType, types.IntType): 481 assert 0, "Time %s in %s wrong type" % (time, time_list)

482

483 - def islocal(self):

484 """ 485 Return true if represents a signature chain in archive_dir 486 """ 487 if self.archive_dir: 488 return True 489 else: 490 return False

491

492 - def add_filename(self, filename, pr = None):

493 """ 494 Add new sig filename to current chain. Return true if fits 495 """ 496 if not pr: 497 pr = file_naming.parse(filename) 498 if not pr: 499 return None 500 501 if self.fullsig: 502 if pr.type != "new-sig": 503 return None 504 if pr.start_time != self.end_time: 505 return None 506 self.inclist.append(filename) 507 self.check_times([pr.end_time]) 508 self.end_time = pr.end_time 509 return 1 510 else: 511 if pr.type != "full-sig": 512 return None 513 self.fullsig = filename 514 self.check_times([pr.time, pr.time]) 515 self.start_time, self.end_time = pr.time, pr.time 516 return 1

517

518 - def get_fileobjs(self, time = None):

519 """ 520 Return ordered list of signature fileobjs opened for reading, 521 optionally at a certain time 522 """ 523 assert self.fullsig 524 if self.archive_dir: # local 525 def filename_to_fileobj(filename): 526 """Open filename in archive_dir, return filtered fileobj""" 527 sig_dp = path.DupPath(self.archive_dir.name, (filename,)) 528 return sig_dp.filtered_open("rb")

529 else: 530 filename_to_fileobj = self.backend.get_fileobj_read 531 return map(filename_to_fileobj, self.get_filenames(time))

532

533 - def delete(self):

534 """ 535 Remove all files in signature set 536 """ 537 # Try to delete in opposite order, so something useful even if aborted 538 if self.archive_dir: 539 for i in range(len(self.inclist)-1, -1, -1): 540 self.archive_dir.append(self.inclist[i]).delete() 541 self.archive_dir.append(self.fullsig).delete() 542 else: 543 assert self.backend 544 inclist_copy = self.inclist[:] 545 inclist_copy.reverse() 546 inclist_copy.append(self.fullsig) 547 self.backend.delete(inclist_copy)

548

549 - def get_filenames(self, time = None):

550 """ 551 Return ordered list of filenames in set, up to a provided time 552 """ 553 if self.fullsig: 554 l = [self.fullsig] 555 else: 556 l = [] 557 558 inclist = self.inclist 559 if time: 560 inclist = filter(lambda n: file_naming.parse(n).end_time <= time, 561 inclist) 562 563 l.extend(inclist) 564 return l

565 566

567 -class CollectionsStatus:

568 """ 569 Hold information about available chains and sets 570 """

571 - def __init__(self, backend, archive_dir):

572 """ 573 Make new object. Does not set values 574 """ 575 self.backend = backend 576 self.archive_dir = archive_dir 577 578 # Will hold (signature chain, backup chain) pair of active 579 # (most recent) chains 580 self.matched_chain_pair = None 581 582 # These should be sorted by end_time 583 self.all_backup_chains = None 584 self.other_backup_chains = None 585 self.all_sig_chains = None 586 587 # Other misc paths and sets which shouldn't be there 588 self.local_orphaned_sig_names = [] 589 self.remote_orphaned_sig_names = [] 590 self.orphaned_backup_sets = None 591 self.incomplete_backup_sets = None 592 593 # True if set_values() below has run 594 self.values_set = None

595

596 - def to_log_info(self):

597 """ 598 Return summary of the collection, suitable for printing to log 599 """ 600 l = ["backend %s" % (self.backend.__class__.__name__,), 601 "archive-dir %s" % (self.archive_dir,)] 602 603 for i in range(len(self.other_backup_chains)): 604 # A bit of a misnomer. Chain might have a sig. 605 l.append("chain-no-sig %d" % (i,)) 606 l += self.other_backup_chains[i].to_log_info(' ') 607 608 if self.matched_chain_pair: 609 l.append("chain-complete") 610 l += self.matched_chain_pair[1].to_log_info(' ') 611 612 l.append("orphaned-sets-num %d" % (len(self.orphaned_backup_sets),)) 613 l.append("incomplete-sets-num %d" % (len(self.incomplete_backup_sets),)) 614 615 return l

616

617 - def __str__(self):

618 """ 619 Return string summary of the collection 620 """ 621 l = [_("Collection Status"), 622 "-----------------", 623 _("Connecting with backend: %s") % 624 (self.backend.__class__.__name__,), 625 _("Archive dir: %s") % (self.archive_dir.name,)] 626 627 l.append("\n" + 628 gettext.ngettext("Found %d secondary backup chain.", 629 "Found %d secondary backup chains.", 630 len(self.other_backup_chains)) 631 % len(self.other_backup_chains)) 632 for i in range(len(self.other_backup_chains)): 633 l.append(_("Secondary chain %d of %d:") % 634 (i+1, len(self.other_backup_chains))) 635 l.append(str(self.other_backup_chains[i])) 636 l.append("") 637 638 if self.matched_chain_pair: 639 l.append("\n" + _("Found primary backup chain with matching " 640 "signature chain:")) 641 l.append(str(self.matched_chain_pair[1])) 642 else: 643 l.append(_("No backup chains with active signatures found")) 644 645 if self.orphaned_backup_sets or self.incomplete_backup_sets: 646 l.append(gettext.ngettext("Also found %d backup set not part of any chain,", 647 "Also found %d backup sets not part of any chain,", 648 len(self.orphaned_backup_sets)) 649 % (len(self.orphaned_backup_sets),)) 650 l.append(gettext.ngettext("and %d incomplete backup set.", 651 "and %d incomplete backup sets.", 652 len(self.incomplete_backup_sets)) 653 % (len(self.incomplete_backup_sets),)) 654 # TRANSL: "cleanup" is a hard-coded command, so do not translate it 655 l.append(_('These may be deleted by running duplicity with the ' 656 '"cleanup" command.')) 657 else: 658 l.append(_("No orphaned or incomplete backup sets found.")) 659 660 return "\n".join(l)

661

662 - def set_values(self, sig_chain_warning = 1):

663 """ 664 Set values from archive_dir and backend. 665 666 Returns self for convenience. If sig_chain_warning is set to None, 667 do not warn about unnecessary sig chains. This is because there may 668 naturally be some unecessary ones after a full backup. 669 """ 670 self.values_set = 1 671 672 # get remote filename list 673 backend_filename_list = self.backend.list() 674 log.Debug(gettext.ngettext("%d file exists on backend", 675 "%d files exist on backend", 676 len(backend_filename_list)) % 677 len(backend_filename_list)) 678 679 # get local filename list 680 local_filename_list = self.archive_dir.listdir() 681 log.Debug(gettext.ngettext("%d file exists in cache", 682 "%d files exist in cache", 683 len(local_filename_list)) % 684 len(local_filename_list)) 685 686 # check for partial backups 687 partials = [] 688 for local_filename in local_filename_list: 689 pr = file_naming.parse(local_filename) 690 if pr and pr.partial: 691 partials.append(local_filename) 692 693 # get various backup sets and chains 694 (backup_chains, self.orphaned_backup_sets, 695 self.incomplete_backup_sets) = \ 696 self.get_backup_chains(partials + backend_filename_list) 697 backup_chains = self.get_sorted_chains(backup_chains) 698 self.all_backup_chains = backup_chains 699 700 assert len(backup_chains) == len(self.all_backup_chains), "get_sorted_chains() did something more than re-ordering" 701 702 local_sig_chains, self.local_orphaned_sig_names = \ 703 self.get_signature_chains(True) 704 remote_sig_chains, self.remote_orphaned_sig_names = \ 705 self.get_signature_chains(False, filelist = backend_filename_list) 706 self.set_matched_chain_pair(local_sig_chains + remote_sig_chains, 707 backup_chains) 708 self.warn(sig_chain_warning) 709 return self

710

711 - def set_matched_chain_pair(self, sig_chains, backup_chains):

712 """ 713 Set self.matched_chain_pair and self.other_sig/backup_chains 714 715 The latest matched_chain_pair will be set. If there are both 716 remote and local signature chains capable of matching the 717 latest backup chain, use the local sig chain (it does not need 718 to be downloaded). 719 """ 720 sig_chains = sig_chains and self.get_sorted_chains(sig_chains) 721 self.all_sig_chains = sig_chains 722 self.other_backup_chains = backup_chains[:] 723 self.matched_chain_pair = None 724 if sig_chains and backup_chains: 725 latest_backup_chain = backup_chains[-1] 726 for i in range(len(sig_chains)-1, -1, -1): 727 if sig_chains[i].end_time == latest_backup_chain.end_time: 728 pass 729 # See if the set before last matches: 730 elif (len(latest_backup_chain.get_all_sets()) >= 2 and 731 sig_chains[i].end_time == latest_backup_chain.get_all_sets()[-2].end_time): 732 # It matches, remove the last backup set: 733 log.Warn(_("Warning, discarding last backup set, because " 734 "of missing signature file.")) 735 self.incomplete_backup_sets.append(latest_backup_chain.incset_list[-1]) 736 latest_backup_chain.incset_list = latest_backup_chain.incset_list[:-1] 737 else: 738 continue 739 740 # Found a matching pair: 741 if self.matched_chain_pair == None: 742 self.matched_chain_pair = (sig_chains[i], latest_backup_chain) 743 744 break 745 746 if self.matched_chain_pair: 747 self.other_backup_chains.remove(self.matched_chain_pair[1])

748

749 - def warn(self, sig_chain_warning):

750 """ 751 Log various error messages if find incomplete/orphaned files 752 """ 753 assert self.values_set 754 755 if self.local_orphaned_sig_names: 756 log.Warn(gettext.ngettext("Warning, found the following local orphaned " 757 "signature file:", 758 "Warning, found the following local orphaned " 759 "signature files:", 760 len(self.local_orphaned_sig_names)) 761 + "\n" + "\n".join(self.local_orphaned_sig_names), 762 log.WarningCode.orphaned_sig) 763 764 if self.remote_orphaned_sig_names: 765 log.Warn(gettext.ngettext("Warning, found the following remote orphaned " 766 "signature file:", 767 "Warning, found the following remote orphaned " 768 "signature files:", 769 len(self.remote_orphaned_sig_names)) 770 + "\n" + "\n".join(self.remote_orphaned_sig_names), 771 log.WarningCode.orphaned_sig) 772 773 if self.all_sig_chains and sig_chain_warning and not self.matched_chain_pair: 774 log.Warn(_("Warning, found signatures but no corresponding " 775 "backup files"), log.WarningCode.unmatched_sig) 776 777 if self.incomplete_backup_sets: 778 log.Warn(_("Warning, found incomplete backup sets, probably left " 779 "from aborted session"), log.WarningCode.incomplete_backup) 780 781 if self.orphaned_backup_sets: 782 log.Warn(gettext.ngettext("Warning, found the following orphaned " 783 "backup file:", 784 "Warning, found the following orphaned " 785 "backup files:", 786 len(self.orphaned_backup_sets)) 787 + "\n" + "\n".join(map(lambda x: str(x), 788 self.orphaned_backup_sets)), 789 log.WarningCode.orphaned_backup)

790

791 - def get_backup_chains(self, filename_list):

792 """ 793 Split given filename_list into chains 794 795 Return value will be tuple (list of chains, list of sets, list 796 of incomplete sets), where the list of sets will comprise sets 797 not fitting into any chain, and the incomplete sets are sets 798 missing files. 799 """ 800 log.Debug(_("Extracting backup chains from list of files: %s") 801 % filename_list) 802 # First put filenames in set form 803 sets = [] 804 def add_to_sets(filename): 805 """ 806 Try adding filename to existing sets, or make new one 807 """ 808 for set in sets: 809 if set.add_filename(filename): 810 log.Debug(_("File %s is part of known set") % (filename,)) 811 break 812 else: 813 log.Debug(_("File %s is not part of a known set; creating new set") % (filename,)) 814 new_set = BackupSet(self.backend) 815 if new_set.add_filename(filename): 816 sets.append(new_set) 817 else: 818 log.Debug(_("Ignoring file (rejected by backup set) '%s'") % filename)

819 map(add_to_sets, filename_list) 820 sets, incomplete_sets = self.get_sorted_sets(sets) 821 822 chains, orphaned_sets = [], [] 823 def add_to_chains(set): 824 """ 825 Try adding set to existing chains, or make new one 826 """ 827 if set.type == "full": 828 new_chain = BackupChain(self.backend) 829 new_chain.set_full(set) 830 chains.append(new_chain) 831 log.Debug(_("Found backup chain %s") % (new_chain.short_desc())) 832 else: 833 assert set.type == "inc" 834 for chain in chains: 835 if chain.add_inc(set): 836 log.Debug(_("Added set %s to pre-existing chain %s") % (set.get_timestr(), 837 chain.short_desc())) 838 break 839 else: 840 log.Debug(_("Found orphaned set %s") % (set.get_timestr(),)) 841 orphaned_sets.append(set)

842 map(add_to_chains, sets) 843 return (chains, orphaned_sets, incomplete_sets) 844

845 - def get_sorted_sets(self, set_list):

846 """ 847 Sort set list by end time, return (sorted list, incomplete) 848 """ 849 time_set_pairs, incomplete_sets = [], [] 850 for set in set_list: 851 if not set.is_complete(): 852 incomplete_sets.append(set) 853 elif set.type == "full": 854 time_set_pairs.append((set.time, set)) 855 else: 856 time_set_pairs.append((set.end_time, set)) 857 time_set_pairs.sort() 858 return (map(lambda p: p[1], time_set_pairs), incomplete_sets)

859

860 - def get_signature_chains(self, local, filelist = None):

861 """ 862 Find chains in archive_dir (if local is true) or backend 863 864 Use filelist if given, otherwise regenerate. Return value is 865 pair (list of chains, list of signature paths not in any 866 chains). 867 """ 868 def get_filelist(): 869 if filelist is not None: 870 return filelist 871 elif local: 872 return self.archive_dir.listdir() 873 else: 874 return self.backend.list()

875 876 def get_new_sigchain(): 877 """ 878 Return new empty signature chain 879 """ 880 if local: 881 return SignatureChain(True, self.archive_dir) 882 else: 883 return SignatureChain(False, self.backend) 884 885 # Build initial chains from full sig filenames 886 chains, new_sig_filenames = [], [] 887 for filename in get_filelist(): 888 pr = file_naming.parse(filename) 889 if pr: 890 if pr.type == "full-sig": 891 new_chain = get_new_sigchain() 892 assert new_chain.add_filename(filename, pr) 893 chains.append(new_chain) 894 elif pr.type == "new-sig": 895 new_sig_filenames.append(filename) 896 897 # compare by file time 898 def by_start_time(a, b): 899 return int(file_naming.parse(a).start_time) - int(file_naming.parse(b).start_time) 900 901 # Try adding new signatures to existing chains 902 orphaned_filenames = [] 903 new_sig_filenames.sort(by_start_time) 904 for sig_filename in new_sig_filenames: 905 for chain in chains: 906 if chain.add_filename(sig_filename): 907 break 908 else: 909 orphaned_filenames.append(sig_filename) 910 return (chains, orphaned_filenames) 911

912 - def get_sorted_chains(self, chain_list):

913 """ 914 Return chains sorted by end_time. If tie, local goes last 915 """ 916 # Build dictionary from end_times to lists of corresponding chains 917 endtime_chain_dict = {} 918 for chain in chain_list: 919 if endtime_chain_dict.has_key(chain.end_time): 920 endtime_chain_dict[chain.end_time].append(chain) 921 else: 922 endtime_chain_dict[chain.end_time] = [chain] 923 924 # Use dictionary to build final sorted list 925 sorted_end_times = endtime_chain_dict.keys() 926 sorted_end_times.sort() 927 sorted_chain_list = [] 928 for end_time in sorted_end_times: 929 chain_list = endtime_chain_dict[end_time] 930 if len(chain_list) == 1: 931 sorted_chain_list.append(chain_list[0]) 932 else: 933 assert len(chain_list) == 2 934 if chain_list[0].backend: # is remote, goes first 935 sorted_chain_list.append(chain_list[0]) 936 sorted_chain_list.append(chain_list[1]) 937 else: # is local, goes second 938 sorted_chain_list.append(chain_list[1]) 939 sorted_chain_list.append(chain_list[0]) 940 941 return sorted_chain_list

942

943 - def get_backup_chain_at_time(self, time):

944 """ 945 Return backup chain covering specified time 946 947 Tries to find the backup chain covering the given time. If 948 there is none, return the earliest chain before, and failing 949 that, the earliest chain. 950 """ 951 if not self.all_backup_chains: 952 raise CollectionsError("No backup chains found") 953 954 covering_chains = filter(lambda c: c.start_time <= time <= c.end_time, 955 self.all_backup_chains) 956 if len(covering_chains) > 1: 957 raise CollectionsError("Two chains cover the given time") 958 elif len(covering_chains) == 1: 959 return covering_chains[0] 960 961 old_chains = filter(lambda c: c.end_time < time, 962 self.all_backup_chains) 963 if old_chains: 964 return old_chains[-1] 965 else: 966 return self.all_backup_chains[0] # no chains are old enough

967

968 - def get_signature_chain_at_time(self, time):

969 """ 970 Return signature chain covering specified time 971 972 Tries to find the signature chain covering the given time. If 973 there is none, return the earliest chain before, and failing 974 that, the earliest chain. 975 """ 976 if not self.all_sig_chains: 977 raise CollectionsError("No signature chains found") 978 979 covering_chains = filter(lambda c: c.start_time <= time <= c.end_time, 980 self.all_sig_chains) 981 if covering_chains: 982 return covering_chains[-1] # prefer local if multiple sig chains 983 984 old_chains = filter(lambda c: c.end_time < time, 985 self.all_sig_chains) 986 if old_chains: 987 return old_chains[-1] 988 else: 989 # no chains are old enough, give oldest and warn user 990 oldest = self.all_sig_chains[0] 991 if time < oldest.start_time: 992 log.Warn(_("No signature chain for the requested time. Using oldest available chain, starting at time %s.") % dup_time.timetopretty(oldest.start_time), log.WarningCode.no_sig_for_time, dup_time.timetostring(oldest.start_time)) 993 return oldest

994

995 - def get_extraneous(self, extra_clean):

996 """ 997 Return list of the names of extraneous duplicity files 998 999 A duplicity file is considered extraneous if it is 1000 recognizable as a duplicity file, but isn't part of some 1001 complete backup set, or current signature chain. 1002 """ 1003 assert self.values_set 1004 local_filenames = [] 1005 remote_filenames = [] 1006 ext_containers = self.orphaned_backup_sets + self.incomplete_backup_sets 1007 if extra_clean: 1008 old_sig_chains = self.all_sig_chains[:] 1009 if self.matched_chain_pair: 1010 matched_sig_chain = self.matched_chain_pair[0] 1011 for sig_chain in self.all_sig_chains: 1012 print sig_chain.start_time, matched_sig_chain.start_time, 1013 print sig_chain.end_time, matched_sig_chain.end_time 1014 if (sig_chain.start_time == matched_sig_chain.start_time and 1015 sig_chain.end_time == matched_sig_chain.end_time): 1016 old_sig_chains.remove(sig_chain) 1017 ext_containers += old_sig_chains 1018 for set_or_chain in ext_containers: 1019 if set_or_chain.backend: 1020 remote_filenames.extend(set_or_chain.get_filenames()) 1021 else: 1022 local_filenames.extend(set_or_chain.get_filenames()) 1023 local_filenames += self.local_orphaned_sig_names 1024 remote_filenames += self.remote_orphaned_sig_names 1025 return local_filenames, remote_filenames

1026

1027 - def sort_sets(self, setlist):

1028 """Return new list containing same elems of setlist, sorted by time""" 1029 pairs = map(lambda s: (s.get_time(), s), setlist) 1030 pairs.sort() 1031 return map(lambda p: p[1], pairs)

1032

1033 - def get_chains_older_than(self, t):

1034 """ 1035 Return a list of chains older than time t 1036 """ 1037 assert self.values_set 1038 return filter(lambda c: c.end_time < t, self.all_backup_chains)

1039

1040 - def get_last_full_backup_time(self):

1041 """ 1042 Return the time of the last full backup, 1043 or 0 if there is none. 1044 """ 1045 return self.get_nth_last_full_backup_time(1)

1046

1047 - def get_nth_last_full_backup_time(self, n):

1048 """ 1049 Return the time of the nth to last full backup, 1050 or 0 if there is none. 1051 """ 1052 chain = self.get_nth_last_backup_chain(n) 1053 if chain is None: 1054 return 0 1055 else: 1056 return chain.get_first().time

1057

1058 - def get_last_backup_chain(self):

1059 """ 1060 Return the last full backup of the collection, 1061 or None if there is no full backup chain. 1062 """ 1063 return self.get_nth_last_backup_chain(1)

1064

1065 - def get_nth_last_backup_chain(self,n):

1066 """ 1067 Return the nth-to-last full backup of the collection, 1068 or None if there is less than n backup chains. 1069 1070 NOTE: n = 1 -> time of latest available chain (n = 0 is not 1071 a valid input). Thus the second-to-last is obtained with n=2 1072 rather than n=1. 1073 """ 1074 def mycmp(x, y): 1075 return cmp(x.get_first().time, y.get_first().time)

1076 1077 assert self.values_set 1078 assert n > 0 1079 1080 if len(self.all_backup_chains) < n: 1081 return None 1082 1083 sorted = self.all_backup_chains[:] 1084 sorted.sort(mycmp) 1085 1086 sorted.reverse() 1087 return sorted[n - 1] 1088

1089 - def get_older_than(self, t):

1090 """ 1091 Returns a list of backup sets older than the given time t 1092 1093 All of the times will be associated with an intact chain. 1094 Furthermore, none of the times will be of a set which a newer 1095 set may depend on. For instance, if set A is a full set older 1096 than t, and set B is an incremental based on A which is newer 1097 than t, then the time of set A will not be returned. 1098 """ 1099 old_sets = [] 1100 for chain in self.get_chains_older_than(t): 1101 if (not self.matched_chain_pair or 1102 chain is not self.matched_chain_pair[1]): 1103 # don't delete the active (matched) chain 1104 old_sets.extend(chain.get_all_sets()) 1105 return self.sort_sets(old_sets)

1106

1107 - def get_older_than_required(self, t):

1108 """ 1109 Returns list of old backup sets required by new sets 1110 1111 This function is similar to the previous one, but it only 1112 returns the times of sets which are old but part of the chains 1113 where the newer end of the chain is newer than t. 1114 """ 1115 assert self.values_set 1116 new_chains = filter(lambda c: c.end_time >= t, self.all_backup_chains) 1117 result_sets = [] 1118 for chain in new_chains: 1119 old_sets = filter(lambda s: s.get_time() < t, chain.get_all_sets()) 1120 result_sets.extend(old_sets) 1121 return self.sort_sets(result_sets)

1122

Source Code for Module duplicity.collections