from __future__ import with_statement import logging import math import operator import os import stat import tempfile import zipfile from json import loads from abl.util import Bunch from abl.vpath.base import URI from abl.vpath.base.exceptions import FileDoesNotExistError from .util import assert_, relpath_parts from .xdelta import apply_xdelta logger = logging.getLogger(__name__) def find_unique_path(base_dir, filename): counter = 1 tmp_fnm = "%s.orig" % filename while True: p = base_dir / tmp_fnm if p.exists(): tmp_fnm = "%s.orig-%d" % (filename, counter) counter = counter + 1 else: return p return None class Updater(object): def __init__(self, app_base_path, copy_app_base_path, autoupdates_dir_base_path): self.app_base_path = app_base_path self.copy_app_base_path = copy_app_base_path self.autoupdates_dir_base_path = autoupdates_dir_base_path self.clone_installation() self.sanity_check_cloned_installation() def apply_delta(self, delta): dest = self.copy_app_base_path created = delta / "created" updated = delta / "updated" deleted = delta / "deleted" def path_in_dest(base, path): parts = [] while path != base: parts.append(path.basename()) path = path.directory() return reduce(operator.div, reversed(parts), dest) if deleted.exists(): assert deleted.isdir(), "deleted must be directory - malformed delta" for root, dirnames, filenames in deleted.walk(followlinks=False): for filename in filenames: file_dest = path_in_dest(deleted, root / filename) try: file_dest.remove(recursive=True) except FileDoesNotExistError: # don't complain on missing files. pass if created.exists(): assert created.isdir(), "created must be directory - malformed delta" for root, dirnames, filenames in created.walk(followlinks=False): for dirname in dirnames: if (root / dirname).islink(): filenames.append(dirname) else: d = path_in_dest(created, root / dirname) # make sure the destination directory exists d.makedirs() for filename in filenames: file_dest = path_in_dest(created, root / filename) if file_dest.isfile() or file_dest.islink(): # this file should not be here, but since it may be # a valueable user added one, move it away logger.error("File already exists: %s" % file_dest) orig_dest = find_unique_path( file_dest.directory(), file_dest.basename() ) file_dest.move(orig_dest) logger.error(" ... moved to: %s" % orig_dest) else: # make sure the destination directory exists file_dest.directory().makedirs() if (root / filename).islink(): (root / filename).readlink().symlink(file_dest) else: (root / filename).move(file_dest) # updating is done by removing the old file and then moving the new # file into position. if updated.exists(): assert updated.isdir(), "updated must be directory - malformed delta" for root, dirnames, filenames in updated.walk(followlinks=False): for filename in filenames: file_dest = path_in_dest(updated, root / filename) try: file_dest.remove(recursive=True) except FileDoesNotExistError: # don't complain on missing files. pass # make sure the destination directory exists file_dest.directory().makedirs() if (root / filename).islink(): (root / filename).readlink().symlink(file_dest) else: (root / filename).move(file_dest) def extract_deltas(self, deltas, delta_dir): padding = int(math.ceil(math.log(len(deltas), 10))) prefix = "d%%0%ii" % padding for i, delta in enumerate(deltas): current_delta_dir = delta_dir / (prefix % i) current_delta_dir.mkdir() df = zipfile.ZipFile(delta.open("rb")) for name in df.namelist(): content = df.read(name) f = reduce(operator.div, name.split(os.sep), current_delta_dir) def create_dirs(d): if d.exists(): return else: create_dirs(d.directory()) d.mkdir() create_dirs(f.directory()) info = df.getinfo(name) mode = info.external_attr >> 16 if ( current_delta_dir.connection.supports_symlinks() and stat.S_IFLNK == stat.S_IFLNK & mode ): URI(content).symlink(f) else: with f.open("wb") as outf: outf.write(content) f.info(dict(mode=mode), followlinks=False) # now post-process the xdeltas for root, dirnames, filenames in current_delta_dir.walk(): for filename in filenames: if filename.endswith(".xdelta"): xdelta = root / filename mode = xdelta.info().mode new_alp = root / filename[: -len(".xdelta")] # cut off the first directory-part, because # that's "updated" alp_rp = relpath_parts(current_delta_dir, new_alp)[1:] old_alp = reduce(operator.div, alp_rp, self.app_base_path) assert old_alp.exists() apply_xdelta(old_alp, xdelta, new_alp) xdelta.remove() new_alp.info(dict(mode=mode)) yield current_delta_dir def load_update_info(self): info_file = self.auto_updates_base() / "info.json" assert_( lambda: info_file.exists() and info_file.isfile(), "no info file %s found" % info_file, ) with info_file.open("r") as inf: return loads(inf.read()) def auto_updates_base(self): dir_ = self.autoupdates_dir_base_path / "_autoupdates" assert_(lambda: dir_.exists(), "auto_update folder %s doesn't exist" % dir_) return dir_ def extraction_dir(self): # pragma: no cover """ Returns an URI-object pointing to the place where next update is being prepared. The directory is guaranteed to be existing, and by default cleaned. """ autoupdate_base = self.auto_updates_base() assert_(lambda: autoupdate_base.exists(), "autoupdate_base doesn't exist") return autoupdate_base / "pending_update" def clone_installation(self): exclude_from_copy = ["_autoupdates"] if self.copy_app_base_path.exists(): logger.debug("Copy wasn't deleted during previous update attempt") self.copy_app_base_path.remove(recursive=True) # make a recursive copy of the original installation, but don't # follow symlinks (i.e. preserve any links we might see) self.app_base_path.copy( self.copy_app_base_path, recursive=True, followlinks=False, ignore=exclude_from_copy, ) def _get_number_of_files(self, path): return sum((len(f) for _, _, f in path.walk(followlinks=False))) def sanity_check_cloned_installation(self): excluded_files_count = 0 if (self.app_base_path / "_autoupdates").exists(): excluded_files_count = self._get_number_of_files( self.app_base_path / "_autoupdates" ) source_files_count = ( self._get_number_of_files(self.app_base_path) - excluded_files_count ) target_files_count = self._get_number_of_files(self.copy_app_base_path) if source_files_count != target_files_count: self.copy_app_base_path.remove(recursive=True) error = ( "failed to clone Live \nNr. of old files: %d \nNr. of new files: %d " % (source_files_count, target_files_count) ) raise Exception(error) def prepare_auto_update(self, v_from, v_to, cleanup=True): auto_updates_base = self.auto_updates_base() deltas_base = auto_updates_base / "deltas" info = self.load_update_info() delta_names = info["versions"][v_from][v_to]["deltas"] extraction_base = self.extraction_dir() # by default, cleanup if cleanup and extraction_base.exists(): extraction_base.remove(recursive=True) if not extraction_base.exists(): extraction_base.mkdir() extracted_deltas_base = extraction_base / "deltas" extracted_deltas_base.mkdir() # now extract the deltas deltas = [deltas_base / name for name in delta_names] for extracted_delta_dir in self.extract_deltas(deltas, extracted_deltas_base): self.apply_delta(extracted_delta_dir) def apply_single_update(self, asu_file): """ Apply the update in the given zip file path. Used for testing. Normally, prepare_auto_update is used instead. """ tempdir = URI(tempfile.mkdtemp()) try: for extracted_delta_dir in self.extract_deltas([asu_file], tempdir): self.apply_delta(extracted_delta_dir) finally: tempdir.remove(recursive=True)