From d5dce0887a9012d30800902f43befe13e2126391 Mon Sep 17 00:00:00 2001 From: lars Date: Tue, 16 Jun 2009 01:58:29 +0000 Subject: [PATCH] * adapt block device handling to linux kernel >v2.6.26 (symlinks in /sys/block/) * turned all attributes of the "Blockdevice" class into properties (postponing evaluation) * unify cache_links for blockdevice information * add default location of config file to preferences handling --- src/cryptobox/core/blockdevice.py | 371 ++++++++++++++++++------------ 1 file changed, 219 insertions(+), 152 deletions(-) diff --git a/src/cryptobox/core/blockdevice.py b/src/cryptobox/core/blockdevice.py index a37135f..4d1edcb 100644 --- a/src/cryptobox/core/blockdevice.py +++ b/src/cryptobox/core/blockdevice.py @@ -130,29 +130,7 @@ class Blockdevice: # invalid device given raise cryptobox.core.exceptions.CBInternalError( "invalid block device requested: %s" % str(major_minor)) - # find the devdir (usually in /sys/block/) - for devdir, one_major_minor in find_blockdevices(self.sysblock_dir).items(): - if major_minor == one_major_minor: - self.devdir = devdir - break - else: - # we did not find a suitable device - raise cryptobox.core.exceptions.CBInternalError( - "could not find blockdevice with the given major/minor: " \ - + "%d/%d" % (self.major, self.minor)) - self.name = os.path.basename(self.devdir) - ## "reset" below will fill these values - self.size = None - self.size_human = None - self.range = None - self.slaves = None - self.holders = None - self.children = None - self.devnodes = None - self.uuid = None - self.type_id = None - self.label = None - self.reset(empty_cache=False) + self.__cache_link = ["blockdevice_info", (self.major, self.minor)] def __cmp__(self, other): @@ -165,7 +143,7 @@ class Blockdevice: return 1 - def reset(self, empty_cache=True): + def reset(self): """reread the data of the device usually we will have to reset the cache, too @@ -174,27 +152,7 @@ class Blockdevice: @type empty_cache: boolean @param empty_cache: Whether to discard the cached information or not. """ - if empty_cache: - CACHE.reset(["blockdevice_info", (self.major, self.minor)]) - self.size = self.__get_size() - self.size_human = self.__get_size_human() - self.range = self.__get_device_range() - self.slaves = self.__get_dev_related("slaves") - self.holders = self.__get_dev_related("holders") - self.children = self.__get_children() - self.devnodes = self.__get_device_nodes() - if self.get_device() is None: - CACHE.reset(["blockdevice_nodes"]) - self.devnodes = self.__get_device_nodes() - if self.get_device() is None: - LOGGER.warning("No suitable device nodes found for: %s" % \ - self.devdir) - # the following code would fail without device node -> skip it - return - attributes = self.__get_blkid_attributes() - self.label = attributes["label"] - self.type_id = attributes["type_id"] - self.uuid = attributes["uuid"] + CACHE.reset(self.__cache_link) def get_device(self): @@ -222,7 +180,8 @@ class Blockdevice: @rtype: boolean @return: 'True' for a valid blockdevice """ - if not self.devnodes: + # return False if no device node for this block device exists + if self.get_device() is None: return False ## check valid major_minor try: @@ -250,7 +209,7 @@ class Blockdevice: @return: 'True' for a device usable as a storage """ ## check the cache first - cache_link = ["blockdevice_info", (self.major, self.minor), "is_storage"] + cache_link = self.__cache_link + ["is_storage"] cached = CACHE.get(cache_link) if not cached is None: return cached @@ -294,7 +253,7 @@ class Blockdevice: """return if the device is a physical volume of a LVM """ ## check the cache first - cache_link = ["blockdevice_info", (self.major, self.minor), "is_lvm_pv"] + cache_link = self.__cache_link + ["is_lvm_pv"] cached = CACHE.get(cache_link) if not cached is None: return cached @@ -312,7 +271,7 @@ class Blockdevice: """return if the device is a logical volume of a LVM """ ## check the cache first - cache_link = ["blockdevice_info", (self.major, self.minor), "is_lvm_lv"] + cache_link = self.__cache_link + ["is_lvm_lv"] cached = CACHE.get(cache_link) if not cached is None: return cached @@ -337,7 +296,7 @@ class Blockdevice: """check if the device is the base of a md raid device """ ## check the cache first - cache_link = ["blockdevice_info", (self.major, self.minor), "is_md_raid"] + cache_link = self.__cache_link + ["is_md_raid"] cached = CACHE.get(cache_link) if not cached is None: return cached @@ -364,7 +323,7 @@ class Blockdevice: """check if the device is a luks container """ ## check the cache first - cache_link = ["blockdevice_info", (self.major, self.minor), "is_luks"] + cache_link = self.__cache_link + ["is_luks"] cached = CACHE.get(cache_link) if not cached is None: return cached @@ -400,7 +359,7 @@ class Blockdevice: """check if the device is marked as 'removable' """ ## check the cache first - cache_link = ["blockdevice_info", (self.major, self.minor), "is_removable"] + cache_link = self.__cache_link + ["is_removable"] cached = CACHE.get(cache_link) if not cached is None: return cached @@ -443,7 +402,8 @@ class Blockdevice: "invalid arguments for 'is_parent_of'") ## recursively go through all the children for child_devname in self.children: - child_dev = get_blockdevice(child_devname) + child_dev = get_blockdevice(child_devname, self.sysblock_dir, + self.devnode_dir) ## direct child? if child_dev == ask_child: return True @@ -454,6 +414,28 @@ class Blockdevice: return False + def __get_devdir(self): + cache_link = self.__cache_link + ["devdir"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + + result = None + # find the devdir (usually in /sys/block/) + for devdir, one_major_minor in find_blockdevices(self.sysblock_dir).items(): + if (self.major, self.minor) == one_major_minor: + result = devdir + break + else: + # we did not find a suitable device + raise cryptobox.core.exceptions.CBInternalError( + "could not find blockdevice with the given major/minor: " \ + + "%d/%d" % (self.major, self.minor)) + + CACHE.set(cache_link, result) + return result + + def __get_dev_related(self, subdir): """return the content of sub directories (e.g. 'holders' or 'slaves') """ @@ -475,15 +457,20 @@ class Blockdevice: def __get_size(self): """return the size (in MB) of the blockdevice """ - default = 0 + cache_link = self.__cache_link + ["size"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + try: size_blocks = int(file(os.path.join(self.devdir, 'size')).read()) ## size is defined as the number of blocks (512 byte each) - return int(size_blocks*512/1024/1024) - except OSError: - return default - except ValueError: - return default + result = int(size_blocks*512/1024/1024) + except (IOError, ValueError): + result = 0 + + CACHE.set(cache_link, result) + return result def __get_device_range(self): @@ -491,15 +478,18 @@ class Blockdevice: partitionable blockdevices have a range > 1 """ - default = 1 + cache_link = self.__cache_link + ["device_range"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + try: - content = file(os.path.join(self.devdir, "range")).read() - except IOError: - return default - try: - return int(content) - except ValueError: - return default + result = int(file(os.path.join(self.devdir, "range")).read()) + except (IOError, ValueError): + result = 1 + + CACHE.set(cache_link, result) + return result def __get_children(self): @@ -507,29 +497,50 @@ class Blockdevice: all holders, subdevices and children of subdevices """ + cache_link = self.__cache_link + ["children"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + direct_children = [ get_blockdevice(major_minor, self.sysblock_dir, self.devnode_dir).name - for major_minor in find_blockdevices(self.devdir).values()] + for major_minor in find_blockdevices(self.devdir, + follow_links=False).values()] direct_children.extend(self.holders[:]) children = direct_children[:] for dchild in direct_children: children.extend(get_blockdevice(dchild, self.sysblock_dir, - self.devnode_dir).children) + self.devnode_dir, follow_links=False).children) + + CACHE.set(cache_link, children) return children def __get_device_nodes(self): """get all device nodes with the major/minor combination of the device """ + cache_link = self.__cache_link + ["device_nodes"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + try: - return find_device_nodes(self.devnode_dir)[(self.major, self.minor)] + result = find_device_nodes(self.devnode_dir)[(self.major, self.minor)] except KeyError: - return [] + result = [] + + CACHE.set(cache_link, result) + return result def __get_uuid_luks(self): """determine the unique identifier of luks devices """ + cache_link = self.__cache_link + ["uuid_luks"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + prefs = _load_preferences() try: proc = subprocess.Popen( @@ -552,15 +563,22 @@ class Blockdevice: error)) return None result = output.strip() - if result: - return result - else: - return None + if not result: + result = None + + CACHE.set(cache_link, result) + return result def __get_uuid_lvm_pv(self): """determine the unique identifier of physical LVM volumes """ + cache_link = self.__cache_link + ["uuid_lvm_pv"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + + result = None if not OPTIONAL_PROGS["lvm"]: ## pvdisplay is not installed - skip it return None @@ -589,9 +607,10 @@ class Blockdevice: for line in output.splitlines(): items = line.strip().split(":") if (len(items) == 12) and (items[0] in self.devnodes): - return items[11] - ## not found - return None + result = items[11] + + CACHE.set(cache_link, result) + return result def __get_blkid_attributes(self): @@ -599,53 +618,64 @@ class Blockdevice: returns a dictionary containing label, type_id and uuid """ + cache_link = self.__cache_link + ["blkid_attributes"] + cached = CACHE.get(cache_link) + if not cached is None: + return cached + result = {"label": None, "type_id": None, "uuid": None} - if not self.is_valid(): - return result - prefs = _load_preferences() - try: - proc = subprocess.Popen( - shell = False, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - args = [ prefs["Programs"]["blkid"], - "-s", "LABEL", - "-s", "TYPE", - "-s", "UUID", - "-c", os.devnull, - "-w", os.devnull, - self.get_device()]) - (output, error) = proc.communicate() - except OSError, err_msg: - LOGGER.warning("Failed to call '%s' to determine label for " \ - % prefs["Programs"]["blkid"] + "'%s': %s" % \ - (self.get_device(), err_msg)) - return result - if proc.returncode == 2: - ## the device does not contain a filesystem (e.g. it is zeroed or - ## it contains a partition table) - return result - if proc.returncode != 0: - LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \ - (prefs["Programs"]["blkid"], self.get_device(), - error.strip())) - return result - # scan the output string for results - # the output string could look like this: - # /dev/hda1: TYPE="ext3" LABEL="neu"de" - pattern = {"LABEL": "label", "TYPE": "type_id", "UUID": "uuid"} - for name, attr in pattern.items(): - match = re.search(r' %s="(.*?)"(?: [A-Z]+="| ?$)' % name, output) - if match: - result[attr] = match.groups()[0] - # check for special attributes of LUKS devices and LVM physical volumes - # In this case the previously retrieved "uuid" value is overwritten. - ## UUIDs of physical LVM volumes can only be determined via pvdisplay - if self.is_lvm_pv(): - result["uuid"] = self.__get_uuid_lvm_pv() - ## UUIDs of luks devices can be determined via luksDump - elif self.is_luks(): - result["uuid"] = self.__get_uuid_luks() + if self.is_valid(): + prefs = _load_preferences() + try: + proc = subprocess.Popen( + shell = False, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + args = [ prefs["Programs"]["blkid"], + "-s", "LABEL", + "-s", "TYPE", + "-s", "UUID", + "-c", os.devnull, + "-w", os.devnull, + self.get_device()]) + (output, error) = proc.communicate() + except OSError, err_msg: + LOGGER.warning("Failed to call '%s' to determine label for " \ + % prefs["Programs"]["blkid"] + "'%s': %s" % \ + (self.get_device(), err_msg)) + else: + if proc.returncode == 2: + ## the device does not contain a filesystem (e.g. it is zeroed or + ## it contains a partition table) + pass + elif proc.returncode != 0: + LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \ + (prefs["Programs"]["blkid"], self.get_device(), + error.strip())) + pass + else: + # scan the output string for results + # the output string could look like this: + # /dev/hda1: TYPE="ext3" LABEL="neu"de" + pattern = {"LABEL": "label", "TYPE": "type_id", "UUID": "uuid"} + for name, attr in pattern.items(): + match = re.search(r' %s="(.*?)"(?: [A-Z]+="| ?$)' % \ + name, output) + if match: + result[attr] = match.groups()[0] + # Check for special attributes of LUKS devices and LVM + # physical volumes. + # In this case the previously retrieved "uuid" value is + # overwritten. + # UUIDs of physical LVM volumes can only be determined via + # pvdisplay. + if self.is_lvm_pv(): + result["uuid"] = self.__get_uuid_lvm_pv() + ## UUIDs of luks devices can be determined via luksDump + elif self.is_luks(): + result["uuid"] = self.__get_uuid_luks() + + CACHE.set(cache_link, result) return result @@ -668,7 +698,7 @@ class Blockdevice: output += "\t%s:\t%s\n" % ("blockdir", self.devdir) output += "\t%s:\t%d/%d\n" % ("major/minor", self.major, self.minor) output += "\t%s:\t\t%s\n" % ("label", self.label) - output += "\t%s:\t\t%s\n" % ("type_id", self.type_id) + output += "\t%s:\t%s\n" % ("type_id", self.type_id) output += "\t%s:\t\t%s\n" % ("UUID", self.uuid) output += "\t%s:\t\t%s\n" % ("range", self.range) output += "\t%s:\t\t%s\n" % ("size", self.size) @@ -689,6 +719,21 @@ class Blockdevice: return output + devdir = property(__get_devdir) + name = property(lambda self: os.path.basename(self.devdir)) + size = property(__get_size) + size_human = property(__get_size_human) + range = property(__get_device_range) + slaves = property(lambda self: self.__get_dev_related("slaves")) + holders = property(lambda self: self.__get_dev_related("holders")) + children = property(__get_children) + devnodes = property(__get_device_nodes) + label = property(lambda self: self.__get_blkid_attributes()["label"]) + type_id = property(lambda self: self.__get_blkid_attributes()["type_id"]) + uuid = property(lambda self: self.__get_blkid_attributes()["uuid"]) + + + class BlockdeviceCache: """manage cached results of blockdevce queries @@ -719,7 +764,7 @@ class BlockdeviceCache: self.set(target, {}) elif isinstance(target, Blockdevice): # we do not reset the expire date - self.set(["blockdevice_info", (target.major, target.minor)], {}) + target.reset() else: LOGGER.log.warn("Invalid argument type for reset: %s" % \ str(type(target))) @@ -828,7 +873,7 @@ def __get_major_minor(dev): def get_blockdevice(dev, sysblock_dir=DEFAULT_SYSBLOCK_DIR, - devnode_dir=DEFAULT_DEVNODE_DIR, retry_once=True): + devnode_dir=DEFAULT_DEVNODE_DIR, retry_once=True, follow_links=True): if dev is None: return None elif isinstance(dev, Blockdevice): @@ -842,7 +887,8 @@ def get_blockdevice(dev, major_minor = __get_major_minor(dev) else: ## the name of a blockdevice (e.g.: 'dm-0') - for one_devdir, one_major_minor in find_blockdevices(sysblock_dir).items(): + for one_devdir, one_major_minor in find_blockdevices(sysblock_dir, + follow_links).items(): if os.path.basename(one_devdir) == dev: major_minor = one_major_minor break @@ -851,7 +897,8 @@ def get_blockdevice(dev, # this is necessary for the "partition" plugin if retry_once and (CACHE.get_age() > CACHE_MINIMUM_AGE_FOR_REBUILD): CACHE.reset() - device = get_blockdevice(dev, sysblock_dir, devnode_dir, False) + device = get_blockdevice(dev, sysblock_dir, devnode_dir, + retry_once=False, follow_links=follow_links) if not device is None: major_minor = (device.major, device.minor) else: @@ -872,33 +919,43 @@ def get_blockdevice(dev, -def find_blockdevices(top_dir): +def find_blockdevices(top_dir, follow_links=True): + # normalize the input directory + top_dir = os.path.realpath(top_dir) cache_link = ["blockdevice_dirs", top_dir] cached = CACHE.get(cache_link) if not cached is None: return cached.copy() dev_dirs = {} + dev_file_name = 'dev' + walk_dirs = [top_dir] + walk_dir_index = 0 - def look4dev_dirs(arg, dirname, fnames): - ## ignore the top level directory to avoid infinite recursion for - ## get_children - if os.path.samefile(dirname, top_dir): - return - ## add directories containing the file 'dev' to the list - dev_file_path = os.path.join(dirname, arg) - if (arg in fnames) and os.path.isfile(dev_file_path): - major_minor = __get_major_minor(dirname) - if not major_minor is None: - dev_dirs[dirname] = major_minor - for fname in fnames: - ## remove symlinks and non-directories - fullname = os.path.join(dirname, fname) - if os.path.islink(fullname) or (not os.path.isdir(fullname)): - fnames.remove(fname) + while walk_dir_index < len(walk_dirs): + for dirname, dirs, fnames in os.walk(walk_dirs[walk_dir_index]): + ## add directories containing the file 'dev' to the list + dev_file_path = os.path.join(dirname, dev_file_name) + # don't include the top-level device itself + if (os.path.realpath(dirname) != top_dir) and \ + (dev_file_name in fnames) and os.path.isfile(dev_file_path): + major_minor = __get_major_minor(dirname) + if (not major_minor is None) and \ + (not major_minor in dev_dirs.values()): + dev_dirs[dirname] = major_minor + # follow symlinks + if follow_links and (walk_dir_index == 0): + for dname in dirs: + fullpath = os.path.join(dirname, dname) + if (os.path.islink(fullpath) and os.path.isdir(fullpath)): + fullpath = os.path.realpath(fullpath) + if not fullpath in walk_dirs: + if not [one_walk for one_walk in walk_dirs + if os.path.samefile(one_walk, fullpath)]: + walk_dirs.append(fullpath) + walk_dir_index += 1 - os.path.walk(top_dir, look4dev_dirs, 'dev') CACHE.set(cache_link, dev_dirs) return dev_dirs.copy() @@ -981,13 +1038,23 @@ def _load_preferences(): return prefs ## we have to load an emergency fallback for proper function ## this is mainly useful for local testing + alternative_locations = [] + # try to use the '/bin/cryptobox.conf' location within a local working + # copy of the subversion repository root_dir = os.path.realpath(os.path.join(globals()["cryptobox"].__path__[0], os.path.pardir, os.path.pardir)) - config_file = os.path.join(root_dir, "bin", "cryptobox.conf") - ## we have to chdir to the 'bin' directory - otherwise the paths in - ## cryptobox.conf do not work - os.chdir(os.path.dirname(config_file)) - return cryptobox.core.settings.CryptoBoxSettings(config_file) + alternative_locations.append( + os.path.join(root_dir, "bin", "cryptobox.conf")) + # try the default config file location + alternative_locations.append('/etc/cryptobox-server/cryptobox.conf') + for config_file in alternative_locations: + if os.path.exists(config_file): + # we have to chdir to the 'bin' directory - otherwise the paths in + # cryptobox.conf do not work + os.chdir(os.path.dirname(config_file)) + return cryptobox.core.settings.CryptoBoxSettings(config_file) + else: + raise CBConfigUnavailableError()