cryptonas blockdevice management:

* significantly reduced the time required for generating the block device list:
 * use a cache for major/minor -> device nodes mapping
 * run 'blkid' only once for LABEL, TYPE and UUID
* added some code for (optional) profiling via python-profiler

src/cryptobox/core/blockdevice.py

@@ -23,12 +23,12 @@ These classes detect and filter available blockdevices.
 '''
 
 
-#TODO: call blkid only once for all devices and scan the result
-
 __revision__ = "$Id$"
 
 
 import os
+import types
+import re
 import subprocess
 import time
 import logging
@@ -57,6 +57,7 @@ CACHE_MONITOR_FILE = '/proc/partitions'
 
 ## useful for manual profiling
 IS_VISIBLE = True
+DO_PROFILE = False
 
 ## caching is quite important for the following implementation
 ## the object will be initializes later below
@@ -73,8 +74,8 @@ class Blockdevices:
         self.sysblock_dir = sysblock_dir
         self.devnode_dir = devnode_dir
         self.devices = []
-        for devdir in find_blockdevices(self.sysblock_dir):
+        for major_minor in find_blockdevices(self.sysblock_dir).values():
-            blockdevice = get_blockdevice(devdir,
+            blockdevice = get_blockdevice(major_minor,
                     self.sysblock_dir, self.devnode_dir)
             if (not blockdevice is None) and blockdevice.is_valid():
                 self.devices.append(blockdevice)
@@ -105,29 +106,34 @@ class Blockdevice:
     use "_get_blockdevice" instead
     """
 
-    def __init__(self, dev,
+    def __init__(self, major_minor,
             sysblock_dir=DEFAULT_SYSBLOCK_DIR,
             devnode_dir=DEFAULT_DEVNODE_DIR):
         """initialize the blockdevice
 
-        @type dev: string
+        @param major_minor: major/minor value of the blockdevice
+        @type major_minor: tuple
         @param dev: The /sys/block/ subdirectory describing a blockdevice
         @type sysblock_dir: string
         @param sysblock_dir: The linux /sys/ directory. Default is '/sys'.
         @type devnode_dir: string
         @param devnode_dir: The linux /dev/ directory. Default is '/dev'.
         """
-        self.devdir = dev
         self.devnode_dir = devnode_dir
         self.sysblock_dir = sysblock_dir
-        ## check if the device is valid
+        self.major, self.minor = major_minor
-        if not os.path.isdir(os.path.join(self.devnode_dir, dev)):
+        # find the devdir (usually in /sys/block/)
+        for devdir, one_major_minor in find_blockdevices(self.sysblock_dir).items():
+            if major_minor == one_major_minor:
+                self.devdir = devdir
+                break
+        else:
+            # we did not find a suitable device
             raise cryptobox.core.exceptions.CBInternalError(
-                    "invalid blockdevice given: %s (%s)" % \
+                    "could not find blockdevice with the given major/minor: " \
-                    (self.devdir, self.sysblock_dir))
+                    + "%d/%d" % (self.major, self.minor))
         self.name = os.path.basename(self.devdir)
         ## "reset" below will fill these values
-        self.devnum = None
         self.size = None
         self.size_human = None
         self.range = None
@@ -136,6 +142,7 @@ class Blockdevice:
         self.children = None
         self.devnodes = None
         self.uuid = None
+        self.type_id = None
         self.label = None
         self.reset(empty_cache=False)
 
@@ -151,7 +158,6 @@ class Blockdevice:
         """
         if empty_cache:
             CACHE.reset(["blockdevice_info", self.name])
-        self.devnum = self.__get_major_minor()
         self.size = self.__get_size()
         self.size_human = self.__get_size_human()
         self.range = self.__get_device_range()
@@ -159,9 +165,10 @@ class Blockdevice:
         self.holders = self.__get_dev_related("holders")
         self.children = self.__get_children()
         self.devnodes = self.__get_device_nodes()
-        self.uuid = self.__get_uuid()
+        attributes = self.__get_blkid_attributes()
-        self.label = self.__get_label()
+        self.label = attributes["label"]
-        self.type_id = self.__get_type_id()
+        self.type_id = attributes["type_id"]
+        self.uuid = attributes["uuid"]
 
 
     def is_valid(self):
@@ -174,17 +181,16 @@ class Blockdevice:
         """
         if not self.devnodes:
             return False
-        ## check valid devnum
+        ## check valid major_minor
         try:
-            major, minor = self.devnum
+            if (self.major == 0) and (self.minor == 0):
-            if (major == 0) and (minor == 0):
                 return False
             ## loop devices are ignored, if they are unused
-            if (major == MAJOR_DEVNUM_LOOP) and (self.size == 0):
+            if (self.major == MAJOR_DEVNUM_LOOP) and (self.size == 0):
                 return False
             ## floppy disks are totally ignored
             ## otherwise we would have a long timeout, while reading the devices
-            if (major == MAJOR_DEVNUM_FLOPPY):
+            if (self.major == MAJOR_DEVNUM_FLOPPY):
                 return False
         except TypeError:
             return False
@@ -298,7 +304,7 @@ class Blockdevice:
         else:
             for hold in self.holders:
                 if get_blockdevice(hold, self.sysblock_dir,
-                        self.devnode_dir).devnum[0] == MAJOR_DEVNUM_MD_RAID:
+                        self.devnode_dir).major == MAJOR_DEVNUM_MD_RAID:
                     result = True
                     break
             else:
@@ -379,7 +385,7 @@ class Blockdevice:
         invalid (None) "ask_child" devices return False
 
         @param ask_child: the blockdevice that is considered to be a possible child
-        @type ask_child: BlockDevice
+        @type ask_child: Blockdevice
         @return: True if the child is (even recursively) part of the blockdevice, otherwise False
         @rtype: bool
         """
@@ -415,7 +421,6 @@ class Blockdevice:
     def __get_size_human(self):
         """return a human readable string representing the size of the device
         """
-        size = self.size
         if self.size > 5120:
             return "%dGB" % int(self.size/1024)
         else:
@@ -436,23 +441,6 @@ class Blockdevice:
             return default
 
 
-    def __get_major_minor(self):
-        """return the major and minor of the device"""
-        default = (0, 0)
-        try:
-            content = file(os.path.join(self.devdir, "dev")).read()
-        except IOError:
-            return default
-        try:
-            major, minor = content.split(":", 1)
-        except TypeError:
-            return default
-        try:
-            return int(major), int(minor)
-        except ValueError:
-            return default
-
-
     def __get_device_range(self):
         """number of possible subdevices
 
@@ -475,8 +463,8 @@ class Blockdevice:
         all holders, subdevices and children of subdevices
         """
         direct_children = [
-                get_blockdevice(child, self.sysblock_dir, self.devnode_dir).name
+                get_blockdevice(major_minor, self.sysblock_dir, self.devnode_dir).name
-                for child in find_blockdevices(self.devdir)]
+                for major_minor in find_blockdevices(self.devdir).values()]
         direct_children.extend(self.holders[:])
         children = direct_children[:]
         for dchild in direct_children:
@@ -488,42 +476,10 @@ class Blockdevice:
     def __get_device_nodes(self):
         """get all device nodes with the major/minor combination of the device
         """
-        result = []
+        try:
-        major, minor = self.devnum
+            return find_device_nodes(self.devnode_dir)[(self.major, self.minor)]
-
+        except KeyError:
-        def find_major_minor(arg, dirname, fnames):
+            return []
-            for fname in fnames:
-                try:
-                    stat = os.stat(os.path.join(dirname, fname))
-                    ## check if it is a blockdevice and compare major/minor
-                    if (stat.st_mode & 060000 == 060000) \
-                            and (os.major(stat.st_rdev) == major) \
-                            and (os.minor(stat.st_rdev) == minor):
-                        result.append(os.path.join(dirname, fname))
-                except OSError:
-                    pass
-
-        os.path.walk(self.devnode_dir, find_major_minor, None)
-        return result
-
-
-    def __get_uuid(self):
-        """determine the unique identifier of this device
-
-        returns None in case of error or for invalid devices (see "is_valid")
-        """
-        if not self.is_valid():
-            return None
-        ## partitionable devices do not have a UUID
-        if self.is_partitionable():
-            return None
-        ## UUIDs of physical LVM volumes can only be determined via pvdisplay
-        if self.is_lvm_pv():
-            return self.__get_uuid_lvm_pv()
-        ## UUIDs of luks devices can be determined via luksDump
-        if self.is_luks():
-            return self.__get_uuid_luks()
-        return self.__get_uuid_default()
 
 
     def __get_uuid_luks(self):
@@ -536,10 +492,10 @@ class Blockdevice:
                     stdout = subprocess.PIPE,
                     stderr = subprocess.PIPE,
                     args = [
-						  		prefs["Programs"]["super"],
+                          		prefs["Programs"]["super"],
-								prefs["Programs"]["CryptoBoxRootActions"],
+                                prefs["Programs"]["CryptoBoxRootActions"],
-						  		"program", "cryptsetup",
+                          		"program", "cryptsetup",
-								"luksUUID", self.devnodes[0] ])
+                                "luksUUID", self.devnodes[0] ])
             (output, error) = proc.communicate()
         except OSError, err_msg:
             LOGGER.warning("Failed to call '%s' to determine UUID: %s" \
@@ -593,53 +549,16 @@ class Blockdevice:
         return None
 
 
-    def __get_uuid_default(self):
+    def __get_blkid_attributes(self):
-        """determine the unique identifier for non-special devices
-
-        luks and lvm_pv devices must be treated differently
-        """
-        prefs = _load_preferences()
-        try:
-            proc = subprocess.Popen(
-                    shell = False,
-                    stdout = subprocess.PIPE,
-                    stderr = subprocess.PIPE,
-                    args = [ prefs["Programs"]["blkid"],
-                            "-s", "UUID",
-                            "-o", "value",
-                            "-c", os.devnull,
-                            "-w", os.devnull,
-                            self.devnodes[0] ])
-            (output, error) = proc.communicate()
-        except OSError, err_msg:
-            LOGGER.warning("Failed to call '%s' to determine UUID: %s" % \
-                    (prefs["Programs"]["blkid"], err_msg))
-            return None
-        if proc.returncode == 2:
-            ## the device does not contain a filesystem (e.g. it is zeroed or
-            ## it contains a partition table)
-            return None
-        if proc.returncode != 0:
-            LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
-                    (prefs["Programs"]["blkid"], self.devnodes[0],
-                        error.strip()))
-            return None
-        result = output.strip()
-        if result:
-            return result
-        else:
-            return None
-
-
-    def __get_label(self):
         """determine the label of a filesystem contained in a device
 
-        return None for errors, empty labels and for luks or non-storage devices
+        returns a dictionary containing label, type_id and uuid
         """
+        result =  {"label": None, "type_id": None, "uuid": None}
         if not self.is_valid():
-            return None
+            return result
         if self.is_luks():
-            return None
+            return result
         prefs = _load_preferences()
         try:
             proc = subprocess.Popen(
@@ -648,7 +567,8 @@ class Blockdevice:
                     stderr = subprocess.PIPE,
                     args = [ prefs["Programs"]["blkid"],
                         "-s", "LABEL",
-                        "-o", "value",
+                        "-s", "TYPE",
+                        "-s", "UUID",
                         "-c", os.devnull,
                         "-w", os.devnull,
                         self.devnodes[0]])
@@ -657,65 +577,33 @@ class Blockdevice:
             LOGGER.warning("Failed to call '%s' to determine label for " \
                     % prefs["Programs"]["blkid"] + "'%s': %s" % \
                     (self.devnodes[0], err_msg))
-            return None
+            return result
         if proc.returncode == 2:
             ## the device does not contain a filesystem (e.g. it is zeroed or
             ## it contains a partition table)
-            return None
+            return result
         if proc.returncode != 0:
             LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
                     (prefs["Programs"]["blkid"], self.devnodes[0],
                         error.strip()))
-            return None
-        result = output.strip()
-        if result:
             return result
-        else:
+        # scan the output string for results
-            return None
+        # the output string could look like this:
-
+        #   /dev/hda1: TYPE="ext3" LABEL="neu"de"
-
+        pattern = {"LABEL": "label", "TYPE": "type_id", "UUID": "uuid"}
-    def __get_type_id(self):
+        for name, attr in pattern.items():
-        """determine the type id of a filesystem contained in a device
+            match = re.search(r' %s="(.*?)"(?: [A-Z]+="|$)' % name, output)
-
+            if match:
-        possible results are: ext2, ext3, vfat, reiserfs, swap, ...
+                result[attr] = match.groups()[0]
-        return None for errors, empty labels and for luks or non-storage devices
+        # check for special attributes of LUKS devices and LVM physical volumes
-        """
+        # In this case the previously retrieved "uuid" value is overwritten.
-        if not self.is_valid():
+        ## UUIDs of physical LVM volumes can only be determined via pvdisplay
-            return None
+        if self.is_lvm_pv():
-        if self.is_luks():
+            result["uuid"] = self.__get_uuid_lvm_pv()
-            return None
+        ## UUIDs of luks devices can be determined via luksDump
-        prefs = _load_preferences()
+        elif self.is_luks():
-        try:
+            result["uuid"] = self.__get_uuid_luks()
-            proc = subprocess.Popen(
+        return result
-                    shell = False,
-                    stdout = subprocess.PIPE,
-                    stderr = subprocess.PIPE,
-                    args = [ prefs["Programs"]["blkid"],
-                        "-s", "TYPE",
-                        "-o", "value",
-                        "-c", os.devnull,
-                        "-w", os.devnull,
-                        self.devnodes[0]])
-            (output, error) = proc.communicate()
-        except OSError, err_msg:
-            LOGGER.warning("Failed to call '%s' to determine type id for" \
-                    % prefs["Programs"]["blkid"] + " '%s': %s" % \
-                    (self.devnodes[0], err_msg))
-            return None
-        if proc.returncode == 2:
-            ## the device does not contain a filesystem (e.g. it is zeroed or
-            ## it contains a partition table)
-            return None
-        if proc.returncode != 0:
-            LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
-                    (prefs["Programs"]["blkid"], self.devnodes[0],
-                        error.strip()))
-            return None
-        result = output.strip()
-        if result:
-            return result
-        else:
-            return None
 
 
     def __eq__(self, device):
@@ -735,7 +623,7 @@ class Blockdevice:
         """
         output = "%s:\n" % self.name
         output += "\t%s:\t%s\n" % ("blockdir", self.devdir)
-        output += "\t%s:\t%s\n" % ("major/minor", self.devnum)
+        output += "\t%s:\t%d/%d\n" % ("major/minor", self.major, self.minor)
         output += "\t%s:\t\t%s\n" % ("label", self.label)
         output += "\t%s:\t\t%s\n" % ("UUID", self.uuid)
         output += "\t%s:\t\t%s\n" % ("range", self.range)
@@ -794,7 +682,7 @@ class BlockdeviceCache:
             if (file(CACHE_MONITOR_FILE).read() != self.partitions_save) or \
                     (self.expires < int(time.time())):
                 return True
-        except IOError:
+        except IOError, err_msg:
             LOGGER.warning("Failed to read '%s': %s" % \
                     (CACHE_MONITOR_FILE, err_msg))
         return False
@@ -838,44 +726,73 @@ class BlockdeviceCache:
             ref = ref[element]
         ## store the item
         ref[link[-1]] = item
-        
+ 
+
+
+def __get_major_minor(dev):
+    """ return a tuple (major/minor) for a device node or for a
+    subdirectory of /sys/block containing a node named 'dev'
+    """
+    if os.path.isdir(dev):
+        # we assume, that is is a subdirectory of /sys/block
+        dev_file_name = os.path.join(dev, 'dev')
+        try:
+            maj_min_string = file(dev_file_name).read().strip()
+        except IOError:
+            return None
+        if (maj_min_string.count(":") == 1):
+            major, minor = maj_min_string.split(":")
+            try:
+                major = int(major)
+                minor = int(minor)
+            except ValueError:
+                return None
+            # everything looks ok
+            return (major, minor)
+        else:
+            return None
+    else:
+        # we assume, that it is a device node (e.g. /dev/hda1)
+        try:
+            # we don't need to check for symlinks: "os.stat" resolves them
+            stat = os.stat(dev)
+        except OSError:
+            # the node does not exist
+            return None
+        # check if it is a block device
+        if (stat.st_mode & 060000 == 060000):
+            major = os.major(stat.st_rdev)
+            minor = os.minor(stat.st_rdev)
+            return (major, minor)
+        else:
+            return None
+
 
 
 def get_blockdevice(dev,
         sysblock_dir=DEFAULT_SYSBLOCK_DIR,
         devnode_dir=DEFAULT_DEVNODE_DIR):
-    if os.path.isabs(dev):
+    if isinstance(dev, Blockdevice):
+        # it is already a blockdevice
+        major_minor = (dev.major, dev.minor)
+    elif type(dev) is types.TupleType:
+		# we assume that the tuple contains major/minor
+        major_minor = dev
+    elif os.path.isabs(dev):
         ## it is an absolute path
-        if dev.startswith(devnode_dir):
+        major_minor = __get_major_minor(dev)
-            ## it is the name of a devicenode (e.g.: '/dev/hda1')
-            ## simplify the path first
-            dev = os.path.realpath(dev)
-            found_dev = [ a_dev for a_dev in Blockdevices(
-                            sysblock_dir, devnode_dir).get_devices()
-                    if dev in a_dev.devnodes ]
-            if found_dev:
-                devdir = found_dev[0].devdir
-            else:
-                return None
-        else:
-            ## it is the path of a /sys/ subdirectory (e.g.: '/sys/block/hda')
-            if os.path.isfile(os.path.join(dev, "dev")):
-                devdir = dev
-            else:
-                return None
     else:
         ## the name of a blockdevice (e.g.: 'dm-0')
-        for one_devdir in find_blockdevices(sysblock_dir):
+        for one_devdir, one_major_minor in find_blockdevices(sysblock_dir).items():
             if os.path.basename(one_devdir) == dev:
-                devdir = one_devdir
+                major_minor = one_major_minor
                 break
         else:
             return None
-    devname = os.path.basename(devdir)
+    cache_link = ["blockdevices", major_minor]
-    cache_link = ["blockdevices", devname]
     dev = CACHE.get(cache_link)
     if dev is None:
-        dev = Blockdevice(devdir, sysblock_dir, devnode_dir)
+        dev = Blockdevice(major_minor, sysblock_dir, devnode_dir)
         CACHE.set(cache_link, dev)
     return dev
 
@@ -885,9 +802,9 @@ def find_blockdevices(top_dir):
     cache_link = ["blockdevice_dirs", top_dir]
     cached = CACHE.get(cache_link)
     if not cached is None:
-        return cached[:]
+        return cached.copy()
 
-    dev_dirs = []
+    dev_dirs = {}
     
     def look4dev_dirs(arg, dirname, fnames):
         ## ignore the top level directory to avoid infinite recursion for
@@ -895,8 +812,11 @@ def find_blockdevices(top_dir):
         if os.path.samefile(dirname, top_dir):
             return
         ## add directories containing the file 'dev' to the list
-        if (arg in fnames) and os.path.isfile(os.path.join(dirname, arg)):
+        dev_file_path = os.path.join(dirname, arg)
-            dev_dirs.append(dirname)
+        if (arg in fnames) and os.path.isfile(dev_file_path):
+            major_minor = __get_major_minor(dirname)
+            if not major_minor is None:
+                dev_dirs[dirname] = major_minor
         for fname in fnames:
             ## remove symlinks and non-directories
             fullname = os.path.join(dirname, fname)
@@ -905,7 +825,38 @@ def find_blockdevices(top_dir):
     
     os.path.walk(top_dir, look4dev_dirs, 'dev')
     CACHE.set(cache_link, dev_dirs)
-    return dev_dirs[:]
+    return dev_dirs.copy()
+
+
+def find_device_nodes(devnode_dir):
+    """find all device nodes with the given major/minor values and
+
+    @param devnode_dir: usually /dev/
+    @type devnode_dir: string
+    @return: list of all found device nodes
+    @rtype: list of strings
+    """
+    cache_link = ["blockdevice_nodes", devnode_dir]
+    cached = CACHE.get(cache_link)
+    if not cached is None:
+        return cached.copy()
+
+    result = {}
+
+    def add_major_minor(arg, dirname, fnames):
+        for fname in fnames:
+            dev_node_path = os.path.join(dirname, fname)
+            if not os.path.isdir(dev_node_path):
+                major_minor = __get_major_minor(dev_node_path)
+                if not major_minor is None:
+                    if result.has_key(major_minor):
+                        result[major_minor].append(dev_node_path)
+                    else:
+                        result[major_minor] = [dev_node_path]
+
+    os.path.walk(devnode_dir, add_major_minor, None)
+    CACHE.set(cache_link, result)
+    return result.copy()
 
 
 def find_lvm_pv():
@@ -988,16 +939,7 @@ def _load_preferences():
 CACHE = BlockdeviceCache()
 
 
-if __name__ == '__main__':
+def show_devices(blocks, show):
-    ## list the properties of all available devices
-    ## this is just for testing purposes
-    blocks = Blockdevices().get_devices()
-
-    ## do we want to show the result?
-    def show(text=""):
-        if IS_VISIBLE:
-            print text
-
     if len(blocks) > 0:
         ## show all devices and their properties
         show("Properties of all devices:")
@@ -1021,3 +963,32 @@ if __name__ == '__main__':
                     pass
             show()
 
+def get_devices_and_show():
+    ## list the properties of all available devices
+    ## this is just for testing purposes
+    blocks = Blockdevices().get_devices()
+    blocks.sort(key=lambda x: x.name)
+
+    ## do we want to show the result?
+    def show(text=""):
+        if IS_VISIBLE:
+            print text
+
+    show_devices(blocks, show)
+
+
+if __name__ == '__main__':
+
+    if DO_PROFILE:
+        # show some profiling information (requires the python-profiler package)
+        import cProfile
+        import pstats
+        IS_VISIBLE = False
+        for index in range(3):
+            print "Run: %d" % index
+            cProfile.run('get_devices_and_show()', 'profinfo')
+            p = pstats.Stats('profinfo')
+            p.sort_stats('cumulative').print_stats(20)
+    else:
+        get_devices_and_show()
+