cryptonas/src/cryptobox/core/blockdevice.py

#
# Copyright 2007 sense.lab e.V.
# 
# This file is part of the CryptoBox.
#
# The CryptoBox is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# The CryptoBox is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with the CryptoBox; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#

'''
These classes detect and filter available blockdevices.
'''


__revision__ = "$Id$"


import os
import types
import re
import subprocess
import time
import logging
import cryptobox.core.settings
import cryptobox.core.exceptions

OPTIONAL_PROGS = { "lvm": True }
"""remember which programs don't exist to avoid calling them in vain

change default values from "True" to "False" to disable the respective program
"""

LOGGER = logging.getLogger("CryptoNAS")

DEFAULT_SYSBLOCK_DIR = '/sys/block'
DEFAULT_DEVNODE_DIR = '/dev'
MINIMUM_STORAGE_SIZE = 10
# defined device numbers: http://www.lanana.org/docs/device-list/devices.txt
MAJOR_DEVNUM_LOOP = 7
MAJOR_DEVNUM_FLOPPY = 2
MAJOR_DEVNUM_RAMDISK = 1
MAJOR_DEVNUM_MD_RAID = 9

## cache settings
CACHE_ENABLED = True
CACHE_EXPIRE_SECONDS = 120
CACHE_MINIMUM_AGE_FOR_REBUILD = 3
CACHE_MONITOR_FILE = '/proc/partitions'

## useful for manual profiling
IS_VISIBLE = True
DO_PROFILE = False

## caching is quite important for the following implementation
## the object will be initializes later below
CACHE = None


class Blockdevices:
    """handle all blockdevices of this system
    """
    
    def __init__(self,
            sysblock_dir=DEFAULT_SYSBLOCK_DIR,
            devnode_dir=DEFAULT_DEVNODE_DIR):
        self.sysblock_dir = sysblock_dir
        self.devnode_dir = devnode_dir
        self.devices = []
        for major_minor in find_blockdevices(self.sysblock_dir).values():
            blockdevice = get_blockdevice(major_minor,
                    self.sysblock_dir, self.devnode_dir)
            if (not blockdevice is None) and blockdevice.is_valid():
                self.devices.append(blockdevice)


    def get_devices(self):
        """return a copy of the device list
        """
        return self.devices[:]


    def get_storage_devices(self):
        """return a list of devices with the 'storage' flag
        """
        return [ dev for dev in self.devices if dev.is_storage() ]


    def get_partitionable_devices(self):
        """return a list of devices with the 'partitionable' flag
        """
        return [ dev for dev in self.devices if dev.is_partitionable() ]



class Blockdevice:
    """don't instantiate this class directly!

    use "_get_blockdevice" instead
    """

    def __init__(self, major_minor,
            sysblock_dir=DEFAULT_SYSBLOCK_DIR,
            devnode_dir=DEFAULT_DEVNODE_DIR):
        """initialize the blockdevice

        @param major_minor: major/minor value of the blockdevice
        @type major_minor: tuple
        @param dev: The /sys/block/ subdirectory describing a blockdevice
        @type sysblock_dir: string
        @param sysblock_dir: The linux /sys/ directory. Default is '/sys'.
        @type devnode_dir: string
        @param devnode_dir: The linux /dev/ directory. Default is '/dev'.
        """
        self.devnode_dir = devnode_dir
        self.sysblock_dir = sysblock_dir
        try:
            self.major, self.minor = major_minor
        except ValueError:
            # invalid device given
            raise cryptobox.core.exceptions.CBInternalError(
                    "invalid block device requested: %s" % str(major_minor))
        self.__cache_link = ["blockdevice_info", (self.major, self.minor)]


    def __cmp__(self, other):
        if (self.major < other.major) or \
                ((self.major == other.major) and (self.minor < other.minor)):
            return -1
        elif (self.major == other.major) and (self.minor == other.minor):
            return 0
        else:
            return 1
                    

    def reset(self):
        """reread the data of the device

        usually we will have to reset the cache, too
        just in case of first-time initialization, this is not necessary

        @type empty_cache: boolean
        @param empty_cache: Whether to discard the cached information or not.
        """
        CACHE.reset(self.__cache_link)


    def get_device(self):
        """Returns the path of a device node representing this device

        e.g.: /dev/hdc1
        """
        # we need to check, which listed device nodes exists
        # This is necessary, since temporary device nodes seem to be created
        # immediately after partitioning a disk (e.g. "/dev/.tmp-22-1").
        for dev in self.devnodes:
            if os.path.exists(dev):
                return dev
        # none of the device nodes exists
        LOGGER.warn("No valid device node found for %s out of %s" % \
                (self.name, str(self.devnodes)))
        return None


    def is_valid(self):
        """check if the device is usable and valid

        causes of invalidity: unused loop device, floppy device

        @rtype: boolean
        @return: 'True' for a valid blockdevice
        """
        # return False if no device node for this block device exists
        if self.get_device() is None:
            return False
        ## check valid major_minor
        try:
            if (self.major == 0) and (self.minor == 0):
                return False
            ## loop devices are ignored, if they are unused
            if (self.major == MAJOR_DEVNUM_LOOP) and (self.size == 0):
                return False
            ## floppy disks are totally ignored
            ## otherwise we would have a long timeout, while reading the devices
            if (self.major == MAJOR_DEVNUM_FLOPPY):
                return False
            # we don't want to store data in ramdisks (e.g. /dev/ram0)
            if (self.major == MAJOR_DEVNUM_RAMDISK):
                return False
        except TypeError:
            return False
        return True


    def is_storage(self):
        """return if this device can be used as a storage

        @rtype: boolean
        @return: 'True' for a device usable as a storage
        """
        ## check the cache first
        cache_link = self.__cache_link + ["is_storage"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached
        result = True

        # always check the current state of "result" to skip useless checks
        if result and (self.range > 1):
            ## partitionable blockdevice
            result = False
        if result and not self.is_valid():
            result = False
        if result and (self.size < MINIMUM_STORAGE_SIZE):
            ## extended partition, unused loop device
            result = False
        ## are we the device mapper of a luks device?
        if result:
            for slave in self.slaves:
                if get_blockdevice(slave, self.sysblock_dir,
                        self.devnode_dir).is_luks():
                    result = False
        if result and self.children:
            ## if we are a luks device with exactly one child, then
            ## we are a storage
            if not ((len(self.children) == 1) and self.is_luks()):
                ## a parent blockdevice
                result = False
        CACHE.set(cache_link, result)
        return result


    def is_partitionable(self):
        """is the device partitionable
        """
        if self.range > 1:
            return True
        else:
            return False
        

    def is_lvm_pv(self):
        """return if the device is a physical volume of a LVM
        """
        ## check the cache first
        cache_link = self.__cache_link + ["is_lvm_pv"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        ## is one of the devnodes of the device a physical volume?
        for one_lvm_pv in find_lvm_pv():
            if one_lvm_pv in self.devnodes:
                CACHE.set(cache_link, True)
                return True
        CACHE.set(cache_link, False)
        return False


    def is_lvm_lv(self):
        """return if the device is a logical volume of a LVM
        """
        ## check the cache first
        cache_link = self.__cache_link + ["is_lvm_lv"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        ## is one of the devnodes of the device a physical volume?
        ## logical LVM volumes always depend on their physical volumes
        if not self.slaves:
            CACHE.set(cache_link, False)
            return False
        ## is one of the LVM physical volumes a device node of our slave(s)?
        for one_lvm_pv in find_lvm_pv():
            for one_slave in self.slaves:
                if one_lvm_pv in get_blockdevice(one_slave,
                        self.sysblock_dir, self.devnode_dir).devnodes:
                    CACHE.set(cache_link, True)
                    return True
        CACHE.set(cache_link, False)
        return False


    def is_md_raid(self):
        """check if the device is the base of a md raid device
        """
        ## check the cache first
        cache_link = self.__cache_link + ["is_md_raid"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        if self.range > 1:
            result = False
        elif self.size < MINIMUM_STORAGE_SIZE:
            result = False
        else:
            for hold in self.holders:
                if get_blockdevice(hold, self.sysblock_dir,
                        self.devnode_dir).major == MAJOR_DEVNUM_MD_RAID:
                    result = True
                    break
            else:
                result = False

        ## store result and return
        CACHE.set(cache_link, result)
        return result


    def is_luks(self):
        """check if the device is a luks container
        """
        ## check the cache first
        cache_link = self.__cache_link + ["is_luks"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        if self.range > 1:
            result = False
        elif self.size < MINIMUM_STORAGE_SIZE:
            result = False
        elif self.is_lvm_pv():
            result = False
        elif self.is_md_raid():
            result = False
        else:
            ## is the device a luks volume?
            prefs = _load_preferences()
            proc = subprocess.Popen(
                    shell = False,
                    stdout = subprocess.PIPE,
                    stderr = subprocess.PIPE,
                    args = [
                prefs["Programs"]["super"],
                prefs["Programs"]["CryptoBoxRootActions"],
                "program", "cryptsetup",
                "isLuks", self.get_device()])            
            proc.wait()
            result = proc.returncode == 0
        ## store result and return
        CACHE.set(cache_link, result)
        return result


    def is_removable(self):
        """check if the device is marked as 'removable'
        """
        ## check the cache first
        cache_link = self.__cache_link + ["is_removable"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        removable_file = os.path.join(self.devdir, "removable")
        if os.path.isfile(removable_file):
            try:
                content = file(removable_file).read().strip()
                if content == "1":
                    return True
                else:
                    return False
            except IOError:
                result = False
        else:
            result = False

        CACHE.set(cache_link, result)
        return result


    def is_parent_of(self, ask_child):
        """check if the blockdevice or any of its children contains the given child

        the result of "foo.is_parent_of(foo)" returns False

        invalid (None) "ask_child" devices return False

        @param ask_child: the blockdevice that is considered to be a possible child
        @type ask_child: Blockdevice
        @return: True if the child is (even recursively) part of the blockdevice, otherwise False
        @rtype: bool
        """
        ## return False for non existing device
        if ask_child is None:
            return False
        ## throw exception for invalid call
        if not isinstance(ask_child, Blockdevice):
            raise cryptobox.core.exceptions.CBInternalError(\
                    "invalid arguments for 'is_parent_of'")
        ## recursively go through all the children
        for child_devname in self.children:
            child_dev = get_blockdevice(child_devname, self.sysblock_dir,
                    self.devnode_dir)
            ## direct child?
            if child_dev == ask_child:
                return True
            ## indirect child?
            if child_dev.is_parent_of(ask_child):
                return True
        ## no matches found
        return False


    def __get_devdir(self):
        cache_link = self.__cache_link + ["devdir"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        result = None
        # find the devdir (usually in /sys/block/)
        for devdir, one_major_minor in find_blockdevices(self.sysblock_dir).items():
            if (self.major, self.minor) == one_major_minor:
                result = devdir
                break
        else:
            # we did not find a suitable device
            raise cryptobox.core.exceptions.CBInternalError(
                    "could not find blockdevice with the given major/minor: " \
                    + "%d/%d" % (self.major, self.minor))

        CACHE.set(cache_link, result)
        return result


    def __get_dev_related(self, subdir):
        """return the content of sub directories (e.g. 'holders' or 'slaves')
        """
        try:
            return os.listdir(os.path.join(self.devdir, subdir))
        except OSError:
            return []


    def __get_size_human(self):
        """return a human readable string representing the size of the device
        """
        if self.size > 5120:
            return "%dGB" % int(self.size/1024)
        else:
            return "%dMB" % self.size


    def __get_size(self):
        """return the size (in MB) of the blockdevice
        """
        cache_link = self.__cache_link + ["size"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        try:
            size_blocks = int(file(os.path.join(self.devdir, 'size')).read())
            ## size is defined as the number of blocks (512 byte each)
            result = int(size_blocks*512/1024/1024)
        except (IOError, ValueError):
            result = 0

        CACHE.set(cache_link, result)
        return result


    def __get_device_range(self):
        """number of possible subdevices

        partitionable blockdevices have a range > 1
        """
        cache_link = self.__cache_link + ["device_range"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        try:
            result = int(file(os.path.join(self.devdir, "range")).read())
        except (IOError, ValueError):
            result = 1

        CACHE.set(cache_link, result)
        return result

    
    def __get_children(self):
        """return all devices depending on the current one
        
        all holders, subdevices and children of subdevices
        """
        cache_link = self.__cache_link + ["children"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        direct_children = [
                get_blockdevice(major_minor, self.sysblock_dir, self.devnode_dir).name
                for major_minor in find_blockdevices(self.devdir,
                    follow_links=False).values()]
        direct_children.extend(self.holders[:])
        children = direct_children[:]
        for dchild in direct_children:
            children.extend(get_blockdevice(dchild, self.sysblock_dir,
                    self.devnode_dir, follow_links=False).children)

        CACHE.set(cache_link, children)
        return children


    def __get_device_nodes(self):
        """get all device nodes with the major/minor combination of the device
        """
        cache_link = self.__cache_link + ["device_nodes"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        try:
            result = find_device_nodes(self.devnode_dir)[(self.major, self.minor)]
        except KeyError:
            result = []

        CACHE.set(cache_link, result)
        return result


    def __get_uuid_luks(self):
        """determine the unique identifier of luks devices
        """
        cache_link = self.__cache_link + ["uuid_luks"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        prefs = _load_preferences()
        try:
            proc = subprocess.Popen(
                    shell = False,
                    stdout = subprocess.PIPE,
                    stderr = subprocess.PIPE,
                    args = [
                          		prefs["Programs"]["super"],
                                prefs["Programs"]["CryptoBoxRootActions"],
                          		"program", "cryptsetup",
                                "luksUUID", self.get_device()])
            (output, error) = proc.communicate()
        except OSError, err_msg:
            LOGGER.warning("Failed to call '%s' to determine UUID: %s" \
                    % (prefs["Programs"]["cryptsetup"], err_msg))
            return None
        if proc.returncode != 0:
            LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
                        (prefs["Programs"]["cryptsetup"], self.get_device(),
                            error))
            return None
        result = output.strip()
        if not result:
            result = None

        CACHE.set(cache_link, result)
        return result


    def __get_uuid_lvm_pv(self):
        """determine the unique identifier of physical LVM volumes
        """
        cache_link = self.__cache_link + ["uuid_lvm_pv"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        result = None
        if not OPTIONAL_PROGS["lvm"]:
            ## pvdisplay is not installed - skip it
            return None
        prefs = _load_preferences()
        try:
            proc = subprocess.Popen(
                    shell = False,
                    stdout = subprocess.PIPE,
                    stderr = subprocess.PIPE,
                    args = [ prefs["Programs"]["super"],
                            prefs["Programs"]["CryptoBoxRootActions"],
                            "program", "pvdisplay" ])
            (output, error) = proc.communicate()
        except OSError, err_msg:
            LOGGER.warning("Failed to call '%s' via 'super' to determine " \
                    % prefs["Programs"]["pvdisplay"] + "UUID: %s" % err_msg)
            return None
        if proc.returncode == 101:
            ## pvdisplay is not installed
            OPTIONAL_PROGS["lvm"] = False
            LOGGER.warning("'lvm' is not installed - I will not try it again")
            return None
        if proc.returncode != 0:
            LOGGER.warning("Execution of 'pvdisplay' failed: %s" % error)
            return None
        for line in output.splitlines():
            items = line.strip().split(":")
            if (len(items) == 12) and (items[0] in self.devnodes):
                result = items[11]

        CACHE.set(cache_link, result)
        return result


    def __get_blkid_attributes(self):
        """determine the label of a filesystem contained in a device

        returns a dictionary containing label, type_id and uuid
        """
        cache_link = self.__cache_link + ["blkid_attributes"]
        cached = CACHE.get(cache_link)
        if not cached is None:
            return cached

        result =  {"label": None, "type_id": None, "uuid": None}
        if self.is_valid():
            prefs = _load_preferences()
            try:
                proc = subprocess.Popen(
                        shell = False,
                        stdout = subprocess.PIPE,
                        stderr = subprocess.PIPE,
                        args = [ prefs["Programs"]["blkid"],
                            "-s", "LABEL",
                            "-s", "TYPE",
                            "-s", "UUID",
                            "-c", os.devnull,
                            "-w", os.devnull,
                            self.get_device()])
                (output, error) = proc.communicate()
            except OSError, err_msg:
                LOGGER.warning("Failed to call '%s' to determine label for " \
                        % prefs["Programs"]["blkid"] + "'%s': %s" % \
                        (self.get_device(), err_msg))
            else:
                if proc.returncode == 2:
                    ## the device does not contain a filesystem (e.g. it is zeroed or
                    ## it contains a partition table)
                    pass
                elif proc.returncode != 0:
                    LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
                            (prefs["Programs"]["blkid"], self.get_device(),
                                error.strip()))
                    pass
                else:
                    # scan the output string for results
                    # the output string could look like this:
                    #   /dev/hda1: TYPE="ext3" LABEL="neu"de"
                    pattern = {"LABEL": "label", "TYPE": "type_id", "UUID": "uuid"}
                    for name, attr in pattern.items():
                        match = re.search(r' %s="(.*?)"(?: [A-Z]+="| ?$)' % \
                                name, output)
                        if match:
                            result[attr] = match.groups()[0]
                    # Check for special attributes of LUKS devices and LVM
                    # physical volumes.
                    # In this case the previously retrieved "uuid" value is
                    # overwritten.
                    # UUIDs of physical LVM volumes can only be determined via
                    # pvdisplay.
                    if self.is_lvm_pv():
                        result["uuid"] = self.__get_uuid_lvm_pv()
                    ## UUIDs of luks devices can be determined via luksDump
                    elif self.is_luks():
                        result["uuid"] = self.__get_uuid_luks()

        CACHE.set(cache_link, result)
        return result


    def __eq__(self, device):
        """compare two blockdevice objects
        """
        return self.name == device.name


    def __str__(self):
        """display the name of the device
        """
        return self.name


    def info(self):
        """display some information about the device
        """
        output = "%s:\n" % self.name
        output += "\t%s:\t%s\n" % ("blockdir", self.devdir)
        output += "\t%s:\t%d/%d\n" % ("major/minor", self.major, self.minor)
        output += "\t%s:\t\t%s\n" % ("label", self.label)
        output += "\t%s:\t%s\n" % ("type_id", self.type_id)
        output += "\t%s:\t\t%s\n" % ("UUID", self.uuid)
        output += "\t%s:\t\t%s\n" % ("range", self.range)
        output += "\t%s:\t\t%s\n" % ("size", self.size)
        output += "\t%s:\t\t%s\n" % ("slaves", self.slaves)
        output += "\t%s:\t%s\n" % ("holders", self.holders)
        output += "\t%s:\t%s\n" % ("children", self.children)
        output += "\t%s:\t%s\n" % ("device nodes", self.devnodes)
        output += "\tflags:\t\t"
        for funcname in [ func for func in dir(self)
                if func.startswith("is_") and callable(getattr(self, func))]:
            try:
                if getattr(self, funcname)():
                    output += "%s " % funcname[3:]
            except TypeError:
                # skip tests that need arguments (e.g. "is_parent_of")
                pass
        output += "\n"
        return output


    devdir = property(__get_devdir)
    name = property(lambda self: os.path.basename(self.devdir))
    size = property(__get_size)
    size_human = property(__get_size_human)
    range = property(__get_device_range)
    slaves = property(lambda self: self.__get_dev_related("slaves"))
    holders = property(lambda self: self.__get_dev_related("holders"))
    children = property(__get_children)
    devnodes = property(__get_device_nodes)
    label = property(lambda self:  self.__get_blkid_attributes()["label"])
    type_id = property(lambda self: self.__get_blkid_attributes()["type_id"])
    uuid = property(lambda self: self.__get_blkid_attributes()["uuid"])



class BlockdeviceCache:
    """manage cached results of blockdevce queries

    the cache expires every 60 seconds or as soon as CACHE_MONITOR_FILE changes
    """

    def __init__(self):
        self.values = {}
        self.expires = None
        self.partitions_save = None
        self.reset()


    def reset(self, target=None):
        """empty the cache and reset the expire time
        """
        if target is None:
            self.values = {}
            try:
                self.partitions_save = file(CACHE_MONITOR_FILE).read()
            except IOError, err_msg:
                LOGGER.warning("Failed to read '%s': %s" % \
                        (CACHE_MONITOR_FILE, err_msg))
                self.partitions_save = ""
            self.expires = int(time.time()) + CACHE_EXPIRE_SECONDS
        elif type(target) == types.ListType:
            # we do not reset the expire date
            self.set(target, {})
        elif isinstance(target, Blockdevice):
            # we do not reset the expire date
            target.reset()
        else:
            LOGGER.log.warn("Invalid argument type for reset: %s" % \
                    str(type(target)))


    def __is_expired(self):
        """check if the cache is expired
        """
        try:
            if (file(CACHE_MONITOR_FILE).read() != self.partitions_save) or \
                    (self.expires < int(time.time())):
                return True
        except IOError, err_msg:
            LOGGER.warning("Failed to read '%s': %s" % \
                    (CACHE_MONITOR_FILE, err_msg))
        return False


    def get(self, link):
        """return a cached value
        
        "link" is an array of the hierachie of the accessed item
        e.g. link = ["blockdevices", "hda"]
        return None if the value is not in the cache or if CACHE_ENABLED is False
        """
        if not CACHE_ENABLED:
            return None
        if self.__is_expired():
            self.reset()
        ## walk down the tree
        ref = self.values
        for element in link:
            if element in ref:
                ref = ref[element]
            else:
                return None
        return ref


    def set(self, link, item):
        """store an item in the cache
        
        "link" is an array of the hierachie of the accessed item
        e.g. link = ["blockdevices", "hda"]
        """
        if not CACHE_ENABLED:
            return
        ## walk down the tree
        ref = self.values
        for element in link[:-1]:
            if not element in ref:
                ## create a non-existing sub element
                ref[element] = {}
            ref = ref[element]
        ## store the item
        ref[link[-1]] = item


    def get_age(self):
        age = CACHE_EXPIRE_SECONDS - (self.expires - int(time.time()))
        if age < 0:
            return 0
        else:
            return age


def __get_major_minor(dev):
    """ return a tuple (major/minor) for a device node or for a
    subdirectory of /sys/block containing a node named 'dev'
    """
    if os.path.isdir(dev):
        # we assume, that is is a subdirectory of /sys/block
        dev_file_name = os.path.join(dev, 'dev')
        try:
            maj_min_string = file(dev_file_name).read().strip()
        except IOError:
            return None
        if (maj_min_string.count(":") == 1):
            major, minor = maj_min_string.split(":")
            try:
                major = int(major)
                minor = int(minor)
            except ValueError:
                return None
            # everything looks ok
            return (major, minor)
        else:
            return None
    else:
        # we assume, that it is a device node (e.g. /dev/hda1)
        try:
            # we don't need to check for symlinks: "os.stat" resolves them
            stat = os.stat(dev)
        except OSError:
            # the node does not exist
            return None
        # check if it is a block device
        if (stat.st_mode & 060000 == 060000):
            major = os.major(stat.st_rdev)
            minor = os.minor(stat.st_rdev)
            return (major, minor)
        else:
            return None



def get_blockdevice(dev,
        sysblock_dir=DEFAULT_SYSBLOCK_DIR,
        devnode_dir=DEFAULT_DEVNODE_DIR, retry_once=True, follow_links=True):
    if dev is None:
        return None
    elif isinstance(dev, Blockdevice):
        # it is already a blockdevice
        major_minor = (dev.major, dev.minor)
    elif type(dev) is types.TupleType:
		# we assume that the tuple contains major/minor
        major_minor = dev
    elif os.path.isabs(dev):
        ## it is an absolute path
        major_minor = __get_major_minor(dev)
    else:
        ## the name of a blockdevice (e.g.: 'dm-0')
        for one_devdir, one_major_minor in find_blockdevices(sysblock_dir,
                follow_links).items():
            if os.path.basename(one_devdir) == dev:
                major_minor = one_major_minor
                break
        else:
            # rebuild the cache if it is rather old and try again
            # this is necessary for the "partition" plugin
            if retry_once and (CACHE.get_age() > CACHE_MINIMUM_AGE_FOR_REBUILD):
                CACHE.reset()
                device = get_blockdevice(dev, sysblock_dir, devnode_dir,
                        retry_once=False, follow_links=follow_links)
                if not device is None:
                    major_minor = (device.major, device.minor)
                else:
                    major_minor = None
            else:
                # it seems like it does really not exist
                major_minor = None
    if major_minor:
        cache_link = ["blockdevices", major_minor]
        dev = CACHE.get(cache_link)
        if dev is None:
            dev = Blockdevice(major_minor, sysblock_dir, devnode_dir)
            if not dev is None:
                CACHE.set(cache_link, dev)
        return dev
    else:
        return None



def find_blockdevices(top_dir, follow_links=True):

    # normalize the input directory
    top_dir = os.path.realpath(top_dir)
    cache_link = ["blockdevice_dirs", top_dir]
    cached = CACHE.get(cache_link)
    if not cached is None:
        return cached.copy()

    dev_dirs = {}
    dev_file_name = 'dev'
    walk_dirs = [top_dir]
    walk_dir_index = 0
    
    while walk_dir_index < len(walk_dirs):
        for dirname, dirs, fnames in os.walk(walk_dirs[walk_dir_index]):
            ## add directories containing the file 'dev' to the list
            dev_file_path = os.path.join(dirname, dev_file_name)
            # don't include the top-level device itself
            if (os.path.realpath(dirname) != top_dir) and \
                    (dev_file_name in fnames) and os.path.isfile(dev_file_path):
                major_minor = __get_major_minor(dirname)
                if (not major_minor is None) and \
                        (not major_minor in dev_dirs.values()):
                    dev_dirs[dirname] = major_minor
            # follow symlinks
            if follow_links and (walk_dir_index == 0):
                for dname in dirs:
                    fullpath = os.path.join(dirname, dname)
                    if (os.path.islink(fullpath) and os.path.isdir(fullpath)):
                        fullpath = os.path.realpath(fullpath)
                        if not fullpath in walk_dirs:
                            if not [one_walk for one_walk in walk_dirs
                                    if os.path.samefile(one_walk, fullpath)]:
                                walk_dirs.append(fullpath)
        walk_dir_index += 1
    
    CACHE.set(cache_link, dev_dirs)
    return dev_dirs.copy()


def find_device_nodes(devnode_dir):
    """find all device nodes with the given major/minor values and

    @param devnode_dir: usually /dev/
    @type devnode_dir: string
    @return: list of all found device nodes
    @rtype: list of strings
    """
    cache_link = ["blockdevice_nodes", devnode_dir]
    cached = CACHE.get(cache_link)
    if not cached is None:
        return cached.copy()

    result = {}

    def add_major_minor(arg, dirname, fnames):
        for fname in fnames:
            dev_node_path = os.path.join(dirname, fname)
            if not os.path.isdir(dev_node_path):
                major_minor = __get_major_minor(dev_node_path)
                if not major_minor is None:
                    if result.has_key(major_minor):
                        result[major_minor].append(dev_node_path)
                    else:
                        result[major_minor] = [dev_node_path]

    os.path.walk(devnode_dir, add_major_minor, None)
    CACHE.set(cache_link, result)
    return result.copy()


def find_lvm_pv():
    """return the blockdevice names of all physical LVM volumes
    """
    if not OPTIONAL_PROGS["lvm"]:
        return []
    cache_link = ["lvm", "pv"]
    cached = CACHE.get(cache_link)
    if not cached is None:
        return cached[:]

    prefs = _load_preferences()
    result = None
    try:
        proc = subprocess.Popen(
                shell = False,
                stdout = subprocess.PIPE,
                stderr = subprocess.PIPE,
                args = [ prefs["Programs"]["super"],
                    prefs["Programs"]["CryptoBoxRootActions"],
                    "program", "pvdisplay" ])
        (output, error) = proc.communicate()
    except OSError, err_msg:
        LOGGER.info("Failed to call 'pvdisplay' via 'super': %s" % err_msg)
        result = []
    if proc.returncode == 101:
        ## pvdisplay is not installed
        OPTIONAL_PROGS["lvm"] = False
        LOGGER.warning("'lvm' is not installed - I will not try it again")
        return []
    if proc.returncode != 0:
        LOGGER.info("Execution of 'pvdisplay' failed: %s" % error.strip())
        result = []
    if result is None:
        result = []
        for line in output.splitlines():
            result.append(line.split(":", 1)[0].strip())
    CACHE.set(cache_link, result)
    return result[:]


def _load_preferences():
    prefs = cryptobox.core.settings.get_current_settings()
    if not prefs is None:
        ## now the preferences are loaded
        return prefs
    ## we have to load an emergency fallback for proper function
    ## this is mainly useful for local testing
    alternative_locations = []
    # try to use the '/bin/cryptobox.conf' location within a local working
    # copy of the subversion repository
    root_dir = os.path.realpath(os.path.join(globals()["cryptobox"].__path__[0],
            os.path.pardir, os.path.pardir))
    alternative_locations.append(
            os.path.join(root_dir, "bin", "cryptobox.conf"))
    # try the default config file location
    alternative_locations.append('/etc/cryptobox-server/cryptobox.conf')
    for config_file in alternative_locations:
        if os.path.exists(config_file):
            # we have to chdir to the 'bin' directory - otherwise the paths in
            # cryptobox.conf do not work
            os.chdir(os.path.dirname(config_file))
            return cryptobox.core.settings.CryptoBoxSettings(config_file)
    else:
        raise CBConfigUnavailableError()



## initialize cache
CACHE = BlockdeviceCache()


def show_devices(blocks):
    result = ""
    if len(blocks) > 0:
        ## show all devices and their properties
        result += "Properties of all devices:\n"
        for device in blocks:
            result += device.info() + "\n"

        ## discover all self-check methods
        example = blocks[0]
        flag_checker = [ method for method in dir(example)
                if callable(getattr(example, method))
                        and method.startswith("is_")]
        ## list all checks and the respective devices
        for check in flag_checker:
            result += "List of '%s' devices:" % check[3:] + "\n"
            for device in blocks:
                try:
                    if getattr(device, check)():
                        result += "\t%s" % device + "\n"
                except TypeError:
                    # ignore tests that need a second argument
                    pass
            result += "\n"
    return result


def get_devices_and_show():
    ## list the properties of all available devices
    ## this is just for testing purposes
    blocks = Blockdevices().get_devices()
    blocks.sort(key=lambda x: x.name)
    return show_devices(blocks)


if __name__ == '__main__':

    if DO_PROFILE:
        # show some profiling information (requires the python-profiler package)
        import cProfile
        import pstats
        for index in range(3):
            print "Run: %d" % index
            cProfile.run('get_devices_and_show()', 'profinfo')
            p = pstats.Stats('profinfo')
            p.sort_stats('cumulative').print_stats(20)
    else:
        print get_devices_and_show()