cryptonas blockdevice management:

* significantly reduced the time required for generating the block device list:
 * use a cache for major/minor -> device nodes mapping
 * run 'blkid' only once for LABEL, TYPE and UUID
* added some code for (optional) profiling via python-profiler
This commit is contained in:
lars 2009-06-10 17:45:58 +00:00
parent 2e8609b0d2
commit 6ad1523510
1 changed files with 189 additions and 218 deletions

View File

@ -23,12 +23,12 @@ These classes detect and filter available blockdevices.
'''
#TODO: call blkid only once for all devices and scan the result
__revision__ = "$Id$"
import os
import types
import re
import subprocess
import time
import logging
@ -57,6 +57,7 @@ CACHE_MONITOR_FILE = '/proc/partitions'
## useful for manual profiling
IS_VISIBLE = True
DO_PROFILE = False
## caching is quite important for the following implementation
## the object will be initializes later below
@ -73,8 +74,8 @@ class Blockdevices:
self.sysblock_dir = sysblock_dir
self.devnode_dir = devnode_dir
self.devices = []
for devdir in find_blockdevices(self.sysblock_dir):
blockdevice = get_blockdevice(devdir,
for major_minor in find_blockdevices(self.sysblock_dir).values():
blockdevice = get_blockdevice(major_minor,
self.sysblock_dir, self.devnode_dir)
if (not blockdevice is None) and blockdevice.is_valid():
self.devices.append(blockdevice)
@ -105,29 +106,34 @@ class Blockdevice:
use "_get_blockdevice" instead
"""
def __init__(self, dev,
def __init__(self, major_minor,
sysblock_dir=DEFAULT_SYSBLOCK_DIR,
devnode_dir=DEFAULT_DEVNODE_DIR):
"""initialize the blockdevice
@type dev: string
@param major_minor: major/minor value of the blockdevice
@type major_minor: tuple
@param dev: The /sys/block/ subdirectory describing a blockdevice
@type sysblock_dir: string
@param sysblock_dir: The linux /sys/ directory. Default is '/sys'.
@type devnode_dir: string
@param devnode_dir: The linux /dev/ directory. Default is '/dev'.
"""
self.devdir = dev
self.devnode_dir = devnode_dir
self.sysblock_dir = sysblock_dir
## check if the device is valid
if not os.path.isdir(os.path.join(self.devnode_dir, dev)):
self.major, self.minor = major_minor
# find the devdir (usually in /sys/block/)
for devdir, one_major_minor in find_blockdevices(self.sysblock_dir).items():
if major_minor == one_major_minor:
self.devdir = devdir
break
else:
# we did not find a suitable device
raise cryptobox.core.exceptions.CBInternalError(
"invalid blockdevice given: %s (%s)" % \
(self.devdir, self.sysblock_dir))
"could not find blockdevice with the given major/minor: " \
+ "%d/%d" % (self.major, self.minor))
self.name = os.path.basename(self.devdir)
## "reset" below will fill these values
self.devnum = None
self.size = None
self.size_human = None
self.range = None
@ -136,6 +142,7 @@ class Blockdevice:
self.children = None
self.devnodes = None
self.uuid = None
self.type_id = None
self.label = None
self.reset(empty_cache=False)
@ -151,7 +158,6 @@ class Blockdevice:
"""
if empty_cache:
CACHE.reset(["blockdevice_info", self.name])
self.devnum = self.__get_major_minor()
self.size = self.__get_size()
self.size_human = self.__get_size_human()
self.range = self.__get_device_range()
@ -159,9 +165,10 @@ class Blockdevice:
self.holders = self.__get_dev_related("holders")
self.children = self.__get_children()
self.devnodes = self.__get_device_nodes()
self.uuid = self.__get_uuid()
self.label = self.__get_label()
self.type_id = self.__get_type_id()
attributes = self.__get_blkid_attributes()
self.label = attributes["label"]
self.type_id = attributes["type_id"]
self.uuid = attributes["uuid"]
def is_valid(self):
@ -174,17 +181,16 @@ class Blockdevice:
"""
if not self.devnodes:
return False
## check valid devnum
## check valid major_minor
try:
major, minor = self.devnum
if (major == 0) and (minor == 0):
if (self.major == 0) and (self.minor == 0):
return False
## loop devices are ignored, if they are unused
if (major == MAJOR_DEVNUM_LOOP) and (self.size == 0):
if (self.major == MAJOR_DEVNUM_LOOP) and (self.size == 0):
return False
## floppy disks are totally ignored
## otherwise we would have a long timeout, while reading the devices
if (major == MAJOR_DEVNUM_FLOPPY):
if (self.major == MAJOR_DEVNUM_FLOPPY):
return False
except TypeError:
return False
@ -298,7 +304,7 @@ class Blockdevice:
else:
for hold in self.holders:
if get_blockdevice(hold, self.sysblock_dir,
self.devnode_dir).devnum[0] == MAJOR_DEVNUM_MD_RAID:
self.devnode_dir).major == MAJOR_DEVNUM_MD_RAID:
result = True
break
else:
@ -379,7 +385,7 @@ class Blockdevice:
invalid (None) "ask_child" devices return False
@param ask_child: the blockdevice that is considered to be a possible child
@type ask_child: BlockDevice
@type ask_child: Blockdevice
@return: True if the child is (even recursively) part of the blockdevice, otherwise False
@rtype: bool
"""
@ -415,7 +421,6 @@ class Blockdevice:
def __get_size_human(self):
"""return a human readable string representing the size of the device
"""
size = self.size
if self.size > 5120:
return "%dGB" % int(self.size/1024)
else:
@ -436,23 +441,6 @@ class Blockdevice:
return default
def __get_major_minor(self):
"""return the major and minor of the device"""
default = (0, 0)
try:
content = file(os.path.join(self.devdir, "dev")).read()
except IOError:
return default
try:
major, minor = content.split(":", 1)
except TypeError:
return default
try:
return int(major), int(minor)
except ValueError:
return default
def __get_device_range(self):
"""number of possible subdevices
@ -475,8 +463,8 @@ class Blockdevice:
all holders, subdevices and children of subdevices
"""
direct_children = [
get_blockdevice(child, self.sysblock_dir, self.devnode_dir).name
for child in find_blockdevices(self.devdir)]
get_blockdevice(major_minor, self.sysblock_dir, self.devnode_dir).name
for major_minor in find_blockdevices(self.devdir).values()]
direct_children.extend(self.holders[:])
children = direct_children[:]
for dchild in direct_children:
@ -488,42 +476,10 @@ class Blockdevice:
def __get_device_nodes(self):
"""get all device nodes with the major/minor combination of the device
"""
result = []
major, minor = self.devnum
def find_major_minor(arg, dirname, fnames):
for fname in fnames:
try:
stat = os.stat(os.path.join(dirname, fname))
## check if it is a blockdevice and compare major/minor
if (stat.st_mode & 060000 == 060000) \
and (os.major(stat.st_rdev) == major) \
and (os.minor(stat.st_rdev) == minor):
result.append(os.path.join(dirname, fname))
except OSError:
pass
os.path.walk(self.devnode_dir, find_major_minor, None)
return result
def __get_uuid(self):
"""determine the unique identifier of this device
returns None in case of error or for invalid devices (see "is_valid")
"""
if not self.is_valid():
return None
## partitionable devices do not have a UUID
if self.is_partitionable():
return None
## UUIDs of physical LVM volumes can only be determined via pvdisplay
if self.is_lvm_pv():
return self.__get_uuid_lvm_pv()
## UUIDs of luks devices can be determined via luksDump
if self.is_luks():
return self.__get_uuid_luks()
return self.__get_uuid_default()
try:
return find_device_nodes(self.devnode_dir)[(self.major, self.minor)]
except KeyError:
return []
def __get_uuid_luks(self):
@ -536,10 +492,10 @@ class Blockdevice:
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
args = [
prefs["Programs"]["super"],
prefs["Programs"]["CryptoBoxRootActions"],
"program", "cryptsetup",
"luksUUID", self.devnodes[0] ])
prefs["Programs"]["super"],
prefs["Programs"]["CryptoBoxRootActions"],
"program", "cryptsetup",
"luksUUID", self.devnodes[0] ])
(output, error) = proc.communicate()
except OSError, err_msg:
LOGGER.warning("Failed to call '%s' to determine UUID: %s" \
@ -593,53 +549,16 @@ class Blockdevice:
return None
def __get_uuid_default(self):
"""determine the unique identifier for non-special devices
luks and lvm_pv devices must be treated differently
"""
prefs = _load_preferences()
try:
proc = subprocess.Popen(
shell = False,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
args = [ prefs["Programs"]["blkid"],
"-s", "UUID",
"-o", "value",
"-c", os.devnull,
"-w", os.devnull,
self.devnodes[0] ])
(output, error) = proc.communicate()
except OSError, err_msg:
LOGGER.warning("Failed to call '%s' to determine UUID: %s" % \
(prefs["Programs"]["blkid"], err_msg))
return None
if proc.returncode == 2:
## the device does not contain a filesystem (e.g. it is zeroed or
## it contains a partition table)
return None
if proc.returncode != 0:
LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
(prefs["Programs"]["blkid"], self.devnodes[0],
error.strip()))
return None
result = output.strip()
if result:
return result
else:
return None
def __get_label(self):
def __get_blkid_attributes(self):
"""determine the label of a filesystem contained in a device
return None for errors, empty labels and for luks or non-storage devices
returns a dictionary containing label, type_id and uuid
"""
result = {"label": None, "type_id": None, "uuid": None}
if not self.is_valid():
return None
return result
if self.is_luks():
return None
return result
prefs = _load_preferences()
try:
proc = subprocess.Popen(
@ -648,7 +567,8 @@ class Blockdevice:
stderr = subprocess.PIPE,
args = [ prefs["Programs"]["blkid"],
"-s", "LABEL",
"-o", "value",
"-s", "TYPE",
"-s", "UUID",
"-c", os.devnull,
"-w", os.devnull,
self.devnodes[0]])
@ -657,65 +577,33 @@ class Blockdevice:
LOGGER.warning("Failed to call '%s' to determine label for " \
% prefs["Programs"]["blkid"] + "'%s': %s" % \
(self.devnodes[0], err_msg))
return None
return result
if proc.returncode == 2:
## the device does not contain a filesystem (e.g. it is zeroed or
## it contains a partition table)
return None
return result
if proc.returncode != 0:
LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
(prefs["Programs"]["blkid"], self.devnodes[0],
error.strip()))
return None
result = output.strip()
if result:
return result
else:
return None
def __get_type_id(self):
"""determine the type id of a filesystem contained in a device
possible results are: ext2, ext3, vfat, reiserfs, swap, ...
return None for errors, empty labels and for luks or non-storage devices
"""
if not self.is_valid():
return None
if self.is_luks():
return None
prefs = _load_preferences()
try:
proc = subprocess.Popen(
shell = False,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
args = [ prefs["Programs"]["blkid"],
"-s", "TYPE",
"-o", "value",
"-c", os.devnull,
"-w", os.devnull,
self.devnodes[0]])
(output, error) = proc.communicate()
except OSError, err_msg:
LOGGER.warning("Failed to call '%s' to determine type id for" \
% prefs["Programs"]["blkid"] + " '%s': %s" % \
(self.devnodes[0], err_msg))
return None
if proc.returncode == 2:
## the device does not contain a filesystem (e.g. it is zeroed or
## it contains a partition table)
return None
if proc.returncode != 0:
LOGGER.warning("Execution of '%s' for '%s' failed: %s" % \
(prefs["Programs"]["blkid"], self.devnodes[0],
error.strip()))
return None
result = output.strip()
if result:
return result
else:
return None
# scan the output string for results
# the output string could look like this:
# /dev/hda1: TYPE="ext3" LABEL="neu"de"
pattern = {"LABEL": "label", "TYPE": "type_id", "UUID": "uuid"}
for name, attr in pattern.items():
match = re.search(r' %s="(.*?)"(?: [A-Z]+="|$)' % name, output)
if match:
result[attr] = match.groups()[0]
# check for special attributes of LUKS devices and LVM physical volumes
# In this case the previously retrieved "uuid" value is overwritten.
## UUIDs of physical LVM volumes can only be determined via pvdisplay
if self.is_lvm_pv():
result["uuid"] = self.__get_uuid_lvm_pv()
## UUIDs of luks devices can be determined via luksDump
elif self.is_luks():
result["uuid"] = self.__get_uuid_luks()
return result
def __eq__(self, device):
@ -735,7 +623,7 @@ class Blockdevice:
"""
output = "%s:\n" % self.name
output += "\t%s:\t%s\n" % ("blockdir", self.devdir)
output += "\t%s:\t%s\n" % ("major/minor", self.devnum)
output += "\t%s:\t%d/%d\n" % ("major/minor", self.major, self.minor)
output += "\t%s:\t\t%s\n" % ("label", self.label)
output += "\t%s:\t\t%s\n" % ("UUID", self.uuid)
output += "\t%s:\t\t%s\n" % ("range", self.range)
@ -794,7 +682,7 @@ class BlockdeviceCache:
if (file(CACHE_MONITOR_FILE).read() != self.partitions_save) or \
(self.expires < int(time.time())):
return True
except IOError:
except IOError, err_msg:
LOGGER.warning("Failed to read '%s': %s" % \
(CACHE_MONITOR_FILE, err_msg))
return False
@ -841,41 +729,70 @@ class BlockdeviceCache:
def __get_major_minor(dev):
""" return a tuple (major/minor) for a device node or for a
subdirectory of /sys/block containing a node named 'dev'
"""
if os.path.isdir(dev):
# we assume, that is is a subdirectory of /sys/block
dev_file_name = os.path.join(dev, 'dev')
try:
maj_min_string = file(dev_file_name).read().strip()
except IOError:
return None
if (maj_min_string.count(":") == 1):
major, minor = maj_min_string.split(":")
try:
major = int(major)
minor = int(minor)
except ValueError:
return None
# everything looks ok
return (major, minor)
else:
return None
else:
# we assume, that it is a device node (e.g. /dev/hda1)
try:
# we don't need to check for symlinks: "os.stat" resolves them
stat = os.stat(dev)
except OSError:
# the node does not exist
return None
# check if it is a block device
if (stat.st_mode & 060000 == 060000):
major = os.major(stat.st_rdev)
minor = os.minor(stat.st_rdev)
return (major, minor)
else:
return None
def get_blockdevice(dev,
sysblock_dir=DEFAULT_SYSBLOCK_DIR,
devnode_dir=DEFAULT_DEVNODE_DIR):
if os.path.isabs(dev):
if isinstance(dev, Blockdevice):
# it is already a blockdevice
major_minor = (dev.major, dev.minor)
elif type(dev) is types.TupleType:
# we assume that the tuple contains major/minor
major_minor = dev
elif os.path.isabs(dev):
## it is an absolute path
if dev.startswith(devnode_dir):
## it is the name of a devicenode (e.g.: '/dev/hda1')
## simplify the path first
dev = os.path.realpath(dev)
found_dev = [ a_dev for a_dev in Blockdevices(
sysblock_dir, devnode_dir).get_devices()
if dev in a_dev.devnodes ]
if found_dev:
devdir = found_dev[0].devdir
else:
return None
else:
## it is the path of a /sys/ subdirectory (e.g.: '/sys/block/hda')
if os.path.isfile(os.path.join(dev, "dev")):
devdir = dev
else:
return None
major_minor = __get_major_minor(dev)
else:
## the name of a blockdevice (e.g.: 'dm-0')
for one_devdir in find_blockdevices(sysblock_dir):
for one_devdir, one_major_minor in find_blockdevices(sysblock_dir).items():
if os.path.basename(one_devdir) == dev:
devdir = one_devdir
major_minor = one_major_minor
break
else:
return None
devname = os.path.basename(devdir)
cache_link = ["blockdevices", devname]
cache_link = ["blockdevices", major_minor]
dev = CACHE.get(cache_link)
if dev is None:
dev = Blockdevice(devdir, sysblock_dir, devnode_dir)
dev = Blockdevice(major_minor, sysblock_dir, devnode_dir)
CACHE.set(cache_link, dev)
return dev
@ -885,9 +802,9 @@ def find_blockdevices(top_dir):
cache_link = ["blockdevice_dirs", top_dir]
cached = CACHE.get(cache_link)
if not cached is None:
return cached[:]
return cached.copy()
dev_dirs = []
dev_dirs = {}
def look4dev_dirs(arg, dirname, fnames):
## ignore the top level directory to avoid infinite recursion for
@ -895,8 +812,11 @@ def find_blockdevices(top_dir):
if os.path.samefile(dirname, top_dir):
return
## add directories containing the file 'dev' to the list
if (arg in fnames) and os.path.isfile(os.path.join(dirname, arg)):
dev_dirs.append(dirname)
dev_file_path = os.path.join(dirname, arg)
if (arg in fnames) and os.path.isfile(dev_file_path):
major_minor = __get_major_minor(dirname)
if not major_minor is None:
dev_dirs[dirname] = major_minor
for fname in fnames:
## remove symlinks and non-directories
fullname = os.path.join(dirname, fname)
@ -905,7 +825,38 @@ def find_blockdevices(top_dir):
os.path.walk(top_dir, look4dev_dirs, 'dev')
CACHE.set(cache_link, dev_dirs)
return dev_dirs[:]
return dev_dirs.copy()
def find_device_nodes(devnode_dir):
"""find all device nodes with the given major/minor values and
@param devnode_dir: usually /dev/
@type devnode_dir: string
@return: list of all found device nodes
@rtype: list of strings
"""
cache_link = ["blockdevice_nodes", devnode_dir]
cached = CACHE.get(cache_link)
if not cached is None:
return cached.copy()
result = {}
def add_major_minor(arg, dirname, fnames):
for fname in fnames:
dev_node_path = os.path.join(dirname, fname)
if not os.path.isdir(dev_node_path):
major_minor = __get_major_minor(dev_node_path)
if not major_minor is None:
if result.has_key(major_minor):
result[major_minor].append(dev_node_path)
else:
result[major_minor] = [dev_node_path]
os.path.walk(devnode_dir, add_major_minor, None)
CACHE.set(cache_link, result)
return result.copy()
def find_lvm_pv():
@ -988,16 +939,7 @@ def _load_preferences():
CACHE = BlockdeviceCache()
if __name__ == '__main__':
## list the properties of all available devices
## this is just for testing purposes
blocks = Blockdevices().get_devices()
## do we want to show the result?
def show(text=""):
if IS_VISIBLE:
print text
def show_devices(blocks, show):
if len(blocks) > 0:
## show all devices and their properties
show("Properties of all devices:")
@ -1021,3 +963,32 @@ if __name__ == '__main__':
pass
show()
def get_devices_and_show():
## list the properties of all available devices
## this is just for testing purposes
blocks = Blockdevices().get_devices()
blocks.sort(key=lambda x: x.name)
## do we want to show the result?
def show(text=""):
if IS_VISIBLE:
print text
show_devices(blocks, show)
if __name__ == '__main__':
if DO_PROFILE:
# show some profiling information (requires the python-profiler package)
import cProfile
import pstats
IS_VISIBLE = False
for index in range(3):
print "Run: %d" % index
cProfile.run('get_devices_and_show()', 'profinfo')
p = pstats.Stats('profinfo')
p.sort_stats('cumulative').print_stats(20)
else:
get_devices_and_show()