[conduit/gsoc09_alexandre: 3/6] Added profiling and cached modules.
- From: Alexandre Rosenfeld <arosenfeld src gnome org>
- To: svn-commits-list gnome org
- Subject: [conduit/gsoc09_alexandre: 3/6] Added profiling and cached modules.
- Date: Sun, 7 Jun 2009 23:41:43 -0400 (EDT)
commit a053a89d69a223736a3e34bbbc9fd9728120100c
Author: Alexandre Rosenfeld <airmind gmail com>
Date: Sun May 31 16:44:16 2009 -0300
Added profiling and cached modules.
---
conduit/Module.py | 147 ++++++++++++++++++++++--------
conduit/ModuleWrapper.py | 42 ++++++++-
conduit/conduit.real | 5 +-
conduit/dataproviders/HalFactory.py | 37 ++++++-
conduit/dataproviders/VolumeFactory.py | 22 ++--
conduit/modules/N800Module/N800Module.py | 2 +-
conduit/modules/iPodModule/iPodModule.py | 47 +---------
7 files changed, 196 insertions(+), 106 deletions(-)
diff --git a/conduit/Module.py b/conduit/Module.py
index a5fd7a1..075e22b 100644
--- a/conduit/Module.py
+++ b/conduit/Module.py
@@ -9,6 +9,10 @@ import gobject
import os, os.path
import traceback
import pydoc
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
import logging
log = logging.getLogger("Module")
@@ -17,6 +21,9 @@ import conduit.ModuleWrapper as ModuleWrapper
import conduit.Knowledge as Knowledge
import conduit.Vfs as Vfs
+class Error(Exception):
+ pass
+
class ModuleManager(gobject.GObject):
"""
Generic dynamic module loader for conduit. Given a path
@@ -56,6 +63,25 @@ class ModuleManager(gobject.GObject):
self.invalidFiles = []
#Keep a ref to dataprovider factories so they are not collected
self.dataproviderFactories = []
+ self.cache_path = os.path.join(conduit.USER_DIR, "modules_cache")
+ self.modules_cache = {}
+ if os.path.exists(self.cache_path):
+ cache_file = open(self.cache_path, "rb")
+ try:
+ self.modules_cache = pickle.load(cache_file)
+ if not isinstance(self.modules_cache, dict):
+ raise Exception()
+ for key, value in self.modules_cache.iteritems():
+ if not isinstance(key, basestring) and not isinstance(value, float):
+ raise Exception()
+ log.critical("Modules cache loaded %s" % self.modules_cache)
+ except:
+ log.warn("Modules cache invalid")
+ self.modules_cache = {}
+ finally:
+ cache_file.close()
+ else:
+ log.critical("No modules cache found")
#scan all dirs for files in the right format (*Module/*Module.py)
self.filelist = self._build_filelist_from_directories(dirs)
@@ -87,7 +113,7 @@ class ModuleManager(gobject.GObject):
directory in which they reside.
This method is automatically invoked by the constructor.
"""
- res = []
+ res = {}
if not directories:
return res
@@ -101,14 +127,17 @@ class ModuleManager(gobject.GObject):
continue
for i in os.listdir(d):
f = os.path.join(d,i)
- if os.path.isfile(f) and self._is_module(f):
- if os.path.basename(f) not in [os.path.basename(j) for j in res]:
- res.append(f)
+ if os.path.isfile(f) and (self._is_module(f) or self._is_factory(f)):
+ if os.path.basename(f) not in [os.path.basename(j) for j in res.keys()]:
+ res[f] = {'mtime': os.stat(f).st_mtime}
elif os.path.isdir(f) and self._is_module_dir(f):
directories.append(f)
except OSError, err:
log.warn("Error reading directory %s, skipping." % (d))
- return res
+ return res
+
+ def _is_factory(self, filename):
+ return filename.endswith("Factory.py")
def _is_module(self, filename):
return filename.endswith("Module.py")
@@ -164,39 +193,72 @@ class ModuleManager(gobject.GObject):
log.warn("Class %s in file %s does define a %s attribute. Skipping." % (modules, filename, i))
raise Exception
return mods
+
+ def _load_module(self, info, *args, **kwargs):
+ mod = self._import_file(info['filename'])
+ #log.critical("Cached imported file %s: %s" % (info['filename'], mod.MODULES.items()))
+ #log.critical(" Looking for %s" % (info,))
+ for modules, infos in mod.MODULES.items():
+ klass = getattr(mod, modules)
+ if getattr(klass, '__name__') == info['classname']:
+ return klass(*args, **kwargs)
+ raise Error("Module %s not found" % info['classname'])
- def _load_modules_in_file(self, filename):
+ def _load_modules_in_file(self, filename, f_data):
"""
Loads all modules in the given file
"""
try:
- mod = self._import_file(filename)
- for modules, infos in mod.MODULES.items():
- try:
- klass = getattr(mod, modules)
- if infos["type"] == "dataprovider" or infos["type"] == "converter":
- mod_wrapper = ModuleWrapper.ModuleWrapper(
- klass=klass,
- initargs=(),
- category=getattr(klass, "_category_", conduit.dataproviders.CATEGORY_TEST)
- )
- #Save the module (signal is emitted in _append_module)
- self._append_module(
- mod_wrapper,
- klass
- )
- elif infos["type"] == "dataprovider-factory":
- # build a dict of kwargs to pass to factories
- kwargs = {
- "moduleManager": self,
- }
- #instantiate and store the factory
- instance = klass(**kwargs)
- self.dataproviderFactories.append(instance)
- else:
- log.warn("Class is an unknown type: %s" % klass)
- except AttributeError:
- log.warn("Could not find module %s in %s\n%s" % (modules,filename,traceback.format_exc()))
+ if self._is_factory(filename) or not (filename in self.modules_cache and self.modules_cache[filename]['mtime'] == os.stat(filename).st_mtime):
+ log.critical("Importing file %s" % filename)
+ mod = self._import_file(filename)
+ modules_cache = []
+ for modules, infos in mod.MODULES.items():
+ try:
+ klass = getattr(mod, modules)
+ if infos["type"] == "dataprovider" or infos["type"] == "converter":
+ mod_wrapper = ModuleWrapper.ModuleWrapper(
+ klass=klass,
+ initargs=(),
+ category=getattr(klass, "_category_", conduit.dataproviders.CATEGORY_TEST)
+ )
+ #Save the module (signal is emitted in _append_module)
+ self._append_module(
+ mod_wrapper,
+ klass
+ )
+ modules_cache.append(mod_wrapper.get_info())
+ #log.critical("Saving to cache %s" % mod_wrapper.get_info())
+ elif infos["type"] == "dataprovider-factory":
+ log.critical("Creating factory %s" % filename)
+ # build a dict of kwargs to pass to factories
+ kwargs = {
+ "moduleManager": self,
+ }
+ #instantiate and store the factory
+ instance = klass(**kwargs)
+ self.dataproviderFactories.append(instance)
+ else:
+ log.warn("Class is an unknown type: %s" % klass)
+ except AttributeError:
+ log.warn("Could not find module %s in %s\n%s" % (modules,filename,traceback.format_exc()))
+ self.filelist[filename]['modules'] = modules_cache
+ self.filelist[filename]['mtime'] = os.stat(filename).st_mtime
+ else:
+ log.critical("File %s in cache" % filename)
+ self.filelist[filename] = self.modules_cache[filename]
+ for module in self.modules_cache[filename]['modules']:
+ module['filename'] = filename
+ mod_wrapper = ModuleWrapper.ModuleWrapper(
+ klass=self._load_module,
+ initargs=(module,),
+ category=module['category'],
+ cached_info=module,
+ )
+ self._append_module(
+ mod_wrapper,
+ module['classname']
+ )
except pydoc.ErrorDuringImport, e:
log.warn("Error loading the file: %s\n%s" % (filename, "".join(traceback.format_exception(e.exc,e.value,e.tb))))
self.invalidFiles.append(os.path.basename(filename))
@@ -211,18 +273,26 @@ class ModuleManager(gobject.GObject):
If whitelist and blacklist are supplied then the name of the file
is tested against them. Default policy is to load all modules unless
"""
- for f in self.filelist:
+ for f, f_data in self.filelist.iteritems():
name, ext = Vfs.uri_get_filename_and_extension(f)
if whitelist:
if name in whitelist:
- self._load_modules_in_file(f)
+ self._load_modules_in_file(f, f_data)
elif blacklist:
if name not in blacklist:
- self._load_modules_in_file(f)
+ self._load_modules_in_file(f, f_data)
else:
- self._load_modules_in_file(f)
+ self._load_modules_in_file(f, f_data)
+
+ self.modules_cache = self.filelist
+ cache_file = open(self.cache_path, "wb")
+ try:
+ pickle.dump(self.filelist, cache_file)
+ finally:
+ cache_file.close()
for i in self.dataproviderFactories:
+ log.critical("Probing %s" % i)
i.connect("dataprovider-removed", self._on_dynamic_dataprovider_removed)
i.connect("dataprovider-added", self._on_dynamic_dataprovider_added)
i.probe()
@@ -258,7 +328,8 @@ class ModuleManager(gobject.GObject):
mod_wrapper = ModuleWrapper.ModuleWrapper(
klass=m.klass,
initargs=m.initargs,
- category=m.category
+ category=m.category,
+ cached_info=m.cached_info,
)
mod_wrapper.instantiate_module()
else:
diff --git a/conduit/ModuleWrapper.py b/conduit/ModuleWrapper.py
index a705bc4..296f6f1 100644
--- a/conduit/ModuleWrapper.py
+++ b/conduit/ModuleWrapper.py
@@ -14,7 +14,7 @@ class ModuleWrapper:
and searching for moldules of certain types, etc.
"""
- def __init__ (self, klass, initargs, category):
+ def __init__ (self, klass, initargs, category, cached_info = None):
"""
Initializes the ModuleWrapper with an uninstantiated class
@@ -40,8 +40,21 @@ class ModuleWrapper:
self.initargs = initargs
self.category = category
+ self.cached_info = cached_info
+
#extract class parameters
- if klass:
+ if cached_info:
+ log.critical("Creating module wrapper cached info %s" % cached_info)
+ self.name = cached_info.get("name", "")
+ self.description = cached_info.get("description", "")
+ self.icon_name = cached_info.get("icon_name", "")
+ self.module_type = cached_info.get("module_type", "")
+ self.in_type = cached_info.get("in_type", "")
+ self.out_type = cached_info.get("out_type", "")
+ self.configurable = cached_info.get("configurable", False)
+ self.classname = cached_info.get("classname", "")
+ elif klass:
+ log.critical("Creating module wrapper for %s" % klass)
self.name = getattr(klass, "_name_", "")
self.description = getattr(klass, "_description_", "")
self.icon_name = getattr(klass, "_icon_", "")
@@ -66,6 +79,17 @@ class ModuleWrapper:
self.icon_path = ""
self.icon = {}
self.descriptiveIcon = None
+
+ def get_info(self):
+ return {'name': self.name,
+ 'description': self.description,
+ 'icon_name': self.icon_name,
+ 'module_type': self.module_type,
+ 'in_type': self.in_type,
+ 'out_type': self.out_type,
+ 'configurable': self.configurable,
+ 'classname': self.classname,
+ 'category': self.category}
def __str__(self):
return "Wrapper: %s %s (UID: %s)" % (self.get_name(), self.module_type, self.get_UID())
@@ -80,6 +104,8 @@ class ModuleWrapper:
#
# 2. They are also serve a way to show the same class in multiple categories
def get_dnd_key(self):
+ if self.cached_info:
+ return self.classname
if self.dndKey:
return self.dndKey
return self.get_key()
@@ -94,8 +120,11 @@ class ModuleWrapper:
I suppose I could have used the builtin __getinitargs__ call used with
pickle but requires less implementation detail on the part of the DP
"""
- if len(self.initargs) > 0:
- return self.classname + ":" + ":".join(self.initargs)
+ initargs = self.initargs
+ if self.cached_info:
+ initargs = initargs[1:]
+ if len(initargs) > 0:
+ return self.classname + ":" + ":".join(initargs)
else:
return self.classname
@@ -252,7 +281,12 @@ class ModuleWrapper:
return self.module.get_configuration_xml()
def instantiate_module(self):
+ #log.critical("Cached %s" % self.cached_info)
+ #if self.cached_info:
+ # self.module = self.klass(self.cached_info, *self.initargs)
+ #else:
self.module = self.klass(*self.initargs)
+ log.critical("Module instantiated: %s (%s)" % (self.name, self.module))
def is_pending(self):
return self.module == None
diff --git a/conduit/conduit.real b/conduit/conduit.real
index a2ac906..bc79351 100755
--- a/conduit/conduit.real
+++ b/conduit/conduit.real
@@ -20,5 +20,8 @@ if os.path.exists(os.path.join(_dirname,"ChangeLog")):
import conduit
import conduit.Main
-app = conduit.Main.Application()
+
+import cProfile
+cProfile.run('app = conduit.Main.Application()', 'prof')
+#app =
diff --git a/conduit/dataproviders/HalFactory.py b/conduit/dataproviders/HalFactory.py
index fa5c0e8..aa3fda2 100644
--- a/conduit/dataproviders/HalFactory.py
+++ b/conduit/dataproviders/HalFactory.py
@@ -1,5 +1,7 @@
import gobject
import dbus
+import logging
+log = logging.getLogger("dataproviders.HalFactory")
import conduit.utils as Utils
import conduit.dataproviders.SimpleFactory as SimpleFactory
@@ -20,9 +22,10 @@ class HalFactory(SimpleFactory.SimpleFactory):
self.hal.connect_to_signal("NewCapability", self._new_capability)
def _maybe_new(self, device_udi):
- props = self._get_properties(device_udi)
- if self.is_interesting(device_udi, props):
- self.item_added(device_udi, **props)
+ def check_interesting(props):
+ if self.is_interesting(device_udi, props):
+ self.item_added(device_udi, **props)
+ self._get_properties(device_udi, handler = check_interesting)
def _device_added(self, device_udi, *args):
self._maybe_new(device_udi)
@@ -34,23 +37,45 @@ class HalFactory(SimpleFactory.SimpleFactory):
def _device_removed(self, device_udi):
self.item_removed(device_udi)
- def _get_properties(self, device):
+ def _get_properties(self, device, handler):
buf = {}
+ #log.critical("Properties for: %s" % device)
+ def properties_handler(props):
+ log.critical("Properties for: %s" % device)
+ for key, value in props.items():
+ buf[str(key)] = value
+ handler(buf)
+ def error_handler(excp):
+ log.warn("Could not get HAL properties for %s" % device_udi)
+ device_dbus_obj = self.bus.get_object("org.freedesktop.Hal" ,device)
+ if handler:
+ device_dbus_obj.GetAllProperties(dbus_interface="org.freedesktop.Hal.Device",
+ reply_handler = properties_handler, error_handler = error_handler)
+ else:
+ for x, y in device_dbus_obj.GetAllProperties(dbus_interface="org.freedesktop.Hal.Device").items():
+ #DBus *still* does not marshal dbus.String to str correctly,
+ #so we force it to
+ buf[str(x)] = y
+ return buf
+ '''
try:
device_dbus_obj = self.bus.get_object("org.freedesktop.Hal" ,device)
- for x, y in device_dbus_obj.GetAllProperties(dbus_interface="org.freedesktop.Hal.Device").items():
+ for x, y in device_dbus_obj.GetAllProperties(dbus_interface="org.freedesktop.Hal.Device",
+ reply_hanlder = properties_handler).items():
#DBus *still* does not marshal dbus.String to str correctly,
#so we force it to
buf[str(x)] = y
except:
log.warn("Could not get HAL properties for %s" % device_udi)
return buf
+ '''
def probe(self):
"""
Enumerate HAL for any entries of interest
"""
- devices = self.hal.GetAllDevices()
+ #devices = self.hal.GetAllDevices()
+
for device in self.hal.GetAllDevices():
self._maybe_new(str(device))
diff --git a/conduit/dataproviders/VolumeFactory.py b/conduit/dataproviders/VolumeFactory.py
index d72e3e0..3312461 100644
--- a/conduit/dataproviders/VolumeFactory.py
+++ b/conduit/dataproviders/VolumeFactory.py
@@ -15,8 +15,7 @@ class VolumeFactory(HalFactory.HalFactory):
"""
def _wait_for_mount(self, udi, props):
- props.update(self._get_properties(udi))
-
+ props.update(self._get_properties(udi, None))
if not props.has_key("volume.is_mounted"):
log.info("Still waiting for HAL to notice: %s" % udi)
return True
@@ -31,15 +30,16 @@ class VolumeFactory(HalFactory.HalFactory):
return False
def _maybe_new(self, device_udi):
- props = self._get_properties(device_udi)
- if "volume" in [str(c) for c in props.get("info.capabilities", [])]:
- #this basically checks if the volume mounting procedure has finished
- if str(props.get("volume.mount_point", "")) == "" or props.has_key("volume.is_mounted") == False:
- log.info("Waiting for HAL to attempt mount")
- gobject.timeout_add(1000, self._wait_for_mount, device_udi, props)
- else:
- if self.is_interesting(device_udi, props):
- self.item_added(device_udi, **props)
+ def properties_handler(props):
+ if "volume" in [str(c) for c in props.get("info.capabilities", [])]:
+ #this basically checks if the volume mounting procedure has finished
+ if str(props.get("volume.mount_point", "")) == "" or props.has_key("volume.is_mounted") == False:
+ log.info("Waiting for HAL to attempt mount")
+ gobject.timeout_add(1000, self._wait_for_mount, device_udi, props)
+ else:
+ if self.is_interesting(device_udi, props):
+ self.item_added(device_udi, **props)
+ self._get_properties(device_udi, handler = properties_handler)
def probe(self):
"""
diff --git a/conduit/modules/N800Module/N800Module.py b/conduit/modules/N800Module/N800Module.py
index f772e98..f594aca 100644
--- a/conduit/modules/N800Module/N800Module.py
+++ b/conduit/modules/N800Module/N800Module.py
@@ -30,7 +30,7 @@ MODULES = {
class N800Factory(VolumeFactory.VolumeFactory):
def is_interesting(self, udi, props):
if props.has_key("info.parent") and props.has_key("info.parent")!="":
- prop2 = self._get_properties(props["info.parent"])
+ prop2 = self._get_properties(props["info.parent"], None)
if prop2.has_key("storage.model") and prop2["storage.model"] in ("N800", "N810"):
if prop2.has_key("storage.removable") and prop2["storage.removable"] == True:
return True
diff --git a/conduit/modules/iPodModule/iPodModule.py b/conduit/modules/iPodModule/iPodModule.py
index 05a8978..5ea2e94 100644
--- a/conduit/modules/iPodModule/iPodModule.py
+++ b/conduit/modules/iPodModule/iPodModule.py
@@ -36,12 +36,11 @@ import conduit.datatypes.Video as Video
from gettext import gettext as _
errormsg = ""
+availiable = False
try:
import gpod
if gpod.version_info >= (0,6,0):
- MODULES = {
- "iPodFactory" : { "type": "dataprovider-factory" },
- }
+ availiable = True
log.info("Module Information: %s" % Utils.get_module_information(gpod, 'version_info'))
except ImportError:
errormsg = "iPod support disabled (python-gpod not availiable)"
@@ -66,48 +65,6 @@ def _string_to_unqiue_file(txt, base_uri, prefix, postfix=''):
temp.set_UID(uri)
return temp.get_rid()
-class iPodFactory(VolumeFactory.VolumeFactory):
-
- def _get_mount_path(self, props):
- return str(props["volume.mount_point"])
-
- def is_interesting(self, udi, props):
- if props.get("info.parent"):
- parent = self._get_properties(props["info.parent"])
- if parent.get("storage.model") == "iPod":
- props.update(parent)
- return True
- return False
-
- def get_category(self, udi, **kwargs):
- label = kwargs['volume.label']
- if not label:
- label = "Apple iPod Music Player"
- return DataProviderCategory.DataProviderCategory(
- label,
- "multimedia-player-ipod-standard-color",
- self._get_mount_path(kwargs))
-
- def get_dataproviders(self, udi, **kwargs):
- #Read information about the ipod, like if it supports
- #photos or not
- d = gpod.itdb_device_new()
- gpod.itdb_device_set_mountpoint(d,self._get_mount_path(kwargs))
- supportsPhotos = gpod.itdb_device_supports_photo(d)
- gpod.itdb_device_free(d)
- if supportsPhotos:
- return [IPodMusicTwoWay, IPodVideoTwoWay, IPodNoteTwoWay, IPodContactsTwoWay, IPodCalendarTwoWay, IPodPhotoSink]
- else:
- log.info("iPod does not report photo support")
- return [IPodMusicTwoWay, IPodVideoTwoWay, IPodNoteTwoWay, IPodContactsTwoWay, IPodCalendarTwoWay]
-
- def get_args(self, udi, **kwargs):
- """
- iPod needs a local path to the DB, not a URI
- """
- kwargs["mount_path"] = self._get_mount_path(kwargs)
- return (kwargs['mount_path'], udi)
-
class IPodBase(DataProvider.TwoWay):
def __init__(self, *args):
DataProvider.TwoWay.__init__(self)
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
