conduit r1377 - in trunk: . conduit conduit/dataproviders test/python-tests

[jstowers svn gnome org]

Author: jstowers
Date: Sun Mar 16 09:07:37 2008
New Revision: 1377
URL: http://svn.gnome.org/viewvc/conduit?rev=1377&view=rev

Log:
2008-03-16  John Stowers  <john stowers gmail com>

	* conduit/Main.py: Add -w and -x switches to the command line arguments.
	These allow you to whitelist/blacklist certain files from being loaded.
	
	* conduit/Module.py:
	* conduit/ModuleWrapper.py:
	* conduit/Vfs.py:
	* conduit/dataproviders/File.py: Strip special characters from filenames
	when synchronizing with legacy filesystems (fat, ntfs). Fixes #522436
	
	* test/python-tests/TestCoreVfs.py:
	* test/python-tests/common.py: Tests for the above



Modified:
   trunk/ChangeLog
   trunk/conduit/Main.py
   trunk/conduit/Module.py
   trunk/conduit/ModuleWrapper.py
   trunk/conduit/Vfs.py
   trunk/conduit/dataproviders/File.py
   trunk/test/python-tests/TestCoreVfs.py
   trunk/test/python-tests/common.py

Modified: trunk/conduit/Main.py
==============================================================================
--- trunk/conduit/Main.py	(original)
+++ trunk/conduit/Main.py	Sun Mar 16 09:07:37 2008
@@ -45,10 +45,12 @@
 
         buildGUI = True
         iconify = False
+        whitelist = None
+        blacklist = None
         self.ui = "gtk"
         try:
-            opts, args = getopt.getopt(sys.argv[1:], "hvs:ciu:", 
-                ["help", "version", "settings=", "console", "iconify", "ui="])
+            opts, args = getopt.getopt(sys.argv[1:], "hvs:ciu:w:x:", 
+                ["help", "version", "settings=", "console", "iconify", "ui=", "with-modules=", "without-modules="])
             #parse args
             for o, a in opts:
                 if o in ("-h", "--help"):
@@ -64,7 +66,12 @@
                 if o in ("-i", "--iconify"):
                     iconify = True
                 if o in ("-u", "--ui"):
-                     self.ui = a
+                    self.ui = a
+                if o in ("-w", "--with-modules"):
+                    whitelist = a.split(",")
+                if o in ("-x", "--without-modules"):
+                    blacklist = a.split(",")
+
         except getopt.GetoptError:
             log.warn("Unknown command line option")
             self._usage()
@@ -113,7 +120,7 @@
         #Initialize all globals variables
         conduit.GLOBALS.app = self
         conduit.GLOBALS.moduleManager = ModuleManager(dirs_to_search)
-        conduit.GLOBALS.moduleManager.load_all()
+        conduit.GLOBALS.moduleManager.load_all(whitelist, blacklist)
         conduit.GLOBALS.typeConverter = TypeConverter(conduit.GLOBALS.moduleManager)
         conduit.GLOBALS.syncManager = SyncManager(conduit.GLOBALS.typeConverter)
         conduit.GLOBALS.mappingDB = MappingDB(self.dbFile)
@@ -160,16 +167,20 @@
     def _usage(self):
         print """Usage: conduit [OPTIONS] - Synchronize things
 OPTIONS:
-    -h, --help          Show this message.
-    -c, --console       Launch with no GUI.
-                        (default=no)
-    -s, --settings=FILE Save settings to FILE.
-                        (default=~/.conduit/settings.xml)
-    -i, --iconify       Iconify on startup.
-                        (default=no)
-    -u, --ui=NAME       Run with the specified UI
-                        (default=gtk)
-    -v, --version       Show version information"""
+    -h, --help              Show this message.
+    -c, --console           Launch with no GUI.
+                            (default=no)
+    -s, --settings=FILE     Save settings to FILE.
+                            (default=$XDG_CONFIG_DIR/.conduit/settings.xml)
+    -i, --iconify           Iconify on startup.
+                            (default=no)
+    -u, --ui=NAME           Run with the specified UI
+                            (default=gtk)
+    -w, --with-modules      Only load modules in the named files
+                            (default=load all modules)
+    -x, --without-modules   Do not load modules in the named files
+                            (default=load all modules)
+    -v, --version           Show version information"""
 
     @dbus.service.method(APPLICATION_DBUS_IFACE, in_signature='', out_signature='b')
     def HasGUI(self):

Modified: trunk/conduit/Module.py
==============================================================================
--- trunk/conduit/Module.py	(original)
+++ trunk/conduit/Module.py	Sun Mar 16 09:07:37 2008
@@ -12,8 +12,9 @@
 import logging
 log = logging.getLogger("Module")
 
-from conduit.dataproviders import CATEGORY_TEST
-from conduit.ModuleWrapper import ModuleWrapper, PendingDataproviderWrapper
+import conduit.dataproviders
+import conduit.ModuleWrapper as ModuleWrapper
+import conduit.Vfs as Vfs
 
 class ModuleManager(gobject.GObject):
     """
@@ -174,10 +175,10 @@
                 try:
                     klass = getattr(mod, modules)
                     if infos["type"] == "dataprovider" or infos["type"] == "converter":
-                        mod_wrapper = ModuleWrapper(
+                        mod_wrapper = ModuleWrapper.ModuleWrapper(
                                         klass=klass,
                                         initargs=(),
-                                        category=getattr(klass, "_category_", CATEGORY_TEST)
+                                        category=getattr(klass, "_category_", conduit.dataproviders.CATEGORY_TEST)
                                         )
                         #Save the module (signal is emitted in _append_module)
                         self._append_module(
@@ -200,12 +201,23 @@
             log.warn("Error loading the file: %s\n%s" % (filename, traceback.format_exc()))
             self.invalidFiles.append(os.path.basename(filename))
 
-    def load_all(self):
+    def load_all(self, whitelist, blacklist):
         """
-        Loads all classes in the configured paths
+        Loads all classes in the configured paths.
+
+        If whitelist and blacklist are supplied then the name of the file
+        is tested against them. Default policy is to load all modules unless
         """
         for f in self.filelist:
-            self._load_modules_in_file(f)
+            name, ext = Vfs.uri_get_filename_and_extension(f)
+            if whitelist:
+                if name in whitelist:
+                    self._load_modules_in_file(f)
+            elif blacklist:
+                if name not in blacklist: 
+                    self._load_modules_in_file(f)
+            else:            
+                self._load_modules_in_file(f)
 
         for i in self.dataproviderFactories:
             i.connect("dataprovider-removed", self._on_dynamic_dataprovider_removed)
@@ -240,7 +252,7 @@
             #Get the existing wrapper
             m = self.moduleWrappers[wrapperKey]
             #Make a copy of it, containing an instantiated module
-            mod_wrapper = ModuleWrapper(
+            mod_wrapper = ModuleWrapper.ModuleWrapper(
                             klass=m.klass,
                             initargs=m.initargs,
                             category=m.category
@@ -248,7 +260,7 @@
             mod_wrapper.instantiate_module()
         else:
             log.warn("Could not find module wrapper: %s" % (wrapperKey))
-            mod_wrapper = PendingDataproviderWrapper(wrapperKey)
+            mod_wrapper = ModuleWrapper.PendingDataproviderWrapper(wrapperKey)
 
         return mod_wrapper
 

Modified: trunk/conduit/ModuleWrapper.py
==============================================================================
--- trunk/conduit/ModuleWrapper.py	(original)
+++ trunk/conduit/ModuleWrapper.py	Sun Mar 16 09:07:37 2008
@@ -2,6 +2,10 @@
 import logging
 log = logging.getLogger("ModuleWrapper")
 
+COMPULSORY_ATTRIBUTES = (
+    "type",
+)
+
 class ModuleWrapper: 
     """
     A generic wrapper for any dynamically loaded module. Wraps the complexity
@@ -9,11 +13,6 @@
     descriptive fields like name and description. Useful for classification 
     and searching for moldules of certain types, etc.
     """
-    
-    NUM_UID_DIGITS = 5
-    COMPULSORY_ATTRIBUTES = [
-                            "type"
-                            ]
     	
     def __init__ (self, klass, initargs, category):
         """

Modified: trunk/conduit/Vfs.py
==============================================================================
--- trunk/conduit/Vfs.py	(original)
+++ trunk/conduit/Vfs.py	Sun Mar 16 09:07:37 2008
@@ -121,6 +121,12 @@
     is it wasnt so slow
     """
     return uri.split(os.sep)[-1]
+
+def uri_get_filename_and_extension(uri):
+    """
+    Returns filename,file_extension
+    """
+    return os.path.splitext(uri_get_filename(uri))
     
 def uri_sanitize_for_filesystem(uri, filesystem=None):
     """
@@ -128,12 +134,27 @@
     given filesystem - particuarly fat and ntfs types
     """
     import string
-    if filesystem in ("vfat","ntfs"):
-        ILLEGAL_CHARS = "\\:*?\"<>|"
+
+    ILLEGAL_CHARS = {
+        "vfat"  :   "\\:*?\"<>|",
+        "ntfs"  :   "\\:*?\"<>|"
+    }
+
+    illegal = ILLEGAL_CHARS.get(filesystem,None)
+    if illegal:
+        #dont escape the scheme part
+        scheme = uri_get_protocol(uri)
+        if scheme:
+            start = uri.index("://")+3
+        else:
+            start = 0    
+        
         #replace illegal chars with a space
-        return uri.translate(string.maketrans(
-                                ILLEGAL_CHARS,
-                                " "*len(ILLEGAL_CHARS)))
+        return scheme+uri[start:].translate(string.maketrans(
+                                                illegal,
+                                                " "*len(illegal)
+                                                )
+                                            )
     return uri
     
 def uri_is_folder(uri):
@@ -283,9 +304,9 @@
                 try:
                     thread.join()
                     joinedThreads += 1
-                except AssertionError: 
+                except RuntimeError: 
                     #deal with not started threads
-                    time.sleep(1)
+                    time.sleep(0.1)
 
     def cancel_all_threads(self):
         """

Modified: trunk/conduit/dataproviders/File.py
==============================================================================
--- trunk/conduit/dataproviders/File.py	(original)
+++ trunk/conduit/dataproviders/File.py	Sun Mar 16 09:07:37 2008
@@ -225,6 +225,7 @@
         self.includeHidden = includeHidden
         self.compareIgnoreMtime = compareIgnoreMtime
 
+        self.fstype = None
         self.files = []
         
     def initialize(self):
@@ -235,11 +236,13 @@
 
     def refresh(self):
         DataProvider.TwoWay.refresh(self)
+        #cache the filesystem type for speed
+        self.fstype = Vfs.uri_get_filesystem_type(self.folder)
+
         #scan the folder
         scanThread = Vfs.FolderScanner(self.folder, self.includeHidden)
         scanThread.start()
         scanThread.join()
-
         self.files = scanThread.get_uris()
 
     def put(self, vfsFile, overwrite, LUID=None):
@@ -279,6 +282,10 @@
                 #unknown. Store in the dir but recreate the group
                 newURI = self.folder+os.sep+os.path.join(vfsFile.group+relpath)
 
+        #escape illegal filesystem characters
+        if self.fstype:
+            newURI = Vfs.uri_sanitize_for_filesystem(newURI, self.fstype)
+
         destFile = File.File(URI=newURI)
         comp = vfsFile.compare(
                         destFile, 

Modified: trunk/test/python-tests/TestCoreVfs.py
==============================================================================
--- trunk/test/python-tests/TestCoreVfs.py	(original)
+++ trunk/test/python-tests/TestCoreVfs.py	Sun Mar 16 09:07:37 2008
@@ -12,7 +12,8 @@
 ok("Dont escape path characters",Vfs.uri_escape(safe+unsafe) == safe+safeunsafe)
 ok("Unescape back to original",Vfs.uri_unescape(safe+safeunsafe) == safe+unsafe)
 ok("Get protocol", Vfs.uri_get_protocol("file:///foo/bar") == "file://")
-ok("Get filename", Vfs.uri_get_filename("file:///foo/bar") == "bar")
+name, ext = Vfs.uri_get_filename_and_extension("file:///foo/bar.ext")
+ok("Get filename (%s,%s)" % (name,ext), name == "bar" and ext == ".ext")
 ok("file:///home exists", Vfs.uri_exists("file:///home") == True)
 ok("/home exists", Vfs.uri_exists("/home") == True)
 ok("/foo/bar does not exist", Vfs.uri_exists("/foo/bar") == False)
@@ -40,7 +41,10 @@
 ntfsUri = get_external_resources('folder')['ntfs-volume']
 fstype = Vfs.uri_get_filesystem_type(ntfsUri)
 ok("Get filesystem type (%s)" % fstype,fstype == "ntfs")
-ok("Escape illegal chars in filenames", Vfs.uri_sanitize_for_filesystem("invalid:|name",fstype).count(":") == 0)
+ok("Escape illegal chars in filenames", 
+        Vfs.uri_sanitize_for_filesystem("invalid:name","ntfs") == "invalid name")
+ok("Escape illegal chars in uris", 
+        Vfs.uri_sanitize_for_filesystem("file:///i:n/i:n","ntfs") == "file:///i n/i n")
 
 localUri = get_external_resources('folder')['folder']
 ok("Local uri --> path", Vfs.uri_to_local_path(localUri) == "/tmp")

Modified: trunk/test/python-tests/common.py
==============================================================================
--- trunk/test/python-tests/common.py	(original)
+++ trunk/test/python-tests/common.py	Sun Mar 16 09:07:37 2008
@@ -272,7 +272,7 @@
 
         self.model = Module.ModuleManager(dirs_to_search)
         conduit.GLOBALS.moduleManager = self.model
-        self.model.load_all()
+        self.model.load_all(whitelist=None, blacklist=None)
         self.type_converter = TypeConverter.TypeConverter(self.model)
         conduit.GLOBALS.typeManager = self.type_converter
         self.sync_manager = Synchronization.SyncManager(self.type_converter)