conduit.dataproviders.File

1 import os.path 2 import logging 3 import ConfigParser 4 log = logging.getLogger("dataproviders.File") 5 6 import conduit 7 import conduit.dataproviders.DataProvider as DataProvider 8 import conduit.datatypes as DataType 9 import conduit.datatypes.File as File 10 import conduit.Vfs as Vfs 11 import conduit.Database as DB 12 import conduit.Exceptions as Exceptions 13 14 TYPE_FILE = "0" 15 TYPE_FOLDER = "1" 16

17 -def is_on_removable_volume(folderUri):

18 return Vfs.uri_is_on_removable_volume(folderUri)

19

20 -def get_removable_volume_info(folderUri):

21 """ 22 Returns the root uri of the volume, and local path of the 23 group config file 24 """ 25 rooturi = Vfs.uri_get_volume_root_uri(folderUri) 26 path = Vfs.uri_join( 27 Vfs.uri_to_local_path(rooturi), 28 ".conduit") 29 return rooturi,path

30

31 -def save_removable_volume_group_file(folderUri, folderGroupName):

32 """ 33 Saves a file on the root of the drive, in ini format, 34 containing the uri and group 35 36 e.g. 37 [DEFAULT] 38 relative/uri/from/volume/root = group name 39 """ 40 if is_on_removable_volume(folderUri): 41 #write to the /volume/root/.conduit file 42 rooturi,path = get_removable_volume_info(folderUri) 43 conf = ConfigParser.SafeConfigParser() 44 conf.read(path) 45 46 log.debug("Saving group (%s = %s) to %s" % (folderUri,folderGroupName,path)) 47 conf.set( 48 "DEFAULT", 49 folderUri.replace(rooturi,""), 50 folderGroupName 51 ) 52 fp = open(path, 'w') 53 conf.write(fp) 54 fp.close() 55 return True 56 return False

57

58 -def read_removable_volume_group_file(folderUri):

59 items = [] 60 if is_on_removable_volume(folderUri): 61 #read from the /volume/root/.conduit file 62 rooturi,path = get_removable_volume_info(folderUri) 63 conf = ConfigParser.SafeConfigParser() 64 conf.read(path) 65 for p,n in conf.items("DEFAULT"): 66 log.debug("Read group (%s = %s)" % (p,n)) 67 #check the path still exists on the volume 68 if Vfs.uri_exists(rooturi+p): 69 items.append((p,n)) 70 return items

71

72 -class FileSource(DataProvider.DataSource, Vfs.FolderScannerThreadManager):

73 74 _category_ = conduit.dataproviders.CATEGORY_FILES 75 _module_type_ = "source" 76 _in_type_ = "file" 77 _out_type_ = "file" 78 _icon_ = "text-x-generic" 79

80 - def __init__(self):

81 DataProvider.DataSource.__init__(self) 82 Vfs.FolderScannerThreadManager.__init__(self) 83 84 #One table stores the top level files and folders (config) 85 #The other stores all files to sync. 86 self.db = DB.ThreadSafeGenericDB() 87 self.db.create( 88 table="config", 89 fields=("URI","TYPE","CONTAINS_NUM_ITEMS","SCAN_COMPLETE","GROUP_NAME") 90 ) 91 self.db.create( 92 table="files", 93 fields=("URI","BASEPATH","GROUPNAME") 94 )

95

96 - def _add_file(self, f):

97 self.db.insert( 98 table="config", 99 values=(f,TYPE_FILE,0,False,"") 100 )

101

102 - def _add_folder(self, f, groupname=""):

103 self.db.insert( 104 table="config", 105 values=(f,TYPE_FOLDER,0,False,groupname) 106 )

107

108 - def initialize(self):

109 return True

110

111 - def uninitialize(self):

112 self.db.close()

113

114 - def refresh(self):

115 DataProvider.DataSource.refresh(self) 116 self.db.execute("DELETE FROM files") 117 #Make a whole bunch of threads to go and scan the directories 118 for oid,uri,groupname in self.db.select("SELECT oid,URI,GROUP_NAME FROM config WHERE TYPE = ?",(TYPE_FOLDER,)): 119 self.make_thread( 120 uri, 121 False, #FIXME: Dont include hidden? 122 self._on_scan_folder_progress, 123 self._on_scan_folder_completed, 124 oid, 125 groupname 126 ) 127 128 #All threads must complete - otherwise we might miss some items 129 self.join_all_threads() 130 131 #now add the single files to the list 132 for oid,uri in self.db.select("SELECT oid,URI FROM config WHERE TYPE = ?",(TYPE_FILE,)): 133 f = File.File(URI=uri) 134 if f.exists(): 135 self.db.insert( 136 table="files", 137 values=(uri,"","") #single files dont have basepath and groupname 138 ) 139 else: 140 self.db.delete( 141 table="config", 142 oid=oid 143 )

144

145 - def get(self, LUID):

146 DataProvider.DataSource.get(self, LUID) 147 basepath,group = self.db.select_one("SELECT BASEPATH,GROUPNAME FROM files WHERE URI = ?", (LUID,)) 148 f = File.File( 149 URI= LUID, 150 basepath= basepath, 151 group= group 152 ) 153 f.set_open_URI(LUID) 154 f.set_UID(LUID) 155 return f

156

157 - def add(self, LUID):

158 f = File.File(URI=LUID) 159 if f.exists(): 160 oid = self.db.select_one("SELECT oid FROM files WHERE URI = ?", (LUID,)) 161 if oid != None: 162 log.debug("Could not add (already added): %s" % LUID) 163 return False 164 165 if f.is_directory(): 166 log.debug("Adding folder: %s" % LUID) 167 self._add_folder(LUID,"FIXME") 168 else: 169 log.debug("Adding file: %s" % LUID) 170 self._add_file(LUID) 171 else: 172 log.warn("Could not add: %s" % LUID) 173 return False 174 return True

175

176 - def get_all(self):

177 #combine the files contained inside dirs with those the user specified 178 files = [f for f, in self.db.select("SELECT URI FROM files")] 179 return files

180

181 - def finish(self, aborted, error, conflict):

182 DataProvider.DataSource.finish(self) 183 self.db.execute("DELETE FROM files")

184

185 - def _on_scan_folder_progress(self, folderScanner, numItems, oid, groupname):

186 """ 187 Called by the folder scanner thread and used to update 188 the estimate of the number of items in the directory 189 """ 190 self.db.update( 191 table="config", 192 oid=oid, 193 CONTAINS_NUM_ITEMS=numItems 194 )

195

196 - def _on_scan_folder_completed(self, folderScanner, oid, groupname):

197 log.debug("Folder scan complete %s" % folderScanner) 198 #Update scan status 199 self.db.update( 200 table="config", 201 oid=oid, 202 SCAN_COMPLETE=True, 203 GROUP_NAME=groupname 204 ) 205 #Put all files into files 206 for f in folderScanner.get_uris(): 207 self.db.insert( 208 table="files", 209 values=(f,folderScanner.baseURI,groupname) 210 )

211

212 -class FolderTwoWay(DataProvider.TwoWay):

213 """ 214 TwoWay dataprovider for synchronizing a folder 215 """ 216 217 _category_ = conduit.dataproviders.CATEGORY_FILES 218 _module_type_ = "twoway" 219 _in_type_ = "file" 220 _out_type_ = "file" 221 _icon_ = "folder" 222

223 - def __init__(self, folder, folderGroupName, includeHidden, compareIgnoreMtime, followSymlinks):

224 DataProvider.TwoWay.__init__(self) 225 self.folder = folder 226 self.folderGroupName = folderGroupName 227 self.includeHidden = includeHidden 228 self.compareIgnoreMtime = compareIgnoreMtime 229 self.followSymlinks = followSymlinks 230 231 self.fstype = None 232 self.files = []

233

234 - def _transfer_file(self, vfsFile, newURI, overwrite):

235 try: 236 vfsFile.transfer(newURI, overwrite) 237 except File.FileTransferError: 238 raise Exceptions.SyncronizeFatalError("Transfer Cancelled")

239

240 - def initialize(self):

241 return True

242

243 - def is_configured(self, isSource, isTwoWay):

244 return Vfs.uri_exists(self.folder)

245

246 - def refresh(self):

247 DataProvider.TwoWay.refresh(self) 248 #cache the filesystem type for speed 249 self.fstype = Vfs.uri_get_filesystem_type(self.folder) 250 251 #scan the folder 252 scanThread = Vfs.FolderScanner(self.folder, self.includeHidden, self.followSymlinks) 253 scanThread.start() 254 scanThread.join() 255 self.files = scanThread.get_uris()

256

257 - def put(self, vfsFile, overwrite, LUID=None):

258 """ 259 Puts vfsFile at the correct location. There are three scenarios 260 1) File came from a foreign DP like tomboy 261 2) File came from another file dp 262 263 Behaviour: 264 1) The foreign DP should have encoded enough information (such as 265 the filename) so that we can go ahead and put the file in the dir 266 2) First we see if the file has a group attribute. 267 a) If so, and the group matches the groupName here then we 268 put the files into the directory. 269 b) If not we put the file in a subdir by the name of the group 270 271 We always retain the relative path for the files 272 """ 273 DataProvider.TwoWay.put(self, vfsFile, overwrite, LUID) 274 newURI = "" 275 if LUID != None: 276 newURI = LUID 277 elif vfsFile.basePath == "": 278 #came from another type of dataprovider such as tomboy 279 #where relative path makes no sense. Could also come from 280 #the FileSource dp when the user has selected a single file 281 log.debug("No basepath. Going to empty dir") 282 newURI = Vfs.uri_join(self.folder,vfsFile.get_filename()) 283 else: 284 #Look for corresponding groups 285 relpath = vfsFile.get_relative_uri() 286 log.debug("Relative path: %s" % relpath) 287 if self.folderGroupName == vfsFile.group: 288 log.debug("Found corresponding group") 289 #put in the folder 290 newURI = Vfs.uri_join(self.folder,relpath) 291 else: 292 log.debug("Recreating group: %s" % vfsFile.group) 293 #unknown. Store in the dir but recreate the group 294 newURI = Vfs.uri_join(self.folder,vfsFile.group,relpath) 295 296 #escape illegal filesystem characters 297 if self.fstype: 298 newURI = Vfs.uri_sanitize_for_filesystem(newURI, self.fstype) 299 300 #overwrite is the easy case, as for it to be true, requires specific user 301 #interaction 302 if overwrite == True: 303 self._transfer_file(vfsFile, newURI, overwrite) 304 else: 305 #check for conflicts 306 destFile = File.File(URI=newURI) 307 if destFile.exists(): 308 comp = vfsFile.compare( 309 destFile, 310 sizeOnly=self.compareIgnoreMtime 311 ) 312 313 if LUID != None and comp == DataType.COMPARISON_NEWER: 314 #we were expecting an existing file, we found it, but 315 #we are newer, so overwrite it 316 self._transfer_file(vfsFile, newURI, True) 317 elif comp == DataType.COMPARISON_EQUAL: 318 #in File.compare, the files are compared based on size, if 319 #their mtimes are the same, so this case is true when 320 # 1) The sizes are the same, and the user told us 321 # to ignore the mtimes 322 # 2) The mtimes and size is the same, and we checked both 323 pass 324 else: 325 raise Exceptions.SynchronizeConflictError(comp, vfsFile, destFile) 326 else: 327 self._transfer_file(vfsFile, newURI, overwrite) 328 329 return self.get(newURI).get_rid()

330

331 - def delete(self, LUID):

332 f = File.File(URI=LUID) 333 if f.exists(): 334 f.delete()

335

336 - def get(self, uid):

337 DataProvider.TwoWay.get(self, uid) 338 f = File.File( 339 URI=uid, 340 basepath=self.folder, 341 group=self.folderGroupName 342 ) 343 f.set_open_URI(uid) 344 f.set_UID(uid) 345 return f

346

347 - def get_all(self):

348 DataProvider.TwoWay.get_all(self) 349 return self.files

350

351 - def finish(self, aborted, error, conflict):

352 DataProvider.TwoWay.finish(self) 353 self.files = [] 354 try: 355 #Save the .group file to the root of this volume (if it is removable) 356 save_removable_volume_group_file(self.folder, self.folderGroupName) 357 except Exception, e: 358 log.warn("Error saving volume group file: %s" % e)

359

360 - def add(self, LUID):

361 f = File.File(URI=LUID) 362 if f.exists() and f.is_directory(): 363 self.folder = f._get_text_uri() 364 return True 365 return False

366

Source Code for Module conduit.dataproviders.File