Changeset 3178

Timestamp:

18 Feb 2008, 16:03:26 (17 years ago)

Author:

joachim

Message:

xxx_imported.csv -> xxx.imported.csv xxx_pending -> xxx.pending
.backup -> .old
tmp-file is removed after run

Location:

WAeUP_SRP/base

Files:

• WAeUPImport.py (modified) (8 diffs)
• WAeUPTool.py (modified) (3 diffs)
• utils.py (modified) (1 diff)

WAeUP_SRP/base/WAeUPImport.py

@@ -64 +64 @@
         self.portal_workflow = getToolByName(waeup_tool, 'portal_workflow')
         self.portal_url = getToolByName(waeup_tool, 'portal_url')
+        self.portal_catalog = waeup_tool.portal_catalog
         self.students_catalog = waeup_tool.students_catalog
         self.courses_catalog = waeup_tool.courses_catalog
@@ -72 +73 @@
         if self.import_method is None:
             errors.append('No importer method %s' % mode)
-        self.pending_path = "%s/import/%s_pending.csv" % (i_home,self.plural_name)
-        self.pending_tmp = "%s/import/%s_pending.csv.tmp" % (i_home,self.plural_name)
-        self.pending_backup = "%s/import/%s_pending.csv.backup" % (i_home,self.plural_name)
+        self.pending_path = "%s/import/%s.pending" % (i_home,self.plural_name)
+        self.pending_tmp = "%s/import/%s.pending.tmp" % (i_home,self.plural_name)
+        self.pending_backup = "%s/import/%s.pending.old" % (i_home,self.plural_name)
         self.pending_fn = os.path.split(self.pending_path)[1]
-        self.imported_path = "%s/import/%s_imported.csv" % (i_home,self.plural_name)
+        self.imported_path = "%s/import/%s.imported" % (i_home,self.plural_name)
         self.imported_fn = os.path.split(self.imported_path)[1]
         iname = "import_%s" % self.name
@@ -97 +98 @@
         info['import_mode'] = mode
         info['error'] = ''
+        info['digest'] = ''
         self.info = info
         self.csv_keys.extend(self.info)
@@ -119 +121 @@
                     pending_digests += digest,
                     pending += item,
+            datafile.close()
             copy2(self.pending_path,self.pending_backup)
-            pending_at_start = len(pending)                
-            datafile.close()
         return pending, pending_digests
 ###)
@@ -442 +443 @@
     name = "department"
     plural_name = "%ss" % name
-    commit_after = 100000
+    commit_after = 1000
     
-
     def create(self,mapping):###(
         "create a department in the correct faculty"
-        fid = mapping['faculty_code']
+        faculty_id = mapping['faculty_code']
         msg = ''
         if getattr(self,'_v_faculties',None) is None:
-            res = self.portal_catalog(portal_type = "Faculty")
+            res = self.portal_catalog(portal_type = "Department")
             self._v_faculties = {}
             for f in res:
                 self._v_faculties[f.getId] = f.getObject()
-        while True:
-            f = self._v_faculties.get(fid,None)
-            if f is None:
-                msg =  "No Faculty with ID: %s" % fid
+        department_id = mapping.get('code','')
+        while True:
+            faculty = self._v_faculties.get(faculty_id,None)
+            if faculty is None:
+                msg =  "No Faculty with ID: %s" % faculty_id
                 break
             else:
-                did = mapping.get('code')
-                d = getattr(f,did,None)
+                d = getattr(faculty,department_id,None)
                 if d is None or d.portal_type == "Faculty":
-                    #logger.info('Creating Department %(code)s, %(title)s' % mapping)
                     try:
-                        f.invokeFactory('Department', did)
+                        faculty.invokeFactory('Department', department_id)
                     except BadRequest,E:
                         msg =  "%s" % E
                         break
-                    d = getattr(f,did)
+                    d = getattr(faculty,department_id)
                     d.invokeFactory('CoursesFolder','courses')
                     courses = getattr(d,'courses')
@@ -480 +479 @@
                     d.getContent().edit(mapping=mapping)
             break
-        return did,msg,mapping
+        return department_id,msg,mapping
     ###)
 
     def edit(self,mapping): ###(
-        "create a department in the correct faculty"
+        "edit a department in the correct faculty"
         academics_folder = self.portal_url.getPortalObject().campus.academics
-        fid = mapping['faculty_code']
-        did = mapping.get('code')
+        faculty_id = mapping['faculty_code']
+        department_id = mapping.get('code','')
         msg = ''
         while True:
             try:
-                d = getattr(getattr(academics_folder,fid),did,None)
+                d = getattr(getattr(academics_folder,faculty_id),department_id,None)
             except KeyError:
-                msg =  "Department %s or Faculty %s wrong" % (did,fid)
-                break
-            else:
-                if d is None or d.portal_type == "Faculty":
-                    #logger.info('Editing Department %(code)s, %(title)s' % mapping)
-                    d.getContent().edit(mapping=mapping)
-            break
-        return did,msg,mapping
+                msg =  "Department %s or Faculty %s wrong" % (department_id,faculty_id)
+                break
+            if d is None or d.portal_type == "Faculty":
+                msg =  "Department %s not found" % (department_id)
+                break
+            d.getContent().edit(mapping=mapping)
+            break
+        return department_id,msg,mapping
     ###)
 ###)
@@ -512 +511 @@
         "create a faculty"
         academics_folder = self.portal_url.getPortalObject().campus.academics
-        fid = mapping['code']
-        msg = ''
-        while True:
-            if getattr(academics_folder,fid,None) is not None:
-                msg =  "Faculty with ID: %s exists" % fid
+        faculty_id = mapping.get('code','')
+        msg = ''
+        while True:
+            if faculty_id in academics_folder.objectIds():
+                msg =  "Faculty with ID: %s exists" % faculty_id
                 break
             logger.info('Creating Faculty %(code)s, %(title)s' % mapping)
             try:
-                academics_folder.invokeFactory('Faculty', fid)
+                academics_folder.invokeFactory('Faculty', faculty_id)
             except BadRequest,E:
                 msg =  "%s" % E
                 break
-            f = getattr(academics_folder,fid,None)
+            f = getattr(academics_folder,faculty_id,None)
             f.getContent().edit(mapping=mapping)
             break
-        return fid,msg,mapping
+        return faculty_id,msg,mapping
         ###)
 
@@ -533 +532 @@
         "edit a faculty"
         academics_folder = self.portal_url.getPortalObject().campus.academics
-        fid = mapping['code']
-        msg = ''
-        while True:
-            f = getattr(academics_folder,fid,None)
+        faculty_id = mapping['code']
+        msg = ''
+        while True:
+            f = getattr(academics_folder,faculty_id,None)
             if f is None:
-                msg =  "Faculty with ID: %s does not exist" % fid
+                msg =  "Faculty with ID: %s does not exist" % faculty_id
             f.getContent().edit(mapping=mapping)
             break
-        return fid,msg,mapping
+        return faculty_id,msg,mapping
     ###)
 ###)

WAeUP_SRP/base/WAeUPTool.py

@@ -1496 +1496 @@
             info['error'] = error
             mapping.update(info)
+            log_list = []
             if error:
                 digest = makeDigest(mapping,data_keys)
                 with_error_count += 1
-                percent_finished = (with_error_count + imported_count)/tti_float*100
+                #percent_finished = (with_error_count + imported_count)/tti_float*100
                 if digest not in pending_digests:
                     pending_digests += digest,
                     pending.append(mapping)
                     if not pending_only:
-                        logger.info("%(percent_finished)6.3f %% done added to %(pending_fn)s %(data_string)s" % vars())
+                        log_list += "added to %(pending_fn)s %(data_string)s" % vars(),
+                        #logger.info("%(percent_finished)6.3f %% done added to %(pending_fn)s %(data_string)s" % vars())
                 else:
                     already_in += 1
@@ -1512 +1514 @@
                 imported_count += 1
                 imported += mapping,
+                log_list += "imported and added to  %(imported_fn)s %(data_string)s" % vars(),
+                #percent_finished = (with_error_count + imported_count)/tti_float*100
+                #logger.info("%(percent_finished)6.3f %% done imported and added to  %(imported_fn)s %(data_string)s" % vars())
+            if log_list:
+                time_till_now = time.time() - elapse
                 percent_finished = (with_error_count + imported_count)/tti_float*100
-                logger.info("%(percent_finished)6.3f %% done  imported and added to  %(imported_fn)s %(data_string)s" % vars())
+                log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2f secs" % vars()),)
+                logger.info(' '.join(log_list))
             finished = count > total_to_import - 1
             must_commit = False
@@ -1557 +1565 @@
         msg += " total %(total_pending)d; " % vars()
         logger.info(msg)
+        os.remove(pending_tmp)
         return msg
     ###)

WAeUP_SRP/base/utils.py

@@ -26 +26 @@
     for key in data_keys:
         d[key] = str(item.get(key,''))
-    return md5.new(str(d)).digest()
+    return md5.new(str(d)).hexdigest()