Changeset 3133

Timestamp:

9 Feb 2008, 09:15:05 (17 years ago)

Author:

joachim

Message:

implement new functionality im importData:

• for verdicts and course_results now records, which return an "id" and

"error" in their mass_edit-function are collected into files named:

verdict_pending.csv for verdicts
or course_result_pending.csv for course_results

These files are checked first, when importing new data, and importable
records are imported and removed from this file.

There are also the options:

Pending Verdicts only
Pending CourseResults? only

to import the pending records only.

The input file is irrelevant then.

File:

• WAeUP_SRP/base/WAeUPTool.py (modified) (4 diffs)

WAeUP_SRP/base/WAeUPTool.py

@@ -1596 +1596 @@
     security.declareProtected(ModifyPortalContent,'mass_edit_course_result') ###(
     def mass_edit_course_result(self,mapping):
-        id_key = ''
-        for id_key in ('student_id','matric_no'):
-            id_field = mapping.get(id_key,None)
-            if id_field is not None:
-                student_id = id_field
+        while True:
+            result_id = mapping.get('id',None)
+            if result_id is not None:
+                mapping['key'] = key = result_id
                 break
-        query = Eq(id_key,id_field)
-        res = self.students_catalog.evalAdvancedQuery(query)
-        if not res:
-            return '', "No student with %(id_key)s %(id_field)s" % vars()
-        if id_field != "student_id":
-            mapping['student_id'] = res[0].id
-        if not res:
-            return '', "No student with %(id_key)s %(id_field)s" % vars()
-        student_rec = res[0]
-        student_id = student_rec.id
-        level_id = mapping['level_id']
-        code = mapping['code']
-        mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
+            for id_key in ('student_id','matric_no'):
+                id_field = mapping.get(id_key,None)
+                if id_field is not None:
+                    student_id = id_field
+                    break
+            query = Eq(id_key,id_field)
+            res = self.students_catalog.evalAdvancedQuery(query)
+            if not res:
+                return '', "No student with %(id_key)s %(id_field)s" % vars()
+            if id_field != "student_id":
+                mapping['student_id'] = res[0].id
+            if not res:
+                return '', "No student with %(id_key)s %(id_field)s" % vars()
+            student_rec = res[0]
+            student_id = student_rec.id
+            level_id = mapping['level_id']
+            code = mapping['code']
+            mapping['key'] = key = "%(student_id)s|%(level_id)s|%(code)s" % vars()
+            break
         try:
             self.course_results.modifyRecord(**mapping)
         except KeyError:
-            return '', "No course result to edit: %s" % key
+            return key, "No course result to edit: %s" % key
         return key,''
     ###)
@@ -1969 +1974 @@
             return '',"no id or matric_no specified"
         student_id = student_record.id
-        student_review_state =  student_record.review_state
-        if student_review_state not in ('courses_validated','returning'):
-            return '',"student %s in wrong review_state %s" % (student_id, student_review_state)
         student_obj = getattr(students_folder,student_id)
         f2t = self.field2types_student
@@ -1980 +1982 @@
         current_session = d['session'] = mapping.get('current_session','')
         if current_session and student_record.session != current_session:
-            return '','student %s: imported session %s does not match current_session %s' % (student_id,
+            return student_id,'student %s: imported session %s does not match current_session %s' % (student_id,
                                                                                         current_session,
                                                                                         student_record.session)
         current_level = mapping.get('current_level','')
+        if not current_level.isdigit():
+            return '','student %s: imported level is empty' % (student_id,)
         if current_level and student_record.level != current_level:
-            return '','student %s: imported level %s does not match current_level %s' % (student_id,
+            return student_id,'student %s: imported level %s does not match current_level %s' % (student_id,
                                                                                     current_level,
                                                                                     student_record.level)
+        student_review_state =  student_record.review_state
+        if student_review_state not in ('courses_validated','returning'):
+            return student_id,"student %s in wrong review_state %s" % (student_id, student_review_state)
         study_course_obj = getattr(student_obj,'study_course',None)
         if study_course_obj is None:
@@ -2020 +2027 @@
     security.declareProtected(ModifyPortalContent,"importData")###(
     def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
+        """load data from CSV values"""
+        import transaction
+        import random
+        member = self.portal_membership.getAuthenticatedMember()
+        logger = logging.getLogger('WAeUPTool.importData')
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        students_folder = self.portal_url.getPortalObject().campus.students
+        start = True
+        tr_count = 0
+        total_imported = 0
+        total_not_imported = 0
+        total = 0
+        pending_only = False
+        pend_str = '_pending'
+        if name.endswith(pend_str):
+            pending_only = True
+            name = name[:-len(pend_str)]
+        iname = "import_%s" % name
+        if name in ('application','course_result',):
+            commit_after = 2000
+        else:
+            commit_after = 100
+        stool = getToolByName(self, 'portal_schemas')
+        ltool = getToolByName(self, 'portal_layouts')
+        schema = stool._getOb(iname)
+        if schema is None:
+            em = 'No such schema %s' % iname
+            logger.error('No such schema %s' % iname)
+            return em
+        layout = ltool._getOb(iname)
+        if layout is None:
+            em = 'No such layout %s' % iname
+            logger.error(em)
+            return em
+        validators = {}
+        for widget in layout.keys():
+            validators[widget] = layout[widget].validate
+        mode = "create"
+        if edit:
+            if filename.endswith('_toDelete'):
+                mode = "delete"
+            else:
+                mode = "edit"
+        importer_name = "mass_%(mode)s_%(name)s" % vars()
+        importer = getattr(self, '%s' % importer_name,None)
+        if importer is None:
+            em = 'No importer function %s' % importer_name
+            logger.error(em)
+            return em
+        pending_fn = "%s/import/%s_pending.csv" % (i_home,name)
+        if pending_only:
+            import_source_fn = pending_fn
+            imported_fn = "%s/import/%s_pending_imported%s.csv" % (i_home,name,current)
+            not_imported_fn = "%s/import/%s_pending_not_imported%s.csv" % (i_home,name,current)
+            attrs = csv.reader(open(pending_fn,"rb")).next()
+        else:
+            import_source_fn = "%s/import/%s.csv" % (i_home,filename)
+            imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current)
+            not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
+            attrs = csv.reader(open(import_source_fn,"rb")).next()
+        import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore') 
+                                                        or k.strip() == 'Error')]
+        diff2schema = set(import_keys).difference(set(schema.keys()))
+        diff2layout = set(import_keys).difference(set(layout.keys()))
+        if diff2schema and diff2schema != set(['id',]):
+            #import pdb;pdb.set_trace()
+            msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema)
+            return msg
+        if mode == "create":
+            required_keys = [layout.getIdUnprefixed(id)
+                                for id,widget in layout.objectItems()
+                                if widget.is_required]
+            if not set(required_keys).issubset(set(import_keys)):
+                diff2import = set(required_keys).difference(set(import_keys))
+                msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import)
+                return msg
+        #
+        # not_imported
+        #
+        not_imported_keys = import_keys + ["Error",]
+        not_imported_file = open(not_imported_fn,"a")
+        not_imported_csv_writer = csv.DictWriter(not_imported_file,
+                                                    not_imported_keys,
+                                                    extrasaction='ignore')
+        mapping = dict((k,k) for k in not_imported_keys)
+        not_imported_csv_writer.writerow(mapping)
+        not_imported = []
+        #
+        # imported
+        #
+        imported_keys = import_keys[:]
+        if 'id' not in imported_keys:
+            imported_keys.insert(0,"id")
+        imported_file = open(imported_fn,"a")
+        imported_csv_writer = csv.DictWriter(imported_file,
+                                             imported_keys,
+                                             extrasaction='ignore')
+        mapping = dict((k,k) for k in imported_keys)
+        imported_csv_writer.writerow(mapping)
+        imported = []
+        #
+        # pending
+        #
+        pending_records = {}
+        imports_pending = 0
+        total_pending = 0
+        format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
+        format_error = format + ',"%(Error)s"'
+        #format = '"%(id)s",'+ format
+        imported = []
+        if name in ('verdict','course_result',):
+            #global pending_records
+            pending_keys = imported_keys[:]
+            pending_keys += "Error",
+            pending_records = {}
+            if os.path.exists(pending_fn):
+                pending_imports_file = open(pending_fn)
+                pending_imports = csv.DictReader(pending_imports_file,
+                                                 fieldnames=pending_keys)
+                if pending_imports:
+                    pending_records = dict((row['id'],row) for row in pending_imports
+                                          if row['id'] != 'id')
+                pending_imports_file.close()
+            new_imports_pending = 0
+            pending_imported = []
+            for item in pending_records.values():
+                id,error = importer(item)
+                if error:
+                    continue
+                else:
+                    if item.has_key('Error'):
+                        del item['Error']
+                    imported.append(item)
+                    msg = format % item
+                    logger.info("imported from %(pending_fn)s %(msg)s" % vars())
+                    pending_imported += id,
+                    continue
+            total_imported = len(pending_imported)
+            logger.info("imported %d pending imports" % len(pending_imported))
+            for id in pending_imported:
+                del pending_records[id]
+            pending_imports_file = open(pending_fn,'w')
+            pending_csv_writer = csv.DictWriter(pending_imports_file,
+                                                pending_keys,
+                                                extrasaction='ignore')
+            mapping = dict((k,k) for k in pending_keys)
+            pending_csv_writer.writerow(mapping)
+        #
+        if pending_only:
+            items = []
+        else:
+            try:
+                items = csv.DictReader(open(import_source_fn,"rb"))
+            except:
+                msg = 'Error reading %s.csv' % filename
+                logger.error(msg)
+                return msg
+            not_imported = []
+        for item in items:
+            item = dict((k.strip(),l.strip()) for (k,l) in item.items())
+            if start:
+                start = False
+                adapters = [MappingStorageAdapter(schema, item)]
+                logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname))
+            dm = DataModel(item, adapters,context=self)
+            ds = DataStructure(data=item,datamodel=dm)
+            error_string = ""
+            total += 1
+            error_count = 0
+            for k in import_keys:
+                if not validators[k](ds,mode=mode):
+                    if error_count:
+                        error_string += ' ++ '
+                    error_string += "%s: %s" % (k,self.translation_service(ds.getError(k),
+                                                                           ds.getErrorMapping(k)))
+                    error_count += 1
+            if error_string:
+                item['Error'] = error_string
+                not_imported.append(item)
+                total_not_imported += 1
+                continue
+            temp_item = item.copy()
+            temp_item.update(dm)
+            id,error = importer(temp_item)
+            if error:
+                item['Error'] = error
+                if id and id not in pending_records.keys():
+                    item['id'] = id
+                    pending_records[id] = item
+                    msg = format_error % item
+                    logger.info("%(id)s added to pending %(msg)s" % vars())
+                    new_imports_pending += 1
+                    continue
+                else:
+                    not_imported.append(item)
+                    total_not_imported += 1
+                    continue
+            item = temp_item
+            item['id'] = id
+            imported.append(item)
+            tr_count += 1
+            total_imported += 1
+            msg = format % item
+            logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars())
+            if total and not total % commit_after:
+                transaction.commit()
+                if len(imported) > 0:
+                    imported_csv_writer.writerows(imported)
+                    imported = []
+                if len(not_imported) > 0:
+                    not_imported_csv_writer.writerows(not_imported)
+                    not_imported = []
+                if len(pending_records) > 0:
+                    pending_csv_writer.writerows(pending_records.values())
+                    total_pending += len(pending_records)
+                    pending_records = {}
+                msg = '%d transactions committed\n' % (tr_count)
+                regs = []
+                logger.info(msg)
+                tr_count = 0
+        if len(imported) > 0:
+            imported_csv_writer.writerows(imported)
+        if len(not_imported) > 0:
+            not_imported_csv_writer.writerows(not_imported)
+        if len(pending_records) > 0:
+            pending_csv_writer.writerows(pending_records.values())
+            total_pending += len(pending_records)
+        msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars()
+        msg += "%(total_imported)d imported, %(total_not_imported)d not imported " % vars()
+        if total_pending:
+            msg += "%(new_imports_pending)d pending added %(total_pending)d total pending in %(pending_fn)s " % vars()
+        msg += "(of total %(total)d)" % vars()
+        logger.info(msg)
+        return msg
+    ###)
+
+    def old_importData(self,filename,name,edit=False,bypass_queue_catalog=False):###(
         """load data from CSV values"""
         import transaction