Changeset 2185

Timestamp:

7 Sep 2007, 12:15:45 (17 years ago)

Author:

joachim

Message:

ignore empty lga, state fields, headings can now start with spaces

Location:

WAeUP_SRP/trunk

Files:

• WAeUPTables.py (modified) (7 diffs)
• Widgets.py (modified) (1 diff)

WAeUP_SRP/trunk/WAeUPTables.py

@@ -167 +167 @@
 ###)
 
-    security.declarePrivate("_import") ###(
-    def _import(self,filename,schema,layout, mode,logger):
+    security.declarePrivate("_import_old") ###(
+    def _import_old(self,filename,schema,layout, mode,logger):
         "import data from csv"
         import transaction
@@ -219 +219 @@
         #     importer = None
         try:
-            items = csv.DictReader(open(import_fn,"rb"))
+            items = csv.DictReader(open(import_fn,"rb"),
+                                   dialect="excel",
+                                   skipinitialspace=True)
         except:
             em = 'Error reading %s.csv' % filename
             logger.error(em)
             return d
+        #import pdb;pdb.set_trace()
         for item in items:
             if start:
@@ -229 +232 @@
                 logger.info('%s starts import from %s.csv' % (member,filename))
                 #import_keys = [k for k in item.keys() if not k.startswith('ignore')]
-                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
+                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb"),
+                                   dialect="excel",
+                                   skipinitialspace=True).next()
                 import_keys = [k for k in attrs if not (k.startswith('ignore') or k.isupper())]
                 diff2schema = set(import_keys).difference(set(schema.keys()))
@@ -269 +274 @@
                 total_imported += 1
             total += 1
-            # if importer and total_imported and not total_imported % 1000:
-            #     transaction.commit()
-            #     if len(not_imported) > 0:
-            #         open(not_imported_fn,"a").write('\n'.join(not_imported) + '\n')
-            #         not_imported = []
-            #     if len(imported) > 0:
-            #         open(imported_fn,"a").write('\n'.join(imported) + '\n')
-            #         imported = []
-            #     em = '%d transactions committed\n' % (tr_count)
-            #     regs = []
-            #     logger.info(em)
-            #     tr_count = 0
         if len(imported) > 0:
             open(imported_fn,"a").write('\n'.join(imported))
@@ -295 +288 @@
         return d
     ###)
+
+    security.declarePrivate("_import") ###(
+    def _import_new(self,csv_items,schema, layout, mode,logger):
+        "import data from csv.Dictreader Instance"
+        start = True
+        tr_count = 1
+        total_imported = 0
+        total_not_imported = 0
+        total = 0
+        iname =  "%s" % filename
+        not_imported = []
+        valid_records = []
+        invalid_records = []
+        duplicate_records = []
+        d = {}
+        d['mode'] = mode
+        d['valid_records'] = valid_records
+        d['invalid_records'] = invalid_records
+        d['invalid_records'] = duplicate_records
+        # d['import_fn'] = import_fn
+        # d['imported_fn'] = imported_fn
+        # d['not_imported_fn'] = not_imported_fn
+        validators = {}
+        for widget in layout.keys():
+            try:
+                validators[widget] = layout[widget].validate
+            except AttributeError:
+                logger.info('%s has no validate attribute' % widget)
+                return d
+        for item in csv_items:
+            if start:
+                start = False
+                logger.info('%s starts import from %s.csv' % (member,filename))
+                #import_keys = [k for k in item.keys() if not k.startswith('ignore')]
+                attrs = csv.reader(open("%s/import/%s.csv" % (i_home,filename),"rb")).next()
+                import_keys = [k for k in attrs if not (k.startswith('ignore') or k.isupper())]
+                diff2schema = set(import_keys).difference(set(schema.keys()))
+                diff2layout = set(import_keys).difference(set(layout.keys()))
+                if diff2layout:
+                    em = "not ignorable key(s) %s found in heading" % diff2layout
+                    logger.info(em)
+                    return d
+                # s = ','.join(['"%s"' % fn for fn in import_keys])
+                # open(not_imported_fn,"a").write(s + ',"Error"'+ '\n')
+                # #s = '"id",' + s
+                # open(imported_fn,"a").write(s + '\n')
+                # format = ','.join(['"%%(%s)s"' % fn for fn in import_keys])
+                # format_error = format + ',"%(Error)s"'
+                # #format = '"%(id)s",'+ format
+                adapters = [MappingStorageAdapter(schema, item)]
+            dm = DataModel(item, adapters,context=self)
+            ds = DataStructure(data=item,datamodel=dm)
+            error_string = ""
+            for k in import_keys:
+                if not validators[k](ds,mode=mode):
+                    error_string += " %s : %s" % (k,ds.getError(k))
+            if error_string:
+                item['Error'] = error_string
+                #invalid_records.append(dm)
+                invalid_records.append(item)
+                total_not_imported += 1
+            else:
+                em = format % item
+                valid_records.append(dm)
+                #logger.info("%(total_imported)d of %(total)d %(em)s" % vars())
+                tr_count += 1
+                total_imported += 1
+            total += 1
+        # if len(imported) > 0:
+        #     open(imported_fn,"a").write('\n'.join(imported))
+        # if len(not_imported) > 0:
+        #     open(not_imported_fn,"a").write('\n'.join(not_imported))
+        #em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
+        d['imported'] = total_imported
+        d['not_imported'] = total_not_imported
+        d['valid_records'] = valid_records
+        d['invalid_records'] = invalid_records
+        return d
+    ###)
+
 ###)
 
@@ -406 +479 @@
         WAeUPTable.__init__(self, name)
 
+    security.declareProtected(ModifyPortalContent,"new_importCSV")###(
+    def new_importCSV(self,filename="JAMB_data",
+                  schema_id="application",
+                  layout_id="application_import",
+                  mode='add'):
+        """ import JAMB data """
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        pm = self.portal_membership
+        member = pm.getAuthenticatedMember()
+        logger = logging.getLogger('WAeUPTables.ApplicantsCatalog.importCSV')
+        lock_fn = "%s/import/%s_import_lock" % (i_home,filename)
+        import_fn = "%s/import/%s.csv" % (i_home,filename)
+        if mode not in ('add','edit'):
+            logger.info("invalid mode: %s" % mode)
+        if os.path.exists(lock_fn):
+            logger.info("import of %(import_fn)s already in progress" % vars())
+            return
+        lock_file = open(lock_fn,"w")
+        lock_file.write("%(current)s \n" % vars())
+        lock_file.close()
+        invalid_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
+        duplicate_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current)
+        stool = getToolByName(self, 'portal_schemas')
+        ltool = getToolByName(self, 'portal_layouts')
+        schema = stool._getOb(schema_id)
+        if schema is None:
+            em = 'No such schema %s' % schema_id
+            logger.error(em)
+            return
+        for postfix in ('_import',''):
+            layout_name = "%(layout_id)s%(postfix)s" % vars()
+            if hasattr(ltool,layout_name):
+                break
+        layout = ltool._getOb(layout_name)
+        if layout is None:
+            em = 'No such layout %s' % layout_id
+            logger.error(em)
+            return
+        try:
+            csv_file = csv.DictReader(open(import_fn,"rb"))
+        except:
+            em = 'Error reading %s.csv' % filename
+            logger.error(em)
+            return 
+        d = self._import_new(csv_items,schema,layout,mode,logger)
+        imported = []
+        edited = []
+        duplicates = []
+        not_found = []
+        if len(d['valid_records']) > 0:
+            for record in d['valid_records']:
+                #import pdb;pdb.set_trace()
+                if mode == "add":
+                    try:
+                        self.addRecord(**dict(record.items()))
+                        imported.append(**dict(record.items()))
+                        logger.info("added %s" % record.items())
+                    except ValueError:
+                        dupplicate.append(**dict(record.items()))
+                        logger.info("duplicate %s" % record.items())
+                elif mode == "edit":
+                    try:
+                        self.modifyRecord(**dict(record.items()))
+                        edited.append(**dict(record.items()))
+                        logger.info("edited %s" % record.items())
+                    except KeyError:
+                        not_found.append(**dict(record.items()))
+                        logger.info("not found %s" % record.items())
+        invalid = d['invalid_records']
+        for itype in ("imported","edited","not_found","duplicate","invalid"):
+            outlist = locals[itype]
+            if len(outlist):
+                d = {}
+                for k in outlist[0].keys():
+                    d[k] = k
+                outlist[0] = d 
+                outfile = open("file_name_%s" % itype,'w')
+                csv.DictWriter(outfile,outlist[0].keys()).writerows(outlist)
+                logger.info("wrote %(itype)s records to %(, written to %(not_imported_fn)s" % d)
+###)
+
     security.declareProtected(ModifyPortalContent,"importCSV")###(
     def importCSV(self,filename="JAMB_data",
@@ -425 +579 @@
             return
         logger = logging.getLogger('WAeUPTables.ApplicantsCatalog.importCSV')
-        d = self._import(filename,schema,layout,mode,logger)
+        d = self._import_old(filename,schema,layout,mode,logger)
         if len(d['valid_records']) > 0:
             for record in d['valid_records']:

WAeUP_SRP/trunk/Widgets.py

@@ -516 +516 @@
         combined,states,lgas = self._getLGAs()
         one_field = False
-        if not self.state_field and not self.lga_field:
+        if len(v) == 0:
+            pass
+        elif not self.state_field and not self.lga_field:
             one_field = True
             if v not in combined.keys():