Changeset 3014

Timestamp:

11 Jan 2008, 20:18:17 (17 years ago)

Author:

joachim

Message:

WAeUP LGA Widget uses new algorithm
ti_441_resolve_part1 + part2 also
part2 can now be used to rename lgas, a list of all not found lgas is
printed at the end.

Location:

WAeUP_SRP/base

Files:

• WAeUPTool.py (modified) (2 diffs)
• Widgets.py (modified) (2 diffs)
• skins/waeup_utilities/ti_441_resolve_part1.py (modified) (2 diffs)
• skins/waeup_utilities/ti_441_resolve_part2.py (modified) (5 diffs)

WAeUP_SRP/base/WAeUPTool.py

@@ -78 +78 @@
     def re_split(self,split_string,string):
         return re.split(split_string,string)
-
+    ###)
 
     security.declareProtected(View,'difference') ###(
     def difference(self,l1,l2):
         return set(l1).difference(set(l2))
+    ###)
 
     def rwrite(self,s): ###(
@@ -144 +145 @@
     ###)
 
+    security.declareProtected(View,'findLga') ###(
+    def findLga(self,words,words_dict):
+        words = re.split('[^a-zA-Z0-9/]',words)
+        lga_words = []
+        for word in words:
+            if word:
+                lga_words += word.strip().lower(),
+        lga_words.sort()
+        state_lga = ''
+        while not state_lga:
+            for k,l in words_dict.items():
+                if lga_words == l:
+                    state_lga = k
+                    break
+            break
+        return state_lga
+    ###)
     security.declareProtected(View,'getAccessInfo') ###(
     def getAccessInfo(self,context):

WAeUP_SRP/base/Widgets.py

@@ -580 +580 @@
 
     def _getLGAs(self):
-        voc = getattr(self.portal_vocabularies,self.vocabulary)
         if getattr(self,'_v_states',None) is not None and\
            getattr(self,'_v_lgas',None) is not None and\
-           getattr(self,'_v_d',None) is not None:
-            return (_v_d,_v_states,_v_lgas)
-        states = []
-        lgas  = []
-        d = {}
-        for k,v in voc.items():
-            parts = v.split(' / ')
-            if len(parts) == 1:
-                state = parts[0].lower()
-                lga = ""
-            elif len(parts) == 2:
-                state = parts[0].lower()
-                lga = "_".join(re.split('\W+',parts[1].lower()))
-            else:
-                continue
-            if state not in states:
-                states.append(state)
-            if lga not in lgas:
-                lgas.append(lga)
-            d[k] = v
-        self._v_d = d
-        self._v_state = state
-        self._v_lga = lga
-        return (d,states,lgas)
+           getattr(self,'_v_d',None) is not None and\
+           getattr(self,'_v_word_dict',None) is not None:
+            return (self._v_d,self._v_states,self._v_lgas,self._v_word_dict)
+        mapping = self.waeup_tool.getStatesLgas()
+        self._v_d = mapping['lga_dict']
+        self._v_states = mapping['states']
+        self._v_lgas = mapping['lgas']
+        self._v_word_dict = mapping['word_dict']
+        return (self._v_d,self._v_states,self._v_lgas,self._v_word_dict)
+
 
     def validate(self, datastructure, **kw):
@@ -619 +604 @@
             return 0
         v = v.lower()
-        combined,states,lgas = self._getLGAs()
+        combined,states,lgas,word_dict = self._getLGAs()
         datamodel = datastructure.getDataModel()
-        if not self.state_field and not self.lga_field:
-            if len(v) == 0 and self.is_required:
-                datastructure.setError(widget_id, "%s required" % widget_id)
+        if self.state_field and self.lga_field:
+            v = ' '.join((datastructure.get(self.state_field,""),
+                              datastructure.get(self.lga_field,"")))
+        state_lga = self.waeup_tool.findLga(v,word_dict)
+        if not state_lga:
+            if self.is_required:
+                datastructure.setError(widget_id, "'%s' not a valid lga key" % v)
+                #datastructure.setError(widget_id, "%s required" % widget_id)
                 return 0
-            elif v not in combined.keys():
-                datastructure.setError(widget_id, "'%s' not a valid lga key" % v)
-                return 0
-            datamodel[self.fields[0]] = v
-        else:
-            state = datastructure.get(self.state_field,"").lower()
-            lga = datastructure.get(self.lga_field,"").lower()
-            if widget_id == self.state_field:
-                if state not in states:
-                    datastructure.setError(widget_id, "'%s' not a valid state" % v)
-                    return 0
-            elif widget_id == self.lga_field:
-                if lga not in lgas:
-                    datastructure.setError(widget_id, "'%s' not a valid lga" % v)
-                    return 0
-            if len(state) == 0 or len(lga) == 0:
-                datastructure.setError(widget_id, "state AND lga must be given")
-                return 0
-            datamodel[self.fields[0]] =   state + "_" + lga
+            else:
+                state_lga = v
+        datamodel[self.fields[0]] = state_lga
         return 1
 

WAeUP_SRP/base/skins/waeup_utilities/ti_441_resolve_part1.py

@@ -57 +57 @@
     jamb_lga = getattr(getattr(students_folder,student_record.id),'application').getContent().jamb_lga
     words = ' '.join((jamb_state, jamb_lga))
-    words = re_split('[^a-zA-Z0-9/]',words)
-    lga_words = []
-    for word in words:
-        if word:
-            lga_words += word.lower(),
-    lga_words.sort()
-    state_lga = ''
-    while not state_lga:
-        for k,l in words_dict.items():
-            if lga_words == l:
-                state_lga = k
-                break
-        break
+    # words = re_split('[^a-zA-Z0-9/]',words)
+    # lga_words = []
+    # for word in words:
+    #     if word:
+    #         lga_words += word.lower(),
+    # lga_words.sort()
+    # state_lga = ''
+    # while not state_lga:
+    #     for k,l in words_dict.items():
+    #         if lga_words == l:
+    #             state_lga = k
+    #             break
+    #     break
+    state_lga = context.waeup_tool.findLga(words,words_dict)
     if state_lga:
         msg = "found %s is %s for %s " % (state_lga,student_record.lga,student_record.id)
@@ -78 +79 @@
             logger.info("found %d to correct state/lga combinations of %d so far" % (len(d),count))
     else:
-        msg = "no '%s' for %s" % (lga_words,student_record.id)
+        msg = "no '%s' for %s" % (words,student_record.id)
         logger.info(msg)
         #rwrite(msg)

WAeUP_SRP/base/skins/waeup_utilities/ti_441_resolve_part2.py

@@ -62 +62 @@
 #     return (d,states,lgas)
 
-difference = context.waeup_tool.difference
+#difference = context.waeup_tool.difference
 re_split = context.waeup_tool.re_split
 #logger.info('')
@@ -69 +69 @@
 logger.info("found %d students" % total)
 count = 0
+correctable = wrong = 1
 commit_after = 100
 logger.info('started to fix %d records in students_catalog' % total)
@@ -76 +77 @@
 words_dict = mapping['word_dict']
 d = {}
+not_found_lgas = []
+not_found_count = {}
+rename_lgas = {}
+rename_lgas['edo_orhionmwon'] = 'edo_ohionmwon'
 for student_record in student_records:
     # skip record without lga value
@@ -87 +92 @@
         continue
     #rwrite("%s: %s" % (student_record.id,student_record.lga))
+    if student_record.lga in rename_lgas.keys():
+        correctable += 1
+        d[student_record.id] = rename_lgas[student_record.lga]
+        if len(d) and not len(d)  % commit_after:
+            logger.info("found %d to correct state/lga combinations of %d so far" % (len(d),count))
+        continue
     words = student_record.lga 
     if len(words.split(' / ')) == 2:
         words = words.replace(' / ',' ')
-    words = re_split('[^a-zA-Z0-9/]',words)
-    lga_words = []
-    for word in words:
-        if word:
-            lga_words += word.lower(),
-    lga_words.sort()
-    # state = sl[0].lower().strip()
-    # state = '_'.join(state.split())
-    # lga = sl[1].lower().strip()
-    # if not lga or not state:
-    #     logger.info("found invalid lga %s of %s" % (student_record.lga,student_record.id))
-    #     continue
-    state_lga = ''
-    while not state_lga:
-        for k,l in words_dict.items():
-            if lga_words == l:
-                state_lga = k
-                break
-        break
+    state_lga = context.waeup_tool.findLga(words,words_dict)
     if state_lga:
         if state_lga != student_record.lga:
+            correctable += 1
             msg = "found %s is %s for %s " % (state_lga,student_record.lga,student_record.id)
             #rwrite(msg)
@@ -119 +113 @@
             msg = "already corrected %s : %s for %s " % (state_lga,student_record.lga,student_record.id)
     else:
-        msg = "no '%s' for %s" % (lga_words,student_record.id)
+        wrong += 1
+        msg = "no '%s' for %s" % (words,student_record.id)
+        if len(words) > 2:
+            if words not in not_found_lgas:
+                not_found_lgas += words,
+                not_found_count[words] = 1
+            else:
+                nfc = not_found_count[words] 
+                nfc += 1
+                not_found_count[words] = nfc 
+        #rwrite(msg)
     # if count > 2000:
     #     break
+not_found_lgas.sort()
+for lga in not_found_lgas:
+    msg = "not found %s count: %d" % (lga,not_found_count[lga])
+    logger.info(msg)
 to_edit = len(d)
-logger.info("found %d correct state/lga combinations of %d" % (to_edit,count))
-
+logger.info("found %d correctable state/lga combinations, not correctable %d total %d" % (correctable,
+                                                                                          wrong,
+                                                                                          count))
 edited = 1
 for student_id,lga in d.items():