Changeset 9209 for main/waeup.kofa/branches/uli-zc-async/tools

Timestamp:

20 Sep 2012, 08:49:37 (13 years ago)

Author:

uli

Message:

Merge changes from update branch (includes trunk changes until r9107).

Location:

main/waeup.kofa/branches/uli-zc-async

Files:

• . (modified) (1 prop)
• tools/copy_media_files.py (modified) (1 diff)
• tools/download_exports.sh (copied) (copied from main/waeup.kofa/branches/uli-async-update/tools/download_exports.sh)
• tools/fix_import_file.py (modified) (21 diffs)
• tools/reload_portraits.py (copied) (copied from main/waeup.kofa/branches/uli-async-update/tools/reload_portraits.py)

main/waeup.kofa/branches/uli-zc-async/tools/copy_media_files.py

@@ -78 +78 @@
     return
 
-def copy_file(src, dst):
+def copy_file(file_src, file_dst):
     create_path(os.path.dirname(dst))
     print "COPY FILE: %s -> %s" % (file_src, file_dst)

main/waeup.kofa/branches/uli-zc-async/tools/fix_import_file.py

@@ -65 +65 @@
     'sex': 'gender',
     'birthday': 'date',
-    'request_date': 'datetime',
     'marit_stat': 'marit_stat',
     'session': 'session',
@@ -85 +84 @@
     'lga': 'lga',
     'order_id': 'no_int',
+    'uniben': 'former',
+    'nysc_year': 'year',
+    'alr_date': 'date',
+    'fst_sit_date': 'date',
+    'scd_sit_date': 'date',
+    'emp_start': 'date',
+    'emp_end': 'date',
+    'emp_start2': 'date',
+    'emp_end2': 'date',
+    'fst_sit_results': 'result',
+    'scd_sit_results': 'result',
+    'alr_results': 'result',
+    'email': 'email',
+    'fst_sit_type': 'sittype',
+    'scd_sit_type': 'sittype',
+    'resp_pay_reference': 'no_int',
+    'type': 'company',
+    'date': 'date',
+    'core_or_elective': 'bool',
+    'category': 'p_category',
+    'reg_transition': 'reg_state',  # we completely change this column,
+                                    # since reg_state import is usually intended
+    'transition': 'reg_transition',
+    'payment_date': 'date',
+    'validation_date': 'date',
     }
 
@@ -90 +114 @@
 COLNAME_MAPPING = {
     # base data
+    'id': 'student_id',
     'reg_state': 'state',
+    'reg_transition': 'state',
     'jamb_reg_no': 'reg_number',
     'matric_no': 'matric_number',
     'birthday': 'date_of_birth',
     'clr_ac_pin': 'clr_code',
+    # clearance
+    'hq_grade': 'hq_degree',
+    'uniben': 'former_matric',
+    'hq_type2': 'hq2_type',
+    'hq_grade2': 'hq2_degree',
+    'hq_school2': 'hq2_school',
+    'hq_matric_no2': 'hq2_matric_no',
+    'hq_session2': 'hq2_session',
+    'hq_disc2': 'hq2_disc',
+    'emp': 'employer',
+    'emp2': 'employer2',
+    'emp_position2': 'emp2_position',
+    'emp_start2': 'emp2_start',
+    'emp_end2': 'emp2_end',
+    'emp_reason2': 'emp2_reason',
     # study course
     'study_course': 'certificate',
@@ -117 +158 @@
     'surcharge': 'surcharge_1',
     'session_id': 'p_session',
+    'type': 'r_company',
     }
 
@@ -131 +173 @@
     'courses_registered': 'courses registered',
     'courses_validated': 'courses validated',
+    'admit': 'admitted',
+    'return': 'returning'
     }
 
@@ -143 +187 @@
     """Replace input fieldnames by fieldnames of COLNAME_MAPPING.
     """
-    header = dict([(name, name) for name in fieldnames])
+    # Remove whitespaces
+    header = dict([(name, name.strip()) for name in fieldnames])
     for in_name, out_name in COLNAME_MAPPING.items():
-        if in_name not in header:
+        if in_name not in header.values():
             continue
-        header[in_name] = out_name
+        # Inverse dictionary lookup
+        key = [key for key,value in header.items() if value==in_name][0]
+        header[key] = out_name
     return header
 
@@ -155 +202 @@
 
     @classmethod
-    def student_id(self, value):
+    def student_id(self, value, row):
         """ 'A123456' --> 'EA123456'
         """
         if len(value) == 7:
-            return 'E' + value
-        return value
-
-    @classmethod
-    def reg_state(self, value):
+            return 'K' + value
+        return value
+
+    @classmethod
+    def reg_state(self, value, row):
         """ 'courses_validated' --> 'courses validated'
         """
@@ -169 +216 @@
 
     @classmethod
-    def level(self, value):
+    def reg_transition(self, value, row):
+        if value == "admitted":
+            return "admit"
+        if value == "returning":
+            return "return"
+        return value
+
+    @classmethod
+    def level(self, value, row):
         """ '000' --> '10'
+        '800' --> '999' if pg student
         """
         try:
@@ -178 +234 @@
         if number == 0:
             return 10
+        if row.get('entry_mode') and row.get('entry_mode').startswith('pg'):
+            return 999
         return number
 
     @classmethod
-    def semester(self, value):
+    def semester(self, value, row):
         """ '0' --> '9'
         """
@@ -193 +251 @@
 
     @classmethod
-    def application_category(self, value):
+    def application_category(self, value, row):
         """ '' --> 'no'
         """
@@ -201 +259 @@
 
     @classmethod
-    def lga(self, value):
+    def lga(self, value, row):
         """ Remove apostrophe
         """
         if value == 'akwa_ibom_uru_offong_oruko':
             return 'akwa_ibom_urue-offong-oruko'
+        if value == 'edo_ohionmwon':
+            return 'edo_orhionmwon'
+
+        if value == 'nassarawa_nassarawa':
+            return 'nassarawa_nassawara'
+
+        if value == 'kogi_mopa-muro-mopi':
+            return 'kogi_mopa-muro'
+
+        if value == 'delta_osimili-north':
+            return 'delta_oshielli-north'
+
+        if value == 'delta_osimili':
+            return 'delta_oshimili'
+
+        if value == 'delta_osimili-south':
+            return 'delta_oshimili-south'
         try:
             value = value.replace("'","")
@@ -220 +295 @@
 
     @classmethod
-    def session(self, value):
+    def session(self, value, row):
         """ '08' --> '2008'
         '2008/2009' --> '2008'
@@ -244 +319 @@
 
     @classmethod
-    def marit_stat(self, value):
+    def former(self, value, row):
+        """ True --> yes
+        '2008/2009' --> '2008'
+        """
+        if value == 'True':
+            return 'yes'
+        return
+
+    @classmethod
+    def bool(self, value, row):
+        """ True --> 1
+        """
+        if value in ('TRUE', 'True'):
+            return '1'
+        elif value in ('FALSE', 'False'):
+            return '0'
+        return
+
+    @classmethod
+    def year(self, value, row):
+        """ '0' --> ''
+        """
+        if value == '0':
+            return
+        if value == 'None':
+            return
+        return value
+
+
+    @classmethod
+    def marit_stat(self, value, row):
         """ 'True'/'False' --> 'married'/'unmarried'
         """
@@ -256 +361 @@
 
     @classmethod
-    def gender(self, value):
+    def gender(self, value, row):
         """ 'True'/'False' --> 'f'/'m'
         """
@@ -268 +373 @@
 
     @classmethod
-    def date(self, value):
+    def date(self, value, row):
         """ 'yyyy/mm/dd' --> 'yyyy-mm-dd'
         """
@@ -283 +388 @@
 
     @classmethod
-    def no_int(self, value):
-        """ Add hash.
+    def no_int(self, value, row):
+        """ Add hash and skip numbers starting with 999999
         """
         # We add the hash symbol to avoid automatic number transformation
         # in Excel and Calc for further processing
-        value += '#'
-        return value
-
-
-    @classmethod
-    def datetime(self, value):
-        """ 'yyyy/mm/dd' --> 'yyyy-mm-dd'
-        """
-        #print  "IN: ", value
-        if value == "None":
-            value = ""
-        elif value == "":
-            value = ""
-        else:
-            #value = datetime.datetime.strptime(value, '%Y/%m/%d')
-            #value = datetime.datetime.strftime(value, '%Y-%m-%d')
+        try:
+            intvalue = int(value)
+            value += '#'
+        except:
             pass
-        #print "OUT: ", value
-        return value
-
-    @classmethod
-    def mode(self, value):
+        if value.startswith('999999'):
+            return
+        return value
+
+    @classmethod
+    def mode(self, value, row):
         if value == "transfer_fulltime":
             return "transfer_ft"
@@ -317 +411 @@
 
     @classmethod
-    def password(self, value):
+    def password(self, value, row):
         if value == "not set":
             return ""
@@ -323 +417 @@
 
     @classmethod
-    def nationality(self, value):
+    def nationality(self, value, row):
         if value == "nigeria":
             return "NG"
-        return value
-
-    @classmethod
-    def phone(self, value):
+        if value == "niger":
+            return "NE"
+        return value
+
+    @classmethod
+    def sittype(self, value, row):
+        if value == "nabtec":
+            return "nabteb"
+        return value
+
+    @classmethod
+    def company(self, value, row):
+        if value == "online":
+            return "interswitch"
+        return value
+
+    @classmethod
+    def p_category(self, value, row):
+        if value == "acceptance":
+            return "clearance"
+        return value
+
+    @classmethod
+    def email(self, value, row):
+        return value.strip()
+
+    @classmethod
+    def phone(self, value, row):
         """ '<num-seq1>-<num-seq2> asd' -> '--<num-seq1><num-seq2>'
 
@@ -335 +453 @@
         of numbers.
         """
+        if not value:
+            return
+        value = value.strip('#')
         value = value.replace('-', '')
         value = value.replace('/', '')
         match = RE_PHONE.match(value)
         phone = match.groups()[0]
-        value = '-%s' % phone
-        return value
+        if value.startswith('234'):
+            value = '+' + value[:3] + '-' + value[3:]
+        else:
+            value = '-%s' % phone
+        return value + '#'
+
+    @classmethod
+    def result(self, value, row):
+        try:
+            liste = eval(value)
+        except:
+            return
+        if isinstance(liste,list):
+            return [(i[0].lower(),i[1]) for i in liste]
+        return
 
 
@@ -357 +491 @@
             header = convert_fieldnames(reader.fieldnames)
             writer.writerow(header)
+        if row.get('reg_state') == 'student_created':
+            # We do not reimport student records which have never been accessed.
+            continue
+        if row.get('status') == 'started':
+            # We do not reimport started payments.
+            continue
         for key, value in row.items():
+            # Remove unwanted whitespaces.
+            row[key] = row[key].strip()
             if not key in OPTIONS.keys():
                 continue
@@ -365 +507 @@
                 print "WARNING: cannot find converter %s" % conv_name
                 continue
-            row[key] = converter(row[key])
+            row[key] = converter(row[key], row)
         try:
             writer.writerow(row)