Index: main/waeup.kofa/branches/uli-zc-async/tools/copy_media_files.py
===================================================================
--- main/waeup.kofa/branches/uli-zc-async/tools/copy_media_files.py	(revision 8786)
+++ main/waeup.kofa/branches/uli-zc-async/tools/copy_media_files.py	(revision 9209)
@@ -78,5 +78,5 @@
     return
 
-def copy_file(src, dst):
+def copy_file(file_src, file_dst):
     create_path(os.path.dirname(dst))
     print "COPY FILE: %s -> %s" % (file_src, file_dst)
Index: main/waeup.kofa/branches/uli-zc-async/tools/download_exports.sh
===================================================================
--- main/waeup.kofa/branches/uli-zc-async/tools/download_exports.sh	(revision 9209)
+++ main/waeup.kofa/branches/uli-zc-async/tools/download_exports.sh	(revision 9209)
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+## We expect at least three arguments...
+if [ $# -ne 3 ]; then
+    echo "Usage: $0 [user name] [password] [exporter]"
+    exit
+fi
+
+wget --save-cookies cookies.txt --keep-session-cookies --post-data "form.login=$1&form.password=$2" https://uniben-kofa.waeup.org/login 
+wget --load-cookies cookies.txt  --output-document $3.csv https://uniben-kofa.waeup.org/datacenter/export.csv?exporter=$3
+tar -czf Kofa$3.tar.gz $3.csv
+rm $3.csv
+mv Kofa$3.tar.gz /home/ftp/uniben
Index: main/waeup.kofa/branches/uli-zc-async/tools/fix_import_file.py
===================================================================
--- main/waeup.kofa/branches/uli-zc-async/tools/fix_import_file.py	(revision 8786)
+++ main/waeup.kofa/branches/uli-zc-async/tools/fix_import_file.py	(revision 9209)
@@ -65,5 +65,4 @@
     'sex': 'gender',
     'birthday': 'date',
-    'request_date': 'datetime',
     'marit_stat': 'marit_stat',
     'session': 'session',
@@ -85,4 +84,29 @@
     'lga': 'lga',
     'order_id': 'no_int',
+    'uniben': 'former',
+    'nysc_year': 'year',
+    'alr_date': 'date',
+    'fst_sit_date': 'date',
+    'scd_sit_date': 'date',
+    'emp_start': 'date',
+    'emp_end': 'date',
+    'emp_start2': 'date',
+    'emp_end2': 'date',
+    'fst_sit_results': 'result',
+    'scd_sit_results': 'result',
+    'alr_results': 'result',
+    'email': 'email',
+    'fst_sit_type': 'sittype',
+    'scd_sit_type': 'sittype',
+    'resp_pay_reference': 'no_int',
+    'type': 'company',
+    'date': 'date',
+    'core_or_elective': 'bool',
+    'category': 'p_category',
+    'reg_transition': 'reg_state',  # we completely change this column,
+                                    # since reg_state import is usually intended
+    'transition': 'reg_transition',
+    'payment_date': 'date',
+    'validation_date': 'date',
     }
 
@@ -90,9 +114,26 @@
 COLNAME_MAPPING = {
     # base data
+    'id': 'student_id',
     'reg_state': 'state',
+    'reg_transition': 'state',
     'jamb_reg_no': 'reg_number',
     'matric_no': 'matric_number',
     'birthday': 'date_of_birth',
     'clr_ac_pin': 'clr_code',
+    # clearance
+    'hq_grade': 'hq_degree',
+    'uniben': 'former_matric',
+    'hq_type2': 'hq2_type',
+    'hq_grade2': 'hq2_degree',
+    'hq_school2': 'hq2_school',
+    'hq_matric_no2': 'hq2_matric_no',
+    'hq_session2': 'hq2_session',
+    'hq_disc2': 'hq2_disc',
+    'emp': 'employer',
+    'emp2': 'employer2',
+    'emp_position2': 'emp2_position',
+    'emp_start2': 'emp2_start',
+    'emp_end2': 'emp2_end',
+    'emp_reason2': 'emp2_reason',
     # study course
     'study_course': 'certificate',
@@ -117,4 +158,5 @@
     'surcharge': 'surcharge_1',
     'session_id': 'p_session',
+    'type': 'r_company',
     }
 
@@ -131,4 +173,6 @@
     'courses_registered': 'courses registered',
     'courses_validated': 'courses validated',
+    'admit': 'admitted',
+    'return': 'returning'
     }
 
@@ -143,9 +187,12 @@
     """Replace input fieldnames by fieldnames of COLNAME_MAPPING.
     """
-    header = dict([(name, name) for name in fieldnames])
+    # Remove whitespaces
+    header = dict([(name, name.strip()) for name in fieldnames])
     for in_name, out_name in COLNAME_MAPPING.items():
-        if in_name not in header:
+        if in_name not in header.values():
             continue
-        header[in_name] = out_name
+        # Inverse dictionary lookup
+        key = [key for key,value in header.items() if value==in_name][0]
+        header[key] = out_name
     return header
 
@@ -155,13 +202,13 @@
 
     @classmethod
-    def student_id(self, value):
+    def student_id(self, value, row):
         """ 'A123456' --> 'EA123456'
         """
         if len(value) == 7:
-            return 'E' + value
-        return value
-
-    @classmethod
-    def reg_state(self, value):
+            return 'K' + value
+        return value
+
+    @classmethod
+    def reg_state(self, value, row):
         """ 'courses_validated' --> 'courses validated'
         """
@@ -169,6 +216,15 @@
 
     @classmethod
-    def level(self, value):
+    def reg_transition(self, value, row):
+        if value == "admitted":
+            return "admit"
+        if value == "returning":
+            return "return"
+        return value
+
+    @classmethod
+    def level(self, value, row):
         """ '000' --> '10'
+        '800' --> '999' if pg student
         """
         try:
@@ -178,8 +234,10 @@
         if number == 0:
             return 10
+        if row.get('entry_mode') and row.get('entry_mode').startswith('pg'):
+            return 999
         return number
 
     @classmethod
-    def semester(self, value):
+    def semester(self, value, row):
         """ '0' --> '9'
         """
@@ -193,5 +251,5 @@
 
     @classmethod
-    def application_category(self, value):
+    def application_category(self, value, row):
         """ '' --> 'no'
         """
@@ -201,9 +259,26 @@
 
     @classmethod
-    def lga(self, value):
+    def lga(self, value, row):
         """ Remove apostrophe
         """
         if value == 'akwa_ibom_uru_offong_oruko':
             return 'akwa_ibom_urue-offong-oruko'
+        if value == 'edo_ohionmwon':
+            return 'edo_orhionmwon'
+
+        if value == 'nassarawa_nassarawa':
+            return 'nassarawa_nassawara'
+
+        if value == 'kogi_mopa-muro-mopi':
+            return 'kogi_mopa-muro'
+
+        if value == 'delta_osimili-north':
+            return 'delta_oshielli-north'
+
+        if value == 'delta_osimili':
+            return 'delta_oshimili'
+
+        if value == 'delta_osimili-south':
+            return 'delta_oshimili-south'
         try:
             value = value.replace("'","")
@@ -220,5 +295,5 @@
 
     @classmethod
-    def session(self, value):
+    def session(self, value, row):
         """ '08' --> '2008'
         '2008/2009' --> '2008'
@@ -244,5 +319,35 @@
 
     @classmethod
-    def marit_stat(self, value):
+    def former(self, value, row):
+        """ True --> yes
+        '2008/2009' --> '2008'
+        """
+        if value == 'True':
+            return 'yes'
+        return
+
+    @classmethod
+    def bool(self, value, row):
+        """ True --> 1
+        """
+        if value in ('TRUE', 'True'):
+            return '1'
+        elif value in ('FALSE', 'False'):
+            return '0'
+        return
+
+    @classmethod
+    def year(self, value, row):
+        """ '0' --> ''
+        """
+        if value == '0':
+            return
+        if value == 'None':
+            return
+        return value
+
+
+    @classmethod
+    def marit_stat(self, value, row):
         """ 'True'/'False' --> 'married'/'unmarried'
         """
@@ -256,5 +361,5 @@
 
     @classmethod
-    def gender(self, value):
+    def gender(self, value, row):
         """ 'True'/'False' --> 'f'/'m'
         """
@@ -268,5 +373,5 @@
 
     @classmethod
-    def date(self, value):
+    def date(self, value, row):
         """ 'yyyy/mm/dd' --> 'yyyy-mm-dd'
         """
@@ -283,31 +388,20 @@
 
     @classmethod
-    def no_int(self, value):
-        """ Add hash.
+    def no_int(self, value, row):
+        """ Add hash and skip numbers starting with 999999
         """
         # We add the hash symbol to avoid automatic number transformation
         # in Excel and Calc for further processing
-        value += '#'
-        return value
-
-
-    @classmethod
-    def datetime(self, value):
-        """ 'yyyy/mm/dd' --> 'yyyy-mm-dd'
-        """
-        #print  "IN: ", value
-        if value == "None":
-            value = ""
-        elif value == "":
-            value = ""
-        else:
-            #value = datetime.datetime.strptime(value, '%Y/%m/%d')
-            #value = datetime.datetime.strftime(value, '%Y-%m-%d')
+        try:
+            intvalue = int(value)
+            value += '#'
+        except:
             pass
-        #print "OUT: ", value
-        return value
-
-    @classmethod
-    def mode(self, value):
+        if value.startswith('999999'):
+            return
+        return value
+
+    @classmethod
+    def mode(self, value, row):
         if value == "transfer_fulltime":
             return "transfer_ft"
@@ -317,5 +411,5 @@
 
     @classmethod
-    def password(self, value):
+    def password(self, value, row):
         if value == "not set":
             return ""
@@ -323,11 +417,35 @@
 
     @classmethod
-    def nationality(self, value):
+    def nationality(self, value, row):
         if value == "nigeria":
             return "NG"
-        return value
-
-    @classmethod
-    def phone(self, value):
+        if value == "niger":
+            return "NE"
+        return value
+
+    @classmethod
+    def sittype(self, value, row):
+        if value == "nabtec":
+            return "nabteb"
+        return value
+
+    @classmethod
+    def company(self, value, row):
+        if value == "online":
+            return "interswitch"
+        return value
+
+    @classmethod
+    def p_category(self, value, row):
+        if value == "acceptance":
+            return "clearance"
+        return value
+
+    @classmethod
+    def email(self, value, row):
+        return value.strip()
+
+    @classmethod
+    def phone(self, value, row):
         """ '<num-seq1>-<num-seq2> asd' -> '--<num-seq1><num-seq2>'
 
@@ -335,10 +453,26 @@
         of numbers.
         """
+        if not value:
+            return
+        value = value.strip('#')
         value = value.replace('-', '')
         value = value.replace('/', '')
         match = RE_PHONE.match(value)
         phone = match.groups()[0]
-        value = '-%s' % phone
-        return value
+        if value.startswith('234'):
+            value = '+' + value[:3] + '-' + value[3:]
+        else:
+            value = '-%s' % phone
+        return value + '#'
+
+    @classmethod
+    def result(self, value, row):
+        try:
+            liste = eval(value)
+        except:
+            return
+        if isinstance(liste,list):
+            return [(i[0].lower(),i[1]) for i in liste]
+        return
 
 
@@ -357,5 +491,13 @@
             header = convert_fieldnames(reader.fieldnames)
             writer.writerow(header)
+        if row.get('reg_state') == 'student_created':
+            # We do not reimport student records which have never been accessed.
+            continue
+        if row.get('status') == 'started':
+            # We do not reimport started payments.
+            continue
         for key, value in row.items():
+            # Remove unwanted whitespaces.
+            row[key] = row[key].strip()
             if not key in OPTIONS.keys():
                 continue
@@ -365,5 +507,5 @@
                 print "WARNING: cannot find converter %s" % conv_name
                 continue
-            row[key] = converter(row[key])
+            row[key] = converter(row[key], row)
         try:
             writer.writerow(row)
Index: main/waeup.kofa/branches/uli-zc-async/tools/reload_portraits.py
===================================================================
--- main/waeup.kofa/branches/uli-zc-async/tools/reload_portraits.py	(revision 9209)
+++ main/waeup.kofa/branches/uli-zc-async/tools/reload_portraits.py	(revision 9209)
@@ -0,0 +1,107 @@
+##
+## This script is called without parameters.
+##
+## It moves passport images from a reload folder to their right place in
+## the media folder.
+##
+## Configuration is done below.
+##
+##
+## Please note that for changing groups of files/directories, you
+## normally have to be root. This script is therefore normally run with
+## sudo.
+##
+
+## ######################################################################
+## CONFIGURATION
+##
+
+## The folder where all source docs can be found (can be any folder)
+#SRC_DIR = "/kofa/uniben/var/datacenter/media/students/reload"
+SRC_DIR = "/zope/instances/unibenpg-import/pictures_admitted_latest"
+
+## The students folder in Kofa where files should go to
+DST_DIR = "/kofa/uniben/var/datacenter/media/students"
+
+
+## Permissions to be set on new files/dirs. Set OWNER and/or GROUP to
+## None to leave them unchanged after creation.
+UMASK = 0664
+OWNER = 'kofa'
+GROUP = 'kofa'
+LETTER = 'B'
+
+##
+## CONFIGURATION END
+## ######################################################################
+
+import csv
+import os
+import shutil
+import sys
+from grp import getgrnam
+from pwd import getpwnam
+
+OWNER_ID = OWNER and getpwnam(OWNER)[2] or -1
+GRP_ID = GROUP and getgrnam(GROUP)[2] or -1
+DIR_UMASK = UMASK | 0111  # directories need extra x-permission
+
+def set_perms(path):
+    # set permissions and ownership for path
+    if os.path.isdir(path):
+        os.chmod(path, DIR_UMASK)
+    else:
+        os.chmod(path, UMASK)
+    os.chown(path, OWNER_ID, GRP_ID)
+    return
+
+def new_and_old_id(filename):
+    try:
+        stud_id, ext = filename.split('.')
+    except:
+        return None, None
+    if ext != 'jpg':
+        return None, None
+    if len(stud_id) == 7:
+        return LETTER + stud_id, stud_id
+    if len(stud_id) == 8:
+        return stud_id, stud_id
+    return None, None
+
+def final_subfolder(stud_id):
+    # compute new folder name from stud_id (old stud_ids or new stud_ids
+    # but not new old stud_ids)
+    num = int(stud_id[1:])
+    num = num / 10000 * 10
+    return '%05d' % num
+
+def copy_file(file_src, file_dst):
+    print "COPY FILE: %s -> %s" % (file_src, file_dst)
+    shutil.copyfile(file_src, file_dst)
+    set_perms(file_dst)
+    return
+
+if not os.path.exists(SRC_DIR):
+    sys.exit("No source folder %s. Skipping." % SRC_DIR)
+
+if len(os.listdir(SRC_DIR)) == 0:
+    sys.exit("Empty source folder for %s. Skipping." % SRC_DIR)
+
+for filename in os.listdir(SRC_DIR):
+    new_stud_id, stud_id = new_and_old_id(filename)
+    if not new_stud_id:
+        print "Wrong filename: %s" % filename
+        continue
+    subfolder = final_subfolder(stud_id)
+    dst_folder = os.path.join(DST_DIR, subfolder, new_stud_id)
+    if not os.path.exists(dst_folder):
+        print "Destination folder does not exist: %s. Skipping" % (dst_folder)
+        continue
+    dst_file = os.path.join(DST_DIR, subfolder, new_stud_id,
+        'passport_%s.jpg' % new_stud_id)
+    if os.path.exists(dst_file):
+        print "Destination file exists: %s. Skipping" % (dst_file)
+        continue
+    file_src = os.path.join(SRC_DIR, filename)
+    file_dst = os.path.join(dst_file)
+    copy_file(file_src, file_dst)
