Changeset 1673 for WAeUP_SRP/branches

Timestamp:

9 Apr 2007, 16:01:51 (18 years ago)

Author:

joachim

Message:

merge up to 1672

Location:

WAeUP_SRP/branches/joachim-event-branch

Files:

• WAeUPTables.py (modified) (1 diff)
• WAeUPTool.py (modified) (1 diff)
• skins/waeup_student/getNewStudentStatistics.py (modified) (1 diff)
• skins/waeup_student/getNewStudentStatistics_old.py (copied) (copied from WAeUP_SRP/trunk/skins/waeup_student/getNewStudentStatistics_old.py)
• skins/waeup_student/list_students.py (modified) (2 diffs)
• skins/waeup_utilities/readOnlinePaymentsFromLog.py (copied) (copied from WAeUP_SRP/trunk/skins/waeup_utilities/readOnlinePaymentsFromLog.py)
• skins/waeup_utilities/ti_182_resolve.py (copied) (copied from WAeUP_SRP/trunk/skins/waeup_utilities/ti_182_resolve.py)

WAeUP_SRP/branches/joachim-event-branch/WAeUPTables.py

@@ -278 +278 @@
 ###)
 
+class OnlinePaymentsImport(WAeUPTable): ###(
+
+    meta_type = 'WAeUP Online Payment Transactions'
+    name = "online_payments_import"
+    key = "order_id"
+    def __init__(self):
+        WAeUPTable.__init__(self, self.name)
+
+
+InitializeClass(CoursesCatalog)
+###)
+
 class ReturningImport(WAeUPTable): ###(
 

WAeUP_SRP/branches/joachim-event-branch/WAeUPTool.py

@@ -532 +532 @@
     ###)
 
+    security.declareProtected(ModifyPortalContent,"getInvalidCallbackTransactions")###(
+    def getCallbacksFromLog(self,filename):
+        """fix Online Payment Transactions from Z2.log entries"""
+        import transaction
+        import random
+        from cgi import parse_qs
+        from urlparse import urlparse
+        #from pdb import set_trace
+        wftool = self.portal_workflow
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        students_folder = self.portal_url.getPortalObject().campus.students
+        s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \['
+        s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "'
+        s += r'(?P<intersw>.*)" "(?P<agent>.*)"'
+        data = re.compile(s)
+        start = True
+        tr_count = 1
+        total = 0
+        #name = 'pume_results'
+        #name = 'epaymentsuccessful_z2log2'
+        name = filename
+        no_import = []
+        imported = []
+        logger = logging.getLogger('WAeUPTool.getFailedTransactions')
+        try:
+            transactions = open("%s/import/%s" % (i_home,name),"rb").readlines()
+        except:
+            logger.error('Error reading %s' % name)
+            return
+        tas = []
+        for line in transactions:
+            dict = {}
+            items = data.search(line)
+            dict['idict'] = idict = items.groupdict()
+            #print idict
+            #from pdb import set_trace;set_trace()
+            urlparsed = urlparse(idict['get'][4:])
+            #print urlparsed
+            path = urlparsed[2].split('/')
+            dict['student_id'] = student_id = path[8]
+            dict['payment_id'] = payment_id = path[10]
+            dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4])
+            tas.append(dict)
+            tr_count += 1
+        return tas
+    ###)
+
+    security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###(
+    def importOnlinePaymentTransactions(self):
+        """load Online Payment Transactions from CSV values"""
+        import transaction
+        import random
+        #from pdb import set_trace
+        current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S")
+        opt = self.online_payments_import
+        students_folder = self.portal_url.getPortalObject().campus.students
+        start = True
+        tr_count = 1
+        total = 0
+        #name = 'pume_results'
+        name = 'OnlineTransactions'
+        no_import = []
+        imported = []
+        logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions')
+        try:
+            transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb"))
+        except:
+            logger.error('Error reading %s.csv' % name)
+            return
+        for pay_transaction in transactions:
+            if start:
+                start = False
+                logger.info('Start loading from %s.csv' % name)
+                s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()])
+                no_import.append('%s,"Error"' % s)
+                format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()])
+                format_error = format + ',"%(Error)s"'
+            data = {}
+
+            # format of the first file sent from Tayo
+            #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date'])
+            #data['student_id'] = student_id = pay_transaction['Payer ID']
+            #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
+            #data['response_code'] = response_code = pay_transaction['Resp Code']
+            #data['amount'] = amount = pay_transaction['Amount']
+
+            # format of the second file sent from Tayo
+            data['datetime'] = date = 0
+            data['student_id'] = student_id = pay_transaction['Payer ID']
+            data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)']
+            data['response_code'] = response_code = '00'
+            data['amount'] = amount = pay_transaction['Amount']
+
+            dup = False
+            if response_code == "12":
+                continue
+            try:
+                opt.addRecord(**data)
+            except ValueError:
+                dup = True
+            #from pdb import set_trace;set_trace()
+            if dup:
+                if response_code == "00":
+                    opt.modifyRecord(**data)
+                else:
+                    pay_transaction['Error'] = "Duplicate order Id"
+                    no_import.append( format_error % pay_transaction)
+                    logger.info("dupplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data)
+                    continue
+            tr_count += 1
+            if tr_count > 1000:
+                if len(no_import) > 0:
+                    open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
+                             '\n'.join(no_import) + '\n')
+                    no_import = []
+                em = '%d transactions committed\n' % (tr_count)
+                transaction.commit()
+                regs = []
+                logger.info(em)
+                total += tr_count
+                tr_count = 0
+        open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write(
+                                                '\n'.join(no_import))
+        return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1'))
+    ###)
+
 InitializeClass(WAeUPTool)

WAeUP_SRP/branches/joachim-event-branch/skins/waeup_student/getNewStudentStatistics.py

@@ -12 +12 @@
 return Student Statistics
 """
+try:
+    from Products.zdb import set_trace
+except:
+    def set_trace():
+        pass
 import logging
 logger = logging.getLogger('Skins.getNewStudentStatistics')
 
 try:
-    from Products.AdvancedQuery import Eq, Between, Le,In, Ge
+    from Products.AdvancedQuery import Eq, Between, Le,In, Ge,MatchRegexp
     aq_portal = context.portal_catalog.evalAdvancedQuery
     aq_students = context.students_catalog.evalAdvancedQuery
 except:
     evalAdvancedQuery = None
+
+def intersect(m,n):
+    return [i for i in m if i in n]
+    
 logger.info('%s invoked statistics' % context.portal_membership.getAuthenticatedMember())
-l = []
 if not context.isStaff():
     return l
-dep = {}
+total_dict = {}
 
-dep['id'] = "All Faculties"
+total_dict['id'] = "All Faculties"
 
-freshquery = Eq('level','100') | (Eq('level','200') & Eq('entry_mode','DE'))
-total_new = aq_students(freshquery)
-total_new = float(len(total_new))
-dep['new'] = "%.0f" % total_new
-
-
-#from Products.zdb import set_trace;set_trace()
-
-adm_res = context.portal_catalog(review_state ='admitted')
-adm_ids = [r.getId for r in adm_res]
-adm = len(adm_res)
-dep['admitted'] = adm
-dep['admitted_percent'] =  "%.0f" % round(adm*100/total_new)
-
-cpe_res = context.portal_catalog(review_state ='clearance_pin_entered')
-cpe_ids = [r.getId for r in cpe_res]
-cpe = len(cpe_res)
-dep['clearance_pin_entered'] = cpe
-dep['clearance_pin_entered_percent'] = "%.0f" % round(cpe*100/total_new)
-
-cr_res = context.portal_catalog(review_state='clearance_requested')
-cr_ids = [r.getId for r in cr_res]
-cr = len(cr_res)
-dep['clearance_requested'] = cr
-dep['clearance_requested_percent'] = "%.0f" % round(cr*100/total_new)
-
-cav_res = context.portal_catalog(review_state ='cleared_and_validated')
-cav_ids = [r.getId for r in cav_res]
-cav = len(cav_res)
-dep['cleared_and_validated'] = cav
-dep['cleared_and_validated_percent'] = "%.0f" % round(cav*100/total_new)
-
-or_res = context.portal_catalog(review_state='objection_raised')
-or_ids = [r.getId for r in or_res]
-ora = len(or_res)
-dep['objection_raised'] = ora
-dep['objection_raised_percent'] = "%.0f" % round(ora*100/total_new)
-
-total_sfp = total_new - (cav + cr + cpe + adm + ora)
-dep['rest'] = int(total_sfp)
-dep['rest_percent'] = "%.0f" % round(total_sfp*100/total_new)
-
-
-l.append(dep)
-fs = context.portal_catalog(portal_type="Faculty")
-for fid in [f.getId for f in fs]:
-    dep = {}
-    dep['id'] = fid
-    fquery = Eq('faculty',fid)
-    fac_res = aq_students(fquery & freshquery)
-    fac_ids = [r.id for r in fac_res]
-    total = float(len(fac_res))
-
-    if total == 0:
-        continue
-
-    dep['new'] = "%.0f" % total
-
-    adm = len([s for s in fac_ids if s in adm_ids])
-    dep['admitted'] = adm
-    dep['admitted_percent'] = 0
-    if total:
-        dep['admitted_percent'] = "%.0f" % round(adm*100/total)
-
-    cpe = len([s for s in fac_ids if s in cpe_ids])
-    dep['clearance_pin_entered'] = cpe
-    dep['clearance_pin_entered_percent'] = 0
-    if total:
-        dep['clearance_pin_entered_percent'] = "%.0f" % round(cpe*100/total)
-    cr = len([s for s in fac_ids if s in cr_ids])
-    dep['clearance_requested'] = cr
-    dep['clearance_requested_percent'] = 0
-    if total:
-        dep['clearance_requested_percent'] = "%.0f" % round(cr*100/total)
-    cav = len([s for s in fac_ids if s in cav_ids])
-    dep['cleared_and_validated'] = cav
-    dep['cleared_and_validated_percent'] = 0
-    if total:
-        dep['cleared_and_validated_percent'] = "%.0f" % round(cav*100/total)
-
-    ora = len([s for s in fac_ids if s in or_ids])
-    dep['objection_raised'] = ora
-    dep['objection_raised_percent'] = 0
-    if total:
-        dep['objection_raised_percent'] = "%.0f" % round(ora*100/total)
-
-    rest = int(total - (cav + cr + cpe + adm + ora))
-    dep['rest'] = rest
-    dep['rest_percent'] = "%.0f" % round(rest*100/total)
-
-    l.append(dep)
+##freshquery = Eq('level','100') | (Eq('level','200') & Eq('entry_mode','DE'))
+##total_new = aq_students(freshquery)
+##total_new = float(len(total_new))
+newquery = Eq('portal_type','StudentApplication') & MatchRegexp('SearchableText',r'^6*')
+total_res = aq_portal(newquery)
+new_ids = [r.getPath().split('/')[-2] for r in total_res]
+total_new = float(len(total_res))
+total_dict['new'] = "%.0f" % total_new
+special_states =  ('admitted',
+                   'clearance_pin_entered',
+                   'clearance_requested',
+                   'cleared_and_validated',
+                   'objection_raised',
+                   )
+state_count = 0 
+state_ids = {}
+for state in special_states:
+    res = context.portal_catalog(review_state = state)
+    ids = [r.getId for r in res]
+    state_ids[state] = ids 
+    count = len(res)
+    state_count += count
+    total_dict[state] = count
+    total_dict['%s_percent' % state] =  "%.0f" % round(count*100/total_new)
+total_dict['rest'] = "%.0f" % (total_new - state_count)
+total_dict['rest_percent'] = "%.0f" % round((total_new - state_count)*100/total_new)
+fac_res = {}
+#fac_res['total'] = total_dict
+faculties = context.portal_catalog(portal_type="Faculty")
+for f in faculties:
+    dict = {}
+    dict['id'] = f.getId
+    dict['title'] = f.Title
+    sid_in_faculty = [s.id for s in context.students_catalog(faculty=f.getId)]
+    new_in_faculty = intersect(new_ids,sid_in_faculty)
+    dict['new'] = rest = len(new_in_faculty)
+    for state in special_states:
+        state_count = len(intersect(state_ids[state], new_in_faculty))
+        rest -= state_count
+        dict[state] = state_count
+    dict['rest'] = rest
+    fac_res[f.getId] = dict
+#set_trace()    
+##for sid,review_state in new_ids:
+##    scat_res = context.students_catalog(id = sid)
+##    if not scat_res:
+##        continue
+##    student = scat_res[0]
+##    faculty = student.faculty
+##    if not fac_res.has_key(faculty):
+##        continue
+##    if review_state in special_states:
+##        fac_res[faculty][review_state] = fac_res[faculty][review_state] + 1
+##    else:
+##        fac_res[faculty]['rest'] = fac_res[faculty]['rest'] + 1
+l = []
+l.append(total_dict)
+fac_ids = [f.getId for f in faculties]
+fac_ids.sort()
+check_dict = {}
+check_dict['id'] = 'Check'
+check_dict['new'] = 0
+check_dict['new_percent'] = 0
+check_dict['rest'] = 0
+check_dict['rest_percent'] = 0
+for state in special_states:
+    check_dict[state] = 0
+    check_dict["%s_percent" % state] = 0
+    
+for f in fac_ids:
+    total_fac = fac_res[f]['new']
+    check_dict['new'] = check_dict['new'] + total_fac
+    check_dict['rest'] = check_dict['rest'] + fac_res[f]['rest']
+    for state in special_states:
+        check_dict[state] = check_dict[state] + fac_res[f][state]
+        if total_fac != 0:
+            fac_res[f]['%s_percent' % state] = "%.0f" % round(fac_res[f][state]*100/total_fac)
+        else:
+            fac_res[f]['%s_percent' % state] = "%.0f" % 0.0
+    if total_fac != 0:
+        fac_res[f]['rest_percent'] = "%.0f" % round(fac_res[f]['rest']*100/total_fac)
+    else:
+        fac_res[f]['rest_percent'] = "%.0f" % 0.0
+    l.append(fac_res[f])
+l.append(check_dict)    
 
 return l

WAeUP_SRP/branches/joachim-event-branch/skins/waeup_student/list_students.py

@@ -12 +12 @@
 export student_list
 """
+try:
+    from Products.zdb import set_trace
+except:
+    def set_trace():
+        pass
+from Products.AdvancedQuery import Eq, Between, Le,In,MatchRegexp
+aq_portal = context.portal_catalog.evalAdvancedQuery
+aq_students = context.students_catalog.evalAdvancedQuery
 request = context.REQUEST
 setheader = request.RESPONSE.setHeader
@@ -33 +41 @@
 lines.append(','.join(fields))
 format = '"%(' + ')s","%('.join(fields) + ')s"'
-cleared = context.portal_catalog(review_state = "cleared_and_validated")
+#cleared = context.portal_catalog(review_state = "cleared_and_validated")
+query = In('review_state',('cleared_and_validated',
+                           'school_fee_paid',
+                           'courses_registered',
+                           'courses_validated'))
+cleared = aq_portal(query)
+newquery = Eq('portal_type','StudentApplication') & MatchRegexp('SearchableText',r'^6*')
+#newquery = MatchRegexp('SearchableText','^5*')
+new_students = aq_portal(newquery)
+new_sids = []
+for ns in new_students:
+    new_sids.append(ns.getPath().split('/')[-2])
 for student in cleared:
+    if student.getId not in new_sids:
+        continue
     erg = scat(id=student.getId)
     if not erg: