1 | # -*- mode: python; mode: fold; -*- |
---|
2 | # (C) Copyright 2005 The WAeUP group <http://www.waeup.org> |
---|
3 | # Author: Joachim Schmitz (js@aixtraware.de) |
---|
4 | # |
---|
5 | # This program is free software; you can redistribute it and/or modify |
---|
6 | # it under the terms of the GNU General Public License version 2 as published |
---|
7 | # by the Free Software Foundation. |
---|
8 | # |
---|
9 | # This program is distributed in the hope that it will be useful, |
---|
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
12 | # GNU General Public License for more details. |
---|
13 | # |
---|
14 | # You should have received a copy of the GNU General Public License |
---|
15 | # along with this program; if not, write to the Free Software |
---|
16 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA |
---|
17 | # 02111-1307, USA. |
---|
18 | # |
---|
19 | # $Id: WAeUPTool.py 5019 2010-02-14 00:18:56Z henrik $ |
---|
20 | """The WAeUP Tool Box. |
---|
21 | """ |
---|
22 | |
---|
23 | from AccessControl import ClassSecurityInfo |
---|
24 | from Acquisition import aq_inner |
---|
25 | from Acquisition import aq_parent |
---|
26 | from Globals import DTMLFile |
---|
27 | from Globals import InitializeClass |
---|
28 | from OFS.SimpleItem import SimpleItem |
---|
29 | from zExceptions import BadRequest |
---|
30 | |
---|
31 | from Products.CMFCore.utils import getToolByName |
---|
32 | from Products.CPSSchemas.DataStructure import DataStructure |
---|
33 | from Products.CPSSchemas.DataModel import DataModel |
---|
34 | from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter |
---|
35 | from Products.CMFCore.ActionProviderBase import ActionProviderBase |
---|
36 | from Products.CMFCore.permissions import View |
---|
37 | from Products.ZCatalog.ZCatalog import ZCatalog |
---|
38 | from Products.CMFCore.permissions import ModifyPortalContent |
---|
39 | from Products.CMFCore.permissions import ManagePortal |
---|
40 | from Products.CMFCore.utils import UniqueObject |
---|
41 | from Products.CMFCore.URLTool import URLTool |
---|
42 | from Products.CMFCore.utils import getToolByName |
---|
43 | from Students import makeCertificateCode |
---|
44 | from Globals import package_home,INSTANCE_HOME |
---|
45 | from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport |
---|
46 | from WAeUPImport import CourseImport,CourseResultImport,StudentStudyLevelImport,PaymentImport |
---|
47 | from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport |
---|
48 | from utils import makeDigest |
---|
49 | import DateTime,time |
---|
50 | import logging |
---|
51 | import transaction |
---|
52 | import csv,re,os,sys |
---|
53 | import md5 |
---|
54 | from shutil import copy2,copy |
---|
55 | from Products.AdvancedQuery import Eq, Between, Le,In |
---|
56 | |
---|
57 | p_home = package_home(globals()) |
---|
58 | i_home = INSTANCE_HOME |
---|
59 | images_base = os.path.join(i_home,"images") |
---|
60 | EMPTY = 'XXX' |
---|
61 | |
---|
62 | def getImagesDir(student_id): |
---|
63 | return os.path.join("%s" % images_base,student_id[0],student_id) |
---|
64 | |
---|
65 | def getObject(object,name): |
---|
66 | if object.hasObject(name): |
---|
67 | return getattr(object,name) |
---|
68 | return None |
---|
69 | |
---|
70 | class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase): |
---|
71 | """WAeUP tool""" |
---|
72 | |
---|
73 | id = 'waeup_tool' |
---|
74 | meta_type = 'WAeUP Tool' |
---|
75 | _actions = () |
---|
76 | security = ClassSecurityInfo() |
---|
77 | security.declareObjectProtected(View) |
---|
78 | manage_options = ( ActionProviderBase.manage_options |
---|
79 | + SimpleItem.manage_options |
---|
80 | ) |
---|
81 | |
---|
82 | security.declareProtected(View,'re_split') ###( |
---|
83 | def re_split(self,split_string,string): |
---|
84 | return re.split(split_string,string) |
---|
85 | ###) |
---|
86 | |
---|
87 | security.declareProtected(View,'difference') ###( |
---|
88 | def difference(self,l1,l2): |
---|
89 | return set(l1).difference(set(l2)) |
---|
90 | ###) |
---|
91 | |
---|
92 | def rwrite(self,s): ###( |
---|
93 | response = self.REQUEST.RESPONSE |
---|
94 | response.setHeader('Content-type','text/html; charset=ISO-8859-15') |
---|
95 | response.write("%s<br />\r\n" % s) |
---|
96 | ###) |
---|
97 | |
---|
98 | def addtodict(self,d,key,item): ###( |
---|
99 | d[key].append(item) |
---|
100 | return d[key] |
---|
101 | ###) |
---|
102 | |
---|
103 | def sleep(self,secs): ###( |
---|
104 | "sleep" |
---|
105 | import time |
---|
106 | time.sleep(secs) |
---|
107 | return |
---|
108 | ###) |
---|
109 | |
---|
110 | security.declareProtected(View,'updateRoleMappingsFor') ###( |
---|
111 | def updateRoleMappingsFor(self,wf_definition,ob): |
---|
112 | "do so for public" |
---|
113 | wf_def = getattr(self.portal_workflow,wf_definition) |
---|
114 | wf_def.updateRoleMappingsFor(ob) |
---|
115 | ###) |
---|
116 | |
---|
117 | security.declareProtected(View,'getStatesLgas') ###( |
---|
118 | def getStatesLgas(self): |
---|
119 | """return lga info""" |
---|
120 | voc = getattr(self.portal_vocabularies,'local_gov_areas') |
---|
121 | states = [] |
---|
122 | lgas = [] |
---|
123 | d = {} |
---|
124 | wd = {} |
---|
125 | for k,v in voc.items(): |
---|
126 | parts = v.split(' / ') |
---|
127 | if len(parts) == 1: |
---|
128 | state = parts[0].lower() |
---|
129 | lga = "" |
---|
130 | elif len(parts) == 2: |
---|
131 | state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower())) |
---|
132 | lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower())) |
---|
133 | else: |
---|
134 | continue |
---|
135 | if state not in states: |
---|
136 | states.append(state) |
---|
137 | if lga not in lgas: |
---|
138 | lgas.append(lga) |
---|
139 | words = re.split('[^a-zA-Z0-9/]',k) |
---|
140 | words.sort() |
---|
141 | wd[k] = words |
---|
142 | d[k] = v |
---|
143 | mapping = {} |
---|
144 | mapping['word_dict'] = wd |
---|
145 | mapping['lga_dict'] = d |
---|
146 | mapping['states'] = states |
---|
147 | mapping['lgas'] = lgas |
---|
148 | return mapping |
---|
149 | ###) |
---|
150 | |
---|
151 | security.declareProtected(View,'findLga') ###( |
---|
152 | def findLga(self,words,words_dict): |
---|
153 | words = re.split('[^a-zA-Z0-9/]',words) |
---|
154 | lga_words = [] |
---|
155 | for word in words: |
---|
156 | if word: |
---|
157 | lga_words += word.strip().lower(), |
---|
158 | lga_words.sort() |
---|
159 | state_lga = '' |
---|
160 | while not state_lga: |
---|
161 | for k,l in words_dict.items(): |
---|
162 | if lga_words == l: |
---|
163 | state_lga = k |
---|
164 | break |
---|
165 | break |
---|
166 | return state_lga |
---|
167 | ###) |
---|
168 | security.declareProtected(View,'getAccessInfo') ###( |
---|
169 | def getAccessInfo(self,context): |
---|
170 | "return a dict with access_info" |
---|
171 | logger = logging.getLogger('WAeUPTool.getAccessInfo') |
---|
172 | mtool = self.portal_membership |
---|
173 | member = mtool.getAuthenticatedMember() |
---|
174 | member_id = str(member) |
---|
175 | info = {} |
---|
176 | is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser() |
---|
177 | is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57 |
---|
178 | is_staff = info['is_staff'] = not is_anonymous and not is_student |
---|
179 | roles = member.getRolesInContext(context) |
---|
180 | info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or |
---|
181 | "SectionManager" in roles or |
---|
182 | "Manager" in roles) |
---|
183 | info['is_clearanceofficer'] = not is_student and ("ClearanceOfficer" in roles) |
---|
184 | #is_allowed = info['is_allowed'] = not is_anonymous |
---|
185 | requested_id = context.getStudentId() |
---|
186 | student_id = None |
---|
187 | if not is_anonymous and requested_id: |
---|
188 | if (is_student and member_id == requested_id) or is_staff: |
---|
189 | student_id = requested_id |
---|
190 | else: |
---|
191 | logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id)) |
---|
192 | info['student_id'] = student_id |
---|
193 | return info |
---|
194 | ###) |
---|
195 | |
---|
196 | security.declareProtected(ModifyPortalContent,'openLog') ###( |
---|
197 | def openLog(self,name): |
---|
198 | """open a log file""" |
---|
199 | version = 1 |
---|
200 | path = "%s/log/%s_%d.log" % (i_home,name,version) |
---|
201 | while os.path.exists(path): |
---|
202 | version += 1 |
---|
203 | path = "%s/log/%s_%d.log" % (i_home,name,version) |
---|
204 | log = open(path,"w") |
---|
205 | return log |
---|
206 | ###) |
---|
207 | |
---|
208 | security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###( |
---|
209 | def bypassQueueCatalog(self,enable=True): |
---|
210 | """bypass the QueueCatalog by setting all indexes to process imediate, |
---|
211 | if enable is True (default) the old settings are restored |
---|
212 | """ |
---|
213 | |
---|
214 | ###) |
---|
215 | |
---|
216 | security.declareProtected(ModifyPortalContent,'measureOaT') ###( |
---|
217 | def measureOaT(self,method="a",probe="1000",nr_pts="1"): |
---|
218 | """measure Object access Time""" |
---|
219 | import random |
---|
220 | if hasattr(self,'portal_catalog_real'): |
---|
221 | aq_portal = self.portal_catalog_real.evalAdvancedQuery |
---|
222 | else: |
---|
223 | aq_portal = self.portal_catalog.evalAdvancedQuery |
---|
224 | nr_pts = int(nr_pts) |
---|
225 | probe = int(probe) |
---|
226 | intervall = probe/10 |
---|
227 | objects = ("application","clearance","personal") |
---|
228 | portal_types = ("StudentApplication","StudentClearance","StudentPersonal") |
---|
229 | #i = random.randrange(num_objects) |
---|
230 | count = 0 |
---|
231 | found = 0 |
---|
232 | not_found = 0 |
---|
233 | t_found = 0 |
---|
234 | t_not_found = 0 |
---|
235 | time_found = time_not_found = 0.0 |
---|
236 | t_time_found = t_time_not_found = 0.0 |
---|
237 | accessed = [] |
---|
238 | t_min = 1000 |
---|
239 | t_max = 0 |
---|
240 | #import pdb;pdb.set_trace() |
---|
241 | students = self.portal_catalog(portal_type="Student") |
---|
242 | num_students = len(students) |
---|
243 | if method == "d": |
---|
244 | query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts]) |
---|
245 | res = aq_portal(query) |
---|
246 | brains = {} |
---|
247 | for r in res: |
---|
248 | sid = r.relative_path.split('/')[-2] |
---|
249 | if brains.has_key(sid): |
---|
250 | brains[sid][r.portal_type] = r |
---|
251 | else: |
---|
252 | brains[sid] = {r.portal_type : r} |
---|
253 | brains_list = brains.keys() |
---|
254 | num_objects = len(brains_list) |
---|
255 | else: |
---|
256 | num_objects = num_students |
---|
257 | print "="*40 |
---|
258 | print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method, |
---|
259 | probe, |
---|
260 | nr_pts, |
---|
261 | num_objects) |
---|
262 | print "nr found/not time found/not min/max" |
---|
263 | elapse = time.time() |
---|
264 | i_elapse = time.time() |
---|
265 | c_elapse = time.clock() |
---|
266 | for c in range(1,probe + 1): |
---|
267 | i = random.randrange(num_objects) |
---|
268 | if method in ('a','b','c'): |
---|
269 | student_brain = students[i] |
---|
270 | elif method == "d": |
---|
271 | #import pdb;pdb.set_trace() |
---|
272 | student_brain = brains[brains_list[i]] |
---|
273 | if method == "c": |
---|
274 | query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts]) |
---|
275 | res = aq_portal(query) |
---|
276 | this_portal_types = [r.portal_type for r in res] |
---|
277 | for i in range(nr_pts): |
---|
278 | oid = objects[i] |
---|
279 | if method == "a": |
---|
280 | try: |
---|
281 | student_path = student_brain.getPath() |
---|
282 | path = "%s/%s" % (student_path,oid) |
---|
283 | doc = self.unrestrictedTraverse(path).getContent() |
---|
284 | found += 1 |
---|
285 | i_time = time.time() - i_elapse |
---|
286 | time_found += i_time |
---|
287 | except: |
---|
288 | not_found += 1 |
---|
289 | i_time = time.time() - i_elapse |
---|
290 | time_not_found += i_time |
---|
291 | pass |
---|
292 | elif method == "b": |
---|
293 | try: |
---|
294 | student_object = student_brain.getObject() |
---|
295 | doc = getattr(student_object,oid).getContent() |
---|
296 | found += 1 |
---|
297 | i_time = time.time() - i_elapse |
---|
298 | time_found += i_time |
---|
299 | except: |
---|
300 | i_time = time.time() - i_elapse |
---|
301 | time_not_found += i_time |
---|
302 | not_found += 1 |
---|
303 | pass |
---|
304 | elif method == "c": |
---|
305 | if portal_types[i] in this_portal_types: |
---|
306 | found += 1 |
---|
307 | doc = res[this_portal_types.index(portal_types[i])].getObject().getContent() |
---|
308 | i_time = time.time() - i_elapse |
---|
309 | time_found += i_time |
---|
310 | else: |
---|
311 | not_found += 1 |
---|
312 | i_time = time.time() - i_elapse |
---|
313 | time_not_found += i_time |
---|
314 | elif method == "d": |
---|
315 | if student_brain.has_key(portal_types[i]): |
---|
316 | found += 1 |
---|
317 | doc = student_brain[portal_types[i]].getObject().getContent() |
---|
318 | i_time = time.time() - i_elapse |
---|
319 | time_found += i_time |
---|
320 | else: |
---|
321 | not_found += 1 |
---|
322 | i_time = time.time() - i_elapse |
---|
323 | time_not_found += i_time |
---|
324 | i_elapse = time.time() |
---|
325 | if c and (c % intervall == 0): |
---|
326 | #i_time = time.time() - i_elapse |
---|
327 | t_per = 0.0 |
---|
328 | if found: |
---|
329 | t_per = time_found/found |
---|
330 | if t_per > t_max: |
---|
331 | t_max = t_per |
---|
332 | if t_per > 0.0 and t_per < t_min: |
---|
333 | t_min = t_per |
---|
334 | itf = 0.0 |
---|
335 | if found: |
---|
336 | itf = time_found/found |
---|
337 | itnf = 0.0 |
---|
338 | if not_found : |
---|
339 | itnf = time_not_found / not_found |
---|
340 | interval_time = time_found + time_not_found |
---|
341 | s = "%(c)d: %(found)d/%(not_found)d " % vars() |
---|
342 | s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() |
---|
343 | s += "%(t_min)6.4f/%(t_max)6.4f" % vars() |
---|
344 | print s |
---|
345 | t_found += found |
---|
346 | t_not_found += not_found |
---|
347 | t_time_found += time_found |
---|
348 | t_time_not_found += time_not_found |
---|
349 | time_found = time_not_found = 0.0 |
---|
350 | found = not_found = 0 |
---|
351 | # t_found += found |
---|
352 | # t_not_found += not_found |
---|
353 | elapse = time.time() - elapse |
---|
354 | itf = 0.0 |
---|
355 | if t_found: |
---|
356 | itf = t_time_found/t_found |
---|
357 | itnf = 0.0 |
---|
358 | if t_not_found: |
---|
359 | itnf = t_time_not_found / t_not_found |
---|
360 | #c_elapse = time.clock() - c_elapse |
---|
361 | s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars() |
---|
362 | s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() |
---|
363 | s += "%(t_min)6.4f/%(t_max)6.4f" % vars() |
---|
364 | print "-"*40 |
---|
365 | print s |
---|
366 | rel_found = float(t_found)/probe |
---|
367 | rel_not_found = float(t_not_found)/probe |
---|
368 | estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf) |
---|
369 | print estimated_total_time |
---|
370 | ###) |
---|
371 | |
---|
372 | security.declareProtected(ModifyPortalContent,'writeLog') ###( |
---|
373 | def writeLog(self,logfile,s): |
---|
374 | """write to the log file""" |
---|
375 | logfile.write(s) |
---|
376 | ###) |
---|
377 | |
---|
378 | def generateStudentId(self,letter): ###( |
---|
379 | import random |
---|
380 | logger = logging.getLogger('WAeUPTool.generateStudentId') |
---|
381 | r = random |
---|
382 | if letter == '?': |
---|
383 | letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY') |
---|
384 | sid = "%c%d" % (letter,r.randint(99999,1000000)) |
---|
385 | students = self.portal_url.getPortalObject().campus.students |
---|
386 | while self.students_catalog(id = sid) or self.waeup_tool.picturePathExists(sid) or self.removed_student_ids(id = sid): |
---|
387 | logger.info('skipped used id %s' % sid) |
---|
388 | sid = "%c%d" % (letter,r.randint(99999,1000000)) |
---|
389 | return sid |
---|
390 | ###) |
---|
391 | |
---|
392 | def generatePassword(self,s=None): ###( |
---|
393 | import random |
---|
394 | r = random |
---|
395 | ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): |
---|
396 | if s is None: |
---|
397 | s = 'abcdefghklmnpqrstuvwxy23456789' |
---|
398 | pw = '' |
---|
399 | while len(pw) < 6: |
---|
400 | pw += r.choice(s) |
---|
401 | return pw |
---|
402 | ###) |
---|
403 | |
---|
404 | security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###( |
---|
405 | def dumpSchoolfeePayments(self): |
---|
406 | "dump paid schoolfees" |
---|
407 | mtool = self.portal_membership |
---|
408 | member = mtool.getAuthenticatedMember() |
---|
409 | logger = logging.getLogger('WAeUPTool.dumpSchoolfees') |
---|
410 | aq_student = self.students_catalog.evalAdvancedQuery |
---|
411 | query = In('review_state',('schoolfee_paid', |
---|
412 | 'courses_registered', |
---|
413 | 'courses_validated', |
---|
414 | )) |
---|
415 | res = aq_student(query) |
---|
416 | #import pdb;pdb.set_trace() |
---|
417 | l = [] |
---|
418 | logger.info("start for %d" % len(res)) |
---|
419 | count = 1 |
---|
420 | log_after = 100 |
---|
421 | for student in res: |
---|
422 | if not count % log_after: |
---|
423 | logger.info("processed %d total %d" % (log_after,count)) |
---|
424 | count += 1 |
---|
425 | fee_dict =self.getSchoolFee(student) |
---|
426 | fulltime = student.mode.endswith('_ft') |
---|
427 | d = {} |
---|
428 | d['student_id'] = student.id |
---|
429 | d['name'] = student.name |
---|
430 | d['amount'] = fee_dict.get(new_returning) |
---|
431 | l += d, |
---|
432 | csv_name = self.dumpListToCSV(l,'payments') |
---|
433 | logger.info('%s dumped payments to %s' % (member,export_file)) |
---|
434 | ###) |
---|
435 | |
---|
436 | security.declarePublic('dumpListToCSV') ###( |
---|
437 | def dumpListToCSV(self,l,filename,fields=None): |
---|
438 | """dump a list of dicts to a CSV file""" |
---|
439 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
---|
440 | export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,) |
---|
441 | if fields is None: |
---|
442 | fields = l[0].keys() |
---|
443 | headline = ','.join(fields) |
---|
444 | out = open(export_file,"wb") |
---|
445 | out.write(headline +'\n') |
---|
446 | out.close() |
---|
447 | out = open(export_file,"a") |
---|
448 | csv_writer = csv.DictWriter(out,fields,) |
---|
449 | csv_writer.writerows(l) |
---|
450 | return export_file |
---|
451 | ###) |
---|
452 | |
---|
453 | security.declareProtected(ManagePortal, 'listMembers') ###( |
---|
454 | def listMembers(self): |
---|
455 | "list all members" |
---|
456 | mtool = self.portal_membership |
---|
457 | member = mtool.getAuthenticatedMember() |
---|
458 | logger = logging.getLogger('WAeUPTool.listMembers') |
---|
459 | if str(member) not in ('admin','joachim'): |
---|
460 | logger.info('%s tried to list members' % (member)) |
---|
461 | return None |
---|
462 | members = self.portal_directories.members |
---|
463 | all = members.listEntryIdsAndTitles() |
---|
464 | l = [] |
---|
465 | for user_id,name in all: |
---|
466 | d = {} |
---|
467 | d['user_id'] = user_id |
---|
468 | d['name'] = name |
---|
469 | d['pw'] = getattr(getattr(members,user_id),'password') |
---|
470 | d['email'] = getattr(getattr(members,user_id),'email') |
---|
471 | d['groups'] = " ".join(getattr(getattr(members,user_id),'groups')) |
---|
472 | d['roles'] = " ".join(getattr(getattr(members,user_id),'roles')) |
---|
473 | l += d, |
---|
474 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
---|
475 | export_file = "%s/export/member_list_%s.csv" % (i_home,current,) |
---|
476 | logger.info('%s dumped member list to %s' % (member,export_file)) |
---|
477 | fields = l[0].keys() |
---|
478 | headline = ','.join(fields) |
---|
479 | out = open(export_file,"wb") |
---|
480 | out.write(headline +'\n') |
---|
481 | out.close() |
---|
482 | out = open(export_file,"a") |
---|
483 | csv_writer = csv.DictWriter(out,fields,) |
---|
484 | csv_writer.writerows(l) |
---|
485 | ###) |
---|
486 | |
---|
487 | security.declareProtected(ManagePortal, 'listStudents') ###( |
---|
488 | def listStudents(self): |
---|
489 | "list all students" |
---|
490 | mtool = self.portal_membership |
---|
491 | member = mtool.getAuthenticatedMember() |
---|
492 | logger = logging.getLogger('WAeUPTool.listStudents') |
---|
493 | if str(member) not in ('admin','joachim'): |
---|
494 | logger.info('%s tried to list students' % (member)) |
---|
495 | return None |
---|
496 | students = self.portal_directories.students |
---|
497 | all = students.listEntryIdsAndTitles() |
---|
498 | l = [] |
---|
499 | for user_id,name in all: |
---|
500 | d = {} |
---|
501 | d['user_id'] = user_id |
---|
502 | d['name'] = name |
---|
503 | d['pw'] = getattr(getattr(students,user_id),'password') |
---|
504 | d['email'] = getattr(getattr(students,user_id),'email') |
---|
505 | d['groups'] = " ".join(getattr(getattr(students,user_id),'groups')) |
---|
506 | d['roles'] = " ".join(getattr(getattr(students,user_id),'roles')) |
---|
507 | l += d, |
---|
508 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
---|
509 | export_file = "%s/export/student_list_%s.csv" % (i_home,current,) |
---|
510 | logger.info('%s dumped student list to %s' % (member,export_file)) |
---|
511 | fields = l[0].keys() |
---|
512 | headline = ','.join(fields) |
---|
513 | out = open(export_file,"wb") |
---|
514 | out.write(headline +'\n') |
---|
515 | out.close() |
---|
516 | out = open(export_file,"a") |
---|
517 | csv_writer = csv.DictWriter(out,fields,) |
---|
518 | csv_writer.writerows(l) |
---|
519 | ###) |
---|
520 | |
---|
521 | security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###( |
---|
522 | def removeDeletedDocIds(self, max=1000): |
---|
523 | """ |
---|
524 | remove deleted docids from repository commit after max |
---|
525 | """ |
---|
526 | logger = logging.getLogger('WAeUPTool.removeDeletedDocIds') |
---|
527 | repository = getToolByName(self, 'portal_repository') |
---|
528 | pxtool = getToolByName(self, 'portal_proxies') |
---|
529 | logger.info('start') |
---|
530 | pxtool_infos = pxtool.getRevisionsUsed() |
---|
531 | logger.info('found %d used revisions ' % (len(pxtool_infos))) |
---|
532 | |
---|
533 | nb_revs = 0 |
---|
534 | docids_d = {} # all docids |
---|
535 | unused_docids_d = {} # all docids that are unused |
---|
536 | ids_unused_revs_docids = [] # ids for revs of unused docids |
---|
537 | unused_ids = [] # ids for unused revs |
---|
538 | total = 0 |
---|
539 | idlist = repository.objectIds() |
---|
540 | to_delete = 0 |
---|
541 | found = False |
---|
542 | for id in idlist: |
---|
543 | docid, rev = repository._splitId(id) |
---|
544 | if docid is None: |
---|
545 | logger.info("invalid doc_id %s" % docid) |
---|
546 | continue |
---|
547 | nb_revs += 1 |
---|
548 | if not pxtool_infos.has_key(docid): |
---|
549 | found = True |
---|
550 | to_delete += 1 |
---|
551 | unused_ids.append(id) |
---|
552 | elif not pxtool_infos[docid].has_key(rev): |
---|
553 | found = True |
---|
554 | to_delete += 1 |
---|
555 | unused_ids.append(id) |
---|
556 | if found and not to_delete % max: |
---|
557 | found = False |
---|
558 | #import pdb;pdb.set_trace() |
---|
559 | repository.manage_delObjects(unused_ids) |
---|
560 | transaction.commit() |
---|
561 | logger.info('removed %d total %d unused docids ' % (max,to_delete)) |
---|
562 | else: |
---|
563 | if unused_ids: |
---|
564 | repository.manage_delObjects(unused_ids) |
---|
565 | transaction.commit() |
---|
566 | logger.info('finished removing %d unused docids ' % (to_delete)) |
---|
567 | ###) |
---|
568 | |
---|
569 | security.declarePublic('getCredential') ###( |
---|
570 | def getCredential(self,student_id): |
---|
571 | student_entry = getattr(self.portal_directories.students,student_id,None) |
---|
572 | #import pdb;pdb.set_trace() |
---|
573 | #if not self.isStaff(): |
---|
574 | # mtool = self.portal_membership |
---|
575 | # member = mtool.getAuthenticatedMember() |
---|
576 | # logger = logging.getLogger('WAeUPTool.getCredential') |
---|
577 | # logger.info('%s tried to access password of %s' % (member,student_id)) |
---|
578 | # return None |
---|
579 | if student_entry is None: |
---|
580 | return "not set" |
---|
581 | return getattr(student_entry,"password","not set") |
---|
582 | ###) |
---|
583 | |
---|
584 | security.declarePublic('checkPassword') ###( |
---|
585 | def checkPassword(self,student_id,password): |
---|
586 | student_entry = getattr(self.portal_directories.students,student_id,None) |
---|
587 | if student_entry is None: |
---|
588 | return False |
---|
589 | return getattr(student_entry,"password","not set") == password |
---|
590 | ###) |
---|
591 | |
---|
592 | security.declarePublic('checkGenericPassword') ###( |
---|
593 | def checkGenericPassword(self,member_id): |
---|
594 | member_entry = getattr(self.portal_directories.members,member_id,None) |
---|
595 | if member_entry is None: |
---|
596 | return False |
---|
597 | ltool = getToolByName(self, 'portal_layouts') |
---|
598 | unsecure_words = ltool._getOb('members')['w__password'].check_words |
---|
599 | password = getattr(member_entry,"password","not set") |
---|
600 | is_unsecure = password in unsecure_words |
---|
601 | if is_unsecure: |
---|
602 | logger = logging.getLogger('WAeUPTool.checkGenericPassword') |
---|
603 | logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password)) |
---|
604 | return is_unsecure |
---|
605 | ###) |
---|
606 | |
---|
607 | security.declarePublic('editPassword') ###( |
---|
608 | def editPassword(self,student_id,password): |
---|
609 | "edit a student password" |
---|
610 | student_entry = getattr(self.portal_directories.students,student_id,None) |
---|
611 | if student_entry is None: |
---|
612 | return |
---|
613 | setattr(student_entry,'password',password) |
---|
614 | ###) |
---|
615 | |
---|
616 | security.declareProtected(ModifyPortalContent,'doCommit') ###( |
---|
617 | def doCommit(self,logger=None): |
---|
618 | "commit some transactions" |
---|
619 | transaction.commit() |
---|
620 | ###) |
---|
621 | |
---|
622 | security.declarePublic('loadStudentFoto') ###( |
---|
623 | def loadStudentFoto(self,student,filename,folder): |
---|
624 | "return a student passport picture" |
---|
625 | #import pdb;pdb.set_trace() |
---|
626 | picture ="%s/import/%s/%s" % (i_home,folder,filename) |
---|
627 | student_id = student.getId() |
---|
628 | images_dir = getImagesDir(student_id) |
---|
629 | if not os.path.exists(images_dir): |
---|
630 | os.mkdir(images_dir) |
---|
631 | image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) |
---|
632 | for extension in ('.jpg','.JPG'): |
---|
633 | fullname = "%(picture)s%(extension)s" % vars() |
---|
634 | if os.path.exists(fullname): |
---|
635 | copy2(fullname,image_name) |
---|
636 | return "successfully copied passport picture" |
---|
637 | return "passport picture not found: %s.jpg or .JPG" % picture |
---|
638 | ###) |
---|
639 | |
---|
640 | |
---|
641 | security.declareProtected(ModifyPortalContent,'createOne') ###( |
---|
642 | def createOne(self,students_folder,student_brain,letter,commit=False): |
---|
643 | sid = self.waeup_tool.generateStudentId(letter) |
---|
644 | students_folder.invokeFactory('Student', sid) |
---|
645 | student = getattr(students_folder,sid) |
---|
646 | self.portal_workflow.doActionFor(student,'return') |
---|
647 | student.manage_setLocalRoles(sid, ['Owner',]) |
---|
648 | matric_no = student_brain.matric_no |
---|
649 | jamb_reg_no = student_brain.Entryregno |
---|
650 | self.students_catalog.addRecord(id = sid, |
---|
651 | matric_no = matric_no, |
---|
652 | jamb_reg_no = jamb_reg_no, |
---|
653 | sex = student_brain.Sex == "F", |
---|
654 | name = "%s %s %s" % (student_brain.Firstname, |
---|
655 | student_brain.Middlename, |
---|
656 | student_brain.Lastname) |
---|
657 | ) |
---|
658 | if commit: |
---|
659 | transaction.commit() |
---|
660 | return sid,jamb_reg_no |
---|
661 | ###) |
---|
662 | |
---|
663 | # Functions without docstring cannot be called from outside (via the URL), |
---|
664 | # thus we can declare addStudent public. |
---|
665 | security.declarePublic('addStudent') ###( |
---|
666 | def addStudent(self,dict): |
---|
667 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
668 | sid = self.waeup_tool.generateStudentId('?') |
---|
669 | students_folder.invokeFactory('Student', sid) |
---|
670 | student_obj = getattr(students_folder,sid) |
---|
671 | f2t = StudentImport.field2types_student |
---|
672 | #from pdb import set_trace; set_trace() |
---|
673 | d = {} |
---|
674 | #d['jamb_sex'] = 'M' |
---|
675 | #if dict.get('sex'): |
---|
676 | # d['jamb_sex'] = 'F' |
---|
677 | |
---|
678 | entry_session = dict.get('entry_session') |
---|
679 | if entry_session == self.getSessionId()[0]: |
---|
680 | wfaction = 'admit' |
---|
681 | wft = 'wf_transition_admit' |
---|
682 | password = None |
---|
683 | else: |
---|
684 | wfaction = 'return' |
---|
685 | wft = 'wf_transition_return' |
---|
686 | password = self.generatePassword() |
---|
687 | self.makeStudentMember(sid,password) |
---|
688 | |
---|
689 | for pt in f2t.keys(): |
---|
690 | student_obj.invokeFactory(pt,f2t[pt]['id']) |
---|
691 | sub_obj = getattr(student_obj,f2t[pt]['id']) |
---|
692 | sub_doc = sub_obj.getContent() |
---|
693 | #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj) |
---|
694 | #d['Title'] = f2t[pt]['title'] |
---|
695 | for field in f2t[pt]['fields']: |
---|
696 | d[field] = dict.get(field,'') |
---|
697 | sub_doc.edit(mapping = d) |
---|
698 | new_state = f2t[pt][wft] |
---|
699 | if new_state != "remain": |
---|
700 | self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) |
---|
701 | self.portal_workflow.doActionFor(student_obj,wfaction) |
---|
702 | student_obj.manage_setLocalRoles(sid, ['Owner',]) |
---|
703 | return sid,password |
---|
704 | ###) |
---|
705 | |
---|
706 | security.declarePublic('getCertificateBrain') ###( |
---|
707 | def getCertificateBrain(self,cert_id): |
---|
708 | "do it" |
---|
709 | res = ZCatalog.searchResults(self.portal_catalog_real, |
---|
710 | {'portal_type':"Certificate", |
---|
711 | 'id': cert_id}) |
---|
712 | if res: |
---|
713 | return res[0] |
---|
714 | return None |
---|
715 | ###) |
---|
716 | |
---|
717 | security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###( |
---|
718 | def get_csv_filenames(self): |
---|
719 | "do it" |
---|
720 | files = [file for file in os.listdir("%s/import/" % (i_home)) |
---|
721 | if file.endswith('.csv') and (file.find('imported') == -1 and |
---|
722 | file.find('pending') == -1)] |
---|
723 | return files |
---|
724 | ###) |
---|
725 | |
---|
726 | security.declarePublic('findStudentByMatricelNo') ###( |
---|
727 | def findStudentByMatricelNo(self,matric_no): |
---|
728 | "do it" |
---|
729 | res = ZCatalog.searchResults(self.portal_catalog_real, |
---|
730 | {'portal_type':"StudentClearance", |
---|
731 | 'SearchableText': matric_no}) |
---|
732 | if res: |
---|
733 | return res[0] |
---|
734 | return None |
---|
735 | ###) |
---|
736 | |
---|
737 | |
---|
738 | security.declarePublic('getOfficerName') ###( |
---|
739 | def getOfficerName(self,mid): |
---|
740 | """get the real name of a member""" |
---|
741 | membership = self.portal_membership |
---|
742 | member_record = membership.getMemberById(mid) |
---|
743 | return member_record.getProperty('fullname',None) |
---|
744 | ###) |
---|
745 | |
---|
746 | |
---|
747 | security.declarePublic('makeStudentMember') ###( |
---|
748 | def makeStudentMember(self,sid,password='uNsEt'): |
---|
749 | """make the student a member""" |
---|
750 | membership = self.portal_membership |
---|
751 | membership.addMember(sid, |
---|
752 | password , |
---|
753 | roles=('Member', |
---|
754 | 'Student', |
---|
755 | ), |
---|
756 | domains='', |
---|
757 | properties = {'memberareaCreationFlag': False, |
---|
758 | 'homeless': True},) |
---|
759 | member = membership.getMemberById(sid) |
---|
760 | self.portal_registration.afterAdd(member, sid, password, None) |
---|
761 | #self.manage_setLocalRoles(sid, ['Owner',]) |
---|
762 | ###) |
---|
763 | |
---|
764 | security.declareProtected(View,'makeStudentData') ###( |
---|
765 | def makeStudentData(self,student_id,email=None,phone_nr=None): |
---|
766 | "create Datastructure for a returning Student" |
---|
767 | #import pdb;pdb.set_trace() |
---|
768 | logger = logging.getLogger('WAeUPTool.makeStudentData') |
---|
769 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
770 | #res = self.students_catalog(id=student_id) |
---|
771 | #if res: |
---|
772 | # st = res[0] |
---|
773 | #res = self.returning_import(matric_no = st.matric_no) |
---|
774 | res = self.returning_import(id = student_id) |
---|
775 | if res: |
---|
776 | student = res[0] |
---|
777 | else: |
---|
778 | logger.info('Id %s not found in returning_import' % student_id) |
---|
779 | return |
---|
780 | logger.info('%s creates data structure' % student_id) |
---|
781 | s_results = self.results_import(matric_no = student.matric_no) |
---|
782 | if s_results: |
---|
783 | lnr = self.getLevelFromResultsCosCode(s_results) |
---|
784 | level = "%d00" % lnr |
---|
785 | verdict,eligible = self.getVerdict(s_results[0].Verdict) |
---|
786 | #if eligible: |
---|
787 | # level = "%d00" % (lnr + 1) |
---|
788 | else: |
---|
789 | logger.info('matric_no %s not found in results_import' % student.matric_no) |
---|
790 | level = '' |
---|
791 | verdict = '' |
---|
792 | #student should not be allowed to perform this transition |
---|
793 | #wftool = self.portal_workflow |
---|
794 | #wftool.doActionFor(student,'return') |
---|
795 | certcode_org = student.Coursemajorcode |
---|
796 | certcode = makeCertificateCode(certcode_org) |
---|
797 | certificate_brain = self.getCertificateBrain(certcode) |
---|
798 | if not certificate_brain: |
---|
799 | em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org) |
---|
800 | logger.info(em) |
---|
801 | matric_no = student.matric_no |
---|
802 | sid = student_id |
---|
803 | student_obj = getattr(students_folder,sid) |
---|
804 | if not getattr(student_obj,'application'): |
---|
805 | student_obj.invokeFactory('StudentApplication','application') |
---|
806 | application = student_obj.application |
---|
807 | self.portal_workflow.doActionFor(application,'open',dest_container=application) |
---|
808 | da = {'Title': 'Application Data'} |
---|
809 | student_obj.invokeFactory('StudentPersonal','personal') |
---|
810 | da['jamb_reg_no'] = student.Entryregno |
---|
811 | em = self.getEntryMode(student.Entryregno) |
---|
812 | da['entry_mode'] = em |
---|
813 | personal = student_obj.personal |
---|
814 | self.portal_workflow.doActionFor(personal,'open',dest_container=personal) |
---|
815 | dp = {'Title': 'Personal Data'} |
---|
816 | student_obj.invokeFactory('StudentClearance','clearance') |
---|
817 | clearance = student_obj.clearance |
---|
818 | self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) |
---|
819 | dc = {'Title': 'Clearance/Eligibility Record'} |
---|
820 | dc['matric_no'] = matric_no |
---|
821 | state = student.State |
---|
822 | lga = student.LGA |
---|
823 | if state and lga: |
---|
824 | lga = state + ' / ' + lga |
---|
825 | else: |
---|
826 | lga = "None" |
---|
827 | da['jamb_lga'] = dc['lga'] = lga |
---|
828 | da['app_email'] = dp['email'] = email |
---|
829 | da['app_mobile'] = dp['phone'] = phone_nr |
---|
830 | dp['firstname'] = student.Firstname |
---|
831 | dp['middlename'] = student.Middlename |
---|
832 | dp['lastname'] = student.Lastname |
---|
833 | da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname) |
---|
834 | da['jamb_sex'] = student.Sex |
---|
835 | dp['sex'] = student.Sex == 'F' |
---|
836 | dp['perm_address'] = student.Permanent_Address |
---|
837 | application.getContent().edit(mapping=da) |
---|
838 | self.portal_workflow.doActionFor(application,'close',dest_container=application) |
---|
839 | personal.getContent().edit(mapping=dp) |
---|
840 | clearance.getContent().edit(mapping=dc) |
---|
841 | self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) |
---|
842 | # |
---|
843 | # Study Course |
---|
844 | # |
---|
845 | student_obj.invokeFactory('StudentStudyCourse','study_course') |
---|
846 | studycourse = student_obj.study_course |
---|
847 | self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) |
---|
848 | dsc = {} |
---|
849 | dsc['study_course'] = certcode |
---|
850 | dsc['current_level'] = level |
---|
851 | dsc['current_verdict'] = verdict |
---|
852 | dsc['current_mode'] = em #no longer used |
---|
853 | dsc['current_session'] = '05' |
---|
854 | studycourse.getContent().edit(mapping=dsc) |
---|
855 | # |
---|
856 | # Level |
---|
857 | # |
---|
858 | # l = getattr(studycourse,level,None) |
---|
859 | # if l is None: |
---|
860 | # studycourse.invokeFactory('StudentStudyLevel', level) |
---|
861 | # l = getattr(studycourse, level) |
---|
862 | # self.portal_workflow.doActionFor(l,'open',dest_container=l) |
---|
863 | # l.getContent().edit(mapping={'Title': "Level %s" % level}) |
---|
864 | ###) |
---|
865 | |
---|
866 | def init_timing(self): ###( |
---|
867 | if self.with_timing: |
---|
868 | if not hasattr(self,'_v_step_times'): |
---|
869 | self._v_timer_count = 0 |
---|
870 | self._v_total = 0 |
---|
871 | self._v_step_times = {} |
---|
872 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
---|
873 | self._v_timer_file = "%s/export/timing_%s.csv" % (i_home,current,) |
---|
874 | self.timer_step = 0 |
---|
875 | self.total_time = 0 |
---|
876 | self.elapse = time.time() |
---|
877 | self.i_elapse = time.time() |
---|
878 | self.c_elapse = time.clock() |
---|
879 | ###) |
---|
880 | |
---|
881 | def do_timing(self): ###( |
---|
882 | if self.with_timing: |
---|
883 | try: |
---|
884 | raise 'dummy' |
---|
885 | except: |
---|
886 | frame = sys.exc_traceback.tb_frame.f_back |
---|
887 | locals = frame.f_locals |
---|
888 | globals = frame.f_globals |
---|
889 | functionname = frame.f_code.co_name |
---|
890 | filename = os.path.basename(frame.f_code.co_filename) |
---|
891 | lineno = frame.f_lineno |
---|
892 | mod_line = "%(functionname)s:%(lineno)s" % vars() |
---|
893 | i_time = time.time() - self.i_elapse |
---|
894 | td = {} |
---|
895 | if self._v_step_times.has_key(mod_line): |
---|
896 | a_time = self._v_step_times[mod_line]['a_time'] + i_time |
---|
897 | td['a_time'] = a_time |
---|
898 | else: |
---|
899 | td['a_time'] = i_time |
---|
900 | td['i_time'] = i_time |
---|
901 | self._v_step_times[mod_line] = td |
---|
902 | self.i_time = i_time |
---|
903 | self.total_time += i_time |
---|
904 | self.timer_step +=1 |
---|
905 | self.i_elapse = time.time() |
---|
906 | ###) |
---|
907 | |
---|
908 | security.declareProtected(ModifyPortalContent,'print_timing') ###( ###( |
---|
909 | def print_timing(self): |
---|
910 | if self.with_timing: |
---|
911 | l = [] |
---|
912 | timer_count = self._v_timer_count + 1 |
---|
913 | mod_lines = self._v_step_times.keys() |
---|
914 | mod_lines.sort(cmp,reverse=0) |
---|
915 | for mod_line in mod_lines: |
---|
916 | td = self._v_step_times[mod_line] |
---|
917 | i_time = td['i_time'] |
---|
918 | a_time = td['a_time']/(self._v_timer_count + 1) |
---|
919 | l += ("%(mod_line)s,%(i_time)6.2f,%(a_time)6.2f,%(timer_count)d" % vars()), |
---|
920 | total_time = self.total_time |
---|
921 | total_avarage = self._v_total / timer_count |
---|
922 | l += ("total,%(total_time)6.4f,%(total_avarage)6.4f,%(timer_count)d" % vars()), |
---|
923 | print "\r\n".join(l) |
---|
924 | out = open(self._v_timer_file,'a') |
---|
925 | out.write("\r\n".join(l)) |
---|
926 | out.close() |
---|
927 | ###) |
---|
928 | |
---|
929 | security.declareProtected(ModifyPortalContent,'get_timing_data') ###( ###( |
---|
930 | def get_timing_data(self): |
---|
931 | if self.with_timing: |
---|
932 | timer_count = self._v_timer_count + 1 |
---|
933 | results = {} |
---|
934 | for k,d in self._v_step_times.items(): |
---|
935 | dd = {} |
---|
936 | dd['a_time'] = d['a_time']/timer_count |
---|
937 | dd['i_time'] = d['i_time'] |
---|
938 | dd['count'] = timer_count |
---|
939 | results[k] = dd |
---|
940 | dd = {} |
---|
941 | dd['a_time'] = self._v_total / timer_count |
---|
942 | dd['i_time'] = self.total_time |
---|
943 | dd['count'] = timer_count |
---|
944 | results["total"] = dd |
---|
945 | return results |
---|
946 | ###) |
---|
947 | |
---|
948 | security.declareProtected(ModifyPortalContent,'admitOneStudent') ###( |
---|
949 | def admitOneStudent(self,brain,entry_session,pin_password,with_timing=False): |
---|
950 | "create Datastructure for an admitted Student" |
---|
951 | #import pdb;pdb.set_trace() |
---|
952 | logger = logging.getLogger('WAeUPTool.admitOneStudent') |
---|
953 | self.with_timing = with_timing |
---|
954 | |
---|
955 | try: |
---|
956 | if brain.screening_type in ('cest','sandwich',) and brain.serial and brain.entry_session: |
---|
957 | if brain.course1: |
---|
958 | reg_no = "%s%s/%s" % (brain.course1[:3],brain.serial,brain.entry_session) |
---|
959 | else: |
---|
960 | reg_no = "%s%s/%s" % (brain.course_admitted[:3],brain.serial,brain.entry_session) |
---|
961 | else: |
---|
962 | reg_no = brain.reg_no |
---|
963 | except: |
---|
964 | logger.info('applicant record %s has errors' % (brain.reg_no)) |
---|
965 | return |
---|
966 | |
---|
967 | #ignore argument entry_session |
---|
968 | if not brain.entry_session: |
---|
969 | logger.info('no entry_session for %s provided' % (reg_no)) |
---|
970 | return |
---|
971 | else: |
---|
972 | es = brain.entry_session |
---|
973 | if len(es) == 1: |
---|
974 | es = '0%c' % es |
---|
975 | |
---|
976 | if not hasattr(self,"_v_certificates"): |
---|
977 | self._v_certificates = self.getCertificatesDict() |
---|
978 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
979 | |
---|
980 | res = self.students_catalog(jamb_reg_no = reg_no) |
---|
981 | if res: |
---|
982 | logger.info('student with reg_no %s exists (%s)' % (reg_no,res[0].id)) |
---|
983 | return |
---|
984 | if brain.status != "admitted": |
---|
985 | logger.info('status of %s is %s' % (reg_no,brain.status)) |
---|
986 | return |
---|
987 | pin_parts = brain.pin.split('-') |
---|
988 | if pin_parts and len(pin_parts) != 3: |
---|
989 | logger.info('invalid pin %s for %s' % (brain.pin,reg_no)) |
---|
990 | return |
---|
991 | if not brain.course_admitted: |
---|
992 | logger.info('no course_admitted provided for %s' % (reg_no)) |
---|
993 | return |
---|
994 | if brain.course_admitted not in self._v_certificates: |
---|
995 | logger.info('certificate %s not found for %s' % (brain.course_admitted,reg_no)) |
---|
996 | return |
---|
997 | if brain.sex not in (True,False): |
---|
998 | logger.info('sex of %s not available' % (reg_no)) |
---|
999 | return |
---|
1000 | self.init_timing() |
---|
1001 | student_id = self.generateStudentId('?') |
---|
1002 | students_folder.invokeFactory('Student', student_id) |
---|
1003 | student_object = getattr(students_folder,student_id) |
---|
1004 | self.do_timing() |
---|
1005 | if pin_password: |
---|
1006 | password = pin_parts[2] |
---|
1007 | self.makeStudentMember(student_id,password = password) |
---|
1008 | student_object.manage_setLocalRoles(student_id, ['Owner',]) |
---|
1009 | self.do_timing() |
---|
1010 | #logger.info("creating %s reg_no %s" % (student_id,reg_no)) |
---|
1011 | # |
---|
1012 | # application |
---|
1013 | # |
---|
1014 | student_object.invokeFactory('StudentApplication','application') |
---|
1015 | application = student_object.application |
---|
1016 | #self.portal_workflow.doActionFor(application,'open',dest_container=application) |
---|
1017 | #self.do_timing() |
---|
1018 | da = {'Title': 'Application Data'} |
---|
1019 | da['jamb_reg_no'] = reg_no |
---|
1020 | |
---|
1021 | sex = 'M' |
---|
1022 | if brain.sex: |
---|
1023 | sex = 'F' |
---|
1024 | da['jamb_sex'] = sex |
---|
1025 | da['jamb_age'] = brain.jamb_age |
---|
1026 | da['app_reg_pin'] = brain.pin |
---|
1027 | da['jamb_lga'] = brain.jamb_lga |
---|
1028 | da['jamb_state'] = brain.jamb_state |
---|
1029 | da['jamb_score'] = brain.aggregate |
---|
1030 | da['app_email'] = brain.email |
---|
1031 | da['app_mobile'] = brain.phone |
---|
1032 | |
---|
1033 | # entry_mode cannot be retrieved from the certtificate |
---|
1034 | # because ug_ft has two different entry modes |
---|
1035 | |
---|
1036 | if brain.entry_mode: # does not happen because import_application |
---|
1037 | da['entry_mode'] = brain.entry_mode # schema has no field entry_mode |
---|
1038 | elif brain.screening_type == 'pume': |
---|
1039 | da['entry_mode'] = 'ume_ft' |
---|
1040 | elif brain.screening_type == 'pde': |
---|
1041 | da['entry_mode'] = 'de_ft' |
---|
1042 | else: |
---|
1043 | da['entry_mode'] = self._v_certificates[brain.course_admitted]['study_mode'] |
---|
1044 | |
---|
1045 | #elif brain.screening_type == 'pce': |
---|
1046 | # da['entry_mode'] = 'pce' |
---|
1047 | #elif brain.screening_type == 'prence': |
---|
1048 | # da['entry_mode'] = 'prence' |
---|
1049 | #else: |
---|
1050 | # da['entry_mode'] = '' |
---|
1051 | |
---|
1052 | |
---|
1053 | #da['entry_session'] = entry_session |
---|
1054 | da['entry_session'] = es |
---|
1055 | da['jamb_lastname'] = brain.lastname |
---|
1056 | da['jamb_middlename'] = brain.middlenames # different field names! |
---|
1057 | da['jamb_firstname'] = brain.firstname |
---|
1058 | da['screening_application_date'] = brain.application_date |
---|
1059 | da['date_of_birth'] = brain.date_of_birth |
---|
1060 | da['jamb_first_cos'] = brain.course1 |
---|
1061 | da['jamb_second_cos'] = brain.course2 |
---|
1062 | da['course3'] = brain.course3 |
---|
1063 | da['screening_type'] = brain.screening_type |
---|
1064 | da['screening_score'] = brain.screening_score |
---|
1065 | da['screening_date'] = brain.screening_date |
---|
1066 | da['hq_type'] = brain.hq_type |
---|
1067 | da['hq_grade'] = brain.hq_grade |
---|
1068 | da['aos'] = brain.aos |
---|
1069 | |
---|
1070 | application.getContent().edit(mapping=da) |
---|
1071 | self.do_timing() |
---|
1072 | #self.portal_workflow.doActionFor(application,'close',dest_container=application) |
---|
1073 | # |
---|
1074 | # personal |
---|
1075 | # |
---|
1076 | student_object.invokeFactory('StudentPersonal','personal') |
---|
1077 | personal = student_object.personal |
---|
1078 | #self.portal_workflow.doActionFor(personal,'open',dest_container=personal) |
---|
1079 | #self.do_timing() |
---|
1080 | dp = {'Title': 'Personal Data'} |
---|
1081 | dp['sex'] = brain.sex |
---|
1082 | dp['email'] = brain.email |
---|
1083 | dp['phone'] = brain.phone |
---|
1084 | dp['lastname'] = brain.lastname |
---|
1085 | dp['middlename'] = brain.middlenames # different field names! |
---|
1086 | dp['firstname'] = brain.firstname |
---|
1087 | personal.getContent().edit(mapping=dp) |
---|
1088 | self.do_timing() |
---|
1089 | # |
---|
1090 | # clearance |
---|
1091 | # |
---|
1092 | student_object.invokeFactory('StudentClearance','clearance') |
---|
1093 | clearance = student_object.clearance |
---|
1094 | #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) |
---|
1095 | dc = {'Title': 'Clearance/Eligibility Record'} |
---|
1096 | dc['lga'] = brain.lga |
---|
1097 | dc['birthday'] = brain.date_of_birth |
---|
1098 | clearance.getContent().edit(mapping=dc) |
---|
1099 | self.do_timing() |
---|
1100 | #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) |
---|
1101 | # |
---|
1102 | # study Course |
---|
1103 | # |
---|
1104 | student_object.invokeFactory('StudentStudyCourse','study_course') |
---|
1105 | studycourse = student_object.study_course |
---|
1106 | #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) |
---|
1107 | #self.do_timing() |
---|
1108 | dsc = {} |
---|
1109 | dsc['study_course'] = brain.course_admitted |
---|
1110 | dsc['current_verdict'] = '' |
---|
1111 | dsc['current_mode'] = da['entry_mode'] # no longer used |
---|
1112 | if da['entry_mode'].startswith('de'): |
---|
1113 | dsc['current_level'] = '200' |
---|
1114 | elif da['entry_mode'].startswith('pre'): |
---|
1115 | dsc['current_level'] = '000' |
---|
1116 | else: |
---|
1117 | dsc['current_level'] = '100' |
---|
1118 | dsc['current_session'] = es |
---|
1119 | studycourse.getContent().edit(mapping=dsc) |
---|
1120 | self.do_timing() |
---|
1121 | # |
---|
1122 | # payments folder |
---|
1123 | student_object.invokeFactory('PaymentsFolder','payments') |
---|
1124 | payments = getattr(student_object,'payments') |
---|
1125 | #self.do_timing() |
---|
1126 | dpay = {} |
---|
1127 | dpay['Title'] = 'Payments' |
---|
1128 | payments.getContent().edit(mapping=dpay) |
---|
1129 | self.portal_workflow.doActionFor(payments,'open') |
---|
1130 | self.do_timing() |
---|
1131 | # |
---|
1132 | # passport foto |
---|
1133 | app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home, |
---|
1134 | brain.screening_type, |
---|
1135 | brain.reg_no) |
---|
1136 | images_dir = getImagesDir(student_id) |
---|
1137 | #images_dir = os.path.join("%s" % images_base,student_id) |
---|
1138 | letter_dir,student_dir = os.path.split(images_dir) |
---|
1139 | if not os.path.exists(letter_dir): |
---|
1140 | os.mkdir(letter_dir) |
---|
1141 | if not os.path.exists(images_dir): |
---|
1142 | os.mkdir(images_dir) |
---|
1143 | image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) |
---|
1144 | if os.path.exists(app_picture): |
---|
1145 | copy2(app_picture,image_name) |
---|
1146 | else: |
---|
1147 | logger.info('passport of %s/%s not found: %s' % (student_id, |
---|
1148 | brain.reg_no, |
---|
1149 | app_picture)) |
---|
1150 | |
---|
1151 | self.do_timing() |
---|
1152 | self.print_timing() |
---|
1153 | if with_timing: |
---|
1154 | self.timer_step = 0 |
---|
1155 | self._v_timer_count += 1 |
---|
1156 | self._v_total += self.total_time |
---|
1157 | return student_id |
---|
1158 | ###) |
---|
1159 | |
---|
1160 | security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###( |
---|
1161 | def makeStudentLevel(self,student_id): |
---|
1162 | "create the StudyLevel for a returning Student" |
---|
1163 | #import pdb;pdb.set_trace() |
---|
1164 | logger = logging.getLogger('WAeUPTool.makeStudentLevel') |
---|
1165 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
1166 | res = self.students_catalog(id=student_id) |
---|
1167 | if res: |
---|
1168 | st = res[0] |
---|
1169 | course = st.course |
---|
1170 | matric_no = st.matric_no |
---|
1171 | level = st.level |
---|
1172 | res = self.results_import(matric_no = matric_no) |
---|
1173 | if res: |
---|
1174 | results = res |
---|
1175 | logger.info('%s creating Level %s' % (student_id,level)) |
---|
1176 | # |
---|
1177 | # Level |
---|
1178 | # |
---|
1179 | student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
---|
1180 | studycourse = getattr(student_obj,"study_course",None) |
---|
1181 | self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse) |
---|
1182 | l = getattr(studycourse,level,None) |
---|
1183 | if l is None: |
---|
1184 | studycourse.invokeFactory('StudentStudyLevel', level) |
---|
1185 | l = getattr(studycourse, level) |
---|
1186 | self.portal_workflow.doActionFor(l,'open',dest_container=l) |
---|
1187 | l.getContent().edit(mapping={'Title': "Level %s" % level}) |
---|
1188 | ###) |
---|
1189 | |
---|
1190 | |
---|
1191 | security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###( |
---|
1192 | def exportAllStudyLevels(self,student_id): |
---|
1193 | "export the StudyLevels for a student" |
---|
1194 | #import pdb;pdb.set_trace() |
---|
1195 | logger = logging.getLogger('WAeUPTool.exportAllStudyLevels') |
---|
1196 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
1197 | res = self.students_catalog(id=student_id) |
---|
1198 | if res: |
---|
1199 | st = res[0] |
---|
1200 | course = st.course |
---|
1201 | matric_no = st.matric_no |
---|
1202 | level = st.level |
---|
1203 | student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
---|
1204 | studycourse = getattr(student_obj,"study_course",None) |
---|
1205 | if studycourse: |
---|
1206 | levels = studycourse.objectIds() |
---|
1207 | else: |
---|
1208 | levels = [] |
---|
1209 | stool = getToolByName(self, 'portal_schemas') |
---|
1210 | schema = stool._getOb('student_study_level') |
---|
1211 | fields = ['student_id','level'] |
---|
1212 | fields.extend(schema.keys()) |
---|
1213 | format = '"%(' + ')s","%('.join(fields) + ')s"' |
---|
1214 | export_file = "%s/export/study_levels_removed.csv" % (i_home) |
---|
1215 | |
---|
1216 | if not os.path.exists(export_file): |
---|
1217 | file_handler = open(export_file,"a") |
---|
1218 | headline = ','.join(fields) |
---|
1219 | file_handler.write(headline +'\n') |
---|
1220 | else: |
---|
1221 | file_handler = open(export_file,"a") |
---|
1222 | |
---|
1223 | for level in levels: |
---|
1224 | level_object = getattr(studycourse,level) |
---|
1225 | level_content = level_object.getContent() |
---|
1226 | d = {'student_id':student_id,'level':level} |
---|
1227 | for field in schema.keys(): |
---|
1228 | d[field] = getattr(level_content,field,'') |
---|
1229 | line = format % d |
---|
1230 | file_handler.write(line +'\n') |
---|
1231 | |
---|
1232 | ###) |
---|
1233 | |
---|
1234 | security.declarePublic('getHallInfo') ###( |
---|
1235 | def getHallInfo(self,bed): |
---|
1236 | """return Hall Info""" |
---|
1237 | info = {} |
---|
1238 | bedsplit = bed.split('_') |
---|
1239 | if len(bedsplit) == 4: |
---|
1240 | hall,block,room,letter = bed.split('_') |
---|
1241 | else: |
---|
1242 | info['maintenance_code'] = 'None' |
---|
1243 | return info |
---|
1244 | res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall) |
---|
1245 | if res and len(res) == 1: |
---|
1246 | hall_brain = res[0] |
---|
1247 | hall_doc = hall_brain.getObject().getContent() |
---|
1248 | else: |
---|
1249 | return info |
---|
1250 | info['hall_title'] = hall_brain.Title |
---|
1251 | info['maintenance_code'] = hall_doc.maintenance_code |
---|
1252 | res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch") |
---|
1253 | batch_doc = None |
---|
1254 | for brain in res: |
---|
1255 | if brain.id.startswith(info['maintenance_code']): |
---|
1256 | batch_doc = brain.getObject().getContent() |
---|
1257 | break |
---|
1258 | if batch_doc is None: |
---|
1259 | info['maintenance_fee'] = '' |
---|
1260 | else: |
---|
1261 | info['maintenance_fee'] = batch_doc.cost |
---|
1262 | return info |
---|
1263 | ###) |
---|
1264 | |
---|
1265 | security.declareProtected(ModifyPortalContent,'removePictureFolder') ###( |
---|
1266 | def removePictureFolder(self,student_id): |
---|
1267 | """remove picture_folder by renaming it""" |
---|
1268 | picture_path = getImagesDir(student_id) |
---|
1269 | dest_path = os.path.join("%s" % images_base,'removed',student_id) |
---|
1270 | dest_path = dest_path + "_removed" |
---|
1271 | if os.path.exists(dest_path) or not os.path.exists(picture_path): |
---|
1272 | return False |
---|
1273 | os.rename(picture_path,dest_path) |
---|
1274 | return True |
---|
1275 | ###) |
---|
1276 | |
---|
1277 | security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###( |
---|
1278 | def restorePictureFolder(self,student_id): |
---|
1279 | """restore picture_folder by renaming it""" |
---|
1280 | picture_path = getImagesDir(student_id) |
---|
1281 | orig_path = os.path.join("%s" % images_base,'removed',student_id) |
---|
1282 | orig_path = orig_path + "_removed" |
---|
1283 | if os.path.exists(picture_path) or not os.path.exists(orig_path): |
---|
1284 | return False |
---|
1285 | os.rename(orig_path,picture_path) |
---|
1286 | return True |
---|
1287 | ###) |
---|
1288 | |
---|
1289 | security.declarePublic('picturesExist') ###( |
---|
1290 | def picturesExist(self, ids,student_id=None): |
---|
1291 | """check if pictures exist in the filesystem""" |
---|
1292 | if student_id is None: |
---|
1293 | student_id = self.getStudentId() |
---|
1294 | if student_id is None: |
---|
1295 | return False |
---|
1296 | picture_path = getImagesDir(student_id) |
---|
1297 | #picture_path = os.path.join(images_base,student_id) |
---|
1298 | if not os.path.exists(picture_path): |
---|
1299 | return False |
---|
1300 | pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] |
---|
1301 | return set(ids).issubset(set(pictures)) |
---|
1302 | ###) |
---|
1303 | |
---|
1304 | security.declarePublic('picturePathExists') ###( |
---|
1305 | def picturePathExists(self, student_id=None): |
---|
1306 | """check if picture path exists in the filesystem""" |
---|
1307 | if student_id is None: |
---|
1308 | student_id = self.getStudentId() |
---|
1309 | if student_id is None: |
---|
1310 | return False |
---|
1311 | picture_path = getImagesDir(student_id) |
---|
1312 | if os.path.exists(picture_path): |
---|
1313 | return True |
---|
1314 | return False |
---|
1315 | ###) |
---|
1316 | |
---|
1317 | |
---|
1318 | security.declareProtected(ModifyPortalContent,'removeUnusedImageFolders') ###( |
---|
1319 | def removeUnusedImageFolders(self): |
---|
1320 | """check if an unused image folders exists in the filesystem""" |
---|
1321 | mtool = self.portal_membership |
---|
1322 | member = mtool.getAuthenticatedMember() |
---|
1323 | member_id = str(member) |
---|
1324 | logger = logging.getLogger('WAeUPTool.removeUnusedImageFolders') |
---|
1325 | abc = os.listdir(images_base) |
---|
1326 | ifolders = [] |
---|
1327 | for i in abc: |
---|
1328 | picture_path = os.path.join(images_base,i) |
---|
1329 | ifolders.extend(os.listdir(picture_path)) |
---|
1330 | unused_ids = [] |
---|
1331 | for id in ifolders: |
---|
1332 | res = self.students_catalog(id=id) |
---|
1333 | if not res: |
---|
1334 | unused_ids.append(id) |
---|
1335 | #import pdb;pdb.set_trace() |
---|
1336 | if not id.endswith('removed'): |
---|
1337 | removed = self.waeup_tool.removePictureFolder(id) |
---|
1338 | if removed: |
---|
1339 | logger.info('%s: image folder %s successfully removed' % (member_id,id)) |
---|
1340 | else: |
---|
1341 | logger.info('%s: image folder %s could not be removed' % (member_id,id)) |
---|
1342 | return |
---|
1343 | |
---|
1344 | ###) |
---|
1345 | |
---|
1346 | security.declarePublic('picturesList') ###( |
---|
1347 | def picturesList(self): |
---|
1348 | """check if pictures exist in the filesystem""" |
---|
1349 | path = 'images' |
---|
1350 | student_id = self.getStudentId() |
---|
1351 | #picture_path = os.path.join(i_home,path,student_id) |
---|
1352 | picture_path = getImagesDir(student_id) |
---|
1353 | if not os.path.exists(picture_path): |
---|
1354 | return [] |
---|
1355 | return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] |
---|
1356 | ###) |
---|
1357 | |
---|
1358 | security.declarePublic('showFsPicture') ###( |
---|
1359 | def showFsPicture(self,path): |
---|
1360 | """return a picture from the filesystem""" |
---|
1361 | #picture_path = os.path.join(i_home,path) |
---|
1362 | picture_path = os.path.join(images_base,path) |
---|
1363 | response = self.REQUEST.RESPONSE |
---|
1364 | #import pdb;pdb.set_trace() |
---|
1365 | registry = getToolByName(self, 'mimetypes_registry') |
---|
1366 | mimetype = str(registry.lookupExtension(path.lower()) or |
---|
1367 | registry.lookupExtension('file.bin')) |
---|
1368 | if os.path.exists(picture_path): |
---|
1369 | response.setHeader('Content-type',mimetype) |
---|
1370 | return open(picture_path).read() |
---|
1371 | picture_path = os.path.join(i_home,'import',path) |
---|
1372 | if os.path.exists(picture_path): |
---|
1373 | return open(picture_path).read() |
---|
1374 | ###) |
---|
1375 | |
---|
1376 | security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###( |
---|
1377 | def deleteAllCourses(self,department="All"): |
---|
1378 | ''' delete the courses''' |
---|
1379 | pm = self.portal_membership |
---|
1380 | member = pm.getAuthenticatedMember() |
---|
1381 | |
---|
1382 | if str(member) not in ("henrik","joachim"): |
---|
1383 | return "not possible" |
---|
1384 | if department == "All": |
---|
1385 | res = self.portal_catalog({'meta_type': 'Department'}) |
---|
1386 | if len(res) < 1: |
---|
1387 | return "No Departments found" |
---|
1388 | |
---|
1389 | deleted = [] |
---|
1390 | for dep in res: |
---|
1391 | cf = dep.getObject().courses |
---|
1392 | if cf: |
---|
1393 | cf.manage_delObjects(ids=cf.objectIds()) |
---|
1394 | deleted.append("deleted Courses in %s" % dep.getId) |
---|
1395 | return "\r".join(deleted) |
---|
1396 | ###) |
---|
1397 | |
---|
1398 | security.declareProtected(ModifyPortalContent,'getLogfileLines') ###( |
---|
1399 | def getLogfileLines(self,filename="event.log",numlines=20): |
---|
1400 | """Get last NUMLINES lines of logfile FILENAME. |
---|
1401 | |
---|
1402 | Return last lines' of a file in the instances logfile directory as |
---|
1403 | a list. The number of returned lines equals `numlines' or less. If |
---|
1404 | less than `numlines' lines are available, the whole file ist |
---|
1405 | returned. If the file can not be opened or some other error |
---|
1406 | occurs, empty list is returend. |
---|
1407 | """ |
---|
1408 | result = [] |
---|
1409 | lines_hit = 0 |
---|
1410 | |
---|
1411 | # We only handle files in instances' log directory... |
---|
1412 | logpath = os.path.join(i_home, "log") |
---|
1413 | filename = str(os.path.abspath( os.path.join( logpath, filename ))) |
---|
1414 | if not filename.startswith( logpath ): |
---|
1415 | # Attempt to access file outside log-dir... |
---|
1416 | return [] |
---|
1417 | |
---|
1418 | try: |
---|
1419 | fd = file( filename, "rb" ) |
---|
1420 | except IOError: |
---|
1421 | return [] |
---|
1422 | if not fd: |
---|
1423 | return [] |
---|
1424 | |
---|
1425 | if os.linesep == None: |
---|
1426 | linesep = '\n' |
---|
1427 | else: |
---|
1428 | linesep = os.linesep |
---|
1429 | |
---|
1430 | # Try to find 'numlines' times a lineseparator, searching from end |
---|
1431 | # and moving to the beginning of file... |
---|
1432 | fd.seek( 0, 2) # Move to end of file... |
---|
1433 | while lines_hit < numlines: |
---|
1434 | if fd.read(1) == linesep[-1]: # This moves filedescriptor |
---|
1435 | # one step forward... |
---|
1436 | lines_hit += 1 |
---|
1437 | try: |
---|
1438 | fd.seek( -2, 1) # Go two bytes back from current pos... |
---|
1439 | except IOError: |
---|
1440 | # We cannot go back two bytes. Maybe the file is too small... |
---|
1441 | break |
---|
1442 | fd.seek(2,1) |
---|
1443 | |
---|
1444 | # Read all lines from current position... |
---|
1445 | result = fd.readlines() |
---|
1446 | # Remove line endings... |
---|
1447 | result = [x.strip() for x in result] |
---|
1448 | fd.close() |
---|
1449 | return result |
---|
1450 | ###) |
---|
1451 | |
---|
1452 | security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###( |
---|
1453 | def getCallbacksFromLog(self,filename): |
---|
1454 | """fix Online Payment Transactions from Z2.log entries""" |
---|
1455 | import transaction |
---|
1456 | import random |
---|
1457 | from cgi import parse_qs |
---|
1458 | from urlparse import urlparse |
---|
1459 | #from pdb import set_trace |
---|
1460 | wftool = self.portal_workflow |
---|
1461 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
---|
1462 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
1463 | s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \[' |
---|
1464 | s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "' |
---|
1465 | s += r'(?P<intersw>.*)" "(?P<agent>.*)"' |
---|
1466 | data = re.compile(s) |
---|
1467 | start = True |
---|
1468 | tr_count = 1 |
---|
1469 | total = 0 |
---|
1470 | #name = 'pume_results' |
---|
1471 | #name = 'epaymentsuccessful_z2log2' |
---|
1472 | name = filename |
---|
1473 | no_import = [] |
---|
1474 | imported = [] |
---|
1475 | logger = logging.getLogger('WAeUPTool.getFailedTransactions') |
---|
1476 | try: |
---|
1477 | transactions = open("%s/import/%s" % (i_home,name),"rb").readlines() |
---|
1478 | except: |
---|
1479 | logger.error('Error reading %s' % name) |
---|
1480 | return |
---|
1481 | tas = [] |
---|
1482 | for line in transactions: |
---|
1483 | dict = {} |
---|
1484 | items = data.search(line) |
---|
1485 | dict['idict'] = idict = items.groupdict() |
---|
1486 | #print idict |
---|
1487 | #from pdb import set_trace;set_trace() |
---|
1488 | urlparsed = urlparse(idict['get'][4:]) |
---|
1489 | #print urlparsed |
---|
1490 | path = urlparsed[2].split('/') |
---|
1491 | dict['student_id'] = student_id = path[8] |
---|
1492 | dict['payment_id'] = payment_id = path[10] |
---|
1493 | dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4]) |
---|
1494 | tas.append(dict) |
---|
1495 | tr_count += 1 |
---|
1496 | return tas |
---|
1497 | ###) |
---|
1498 | |
---|
1499 | security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###( |
---|
1500 | def importOnlinePaymentTransactions(self): |
---|
1501 | """load Online Payment Transactions from CSV values""" |
---|
1502 | import transaction |
---|
1503 | import random |
---|
1504 | #from pdb import set_trace |
---|
1505 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
---|
1506 | opt = self.online_payments_import |
---|
1507 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
1508 | start = True |
---|
1509 | tr_count = 1 |
---|
1510 | total = 0 |
---|
1511 | #name = 'pume_results' |
---|
1512 | name = 'OnlineTransactions' |
---|
1513 | no_import = [] |
---|
1514 | imported = [] |
---|
1515 | logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions') |
---|
1516 | try: |
---|
1517 | transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) |
---|
1518 | except: |
---|
1519 | logger.error('Error reading %s.csv' % name) |
---|
1520 | return |
---|
1521 | for pay_transaction in transactions: |
---|
1522 | if start: |
---|
1523 | start = False |
---|
1524 | logger.info('Start loading from %s.csv' % name) |
---|
1525 | s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()]) |
---|
1526 | no_import.append('%s,"Error"' % s) |
---|
1527 | format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()]) |
---|
1528 | format_error = format + ',"%(Error)s"' |
---|
1529 | data = {} |
---|
1530 | |
---|
1531 | # format of the first file sent by Tayo |
---|
1532 | #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date']) |
---|
1533 | #data['student_id'] = student_id = pay_transaction['Payer ID'] |
---|
1534 | #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] |
---|
1535 | #data['response_code'] = response_code = pay_transaction['Resp Code'] |
---|
1536 | #data['amount'] = amount = pay_transaction['Amount'] |
---|
1537 | |
---|
1538 | # format of the second file sent by Tayo |
---|
1539 | #data['datetime'] = date = 0 |
---|
1540 | #data['student_id'] = student_id = pay_transaction['Payer ID'] |
---|
1541 | #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] |
---|
1542 | #data['response_code'] = response_code = '00' |
---|
1543 | #data['amount'] = amount = pay_transaction['Amount'] |
---|
1544 | |
---|
1545 | # format of the third file sent by Kehinde |
---|
1546 | data['datetime'] = date = 0 |
---|
1547 | data['student_id'] = student_id = pay_transaction['customer_id'] |
---|
1548 | data['order_id'] = order_id = pay_transaction['merchant_reference'] |
---|
1549 | data['response_code'] = response_code = '00' |
---|
1550 | data['amount'] = amount = pay_transaction['Amount'] |
---|
1551 | |
---|
1552 | dup = False |
---|
1553 | if response_code == "12": |
---|
1554 | continue |
---|
1555 | try: |
---|
1556 | opt.addRecord(**data) |
---|
1557 | except ValueError: |
---|
1558 | dup = True |
---|
1559 | #from pdb import set_trace;set_trace() |
---|
1560 | if dup: |
---|
1561 | if response_code == "00": |
---|
1562 | try: |
---|
1563 | opt.modifyRecord(**data) |
---|
1564 | except: |
---|
1565 | logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data) |
---|
1566 | continue |
---|
1567 | else: |
---|
1568 | pay_transaction['Error'] = "Duplicate order_id" |
---|
1569 | no_import.append( format_error % pay_transaction) |
---|
1570 | logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data) |
---|
1571 | continue |
---|
1572 | tr_count += 1 |
---|
1573 | if tr_count > 1000: |
---|
1574 | if len(no_import) > 0: |
---|
1575 | open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( |
---|
1576 | '\n'.join(no_import) + '\n') |
---|
1577 | no_import = [] |
---|
1578 | em = '%d transactions committed\n' % (tr_count) |
---|
1579 | transaction.commit() |
---|
1580 | regs = [] |
---|
1581 | logger.info(em) |
---|
1582 | total += tr_count |
---|
1583 | tr_count = 0 |
---|
1584 | open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( |
---|
1585 | '\n'.join(no_import)) |
---|
1586 | return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) |
---|
1587 | ###) |
---|
1588 | |
---|
1589 | security.declareProtected(ModifyPortalContent,"importData")###( |
---|
1590 | def importData(self,filename,name,edit=False,bypass_queue_catalog=False): |
---|
1591 | """load data from CSV values""" |
---|
1592 | import transaction |
---|
1593 | import random |
---|
1594 | students_folder = self.portal_url.getPortalObject().campus.students |
---|
1595 | uploads_folder = self.portal_url.getPortalObject().campus.uploads |
---|
1596 | pending_only = False |
---|
1597 | pend_str = '--' |
---|
1598 | elapse = time.time() |
---|
1599 | # |
---|
1600 | # preparations |
---|
1601 | # |
---|
1602 | if filename == pend_str: |
---|
1603 | pending_only = True |
---|
1604 | importer_name = ''.join([part.capitalize() for part in name.split('_')]) |
---|
1605 | importer = eval("%sImport" % importer_name)(self) |
---|
1606 | logger = importer.logger |
---|
1607 | if importer.init_errors: |
---|
1608 | logger.info(importer.init_errors) |
---|
1609 | return importer.init_errors |
---|
1610 | member = importer.member |
---|
1611 | #current = importer.current |
---|
1612 | import_date = importer.import_date |
---|
1613 | # |
---|
1614 | # not_imported |
---|
1615 | # |
---|
1616 | info = importer.info |
---|
1617 | data_keys = importer.data_keys |
---|
1618 | csv_keys = importer.csv_keys |
---|
1619 | #csv_keys.extend(info.keys()) |
---|
1620 | headline_mapping = dict((k,k) for k in csv_keys) |
---|
1621 | # |
---|
1622 | # pending |
---|
1623 | # |
---|
1624 | pending_path = importer.pending_path |
---|
1625 | pending_tmp = importer.pending_tmp |
---|
1626 | pending_backup = importer.pending_backup |
---|
1627 | pending_fn = importer.pending_fn |
---|
1628 | imported_path = importer.imported_path |
---|
1629 | imported_fn = importer.imported_fn |
---|
1630 | commit_after = importer.commit_after |
---|
1631 | pending = [] |
---|
1632 | pending_digests = [] |
---|
1633 | #total_added_to_pending = 0 |
---|
1634 | if not pending_only: |
---|
1635 | pending,pending_digests = importer.makeIdLists() |
---|
1636 | pending_at_start = len(pending) |
---|
1637 | datafile = open(pending_tmp,"w") |
---|
1638 | pending_csv_writer = csv.DictWriter(datafile, |
---|
1639 | csv_keys, |
---|
1640 | extrasaction='ignore') |
---|
1641 | pending_csv_writer.writerow(headline_mapping) |
---|
1642 | datafile.close() |
---|
1643 | # |
---|
1644 | # imported |
---|
1645 | # |
---|
1646 | if not os.path.exists(imported_path): |
---|
1647 | datafile = open(imported_path,"w") |
---|
1648 | imported_csv_writer = csv.DictWriter(datafile, |
---|
1649 | csv_keys, |
---|
1650 | extrasaction='ignore') |
---|
1651 | imported_csv_writer.writerow(headline_mapping) |
---|
1652 | datafile.close() |
---|
1653 | start = True |
---|
1654 | tr_count = 0 |
---|
1655 | total = 0 |
---|
1656 | total_added_to_imported = 0 |
---|
1657 | total_pending = 0 |
---|
1658 | import_source_done = "" |
---|
1659 | if pending_only: |
---|
1660 | import_source_path = pending_path |
---|
1661 | else: |
---|
1662 | import_source_path = "%s/import/%s.csv" % (i_home,filename) |
---|
1663 | import_source_done = "%s/import/%s.done" % (i_home,filename) |
---|
1664 | if not os.path.exists(import_source_path): |
---|
1665 | fn = os.path.split(import_source_path)[1] |
---|
1666 | em = 'no import file %(fn)s' % vars() |
---|
1667 | return em |
---|
1668 | import_source_fn = os.path.split(import_source_path)[1] |
---|
1669 | if not pending_only: |
---|
1670 | info['imported_from'] = import_source_fn |
---|
1671 | headline = csv.reader(open(import_source_path,"rb")).next() |
---|
1672 | if "import_mode" not in headline: |
---|
1673 | msg = 'import_mode must be in heading' |
---|
1674 | return msg |
---|
1675 | invalid_keys = importer.checkHeadline(headline) |
---|
1676 | if invalid_keys: |
---|
1677 | return 'not ignorable key(s): "%s" found in heading' % ", ".join(invalid_keys) |
---|
1678 | |
---|
1679 | import_keys = [k.strip() for k in headline if not (k.strip().startswith('ignore') |
---|
1680 | or k.strip() in info.keys())] |
---|
1681 | # diff2schema = set(import_keys).difference(set(importer.schema.keys())) |
---|
1682 | # diff2layout = set(import_keys).difference(set(importer.layout.keys())) |
---|
1683 | # if diff2schema and diff2schema != set(['id',]): |
---|
1684 | # msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema) |
---|
1685 | # return msg |
---|
1686 | # |
---|
1687 | # start importing |
---|
1688 | # |
---|
1689 | try: |
---|
1690 | reader = csv.DictReader(open(import_source_path,"rb")) |
---|
1691 | except: |
---|
1692 | msg = 'Error reading %s.csv' % filename |
---|
1693 | logger.error(msg) |
---|
1694 | return msg |
---|
1695 | items = [item for item in reader] |
---|
1696 | total_to_import = len(items) |
---|
1697 | tti_float = float(total_to_import) |
---|
1698 | if pending_only: |
---|
1699 | pending_at_start = total_to_import |
---|
1700 | count = 0 |
---|
1701 | imported = [] |
---|
1702 | old_commit_count = 0 |
---|
1703 | error_count = imported_count = 0 |
---|
1704 | already_in = 0 |
---|
1705 | for record in items: |
---|
1706 | item = {} |
---|
1707 | empty_value_keys = [] |
---|
1708 | for k,v in record.items(): |
---|
1709 | if k is None: |
---|
1710 | continue |
---|
1711 | if v: |
---|
1712 | if v == EMPTY: |
---|
1713 | empty_value_keys += k, |
---|
1714 | v = '' |
---|
1715 | item[k.strip()] = v.strip() |
---|
1716 | count += 1 |
---|
1717 | if start: |
---|
1718 | start = False |
---|
1719 | adapters = [MappingStorageAdapter(importer.schema, item)] |
---|
1720 | logger.info('%(member)s starts import from %(import_source_fn)s' % vars()) |
---|
1721 | dm = DataModel(item, adapters,context=self) |
---|
1722 | ds = DataStructure(data=item,datamodel=dm) |
---|
1723 | error_string = "" |
---|
1724 | total += 1 |
---|
1725 | import_mode = item.get('import_mode','') |
---|
1726 | import_method = getattr(importer, '%(import_mode)s' % vars(),None ) |
---|
1727 | if import_method is None: |
---|
1728 | error_string += "import_mode '%(import_mode)s' is invalid" % vars() |
---|
1729 | elif (import_mode in importer.required_modes and |
---|
1730 | not set(importer.required_keys[import_mode]).issubset(set(item.keys()))): |
---|
1731 | diff2import = set(importer.required_keys[import_mode]).difference(set(item.keys())) |
---|
1732 | error_string += 'required key(s): "%s" not found in record' % ", ".join(diff2import) |
---|
1733 | else: |
---|
1734 | for k in import_keys: |
---|
1735 | if k not in item.keys() or k not in importer.validators.keys(): |
---|
1736 | continue |
---|
1737 | if not importer.validators[k](ds,mode=import_mode): |
---|
1738 | error_string += ' ++ ' |
---|
1739 | error_string += "%s: %s" % (k,self.translation_service(ds.getError(k), |
---|
1740 | ds.getErrorMapping(k))) |
---|
1741 | if error_string: |
---|
1742 | error = error_string |
---|
1743 | id = '' |
---|
1744 | mapping = item |
---|
1745 | else: |
---|
1746 | temp_item = item.copy() |
---|
1747 | temp_item.update(dm) |
---|
1748 | results = import_method(temp_item) |
---|
1749 | id = results[0] |
---|
1750 | error = results[1] |
---|
1751 | mapping = results[2] |
---|
1752 | # |
---|
1753 | # restore some values |
---|
1754 | # |
---|
1755 | for k in empty_value_keys: |
---|
1756 | mapping[k] = EMPTY |
---|
1757 | if mapping.has_key('sex'): |
---|
1758 | #import pdb;pdb.set_trace() |
---|
1759 | if mapping['sex'] == True: |
---|
1760 | mapping['sex'] = 'F' |
---|
1761 | elif mapping['sex'] == False: |
---|
1762 | mapping['sex'] = 'M' |
---|
1763 | if pending_only: |
---|
1764 | info['imported_from'] = item['imported_from'] |
---|
1765 | data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items()) |
---|
1766 | info['error'] = error |
---|
1767 | info['import_record_no'] = count + 1 |
---|
1768 | mapping.update(info) |
---|
1769 | log_list = [] |
---|
1770 | if error: |
---|
1771 | error_count += 1 |
---|
1772 | digest = makeDigest(mapping,data_keys) |
---|
1773 | if digest not in pending_digests: |
---|
1774 | pending_digests += digest, |
---|
1775 | pending.append(mapping) |
---|
1776 | if not pending_only: |
---|
1777 | log_list += "record from %(import_source_fn)s added to %(pending_fn)s, %(data_string)s, %(error)s" % vars(), |
---|
1778 | else: |
---|
1779 | already_in += 1 |
---|
1780 | pass |
---|
1781 | else: |
---|
1782 | imported_count += 1 |
---|
1783 | imported += mapping, |
---|
1784 | log_list += "record imported and added to %(imported_fn)s from %(import_source_fn)s, %(data_string)s" % vars(), |
---|
1785 | if log_list: |
---|
1786 | time_till_now = time.time() - elapse |
---|
1787 | percent_finished = (error_count + imported_count)/tti_float*100 |
---|
1788 | log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),) |
---|
1789 | logger.info(' '.join(log_list)) |
---|
1790 | finished = count > total_to_import - 1 |
---|
1791 | must_commit = (imported_count != old_commit_count) and (not imported_count % commit_after) |
---|
1792 | if must_commit: |
---|
1793 | old_commit_count = imported_count |
---|
1794 | |
---|
1795 | if must_commit or finished: |
---|
1796 | if len(imported): |
---|
1797 | transaction.commit() |
---|
1798 | datafile = open(imported_path,"a") |
---|
1799 | writer = csv.DictWriter(datafile, |
---|
1800 | csv_keys, |
---|
1801 | extrasaction='ignore') |
---|
1802 | writer.writerows(imported) |
---|
1803 | datafile.close() |
---|
1804 | total_added_to_imported += len(imported) |
---|
1805 | imported = [] |
---|
1806 | if len(pending) > 0: |
---|
1807 | datafile = open(pending_tmp,"a") |
---|
1808 | writer = csv.DictWriter(datafile, |
---|
1809 | csv_keys, |
---|
1810 | extrasaction='ignore') |
---|
1811 | writer.writerows(pending) |
---|
1812 | datafile.close() |
---|
1813 | total_pending += len(pending) |
---|
1814 | #total_added_to_pending += len(pending) |
---|
1815 | pending = [] |
---|
1816 | if not finished: |
---|
1817 | msg = '%(commit_after)d records imported and committed of total %(total_added_to_imported)d\n' % vars() |
---|
1818 | logger.info(msg) |
---|
1819 | elapse = time.time() - elapse |
---|
1820 | if os.path.exists(pending_path): |
---|
1821 | copy2(pending_path,pending_backup) |
---|
1822 | copy2(pending_tmp,pending_path) |
---|
1823 | msg = "finished importing from %(import_source_fn)s in %(elapse).2f seconds, " % vars() |
---|
1824 | msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars() |
---|
1825 | if pending_only: |
---|
1826 | removed_pending = pending_at_start - total_pending |
---|
1827 | msg += "%(removed_pending)d removed from %(pending_fn)s" % vars() |
---|
1828 | else: |
---|
1829 | added_pending = total_pending - pending_at_start |
---|
1830 | msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars() |
---|
1831 | #msg += "%(total_pending)d totally written" % vars() # this line does not make any sense |
---|
1832 | logger.info(msg) |
---|
1833 | if import_source_done: |
---|
1834 | copy(import_source_path,import_source_done) |
---|
1835 | os.remove(import_source_path) |
---|
1836 | upload = getattr(uploads_folder,os.path.split(import_source_path)[1],None) |
---|
1837 | if upload is not None: |
---|
1838 | upload_doc = upload.getContent() |
---|
1839 | mapping = {} |
---|
1840 | #mapping['import_date'] = DateTime.DateTime() |
---|
1841 | mapping['import_date'] = import_date |
---|
1842 | mapping['imported_by'] = importer.imported_by |
---|
1843 | mapping['import_message'] = msg |
---|
1844 | upload_doc.edit(mapping = mapping) |
---|
1845 | os.remove(pending_tmp) |
---|
1846 | return msg |
---|
1847 | ###) |
---|
1848 | |
---|
1849 | security.declareProtected(ModifyPortalContent,"moveImagesToFS")###( |
---|
1850 | def moveImagesToFS(self,student_id="O738726"): |
---|
1851 | "move the images to the filesystem" |
---|
1852 | images_dir = getImagesDir(student_id) |
---|
1853 | #images_dir = os.path.join("%s" % images_base,student_id) |
---|
1854 | student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
---|
1855 | stool = getToolByName(self, 'portal_schemas') |
---|
1856 | schemas = ['student_application', |
---|
1857 | 'student_clearance', |
---|
1858 | ] |
---|
1859 | created = False |
---|
1860 | for schema_id in schemas: |
---|
1861 | schema = stool._getOb(schema_id) |
---|
1862 | object = getattr(student_folder,schema_id[len('student_'):],None) |
---|
1863 | if object is None: |
---|
1864 | continue |
---|
1865 | doc = object.getContent() |
---|
1866 | for key in schema.keys(): |
---|
1867 | if schema[key].meta_type != "CPS Image Field": |
---|
1868 | continue |
---|
1869 | #import pdb;pdb.set_trace() |
---|
1870 | image = getattr(doc,key,None) |
---|
1871 | if not image or not hasattr(image,"data"): |
---|
1872 | continue |
---|
1873 | if not created: |
---|
1874 | if not os.path.exists(images_dir): |
---|
1875 | os.mkdir(images_dir) |
---|
1876 | created = True |
---|
1877 | filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars()) |
---|
1878 | open(filename,"wb").write(str(image.data)) |
---|
1879 | ###) |
---|
1880 | |
---|
1881 | security.declareProtected(ModifyPortalContent,"movePassportToFS")###( |
---|
1882 | def movePassportToFS(self,student_id="O738726"): |
---|
1883 | "move the passports to the filesystem" |
---|
1884 | images_dir = os.path.join("%s" % i_home,'passports') |
---|
1885 | student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
---|
1886 | stool = getToolByName(self, 'portal_schemas') |
---|
1887 | schemas = ['student_application', |
---|
1888 | #'student_clearance', |
---|
1889 | ] |
---|
1890 | created = False |
---|
1891 | for schema_id in schemas: |
---|
1892 | schema = stool._getOb(schema_id) |
---|
1893 | object = getattr(student_folder,schema_id[len('student_'):],None) |
---|
1894 | if object is None: |
---|
1895 | continue |
---|
1896 | doc = object.getContent() |
---|
1897 | for key in schema.keys(): |
---|
1898 | if schema[key].meta_type != "CPS Image Field": |
---|
1899 | continue |
---|
1900 | #import pdb;pdb.set_trace() |
---|
1901 | image = getattr(doc,key) |
---|
1902 | if not hasattr(image,"data"): |
---|
1903 | continue |
---|
1904 | if not created: |
---|
1905 | if not os.path.exists(images_dir): |
---|
1906 | os.mkdir(images_dir) |
---|
1907 | created = True |
---|
1908 | filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars()) |
---|
1909 | open(filename,"wb").write(str(image.data)) |
---|
1910 | ###) |
---|
1911 | |
---|
1912 | |
---|
1913 | security.declarePublic('getConfigParams') |
---|
1914 | def getConfigParams(self,conf_id="configuration"): |
---|
1915 | conf = getattr(self.portal_url.getPortalObject().campus,conf_id) |
---|
1916 | conf_obj = conf.getContent() |
---|
1917 | stool = getToolByName(self, 'portal_schemas') |
---|
1918 | schema = stool._getOb('configuration') |
---|
1919 | d = {} |
---|
1920 | for key in schema.keys(): |
---|
1921 | d[key] = getattr(conf_obj,key,None) |
---|
1922 | return d |
---|
1923 | |
---|
1924 | |
---|
1925 | InitializeClass(WAeUPTool) |
---|