| [1991] | 1 | #-*- mode: python; mode: fold -*- |
|---|
| [197] | 2 | # (C) Copyright 2005 The WAeUP group <http://www.waeup.org> |
|---|
| 3 | # Author: Joachim Schmitz (js@aixtraware.de) |
|---|
| 4 | # |
|---|
| 5 | # This program is free software; you can redistribute it and/or modify |
|---|
| 6 | # it under the terms of the GNU General Public License version 2 as published |
|---|
| 7 | # by the Free Software Foundation. |
|---|
| 8 | # |
|---|
| 9 | # This program is distributed in the hope that it will be useful, |
|---|
| 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 12 | # GNU General Public License for more details. |
|---|
| 13 | # |
|---|
| 14 | # You should have received a copy of the GNU General Public License |
|---|
| 15 | # along with this program; if not, write to the Free Software |
|---|
| 16 | # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA |
|---|
| 17 | # 02111-1307, USA. |
|---|
| 18 | # |
|---|
| 19 | # $Id: WAeUPTool.py 3181 2008-02-18 20:28:57Z henrik $ |
|---|
| [1174] | 20 | """The WAeUP Tool Box. |
|---|
| [197] | 21 | """ |
|---|
| 22 | |
|---|
| 23 | from AccessControl import ClassSecurityInfo |
|---|
| [828] | 24 | from Acquisition import aq_inner |
|---|
| 25 | from Acquisition import aq_parent |
|---|
| 26 | from Globals import DTMLFile |
|---|
| 27 | from Globals import InitializeClass |
|---|
| 28 | from OFS.SimpleItem import SimpleItem |
|---|
| [2283] | 29 | from zExceptions import BadRequest |
|---|
| [197] | 30 | |
|---|
| [1890] | 31 | from Products.CMFCore.utils import getToolByName |
|---|
| [1747] | 32 | from Products.CPSSchemas.DataStructure import DataStructure |
|---|
| 33 | from Products.CPSSchemas.DataModel import DataModel |
|---|
| 34 | from Products.CPSSchemas.StorageAdapter import MappingStorageAdapter |
|---|
| [828] | 35 | from Products.CMFCore.ActionProviderBase import ActionProviderBase |
|---|
| 36 | from Products.CMFCore.permissions import View |
|---|
| 37 | from Products.ZCatalog.ZCatalog import ZCatalog |
|---|
| 38 | from Products.CMFCore.permissions import ModifyPortalContent |
|---|
| [1890] | 39 | from Products.CMFCore.permissions import ManagePortal |
|---|
| [197] | 40 | from Products.CMFCore.utils import UniqueObject |
|---|
| [1194] | 41 | from Products.CMFCore.URLTool import URLTool |
|---|
| [1747] | 42 | from Products.CMFCore.utils import getToolByName |
|---|
| [1151] | 43 | from Students import makeCertificateCode |
|---|
| [1285] | 44 | from Globals import package_home,INSTANCE_HOME |
|---|
| [3172] | 45 | from WAeUPImport import ApplicationImport,CertificateImport,CertificateCourseImport |
|---|
| 46 | from WAeUPImport import CourseImport,CourseResultImport |
|---|
| 47 | from WAeUPImport import DepartmentImport,FacultyImport,StudentImport,VerdictImport |
|---|
| 48 | from utils import makeDigest |
|---|
| [2094] | 49 | import DateTime,time |
|---|
| [1620] | 50 | import logging |
|---|
| [1170] | 51 | import transaction |
|---|
| [2193] | 52 | import csv,re,os,sys |
|---|
| [3172] | 53 | import md5 |
|---|
| [2413] | 54 | from shutil import copy2 |
|---|
| [1707] | 55 | from Products.AdvancedQuery import Eq, Between, Le,In |
|---|
| [197] | 56 | |
|---|
| [2413] | 57 | p_home = package_home(globals()) |
|---|
| 58 | i_home = INSTANCE_HOME |
|---|
| 59 | images_base = os.path.join(i_home,"images") |
|---|
| 60 | |
|---|
| [2675] | 61 | def getImagesDir(student_id): |
|---|
| 62 | return os.path.join("%s" % images_base,student_id[0],student_id) |
|---|
| [2413] | 63 | |
|---|
| [1707] | 64 | def getObject(object,name): |
|---|
| 65 | if object.hasObject(name): |
|---|
| 66 | return getattr(object,name) |
|---|
| 67 | return None |
|---|
| [1720] | 68 | |
|---|
| [828] | 69 | class WAeUPTool(UniqueObject, SimpleItem, ActionProviderBase): |
|---|
| [197] | 70 | """WAeUP tool""" |
|---|
| 71 | |
|---|
| [828] | 72 | id = 'waeup_tool' |
|---|
| [197] | 73 | meta_type = 'WAeUP Tool' |
|---|
| [828] | 74 | _actions = () |
|---|
| [197] | 75 | security = ClassSecurityInfo() |
|---|
| [828] | 76 | security.declareObjectProtected(View) |
|---|
| 77 | manage_options = ( ActionProviderBase.manage_options |
|---|
| 78 | + SimpleItem.manage_options |
|---|
| 79 | ) |
|---|
| 80 | |
|---|
| [2999] | 81 | security.declareProtected(View,'re_split') ###( |
|---|
| 82 | def re_split(self,split_string,string): |
|---|
| 83 | return re.split(split_string,string) |
|---|
| [3014] | 84 | ###) |
|---|
| [2999] | 85 | |
|---|
| 86 | security.declareProtected(View,'difference') ###( |
|---|
| 87 | def difference(self,l1,l2): |
|---|
| 88 | return set(l1).difference(set(l2)) |
|---|
| [3014] | 89 | ###) |
|---|
| [2999] | 90 | |
|---|
| [1818] | 91 | def rwrite(self,s): ###( |
|---|
| [1707] | 92 | response = self.REQUEST.RESPONSE |
|---|
| 93 | response.setHeader('Content-type','text/html; charset=ISO-8859-15') |
|---|
| 94 | response.write("%s<br />\r\n" % s) |
|---|
| [1818] | 95 | ###) |
|---|
| [1174] | 96 | |
|---|
| [2695] | 97 | def addtodict(self,d,key,item): ###( |
|---|
| 98 | d[key].append(item) |
|---|
| 99 | return d[key] |
|---|
| 100 | ###) |
|---|
| 101 | |
|---|
| [1818] | 102 | def sleep(self,secs): ###( |
|---|
| 103 | "sleep" |
|---|
| 104 | import time |
|---|
| 105 | time.sleep(secs) |
|---|
| 106 | return |
|---|
| [1827] | 107 | |
|---|
| [1818] | 108 | ###) |
|---|
| 109 | |
|---|
| [2695] | 110 | security.declareProtected(View,'updateRoleMappingsFor') ###( |
|---|
| 111 | def updateRoleMappingsFor(self,wf_definition,ob): |
|---|
| 112 | "do so for public" |
|---|
| 113 | wf_def = getattr(self.portal_workflow,wf_definition) |
|---|
| 114 | wf_def.updateRoleMappingsFor(ob) |
|---|
| 115 | ###) |
|---|
| [2699] | 116 | |
|---|
| [2999] | 117 | security.declareProtected(View,'getStatesLgas') ###( |
|---|
| 118 | def getStatesLgas(self): |
|---|
| 119 | """return lga info""" |
|---|
| 120 | voc = getattr(self.portal_vocabularies,'local_gov_areas') |
|---|
| 121 | states = [] |
|---|
| 122 | lgas = [] |
|---|
| 123 | d = {} |
|---|
| 124 | wd = {} |
|---|
| 125 | for k,v in voc.items(): |
|---|
| 126 | parts = v.split(' / ') |
|---|
| 127 | if len(parts) == 1: |
|---|
| 128 | state = parts[0].lower() |
|---|
| 129 | lga = "" |
|---|
| 130 | elif len(parts) == 2: |
|---|
| 131 | state = "_".join(re.split('[^a-zA-Z0-9/]',parts[0].lower())) |
|---|
| 132 | lga = "-".join(re.split('[^a-zA-Z0-9/]',parts[1].lower())) |
|---|
| 133 | else: |
|---|
| 134 | continue |
|---|
| 135 | if state not in states: |
|---|
| 136 | states.append(state) |
|---|
| 137 | if lga not in lgas: |
|---|
| 138 | lgas.append(lga) |
|---|
| 139 | words = re.split('[^a-zA-Z0-9/]',k) |
|---|
| 140 | words.sort() |
|---|
| 141 | wd[k] = words |
|---|
| 142 | d[k] = v |
|---|
| 143 | mapping = {} |
|---|
| 144 | mapping['word_dict'] = wd |
|---|
| 145 | mapping['lga_dict'] = d |
|---|
| 146 | mapping['states'] = states |
|---|
| 147 | mapping['lgas'] = lgas |
|---|
| 148 | return mapping |
|---|
| 149 | ###) |
|---|
| 150 | |
|---|
| [3014] | 151 | security.declareProtected(View,'findLga') ###( |
|---|
| 152 | def findLga(self,words,words_dict): |
|---|
| 153 | words = re.split('[^a-zA-Z0-9/]',words) |
|---|
| 154 | lga_words = [] |
|---|
| 155 | for word in words: |
|---|
| 156 | if word: |
|---|
| 157 | lga_words += word.strip().lower(), |
|---|
| 158 | lga_words.sort() |
|---|
| 159 | state_lga = '' |
|---|
| 160 | while not state_lga: |
|---|
| 161 | for k,l in words_dict.items(): |
|---|
| 162 | if lga_words == l: |
|---|
| 163 | state_lga = k |
|---|
| 164 | break |
|---|
| 165 | break |
|---|
| 166 | return state_lga |
|---|
| 167 | ###) |
|---|
| [2975] | 168 | security.declareProtected(View,'getAccessInfo') ###( |
|---|
| 169 | def getAccessInfo(self,context): |
|---|
| 170 | "return a dict with access_info" |
|---|
| 171 | logger = logging.getLogger('WAeUPTool.getAccessInfo') |
|---|
| 172 | mtool = self.portal_membership |
|---|
| 173 | member = mtool.getAuthenticatedMember() |
|---|
| 174 | member_id = str(member) |
|---|
| 175 | info = {} |
|---|
| 176 | is_anonymous = info['is_anonymous'] = mtool.isAnonymousUser() |
|---|
| 177 | is_student = info['is_student'] = ord(member_id[1]) > 48 and ord(member_id[1]) <= 57 |
|---|
| 178 | is_staff = info['is_staff'] = not is_anonymous and not is_student |
|---|
| 179 | roles = member.getRolesInContext(context) |
|---|
| 180 | is_sectionofficer = info['is_sectionofficer'] = not is_student and ("SectionOfficer" in roles or |
|---|
| 181 | "SectionManager" in roles or |
|---|
| 182 | "Manager" in roles) |
|---|
| 183 | is_allowed = info['is_allowed'] = not is_anonymous |
|---|
| 184 | requested_id = context.getStudentId() |
|---|
| 185 | student_id = None |
|---|
| [3025] | 186 | if is_allowed: |
|---|
| 187 | if not is_student and requested_id: |
|---|
| 188 | student_id = requested_id |
|---|
| 189 | elif is_anonymous or member_id != requested_id: |
|---|
| 190 | logger.info('%s tried to access %s of %s' % (member_id,context.portal_type,requested_id)) |
|---|
| 191 | else: |
|---|
| 192 | student_id = member_id |
|---|
| [2975] | 193 | info['student_id'] = student_id |
|---|
| 194 | return info |
|---|
| 195 | ###) |
|---|
| 196 | |
|---|
| [1818] | 197 | security.declareProtected(ModifyPortalContent,'openLog') ###( |
|---|
| [1716] | 198 | def openLog(self,name): |
|---|
| 199 | """open a log file""" |
|---|
| 200 | version = 1 |
|---|
| 201 | path = "%s/log/%s_%d.log" % (i_home,name,version) |
|---|
| 202 | while os.path.exists(path): |
|---|
| 203 | version += 1 |
|---|
| 204 | path = "%s/log/%s_%d.log" % (i_home,name,version) |
|---|
| 205 | log = open(path,"w") |
|---|
| 206 | return log |
|---|
| [2259] | 207 | ###) |
|---|
| [1716] | 208 | |
|---|
| [2259] | 209 | security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###( |
|---|
| 210 | def bypassQueueCatalog(self,enable=True): |
|---|
| 211 | """bypass the QueueCatalog by setting all indexes to process imediate, |
|---|
| 212 | if enable is True (default) the old settings are restored |
|---|
| 213 | """ |
|---|
| [1818] | 214 | |
|---|
| [2259] | 215 | ###) |
|---|
| 216 | |
|---|
| [2094] | 217 | security.declareProtected(ModifyPortalContent,'measureOaT') ###( |
|---|
| 218 | def measureOaT(self,method="a",probe="1000",nr_pts="1"): |
|---|
| 219 | """measure Object access Time""" |
|---|
| 220 | import random |
|---|
| 221 | if hasattr(self,'portal_catalog_real'): |
|---|
| 222 | aq_portal = self.portal_catalog_real.evalAdvancedQuery |
|---|
| 223 | else: |
|---|
| 224 | aq_portal = self.portal_catalog.evalAdvancedQuery |
|---|
| 225 | nr_pts = int(nr_pts) |
|---|
| 226 | probe = int(probe) |
|---|
| 227 | intervall = probe/10 |
|---|
| 228 | objects = ("application","clearance","personal") |
|---|
| 229 | portal_types = ("StudentApplication","StudentClearance","StudentPersonal") |
|---|
| 230 | #i = random.randrange(num_objects) |
|---|
| 231 | count = 0 |
|---|
| 232 | found = 0 |
|---|
| 233 | not_found = 0 |
|---|
| 234 | t_found = 0 |
|---|
| 235 | t_not_found = 0 |
|---|
| 236 | time_found = time_not_found = 0.0 |
|---|
| 237 | t_time_found = t_time_not_found = 0.0 |
|---|
| 238 | accessed = [] |
|---|
| 239 | t_min = 1000 |
|---|
| 240 | t_max = 0 |
|---|
| 241 | #import pdb;pdb.set_trace() |
|---|
| 242 | students = self.portal_catalog(portal_type="Student") |
|---|
| 243 | num_students = len(students) |
|---|
| 244 | if method == "d": |
|---|
| 245 | query = Eq('path','/uniben/campus/students') & In('portal_type',portal_types[:nr_pts]) |
|---|
| 246 | res = aq_portal(query) |
|---|
| 247 | brains = {} |
|---|
| 248 | for r in res: |
|---|
| 249 | sid = r.relative_path.split('/')[-2] |
|---|
| 250 | if brains.has_key(sid): |
|---|
| 251 | brains[sid][r.portal_type] = r |
|---|
| 252 | else: |
|---|
| 253 | brains[sid] = {r.portal_type : r} |
|---|
| 254 | brains_list = brains.keys() |
|---|
| 255 | num_objects = len(brains_list) |
|---|
| 256 | else: |
|---|
| 257 | num_objects = num_students |
|---|
| 258 | print "="*40 |
|---|
| 259 | print "method: %s probes: %d nr_pts: %d num_objects: %d" % (method, |
|---|
| 260 | probe, |
|---|
| 261 | nr_pts, |
|---|
| 262 | num_objects) |
|---|
| 263 | print "nr found/not time found/not min/max" |
|---|
| 264 | elapse = time.time() |
|---|
| 265 | i_elapse = time.time() |
|---|
| 266 | c_elapse = time.clock() |
|---|
| 267 | for c in range(1,probe + 1): |
|---|
| 268 | i = random.randrange(num_objects) |
|---|
| 269 | if method in ('a','b','c'): |
|---|
| 270 | student_brain = students[i] |
|---|
| 271 | elif method == "d": |
|---|
| 272 | #import pdb;pdb.set_trace() |
|---|
| 273 | student_brain = brains[brains_list[i]] |
|---|
| 274 | if method == "c": |
|---|
| 275 | query = Eq('path',student_brain.getPath()) & In('portal_type',portal_types[:nr_pts]) |
|---|
| 276 | res = aq_portal(query) |
|---|
| 277 | this_portal_types = [r.portal_type for r in res] |
|---|
| 278 | for i in range(nr_pts): |
|---|
| 279 | oid = objects[i] |
|---|
| 280 | if method == "a": |
|---|
| 281 | try: |
|---|
| 282 | student_path = student_brain.getPath() |
|---|
| 283 | path = "%s/%s" % (student_path,oid) |
|---|
| 284 | doc = self.unrestrictedTraverse(path).getContent() |
|---|
| 285 | found += 1 |
|---|
| 286 | i_time = time.time() - i_elapse |
|---|
| 287 | time_found += i_time |
|---|
| 288 | except: |
|---|
| 289 | not_found += 1 |
|---|
| 290 | i_time = time.time() - i_elapse |
|---|
| 291 | time_not_found += i_time |
|---|
| 292 | pass |
|---|
| 293 | elif method == "b": |
|---|
| 294 | try: |
|---|
| 295 | student_object = student_brain.getObject() |
|---|
| 296 | doc = getattr(student_object,oid).getContent() |
|---|
| 297 | found += 1 |
|---|
| 298 | i_time = time.time() - i_elapse |
|---|
| 299 | time_found += i_time |
|---|
| 300 | except: |
|---|
| 301 | i_time = time.time() - i_elapse |
|---|
| 302 | time_not_found += i_time |
|---|
| 303 | not_found += 1 |
|---|
| 304 | pass |
|---|
| 305 | elif method == "c": |
|---|
| 306 | if portal_types[i] in this_portal_types: |
|---|
| 307 | found += 1 |
|---|
| 308 | doc = res[this_portal_types.index(portal_types[i])].getObject().getContent() |
|---|
| 309 | i_time = time.time() - i_elapse |
|---|
| 310 | time_found += i_time |
|---|
| 311 | else: |
|---|
| 312 | not_found += 1 |
|---|
| 313 | i_time = time.time() - i_elapse |
|---|
| 314 | time_not_found += i_time |
|---|
| 315 | elif method == "d": |
|---|
| 316 | if student_brain.has_key(portal_types[i]): |
|---|
| 317 | found += 1 |
|---|
| 318 | doc = student_brain[portal_types[i]].getObject().getContent() |
|---|
| 319 | i_time = time.time() - i_elapse |
|---|
| 320 | time_found += i_time |
|---|
| 321 | else: |
|---|
| 322 | not_found += 1 |
|---|
| 323 | i_time = time.time() - i_elapse |
|---|
| 324 | time_not_found += i_time |
|---|
| 325 | i_elapse = time.time() |
|---|
| 326 | if c and (c % intervall == 0): |
|---|
| 327 | #i_time = time.time() - i_elapse |
|---|
| 328 | t_per = 0.0 |
|---|
| 329 | if found: |
|---|
| 330 | t_per = time_found/found |
|---|
| 331 | if t_per > t_max: |
|---|
| 332 | t_max = t_per |
|---|
| 333 | if t_per > 0.0 and t_per < t_min: |
|---|
| [2119] | 334 | t_min = t_per |
|---|
| [2094] | 335 | itf = 0.0 |
|---|
| 336 | if found: |
|---|
| 337 | itf = time_found/found |
|---|
| 338 | itnf = 0.0 |
|---|
| 339 | if not_found : |
|---|
| 340 | itnf = time_not_found / not_found |
|---|
| [2119] | 341 | interval_time = time_found + time_not_found |
|---|
| 342 | s = "%(c)d: %(found)d/%(not_found)d " % vars() |
|---|
| [2094] | 343 | s += "%(interval_time)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() |
|---|
| 344 | s += "%(t_min)6.4f/%(t_max)6.4f" % vars() |
|---|
| 345 | print s |
|---|
| 346 | t_found += found |
|---|
| 347 | t_not_found += not_found |
|---|
| 348 | t_time_found += time_found |
|---|
| 349 | t_time_not_found += time_not_found |
|---|
| 350 | time_found = time_not_found = 0.0 |
|---|
| 351 | found = not_found = 0 |
|---|
| 352 | # t_found += found |
|---|
| 353 | # t_not_found += not_found |
|---|
| 354 | elapse = time.time() - elapse |
|---|
| 355 | itf = 0.0 |
|---|
| 356 | if t_found: |
|---|
| 357 | itf = t_time_found/t_found |
|---|
| 358 | itnf = 0.0 |
|---|
| 359 | if t_not_found: |
|---|
| 360 | itnf = t_time_not_found / t_not_found |
|---|
| 361 | #c_elapse = time.clock() - c_elapse |
|---|
| [2119] | 362 | s = "%(probe)d: %(t_found)d/%(t_not_found)d " % vars() |
|---|
| [2094] | 363 | s += "%(elapse)6.2f %(itf)6.4f/%(itnf)6.4f " % vars() |
|---|
| 364 | s += "%(t_min)6.4f/%(t_max)6.4f" % vars() |
|---|
| 365 | print "-"*40 |
|---|
| 366 | print s |
|---|
| 367 | rel_found = float(t_found)/probe |
|---|
| 368 | rel_not_found = float(t_not_found)/probe |
|---|
| 369 | estimated_total_time = num_objects*(rel_found*itf + rel_not_found*itnf) |
|---|
| 370 | print estimated_total_time |
|---|
| 371 | ###) |
|---|
| 372 | |
|---|
| [1818] | 373 | security.declareProtected(ModifyPortalContent,'writeLog') ###( |
|---|
| [1716] | 374 | def writeLog(self,logfile,s): |
|---|
| 375 | """write to the log file""" |
|---|
| 376 | logfile.write(s) |
|---|
| 377 | |
|---|
| [1818] | 378 | ###) |
|---|
| [1720] | 379 | |
|---|
| [1151] | 380 | def generateStudentId(self,letter): ###( |
|---|
| 381 | import random |
|---|
| 382 | r = random |
|---|
| [1194] | 383 | ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): |
|---|
| 384 | if letter == '?': |
|---|
| [2695] | 385 | letter= r.choice('ABCDEFGHKLMNPQRSTUVWXY') |
|---|
| [1151] | 386 | sid = "%c%d" % (letter,r.randint(99999,1000000)) |
|---|
| [1720] | 387 | students = self.portal_url.getPortalObject().campus.students |
|---|
| [1721] | 388 | ## while hasattr(students, sid): |
|---|
| 389 | ## sid = "%c%d" % (letter,r.randint(99999,1000000)) |
|---|
| 390 | while self.students_catalog(id = sid): |
|---|
| [1720] | 391 | sid = "%c%d" % (letter,r.randint(99999,1000000)) |
|---|
| [1151] | 392 | return sid |
|---|
| [1460] | 393 | ###) |
|---|
| [1415] | 394 | |
|---|
| 395 | def generatePassword(self,s=None): ###( |
|---|
| 396 | import random |
|---|
| 397 | r = random |
|---|
| 398 | ##if letter not in ('ABCDEFGIHKLMNOPQRSTUVWXY'): |
|---|
| 399 | if s is None: |
|---|
| 400 | s = 'abcdefghklmnpqrstuvwxy23456789' |
|---|
| 401 | pw = '' |
|---|
| 402 | while len(pw) < 6: |
|---|
| 403 | pw += r.choice(s) |
|---|
| 404 | return pw |
|---|
| [1151] | 405 | ###) |
|---|
| [828] | 406 | |
|---|
| [2645] | 407 | security.declareProtected(ModifyPortalContent, 'dumpSchoolfeePayments') ###( |
|---|
| 408 | def dumpSchoolfeePayments(self): |
|---|
| 409 | "dump paid schoolfees" |
|---|
| [2666] | 410 | mtool = self.portal_membership |
|---|
| 411 | member = mtool.getAuthenticatedMember() |
|---|
| [2645] | 412 | logger = logging.getLogger('WAeUPTool.dumpSchoolfees') |
|---|
| 413 | aq_student = self.students_catalog.evalAdvancedQuery |
|---|
| 414 | query = In('review_state',('schoolfee_paid', |
|---|
| 415 | 'courses_registered', |
|---|
| 416 | 'courses_validated', |
|---|
| 417 | )) |
|---|
| 418 | res = aq_student(query) |
|---|
| [2666] | 419 | #import pdb;pdb.set_trace() |
|---|
| [2645] | 420 | l = [] |
|---|
| 421 | logger.info("start for %d" % len(res)) |
|---|
| 422 | count = 1 |
|---|
| 423 | log_after = 100 |
|---|
| 424 | for student in res: |
|---|
| [2656] | 425 | if not count % log_after: |
|---|
| [2645] | 426 | logger.info("processed %d total %d" % (log_after,count)) |
|---|
| 427 | count += 1 |
|---|
| 428 | fee_dict =self.getSchoolFee(student) |
|---|
| 429 | fulltime = student.mode.endswith('_ft') |
|---|
| 430 | d = {} |
|---|
| 431 | d['student_id'] = student.id |
|---|
| [2666] | 432 | d['name'] = student.name |
|---|
| 433 | d['amount'] = fee_dict.get(new_returning) |
|---|
| [2645] | 434 | l += d, |
|---|
| [2666] | 435 | csv_name = self.dumpListToCSV(l,'payments') |
|---|
| 436 | logger.info('%s dumped payments to %s' % (member,export_file)) |
|---|
| [2645] | 437 | ###) |
|---|
| 438 | |
|---|
| [2666] | 439 | security.declarePublic('dumpListToCSV') ###( |
|---|
| 440 | def dumpListToCSV(self,l,filename,fields=None): |
|---|
| 441 | """dump a list of dicts to a CSV file""" |
|---|
| 442 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
|---|
| 443 | export_file = "%s/export/%s_%s.csv" % (i_home,filename,current,) |
|---|
| 444 | if fields is None: |
|---|
| 445 | fields = l[0].keys() |
|---|
| 446 | headline = ','.join(fields) |
|---|
| 447 | out = open(export_file,"wb") |
|---|
| 448 | out.write(headline +'\n') |
|---|
| 449 | out.close() |
|---|
| 450 | out = open(export_file,"a") |
|---|
| 451 | csv_writer = csv.DictWriter(out,fields,) |
|---|
| 452 | csv_writer.writerows(l) |
|---|
| 453 | return export_file |
|---|
| 454 | ###) |
|---|
| 455 | |
|---|
| [2643] | 456 | security.declareProtected(ManagePortal, 'listMembers') ###( |
|---|
| 457 | def listMembers(self): |
|---|
| 458 | "list all members" |
|---|
| 459 | mtool = self.portal_membership |
|---|
| 460 | member = mtool.getAuthenticatedMember() |
|---|
| 461 | logger = logging.getLogger('WAeUPTool.listMembers') |
|---|
| 462 | if str(member) not in ('admin','joachim'): |
|---|
| 463 | logger.info('%s tried to list members' % (member)) |
|---|
| 464 | return None |
|---|
| 465 | members = self.portal_directories.members |
|---|
| 466 | all = members.listEntryIdsAndTitles() |
|---|
| 467 | l = [] |
|---|
| 468 | for user_id,name in all: |
|---|
| 469 | d = {} |
|---|
| 470 | d['user_id'] = user_id |
|---|
| 471 | d['name'] = name |
|---|
| 472 | d['pw'] = getattr(getattr(members,user_id),'password') |
|---|
| [2645] | 473 | d['email'] = getattr(getattr(members,user_id),'email') |
|---|
| 474 | d['groups'] = " ".join(getattr(getattr(members,user_id),'groups')) |
|---|
| 475 | d['roles'] = " ".join(getattr(getattr(members,user_id),'roles')) |
|---|
| [2643] | 476 | l += d, |
|---|
| 477 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
|---|
| 478 | export_file = "%s/export/member_list_%s.csv" % (i_home,current,) |
|---|
| 479 | logger.info('%s dumped memberlist to %s' % (member,export_file)) |
|---|
| 480 | fields = l[0].keys() |
|---|
| 481 | headline = ','.join(fields) |
|---|
| 482 | out = open(export_file,"wb") |
|---|
| 483 | out.write(headline +'\n') |
|---|
| 484 | out.close() |
|---|
| 485 | out = open(export_file,"a") |
|---|
| 486 | csv_writer = csv.DictWriter(out,fields,) |
|---|
| 487 | csv_writer.writerows(l) |
|---|
| 488 | ###) |
|---|
| [2656] | 489 | |
|---|
| [1890] | 490 | security.declareProtected(ManagePortal, 'removeDeletedDocIds') ###( |
|---|
| 491 | def removeDeletedDocIds(self, max=1000): |
|---|
| 492 | """ |
|---|
| 493 | remove deleted docids from repository commit after max |
|---|
| 494 | """ |
|---|
| 495 | logger = logging.getLogger('WAeUPTool.removeDeletedDocIds') |
|---|
| 496 | repository = getToolByName(self, 'portal_repository') |
|---|
| 497 | pxtool = getToolByName(self, 'portal_proxies') |
|---|
| [2632] | 498 | logger.info('start') |
|---|
| [1890] | 499 | pxtool_infos = pxtool.getRevisionsUsed() |
|---|
| [2632] | 500 | logger.info('found %d used revisions ' % (len(pxtool_infos))) |
|---|
| [1895] | 501 | |
|---|
| [1890] | 502 | nb_revs = 0 |
|---|
| 503 | docids_d = {} # all docids |
|---|
| 504 | unused_docids_d = {} # all docids that are unused |
|---|
| 505 | ids_unused_revs_docids = [] # ids for revs of unused docids |
|---|
| [2632] | 506 | unused_ids = [] # ids for unused revs |
|---|
| [1890] | 507 | total = 0 |
|---|
| [1895] | 508 | idlist = repository.objectIds() |
|---|
| [2632] | 509 | to_delete = 0 |
|---|
| 510 | found = False |
|---|
| [1895] | 511 | for id in idlist: |
|---|
| [1890] | 512 | docid, rev = repository._splitId(id) |
|---|
| 513 | if docid is None: |
|---|
| 514 | logger.info("invalid doc_id %s" % docid) |
|---|
| 515 | continue |
|---|
| 516 | nb_revs += 1 |
|---|
| 517 | if not pxtool_infos.has_key(docid): |
|---|
| [2632] | 518 | found = True |
|---|
| 519 | to_delete += 1 |
|---|
| 520 | unused_ids.append(id) |
|---|
| [1890] | 521 | elif not pxtool_infos[docid].has_key(rev): |
|---|
| [2632] | 522 | found = True |
|---|
| 523 | to_delete += 1 |
|---|
| 524 | unused_ids.append(id) |
|---|
| 525 | if found and not to_delete % max: |
|---|
| 526 | found = False |
|---|
| [1890] | 527 | #import pdb;pdb.set_trace() |
|---|
| [2632] | 528 | repository.manage_delObjects(unused_ids) |
|---|
| [1890] | 529 | transaction.commit() |
|---|
| [2632] | 530 | logger.info('removed %d total %d unused docids ' % (max,to_delete)) |
|---|
| 531 | else: |
|---|
| 532 | if unused_ids: |
|---|
| 533 | repository.manage_delObjects(unused_ids) |
|---|
| 534 | transaction.commit() |
|---|
| 535 | logger.info('finished removing %d unused docids ' % (to_delete)) |
|---|
| [1890] | 536 | |
|---|
| [1895] | 537 | |
|---|
| [1890] | 538 | ###) |
|---|
| 539 | |
|---|
| [2562] | 540 | security.declareProtected(View,'getCredential') ###( |
|---|
| [1250] | 541 | def getCredential(self,student_id): |
|---|
| 542 | student_entry = getattr(self.portal_directories.students,student_id,None) |
|---|
| [2555] | 543 | if not self.isStaff(): |
|---|
| [2563] | 544 | mtool = self.portal_membership |
|---|
| 545 | member = mtool.getAuthenticatedMember() |
|---|
| 546 | logger = logging.getLogger('WAeUPTool.getCredential') |
|---|
| 547 | logger.info('%s tried to access password of %s' % (member,student_id)) |
|---|
| [2555] | 548 | return None |
|---|
| [1250] | 549 | if student_entry is None: |
|---|
| 550 | return None |
|---|
| [1263] | 551 | return getattr(student_entry,"password","not set") |
|---|
| [1261] | 552 | ###) |
|---|
| [1250] | 553 | |
|---|
| [2747] | 554 | security.declarePublic('checkPassword') ###( |
|---|
| [1467] | 555 | def checkPassword(self,student_id,password): |
|---|
| [1460] | 556 | student_entry = getattr(self.portal_directories.students,student_id,None) |
|---|
| 557 | if student_entry is None: |
|---|
| 558 | return False |
|---|
| 559 | return getattr(student_entry,"password","not set") == password |
|---|
| [2747] | 560 | ###) |
|---|
| [2712] | 561 | |
|---|
| [2747] | 562 | security.declarePublic('checkGenericPassword') ###( |
|---|
| [2710] | 563 | def checkGenericPassword(self,member_id): |
|---|
| 564 | member_entry = getattr(self.portal_directories.members,member_id,None) |
|---|
| 565 | if member_entry is None: |
|---|
| 566 | return False |
|---|
| [2712] | 567 | ltool = getToolByName(self, 'portal_layouts') |
|---|
| [2710] | 568 | unsecure_words = ltool._getOb('members')['w__password'].check_words |
|---|
| [2712] | 569 | password = getattr(member_entry,"password","not set") |
|---|
| 570 | is_unsecure = password in unsecure_words |
|---|
| 571 | if is_unsecure: |
|---|
| [2713] | 572 | logger = logging.getLogger('WAeUPTool.checkGenericPassword') |
|---|
| [2712] | 573 | logger.info('Member %s tried to log in with unsecure password %s' %(member_id,password)) |
|---|
| 574 | return is_unsecure |
|---|
| [2747] | 575 | ###) |
|---|
| [1460] | 576 | |
|---|
| [2556] | 577 | security.declareProtected(ModifyPortalContent,'editPassword') ###( |
|---|
| [1467] | 578 | def editPassword(self,student_id,password): |
|---|
| 579 | "edit a student password" |
|---|
| 580 | student_entry = getattr(self.portal_directories.students,student_id,None) |
|---|
| 581 | if student_entry is None: |
|---|
| [1571] | 582 | return |
|---|
| [1467] | 583 | setattr(student_entry,'password',password) |
|---|
| 584 | ###) |
|---|
| 585 | |
|---|
| [2555] | 586 | security.declareProtected(ModifyPortalContent,'doCommit') ###( |
|---|
| [1401] | 587 | def doCommit(self,logger=None): |
|---|
| 588 | "commit some transactions" |
|---|
| 589 | transaction.commit() |
|---|
| 590 | ###) |
|---|
| 591 | |
|---|
| [1285] | 592 | security.declarePublic('loadStudentFoto') ###( |
|---|
| [1813] | 593 | def loadStudentFoto(self,student,filename,folder): |
|---|
| [1285] | 594 | "return a student passport picture" |
|---|
| [2413] | 595 | #import pdb;pdb.set_trace() |
|---|
| 596 | picture ="%s/import/%s/%s" % (i_home,folder,filename) |
|---|
| 597 | student_id = student.getId() |
|---|
| [2675] | 598 | images_dir = getImagesDir(student_id) |
|---|
| [2413] | 599 | if not os.path.exists(images_dir): |
|---|
| 600 | os.mkdir(images_dir) |
|---|
| 601 | image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) |
|---|
| 602 | for extension in ('.jpg','.JPG'): |
|---|
| 603 | fullname = "%(picture)s%(extension)s" % vars() |
|---|
| 604 | if os.path.exists(fullname): |
|---|
| 605 | copy2(fullname,image_name) |
|---|
| 606 | return "successfully copied passport picture" |
|---|
| [2467] | 607 | return "passport picture not found: %s.jpg or .JPG" % picture |
|---|
| [2413] | 608 | ###) |
|---|
| 609 | |
|---|
| [2420] | 610 | def old____loadStudentFoto(self,student,filename,folder): ###( |
|---|
| [2413] | 611 | "return a student passport picture" |
|---|
| [1921] | 612 | app = student.application |
|---|
| 613 | app_doc = app.getContent() |
|---|
| [1813] | 614 | #clear = student.clearance |
|---|
| 615 | #clear_doc = clear.getContent() |
|---|
| 616 | #matric_no = clear_doc.matric_no.upper() |
|---|
| 617 | picture1 ="%s/import/%s/%s.jpg" % (i_home,folder,filename) |
|---|
| 618 | picture2 ="%s/import/%s/%s.JPG" % (i_home,folder,filename) |
|---|
| [1285] | 619 | #import pdb;pdb.set_trace() |
|---|
| [1286] | 620 | if os.path.exists(picture1): |
|---|
| 621 | file = open(picture1) |
|---|
| 622 | elif os.path.exists(picture2): |
|---|
| [1287] | 623 | file = open(picture2) |
|---|
| [1286] | 624 | else: |
|---|
| [1287] | 625 | return "passport picture not found %s" % picture1 |
|---|
| [1921] | 626 | reopened = False |
|---|
| 627 | if self.portal_workflow.getInfoFor(app,'review_state',None) !='opened': |
|---|
| 628 | self.portal_workflow.doActionFor(app,'open') |
|---|
| 629 | reopened = True |
|---|
| [1285] | 630 | outfile = file.read() |
|---|
| 631 | app_doc.manage_addFile('passport', |
|---|
| 632 | file=outfile, |
|---|
| [1818] | 633 | title="%s.jpg" % filename) |
|---|
| [1921] | 634 | if reopened: |
|---|
| 635 | self.portal_workflow.doActionFor(app,'close') |
|---|
| [1286] | 636 | return "successfully loaded passport picture" |
|---|
| [1285] | 637 | ###) |
|---|
| 638 | |
|---|
| [1170] | 639 | security.declareProtected(ModifyPortalContent,'createOne') ###( |
|---|
| [1194] | 640 | def createOne(self,students_folder,student_brain,letter,commit=False): |
|---|
| 641 | sid = self.waeup_tool.generateStudentId(letter) |
|---|
| [1170] | 642 | students_folder.invokeFactory('Student', sid) |
|---|
| 643 | student = getattr(students_folder,sid) |
|---|
| 644 | self.portal_workflow.doActionFor(student,'return') |
|---|
| 645 | student.manage_setLocalRoles(sid, ['Owner',]) |
|---|
| 646 | matric_no = student_brain.matric_no |
|---|
| 647 | jamb_reg_no = student_brain.Entryregno |
|---|
| 648 | self.students_catalog.addRecord(id = sid, |
|---|
| 649 | matric_no = matric_no, |
|---|
| 650 | jamb_reg_no = jamb_reg_no, |
|---|
| 651 | sex = student_brain.Sex == "F", |
|---|
| 652 | name = "%s %s %s" % (student_brain.Firstname, |
|---|
| 653 | student_brain.Middlename, |
|---|
| 654 | student_brain.Lastname) |
|---|
| 655 | ) |
|---|
| 656 | if commit: |
|---|
| 657 | transaction.commit() |
|---|
| 658 | return sid,jamb_reg_no |
|---|
| 659 | ###) |
|---|
| 660 | |
|---|
| [1752] | 661 | security.declareProtected(ModifyPortalContent,'addStudent') ###( |
|---|
| 662 | def addStudent(self,dict): |
|---|
| [1415] | 663 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| 664 | sid = self.waeup_tool.generateStudentId('?') |
|---|
| 665 | students_folder.invokeFactory('Student', sid) |
|---|
| 666 | student_obj = getattr(students_folder,sid) |
|---|
| [3027] | 667 | f2t = self.field2types_student |
|---|
| [1772] | 668 | #from pdb import set_trace; set_trace() |
|---|
| 669 | d = {} |
|---|
| [2297] | 670 | #d['jamb_sex'] = 'M' |
|---|
| 671 | #if dict.get('sex'): |
|---|
| 672 | # d['jamb_sex'] = 'F' |
|---|
| [1904] | 673 | |
|---|
| [1898] | 674 | entry_session = dict.get('entry_session') |
|---|
| [2454] | 675 | if entry_session == self.getSessionId()[0]: |
|---|
| [1898] | 676 | wfaction = 'admit' |
|---|
| [1991] | 677 | wft = 'wf_transition_admit' |
|---|
| [1900] | 678 | password = None |
|---|
| [1904] | 679 | else: |
|---|
| [1900] | 680 | wfaction = 'return' |
|---|
| [1991] | 681 | wft = 'wf_transition_return' |
|---|
| [1900] | 682 | password = self.generatePassword() |
|---|
| [1904] | 683 | self.makeStudentMember(sid,password) |
|---|
| 684 | |
|---|
| [1728] | 685 | for pt in f2t.keys(): |
|---|
| 686 | student_obj.invokeFactory(pt,f2t[pt]['id']) |
|---|
| 687 | sub_obj = getattr(student_obj,f2t[pt]['id']) |
|---|
| 688 | sub_doc = sub_obj.getContent() |
|---|
| [1898] | 689 | #self.portal_workflow.doActionFor(sub_obj,'open',dest_container=sub_obj) |
|---|
| [1772] | 690 | d['Title'] = f2t[pt]['title'] |
|---|
| [1728] | 691 | for field in f2t[pt]['fields']: |
|---|
| [1749] | 692 | d[field] = dict.get(field,'') |
|---|
| [1728] | 693 | sub_doc.edit(mapping = d) |
|---|
| [1898] | 694 | new_state = f2t[pt][wft] |
|---|
| [1772] | 695 | if new_state != "remain": |
|---|
| 696 | self.portal_workflow.doActionFor(sub_obj,new_state,dest_container=sub_obj) |
|---|
| [1415] | 697 | self.portal_workflow.doActionFor(student_obj,wfaction) |
|---|
| 698 | student_obj.manage_setLocalRoles(sid, ['Owner',]) |
|---|
| 699 | return sid,password |
|---|
| 700 | ###) |
|---|
| 701 | |
|---|
| [1151] | 702 | security.declarePublic('getCertificateBrain') ###( |
|---|
| 703 | def getCertificateBrain(self,cert_id): |
|---|
| 704 | "do it" |
|---|
| [1849] | 705 | res = ZCatalog.searchResults(self.portal_catalog_real, |
|---|
| [1151] | 706 | {'portal_type':"Certificate", |
|---|
| 707 | 'id': cert_id}) |
|---|
| 708 | if res: |
|---|
| 709 | return res[0] |
|---|
| 710 | return None |
|---|
| 711 | ###) |
|---|
| [1160] | 712 | |
|---|
| [1756] | 713 | security.declareProtected(ModifyPortalContent,'get_csv_filenames') ###( |
|---|
| 714 | def get_csv_filenames(self): |
|---|
| 715 | "do it" |
|---|
| [1759] | 716 | files = [file for file in os.listdir("%s/import/" % (i_home)) |
|---|
| [3143] | 717 | if file.endswith('.csv') and (file.find('imported') == -1 and |
|---|
| 718 | file.find('pending') == -1)] |
|---|
| [1756] | 719 | return files |
|---|
| 720 | ###) |
|---|
| 721 | |
|---|
| [1151] | 722 | security.declarePublic('findStudentByMatricelNo') ###( |
|---|
| 723 | def findStudentByMatricelNo(self,matric_no): |
|---|
| 724 | "do it" |
|---|
| [1849] | 725 | res = ZCatalog.searchResults(self.portal_catalog_real, |
|---|
| [1151] | 726 | {'portal_type':"StudentClearance", |
|---|
| 727 | 'SearchableText': matric_no}) |
|---|
| 728 | if res: |
|---|
| 729 | return res[0] |
|---|
| 730 | return None |
|---|
| 731 | ###) |
|---|
| 732 | |
|---|
| [2557] | 733 | security.declarePublic('makeStudentMember') ###( |
|---|
| [1151] | 734 | def makeStudentMember(self,sid,password='uNsEt'): |
|---|
| 735 | """make the student a member""" |
|---|
| 736 | membership = self.portal_membership |
|---|
| 737 | membership.addMember(sid, |
|---|
| 738 | password , |
|---|
| 739 | roles=('Member', |
|---|
| 740 | 'Student', |
|---|
| 741 | ), |
|---|
| 742 | domains='', |
|---|
| 743 | properties = {'memberareaCreationFlag': False, |
|---|
| 744 | 'homeless': True},) |
|---|
| 745 | member = membership.getMemberById(sid) |
|---|
| 746 | self.portal_registration.afterAdd(member, sid, password, None) |
|---|
| [1261] | 747 | #self.manage_setLocalRoles(sid, ['Owner',]) |
|---|
| [1151] | 748 | ###) |
|---|
| [1160] | 749 | |
|---|
| [2560] | 750 | security.declareProtected(View,'makeStudentData') ###( |
|---|
| [1158] | 751 | def makeStudentData(self,student_id,email=None,phone_nr=None): |
|---|
| [1151] | 752 | "create Datastructure for a returning Student" |
|---|
| [1406] | 753 | #import pdb;pdb.set_trace() |
|---|
| [1571] | 754 | logger = logging.getLogger('WAeUPTool.makeStudentData') |
|---|
| [1151] | 755 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| [1794] | 756 | #res = self.students_catalog(id=student_id) |
|---|
| 757 | #if res: |
|---|
| 758 | # st = res[0] |
|---|
| 759 | #res = self.returning_import(matric_no = st.matric_no) |
|---|
| [1798] | 760 | res = self.returning_import(id = student_id) |
|---|
| [1151] | 761 | if res: |
|---|
| [1160] | 762 | student = res[0] |
|---|
| [1794] | 763 | else: |
|---|
| [1816] | 764 | logger.info('Id %s not found in returning_import' % student_id) |
|---|
| [1794] | 765 | return |
|---|
| [1580] | 766 | logger.info('%s creates data structure' % student_id) |
|---|
| [1794] | 767 | s_results = self.results_import(matric_no = student.matric_no) |
|---|
| [1816] | 768 | if s_results: |
|---|
| 769 | lnr = self.getLevelFromResultsCosCode(s_results) |
|---|
| 770 | level = "%d00" % lnr |
|---|
| 771 | verdict,eligible = self.getVerdict(s_results[0].Verdict) |
|---|
| [2454] | 772 | #if eligible: |
|---|
| 773 | # level = "%d00" % (lnr + 1) |
|---|
| [1816] | 774 | else: |
|---|
| 775 | logger.info('matric_no %s not found in results_import' % student.matric_no) |
|---|
| [2656] | 776 | level = '' |
|---|
| 777 | verdict = '' |
|---|
| [1171] | 778 | #student should not be allowed to perform this transition |
|---|
| [1174] | 779 | #wftool = self.portal_workflow |
|---|
| 780 | #wftool.doActionFor(student,'return') |
|---|
| [1151] | 781 | certcode_org = student.Coursemajorcode |
|---|
| 782 | certcode = makeCertificateCode(certcode_org) |
|---|
| 783 | certificate_brain = self.getCertificateBrain(certcode) |
|---|
| 784 | if not certificate_brain: |
|---|
| 785 | em = 'Certificate %s org-code %s not found\n' % (certcode, certcode_org) |
|---|
| 786 | logger.info(em) |
|---|
| 787 | matric_no = student.matric_no |
|---|
| 788 | sid = student_id |
|---|
| 789 | student_obj = getattr(students_folder,sid) |
|---|
| [3114] | 790 | if not getattr(student_obj,'application'): |
|---|
| 791 | student_obj.invokeFactory('StudentApplication','application') |
|---|
| [1151] | 792 | application = student_obj.application |
|---|
| [1169] | 793 | self.portal_workflow.doActionFor(application,'open',dest_container=application) |
|---|
| [1151] | 794 | da = {'Title': 'Application Data'} |
|---|
| 795 | student_obj.invokeFactory('StudentPersonal','personal') |
|---|
| 796 | da['jamb_reg_no'] = student.Entryregno |
|---|
| [1462] | 797 | em = self.getEntryMode(student.Entryregno) |
|---|
| [1401] | 798 | da['entry_mode'] = em |
|---|
| [1151] | 799 | personal = student_obj.personal |
|---|
| 800 | self.portal_workflow.doActionFor(personal,'open',dest_container=personal) |
|---|
| 801 | dp = {'Title': 'Personal Data'} |
|---|
| 802 | student_obj.invokeFactory('StudentClearance','clearance') |
|---|
| 803 | clearance = student_obj.clearance |
|---|
| [1169] | 804 | self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) |
|---|
| [1151] | 805 | dc = {'Title': 'Clearance/Eligibility Record'} |
|---|
| 806 | dc['matric_no'] = matric_no |
|---|
| 807 | state = student.State |
|---|
| 808 | lga = student.LGA |
|---|
| 809 | if state and lga: |
|---|
| 810 | lga = state + ' / ' + lga |
|---|
| 811 | else: |
|---|
| 812 | lga = "None" |
|---|
| [1174] | 813 | da['jamb_lga'] = dc['lga'] = lga |
|---|
| [1173] | 814 | da['app_email'] = dp['email'] = email |
|---|
| 815 | da['app_mobile'] = dp['phone'] = phone_nr |
|---|
| [1411] | 816 | dp['firstname'] = student.Firstname |
|---|
| 817 | dp['middlename'] = student.Middlename |
|---|
| 818 | dp['lastname'] = student.Lastname |
|---|
| 819 | da['jamb_lastname'] = "%s %s %s" % (student.Firstname,student.Middlename,student.Lastname) |
|---|
| [1174] | 820 | da['jamb_sex'] = student.Sex |
|---|
| [1151] | 821 | dp['sex'] = student.Sex == 'F' |
|---|
| 822 | dp['perm_address'] = student.Permanent_Address |
|---|
| 823 | application.getContent().edit(mapping=da) |
|---|
| [1169] | 824 | self.portal_workflow.doActionFor(application,'close',dest_container=application) |
|---|
| [1151] | 825 | personal.getContent().edit(mapping=dp) |
|---|
| 826 | clearance.getContent().edit(mapping=dc) |
|---|
| [1169] | 827 | self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) |
|---|
| [1151] | 828 | # |
|---|
| 829 | # Study Course |
|---|
| 830 | # |
|---|
| 831 | student_obj.invokeFactory('StudentStudyCourse','study_course') |
|---|
| 832 | studycourse = student_obj.study_course |
|---|
| 833 | self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) |
|---|
| 834 | dsc = {} |
|---|
| 835 | dsc['study_course'] = certcode |
|---|
| [1401] | 836 | dsc['current_level'] = level |
|---|
| 837 | dsc['current_verdict'] = verdict |
|---|
| [1827] | 838 | dsc['current_mode'] = em |
|---|
| [2466] | 839 | dsc['current_session'] = '05' |
|---|
| [1151] | 840 | studycourse.getContent().edit(mapping=dsc) |
|---|
| 841 | # |
|---|
| 842 | # Level |
|---|
| 843 | # |
|---|
| [1198] | 844 | ## l = getattr(studycourse,level,None) |
|---|
| 845 | ## if l is None: |
|---|
| 846 | ## studycourse.invokeFactory('StudentStudyLevel', level) |
|---|
| 847 | ## l = getattr(studycourse, level) |
|---|
| 848 | ## self.portal_workflow.doActionFor(l,'open',dest_container=l) |
|---|
| 849 | ## l.getContent().edit(mapping={'Title': "Level %s" % level}) |
|---|
| [1194] | 850 | ###) |
|---|
| [1160] | 851 | |
|---|
| [2556] | 852 | security.declareProtected(ModifyPortalContent,'admitOneStudent') ###( |
|---|
| [2576] | 853 | def admitOneStudent(self,brain,entry_session,pin_password): |
|---|
| [2514] | 854 | "create Datastructure for an admitted Student" |
|---|
| 855 | #import pdb;pdb.set_trace() |
|---|
| [2727] | 856 | if not hasattr(self,"_v_certificates"): |
|---|
| 857 | self._v_certificates = [entry[0] for entry in self.getCertificatesVoc()] |
|---|
| [2514] | 858 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| [2716] | 859 | logger = logging.getLogger('WAeUPTool.admitOneStudent') |
|---|
| [2514] | 860 | if brain.status != "admitted": |
|---|
| 861 | logger.info('status of %s is %s' % (brain.reg_no,brain.status)) |
|---|
| [2540] | 862 | return |
|---|
| [2514] | 863 | pin_parts = brain.pin.split('-') |
|---|
| [2755] | 864 | if pin_parts and len(pin_parts) != 3: |
|---|
| [2695] | 865 | logger.info('invalid pin %s for %s' % (brain.pin,brain.reg_no)) |
|---|
| [2540] | 866 | return |
|---|
| [2727] | 867 | if brain.course_admitted not in self._v_certificates: |
|---|
| 868 | logger.info('certificate %s not found for %s' % (brain.course_admitted,brain.reg_no)) |
|---|
| 869 | return |
|---|
| [2959] | 870 | if brain.sex not in (True,False): |
|---|
| 871 | logger.info('sex of %s not available' % (brain.reg_no)) |
|---|
| [3002] | 872 | return |
|---|
| [2540] | 873 | student_id = self.generateStudentId('?') |
|---|
| [2514] | 874 | students_folder.invokeFactory('Student', student_id) |
|---|
| 875 | student_object = getattr(students_folder,student_id) |
|---|
| [2576] | 876 | if pin_password: |
|---|
| 877 | password = pin_parts[2] |
|---|
| 878 | self.makeStudentMember(student_id,password = password) |
|---|
| [2540] | 879 | student_object.manage_setLocalRoles(student_id, ['Owner',]) |
|---|
| 880 | #logger.info("creating %s reg_no %s" % (student_id,brain.reg_no)) |
|---|
| [2514] | 881 | # |
|---|
| 882 | # application |
|---|
| 883 | # |
|---|
| [2540] | 884 | student_object.invokeFactory('StudentApplication','application') |
|---|
| 885 | application = student_object.application |
|---|
| [2575] | 886 | #self.portal_workflow.doActionFor(application,'open',dest_container=application) |
|---|
| [2514] | 887 | da = {'Title': 'Application Data'} |
|---|
| 888 | da['jamb_reg_no'] = brain.reg_no |
|---|
| [2576] | 889 | |
|---|
| [2540] | 890 | sex = 'M' |
|---|
| 891 | if brain.sex: |
|---|
| 892 | sex = 'F' |
|---|
| 893 | da['jamb_sex'] = sex |
|---|
| [2670] | 894 | da['jamb_age'] = brain.jamb_age |
|---|
| [2575] | 895 | #da['app_ac_pin'] = brain.pin |
|---|
| [2670] | 896 | da['jamb_lga'] = brain.jamb_lga |
|---|
| 897 | da['jamb_state'] = brain.jamb_state |
|---|
| [2575] | 898 | da['jamb_score'] = brain.aggregate |
|---|
| [2514] | 899 | da['app_email'] = brain.email |
|---|
| [2540] | 900 | da['app_mobile'] = brain.phone |
|---|
| [2575] | 901 | if brain.entry_mode: |
|---|
| 902 | da['entry_mode'] = brain.entry_mode |
|---|
| 903 | elif brain.screening_type == 'pume': |
|---|
| 904 | da['entry_mode'] = 'ume_ft' |
|---|
| 905 | elif brain.screening_type == 'pde': |
|---|
| 906 | da['entry_mode'] = 'de_ft' |
|---|
| [2950] | 907 | elif brain.screening_type == 'pce': |
|---|
| [2959] | 908 | da['entry_mode'] = 'pce' |
|---|
| [2950] | 909 | elif brain.screening_type == 'prence': |
|---|
| [2959] | 910 | da['entry_mode'] = 'prence' |
|---|
| [2575] | 911 | else: |
|---|
| [2921] | 912 | da['entry_mode'] = '' |
|---|
| [2575] | 913 | da['entry_session'] = entry_session |
|---|
| 914 | da['jamb_lastname'] = brain.lastname |
|---|
| 915 | da['jamb_middlename'] = brain.middlenames # different field names! |
|---|
| [2670] | 916 | da['jamb_firstname'] = brain.firstname |
|---|
| [2575] | 917 | da['screening_application_date'] = brain.application_date |
|---|
| 918 | da['date_of_birth'] = brain.date_of_birth |
|---|
| 919 | da['jamb_first_cos'] = brain.course1 |
|---|
| 920 | da['jamb_second_cos'] = brain.course2 |
|---|
| 921 | da['course3'] = brain.course3 |
|---|
| 922 | da['screening_type'] = brain.screening_type |
|---|
| 923 | da['screening_score'] = brain.screening_score |
|---|
| 924 | da['screening_date'] = brain.screening_date |
|---|
| 925 | da['hq_type'] = brain.hq_type |
|---|
| 926 | da['hq_grade'] = brain.hq_grade |
|---|
| 927 | da['aos'] = brain.aos |
|---|
| [2576] | 928 | |
|---|
| [2514] | 929 | application.getContent().edit(mapping=da) |
|---|
| [2575] | 930 | #self.portal_workflow.doActionFor(application,'close',dest_container=application) |
|---|
| [2514] | 931 | # |
|---|
| 932 | # personal |
|---|
| 933 | # |
|---|
| [2540] | 934 | student_object.invokeFactory('StudentPersonal','personal') |
|---|
| 935 | personal = student_object.personal |
|---|
| [2575] | 936 | #self.portal_workflow.doActionFor(personal,'open',dest_container=personal) |
|---|
| [2514] | 937 | dp = {'Title': 'Personal Data'} |
|---|
| 938 | dp['sex'] = brain.sex |
|---|
| 939 | dp['email'] = brain.email |
|---|
| [2540] | 940 | dp['phone'] = brain.phone |
|---|
| [2514] | 941 | dp['lastname'] = brain.lastname |
|---|
| [2575] | 942 | dp['middlename'] = brain.middlenames # different field names! |
|---|
| [2576] | 943 | dp['firstname'] = brain.firstname |
|---|
| [2514] | 944 | personal.getContent().edit(mapping=dp) |
|---|
| 945 | # |
|---|
| 946 | # clearance |
|---|
| 947 | # |
|---|
| [2540] | 948 | student_object.invokeFactory('StudentClearance','clearance') |
|---|
| 949 | clearance = student_object.clearance |
|---|
| [2575] | 950 | #self.portal_workflow.doActionFor(clearance,'open',dest_container=clearance) |
|---|
| [2514] | 951 | dc = {'Title': 'Clearance/Eligibility Record'} |
|---|
| 952 | dc['lga'] = brain.lga |
|---|
| [2670] | 953 | dc['birthday'] = brain.date_of_birth |
|---|
| [2514] | 954 | clearance.getContent().edit(mapping=dc) |
|---|
| [2575] | 955 | #self.portal_workflow.doActionFor(clearance,'close',dest_container=clearance) |
|---|
| [2514] | 956 | # |
|---|
| 957 | # study Course |
|---|
| 958 | # |
|---|
| [2540] | 959 | student_object.invokeFactory('StudentStudyCourse','study_course') |
|---|
| 960 | studycourse = student_object.study_course |
|---|
| [2575] | 961 | #self.portal_workflow.doActionFor(studycourse,'open',dest_container=studycourse) |
|---|
| [2514] | 962 | dsc = {} |
|---|
| 963 | dsc['study_course'] = brain.course_admitted |
|---|
| 964 | dsc['current_verdict'] = '' |
|---|
| [2575] | 965 | dsc['current_mode'] = da['entry_mode'] |
|---|
| 966 | if da['entry_mode'].startswith('de'): |
|---|
| 967 | dsc['current_level'] = '200' |
|---|
| [3040] | 968 | elif da['entry_mode'].startswith('pre'): |
|---|
| 969 | dsc['current_level'] = '000' |
|---|
| [2575] | 970 | else: |
|---|
| 971 | dsc['current_level'] = '100' |
|---|
| 972 | dsc['current_session'] = entry_session |
|---|
| [2514] | 973 | studycourse.getContent().edit(mapping=dsc) |
|---|
| 974 | # |
|---|
| 975 | # payments folder |
|---|
| [2540] | 976 | student_object.invokeFactory('PaymentsFolder','payments') |
|---|
| 977 | payments = getattr(student_object,'payments') |
|---|
| [2514] | 978 | dpay = {} |
|---|
| 979 | dpay['Title'] = 'Payments' |
|---|
| 980 | payments.getContent().edit(mapping=dpay) |
|---|
| [2540] | 981 | self.portal_workflow.doActionFor(payments,'open') |
|---|
| [2580] | 982 | # |
|---|
| 983 | # passport foto |
|---|
| 984 | app_picture ="%s/import/images/%s/%s_passport.jpg" % (i_home, |
|---|
| 985 | brain.screening_type, |
|---|
| 986 | brain.reg_no) |
|---|
| [2675] | 987 | images_dir = getImagesDir(student_id) |
|---|
| 988 | #images_dir = os.path.join("%s" % images_base,student_id) |
|---|
| 989 | letter_dir,student_dir = os.path.split(images_dir) |
|---|
| 990 | if not os.path.exists(letter_dir): |
|---|
| 991 | os.mkdir(letter_dir) |
|---|
| [2580] | 992 | if not os.path.exists(images_dir): |
|---|
| 993 | os.mkdir(images_dir) |
|---|
| 994 | image_name = os.path.join(images_dir,"passport_%(student_id)s.jpg" % vars()) |
|---|
| 995 | if os.path.exists(app_picture): |
|---|
| 996 | copy2(app_picture,image_name) |
|---|
| 997 | else: |
|---|
| 998 | logger.info('passport of %s/%s not found: %s' % (student_id, |
|---|
| 999 | brain.reg_no, |
|---|
| 1000 | app_picture)) |
|---|
| [2656] | 1001 | |
|---|
| [2514] | 1002 | return student_id |
|---|
| 1003 | ###) |
|---|
| 1004 | |
|---|
| [2556] | 1005 | security.declareProtected(ModifyPortalContent,'makeStudentLevel') ###( |
|---|
| [1194] | 1006 | def makeStudentLevel(self,student_id): |
|---|
| 1007 | "create the StudyLevel for a returning Student" |
|---|
| 1008 | #import pdb;pdb.set_trace() |
|---|
| [1571] | 1009 | logger = logging.getLogger('WAeUPTool.makeStudentLevel') |
|---|
| [1194] | 1010 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| 1011 | res = self.students_catalog(id=student_id) |
|---|
| 1012 | if res: |
|---|
| 1013 | st = res[0] |
|---|
| 1014 | course = st.course |
|---|
| 1015 | matric_no = st.matric_no |
|---|
| 1016 | level = st.level |
|---|
| 1017 | res = self.results_import(matric_no = matric_no) |
|---|
| 1018 | if res: |
|---|
| 1019 | results = res |
|---|
| [1571] | 1020 | logger.info('%s creating Level %s' % (student_id,level)) |
|---|
| [1194] | 1021 | # |
|---|
| 1022 | # Level |
|---|
| 1023 | # |
|---|
| 1024 | student_obj = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
|---|
| 1025 | studycourse = getattr(student_obj,"study_course",None) |
|---|
| 1026 | self.portal_workflow.doActionFor(studycourse,'close_for_edit',dest_container=studycourse) |
|---|
| 1027 | l = getattr(studycourse,level,None) |
|---|
| 1028 | if l is None: |
|---|
| 1029 | studycourse.invokeFactory('StudentStudyLevel', level) |
|---|
| 1030 | l = getattr(studycourse, level) |
|---|
| 1031 | self.portal_workflow.doActionFor(l,'open',dest_container=l) |
|---|
| 1032 | l.getContent().edit(mapping={'Title': "Level %s" % level}) |
|---|
| 1033 | ###) |
|---|
| 1034 | |
|---|
| [2000] | 1035 | security.declarePublic('getHallInfo') ###( |
|---|
| 1036 | def getHallInfo(self,bed): |
|---|
| 1037 | """return Hall Info""" |
|---|
| [828] | 1038 | info = {} |
|---|
| 1039 | hall,block,room,letter = bed.split('_') |
|---|
| [1846] | 1040 | res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="AccoHall",id=hall) |
|---|
| [828] | 1041 | if res and len(res) == 1: |
|---|
| 1042 | hall_brain = res[0] |
|---|
| 1043 | hall_doc = hall_brain.getObject().getContent() |
|---|
| 1044 | else: |
|---|
| 1045 | return info |
|---|
| 1046 | info['hall_title'] = hall_brain.Title |
|---|
| 1047 | info['maintenance_code'] = hall_doc.maintenance_code |
|---|
| [1846] | 1048 | res = ZCatalog.searchResults(self.portal_catalog_real,portal_type="ScratchCardBatch") |
|---|
| [828] | 1049 | batch_doc = None |
|---|
| 1050 | for brain in res: |
|---|
| 1051 | if brain.id.startswith(info['maintenance_code']): |
|---|
| 1052 | batch_doc = brain.getObject().getContent() |
|---|
| 1053 | break |
|---|
| 1054 | if batch_doc is None: |
|---|
| [2913] | 1055 | info['maintenance_fee'] = '' |
|---|
| [828] | 1056 | else: |
|---|
| 1057 | info['maintenance_fee'] = batch_doc.cost |
|---|
| 1058 | return info |
|---|
| [1151] | 1059 | ###) |
|---|
| [828] | 1060 | |
|---|
| [2396] | 1061 | security.declareProtected(ModifyPortalContent,'removePictureFolder') ###( |
|---|
| 1062 | def removePictureFolder(self,student_id): |
|---|
| 1063 | """remove picture_folder by renaming it""" |
|---|
| 1064 | path = 'images' |
|---|
| 1065 | picture_path = os.path.join(i_home,path,student_id) |
|---|
| 1066 | if not os.path.exists(picture_path): |
|---|
| 1067 | return False |
|---|
| 1068 | os.rename(picture_path,picture_path + "_removed") |
|---|
| 1069 | return True |
|---|
| 1070 | ###) |
|---|
| 1071 | |
|---|
| 1072 | security.declareProtected(ModifyPortalContent,'restorePictureFolder') ###( |
|---|
| 1073 | def restorePictureFolder(self,student_id): |
|---|
| 1074 | """restore picture_folder by renaming it""" |
|---|
| 1075 | path = 'images' |
|---|
| 1076 | picture_path = os.path.join(i_home,path,student_id) |
|---|
| 1077 | if not os.path.exists(picture_path + "_removed"): |
|---|
| 1078 | return False |
|---|
| 1079 | os.rename(picture_path + "_removed",picture_path) |
|---|
| 1080 | return True |
|---|
| 1081 | ###) |
|---|
| 1082 | |
|---|
| [2363] | 1083 | security.declarePublic('picturesExist') ###( |
|---|
| [2420] | 1084 | def picturesExist(self, ids,student_id=None): |
|---|
| [2363] | 1085 | """check if pictures exist in the filesystem""" |
|---|
| [2420] | 1086 | if student_id is None: |
|---|
| 1087 | student_id = self.getStudentId() |
|---|
| 1088 | if student_id is None: |
|---|
| 1089 | return False |
|---|
| [2675] | 1090 | picture_path = getImagesDir(student_id) |
|---|
| 1091 | #picture_path = os.path.join(images_base,student_id) |
|---|
| [2363] | 1092 | if not os.path.exists(picture_path): |
|---|
| 1093 | return False |
|---|
| 1094 | pictures = [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] |
|---|
| 1095 | return set(ids).issubset(set(pictures)) |
|---|
| 1096 | ###) |
|---|
| 1097 | |
|---|
| [2364] | 1098 | security.declarePublic('picturesList') ###( |
|---|
| 1099 | def picturesList(self): |
|---|
| 1100 | """check if pictures exist in the filesystem""" |
|---|
| 1101 | path = 'images' |
|---|
| 1102 | student_id = self.getStudentId() |
|---|
| [2677] | 1103 | #picture_path = os.path.join(i_home,path,student_id) |
|---|
| 1104 | picture_path = getImagesDir(student_id) |
|---|
| [2364] | 1105 | if not os.path.exists(picture_path): |
|---|
| 1106 | return [] |
|---|
| 1107 | return [picture[:picture.rfind('_')] for picture in os.listdir(picture_path)] |
|---|
| 1108 | ###) |
|---|
| 1109 | |
|---|
| [2114] | 1110 | security.declarePublic('showFsPicture') ###( |
|---|
| 1111 | def showFsPicture(self,path): |
|---|
| 1112 | """return a picture from the filesystem""" |
|---|
| [2675] | 1113 | #picture_path = os.path.join(i_home,path) |
|---|
| 1114 | picture_path = os.path.join(images_base,path) |
|---|
| [2335] | 1115 | response = self.REQUEST.RESPONSE |
|---|
| [2342] | 1116 | #import pdb;pdb.set_trace() |
|---|
| [2354] | 1117 | registry = getToolByName(self, 'mimetypes_registry') |
|---|
| 1118 | mimetype = str(registry.lookupExtension(path.lower()) or |
|---|
| 1119 | registry.lookupExtension('file.bin')) |
|---|
| [2114] | 1120 | if os.path.exists(picture_path): |
|---|
| [2354] | 1121 | response.setHeader('Content-type',mimetype) |
|---|
| [2114] | 1122 | return open(picture_path).read() |
|---|
| [2335] | 1123 | picture_path = os.path.join(i_home,'import',path) |
|---|
| 1124 | if os.path.exists(picture_path): |
|---|
| 1125 | return open(picture_path).read() |
|---|
| [2114] | 1126 | ###) |
|---|
| 1127 | |
|---|
| [1151] | 1128 | security.declareProtected(ModifyPortalContent,'deleteAllCourses') ###( |
|---|
| 1129 | def deleteAllCourses(self,department="All"): |
|---|
| 1130 | ''' delete the courses''' |
|---|
| 1131 | pm = self.portal_membership |
|---|
| 1132 | member = pm.getAuthenticatedMember() |
|---|
| [1160] | 1133 | |
|---|
| [1151] | 1134 | if str(member) not in ("henrik","joachim"): |
|---|
| 1135 | return "not possible" |
|---|
| 1136 | if department == "All": |
|---|
| 1137 | res = self.portal_catalog({'meta_type': 'Department'}) |
|---|
| 1138 | if len(res) < 1: |
|---|
| 1139 | return "No Departments found" |
|---|
| [1160] | 1140 | |
|---|
| [1151] | 1141 | deleted = [] |
|---|
| 1142 | for dep in res: |
|---|
| 1143 | cf = dep.getObject().courses |
|---|
| 1144 | if cf: |
|---|
| 1145 | cf.manage_delObjects(ids=cf.objectIds()) |
|---|
| 1146 | deleted.append("deleted Courses in %s" % dep.getId) |
|---|
| 1147 | return "\r".join(deleted) |
|---|
| [1160] | 1148 | ###) |
|---|
| [1151] | 1149 | |
|---|
| [1572] | 1150 | security.declareProtected(ModifyPortalContent,'getLogfileLines') ###( |
|---|
| 1151 | def getLogfileLines(self,filename="event.log",numlines=20): |
|---|
| 1152 | """Get last NUMLINES lines of logfile FILENAME. |
|---|
| [1160] | 1153 | |
|---|
| [1572] | 1154 | Return last lines' of a file in the instances logfile directory as |
|---|
| 1155 | a list. The number of returned lines equals `numlines' or less. If |
|---|
| 1156 | less than `numlines' lines are available, the whole file ist |
|---|
| 1157 | returned. If the file can not be opened or some other error |
|---|
| 1158 | occurs, empty list is returend. |
|---|
| 1159 | """ |
|---|
| 1160 | result = [] |
|---|
| 1161 | lines_hit = 0 |
|---|
| 1162 | |
|---|
| 1163 | # We only handle files in instances' log directory... |
|---|
| 1164 | logpath = os.path.join(i_home, "log") |
|---|
| 1165 | filename = str(os.path.abspath( os.path.join( logpath, filename ))) |
|---|
| 1166 | if not filename.startswith( logpath ): |
|---|
| 1167 | # Attempt to access file outside log-dir... |
|---|
| 1168 | return [] |
|---|
| 1169 | |
|---|
| 1170 | try: |
|---|
| 1171 | fd = file( filename, "rb" ) |
|---|
| 1172 | except IOError: |
|---|
| 1173 | return [] |
|---|
| 1174 | if not fd: |
|---|
| 1175 | return [] |
|---|
| 1176 | |
|---|
| 1177 | if os.linesep == None: |
|---|
| 1178 | linesep = '\n' |
|---|
| 1179 | else: |
|---|
| 1180 | linesep = os.linesep |
|---|
| 1181 | |
|---|
| 1182 | # Try to find 'numlines' times a lineseparator, searching from end |
|---|
| 1183 | # and moving to the beginning of file... |
|---|
| 1184 | fd.seek( 0, 2) # Move to end of file... |
|---|
| 1185 | while lines_hit < numlines: |
|---|
| 1186 | if fd.read(1) == linesep[-1]: # This moves filedescriptor |
|---|
| 1187 | # one step forward... |
|---|
| 1188 | lines_hit += 1 |
|---|
| 1189 | try: |
|---|
| 1190 | fd.seek( -2, 1) # Go two bytes back from current pos... |
|---|
| 1191 | except IOError: |
|---|
| 1192 | # We cannot go back two bytes. Maybe the file is too small... |
|---|
| 1193 | break |
|---|
| 1194 | fd.seek(2,1) |
|---|
| 1195 | |
|---|
| 1196 | # Read all lines from current position... |
|---|
| 1197 | result = fd.readlines() |
|---|
| 1198 | # Remove line endings... |
|---|
| 1199 | result = [x.strip() for x in result] |
|---|
| 1200 | fd.close() |
|---|
| 1201 | return result |
|---|
| 1202 | ###) |
|---|
| 1203 | |
|---|
| [1700] | 1204 | security.declareProtected(ModifyPortalContent,"getCallbacksFromLog")###( |
|---|
| [1665] | 1205 | def getCallbacksFromLog(self,filename): |
|---|
| 1206 | """fix Online Payment Transactions from Z2.log entries""" |
|---|
| 1207 | import transaction |
|---|
| 1208 | import random |
|---|
| 1209 | from cgi import parse_qs |
|---|
| 1210 | from urlparse import urlparse |
|---|
| 1211 | #from pdb import set_trace |
|---|
| 1212 | wftool = self.portal_workflow |
|---|
| 1213 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
|---|
| 1214 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| 1215 | s = r'(?P<client_ip>\S+) - (?P<member_id>\S+) \[' |
|---|
| 1216 | s += r'(?P<date>.*)\] "(?P<get>.*)" (?P<codes>\d+ \d+) "' |
|---|
| 1217 | s += r'(?P<intersw>.*)" "(?P<agent>.*)"' |
|---|
| 1218 | data = re.compile(s) |
|---|
| 1219 | start = True |
|---|
| 1220 | tr_count = 1 |
|---|
| 1221 | total = 0 |
|---|
| 1222 | #name = 'pume_results' |
|---|
| 1223 | #name = 'epaymentsuccessful_z2log2' |
|---|
| 1224 | name = filename |
|---|
| 1225 | no_import = [] |
|---|
| 1226 | imported = [] |
|---|
| 1227 | logger = logging.getLogger('WAeUPTool.getFailedTransactions') |
|---|
| 1228 | try: |
|---|
| 1229 | transactions = open("%s/import/%s" % (i_home,name),"rb").readlines() |
|---|
| 1230 | except: |
|---|
| 1231 | logger.error('Error reading %s' % name) |
|---|
| 1232 | return |
|---|
| 1233 | tas = [] |
|---|
| 1234 | for line in transactions: |
|---|
| 1235 | dict = {} |
|---|
| 1236 | items = data.search(line) |
|---|
| 1237 | dict['idict'] = idict = items.groupdict() |
|---|
| 1238 | #print idict |
|---|
| 1239 | #from pdb import set_trace;set_trace() |
|---|
| 1240 | urlparsed = urlparse(idict['get'][4:]) |
|---|
| 1241 | #print urlparsed |
|---|
| 1242 | path = urlparsed[2].split('/') |
|---|
| 1243 | dict['student_id'] = student_id = path[8] |
|---|
| 1244 | dict['payment_id'] = payment_id = path[10] |
|---|
| 1245 | dict['qs_dict'] = qs_dict = parse_qs(urlparsed[4]) |
|---|
| 1246 | tas.append(dict) |
|---|
| 1247 | tr_count += 1 |
|---|
| 1248 | return tas |
|---|
| 1249 | ###) |
|---|
| 1250 | |
|---|
| [1620] | 1251 | security.declareProtected(ModifyPortalContent,"importOnlinePaymentTransactions")###( |
|---|
| 1252 | def importOnlinePaymentTransactions(self): |
|---|
| 1253 | """load Online Payment Transactions from CSV values""" |
|---|
| 1254 | import transaction |
|---|
| 1255 | import random |
|---|
| 1256 | #from pdb import set_trace |
|---|
| 1257 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
|---|
| [1625] | 1258 | opt = self.online_payments_import |
|---|
| [1620] | 1259 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| 1260 | start = True |
|---|
| 1261 | tr_count = 1 |
|---|
| 1262 | total = 0 |
|---|
| 1263 | #name = 'pume_results' |
|---|
| 1264 | name = 'OnlineTransactions' |
|---|
| 1265 | no_import = [] |
|---|
| 1266 | imported = [] |
|---|
| 1267 | logger = logging.getLogger('WAeUPTool.importOnlinePaymentTransactions') |
|---|
| 1268 | try: |
|---|
| 1269 | transactions = csv.DictReader(open("%s/import/%s.csv" % (i_home,name),"rb")) |
|---|
| 1270 | except: |
|---|
| 1271 | logger.error('Error reading %s.csv' % name) |
|---|
| 1272 | return |
|---|
| 1273 | for pay_transaction in transactions: |
|---|
| 1274 | if start: |
|---|
| 1275 | start = False |
|---|
| 1276 | logger.info('Start loading from %s.csv' % name) |
|---|
| 1277 | s = ','.join(['"%s"' % fn for fn in pay_transaction.keys()]) |
|---|
| 1278 | no_import.append('%s,"Error"' % s) |
|---|
| 1279 | format = ','.join(['"%%(%s)s"' % fn for fn in pay_transaction.keys()]) |
|---|
| 1280 | format_error = format + ',"%(Error)s"' |
|---|
| 1281 | data = {} |
|---|
| [1644] | 1282 | |
|---|
| [1781] | 1283 | # format of the first file sent by Tayo |
|---|
| [1643] | 1284 | #data['datetime'] = date = DateTime.DateTime(pay_transaction['Date']) |
|---|
| 1285 | #data['student_id'] = student_id = pay_transaction['Payer ID'] |
|---|
| 1286 | #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] |
|---|
| 1287 | #data['response_code'] = response_code = pay_transaction['Resp Code'] |
|---|
| 1288 | #data['amount'] = amount = pay_transaction['Amount'] |
|---|
| [1644] | 1289 | |
|---|
| [1781] | 1290 | # format of the second file sent by Tayo |
|---|
| [1798] | 1291 | #data['datetime'] = date = 0 |
|---|
| 1292 | #data['student_id'] = student_id = pay_transaction['Payer ID'] |
|---|
| 1293 | #data['order_id'] = order_id = pay_transaction['Order ID (Tranx Ref)'] |
|---|
| 1294 | #data['response_code'] = response_code = '00' |
|---|
| 1295 | #data['amount'] = amount = pay_transaction['Amount'] |
|---|
| [1813] | 1296 | |
|---|
| [1798] | 1297 | # format of the third file sent by Kehinde |
|---|
| [1643] | 1298 | data['datetime'] = date = 0 |
|---|
| [1798] | 1299 | data['student_id'] = student_id = pay_transaction['customer_id'] |
|---|
| 1300 | data['order_id'] = order_id = pay_transaction['merchant_reference'] |
|---|
| [1644] | 1301 | data['response_code'] = response_code = '00' |
|---|
| [1813] | 1302 | data['amount'] = amount = pay_transaction['Amount'] |
|---|
| [1644] | 1303 | |
|---|
| [1620] | 1304 | dup = False |
|---|
| 1305 | if response_code == "12": |
|---|
| 1306 | continue |
|---|
| 1307 | try: |
|---|
| 1308 | opt.addRecord(**data) |
|---|
| 1309 | except ValueError: |
|---|
| 1310 | dup = True |
|---|
| 1311 | #from pdb import set_trace;set_trace() |
|---|
| 1312 | if dup: |
|---|
| 1313 | if response_code == "00": |
|---|
| [1798] | 1314 | try: |
|---|
| 1315 | opt.modifyRecord(**data) |
|---|
| 1316 | except: |
|---|
| 1317 | logger.info("duplicate uid, order_id %(order_id)s, student_id %(student_id)s, response_code %(response_code)s" % data) |
|---|
| 1318 | continue |
|---|
| [1620] | 1319 | else: |
|---|
| [1674] | 1320 | pay_transaction['Error'] = "Duplicate order_id" |
|---|
| [1620] | 1321 | no_import.append( format_error % pay_transaction) |
|---|
| [1674] | 1322 | logger.info("duplicate order_id %(order_id)s for %(student_id)s %(response_code)s" % data) |
|---|
| [1620] | 1323 | continue |
|---|
| 1324 | tr_count += 1 |
|---|
| 1325 | if tr_count > 1000: |
|---|
| 1326 | if len(no_import) > 0: |
|---|
| 1327 | open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( |
|---|
| 1328 | '\n'.join(no_import) + '\n') |
|---|
| 1329 | no_import = [] |
|---|
| [1645] | 1330 | em = '%d transactions committed\n' % (tr_count) |
|---|
| [1620] | 1331 | transaction.commit() |
|---|
| 1332 | regs = [] |
|---|
| 1333 | logger.info(em) |
|---|
| 1334 | total += tr_count |
|---|
| 1335 | tr_count = 0 |
|---|
| 1336 | open("%s/import/%s_not_imported%s.csv" % (i_home,name,current),"a").write( |
|---|
| 1337 | '\n'.join(no_import)) |
|---|
| 1338 | return self.REQUEST.RESPONSE.redirect("%s" % self.REQUEST.get('URL1')) |
|---|
| 1339 | ###) |
|---|
| [2237] | 1340 | |
|---|
| [3172] | 1341 | security.declareProtected(ModifyPortalContent,"importData")###( |
|---|
| 1342 | def importData(self,filename,name,edit=False,bypass_queue_catalog=False): |
|---|
| 1343 | """load data from CSV values""" |
|---|
| 1344 | import transaction |
|---|
| 1345 | import random |
|---|
| 1346 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| 1347 | pending_only = False |
|---|
| [3177] | 1348 | pend_str = '--' |
|---|
| [3172] | 1349 | elapse = time.time() |
|---|
| 1350 | # |
|---|
| 1351 | # preparations |
|---|
| 1352 | # |
|---|
| [3177] | 1353 | if filename == pend_str: |
|---|
| [3172] | 1354 | pending_only = True |
|---|
| 1355 | mode = "create" |
|---|
| 1356 | if edit: |
|---|
| 1357 | if filename.endswith('_toDelete'): |
|---|
| 1358 | mode = "delete" |
|---|
| 1359 | else: |
|---|
| 1360 | mode = "edit" |
|---|
| 1361 | importer_name = ''.join([part.capitalize() for part in name.split('_')]) |
|---|
| 1362 | importer = eval("%sImport" % importer_name)(mode,self) |
|---|
| 1363 | logger = importer.logger |
|---|
| 1364 | if importer.init_errors: |
|---|
| 1365 | logger.info(self.init_errors) |
|---|
| 1366 | return self.init_errors |
|---|
| 1367 | member = importer.member |
|---|
| 1368 | current = importer.current |
|---|
| 1369 | import_date = importer.import_date |
|---|
| 1370 | # |
|---|
| 1371 | # not_imported |
|---|
| 1372 | # |
|---|
| 1373 | info = importer.info |
|---|
| 1374 | data_keys = importer.data_keys |
|---|
| [3181] | 1375 | csv_keys = importer.csv_keys |
|---|
| [3172] | 1376 | #csv_keys.extend(info.keys()) |
|---|
| 1377 | headline_mapping = dict((k,k) for k in csv_keys) |
|---|
| 1378 | # |
|---|
| 1379 | # pending |
|---|
| 1380 | # |
|---|
| 1381 | pending_path = importer.pending_path |
|---|
| 1382 | pending_tmp = importer.pending_tmp |
|---|
| 1383 | pending_backup = importer.pending_backup |
|---|
| 1384 | pending_fn = importer.pending_fn |
|---|
| 1385 | imported_path = importer.imported_path |
|---|
| 1386 | imported_fn = importer.imported_fn |
|---|
| [3180] | 1387 | commit_after = importer.commit_after |
|---|
| [3172] | 1388 | pending = [] |
|---|
| 1389 | pending_digests = [] |
|---|
| 1390 | #total_added_to_pending = 0 |
|---|
| 1391 | if not pending_only: |
|---|
| 1392 | pending,pending_digests = importer.makeIdLists() |
|---|
| 1393 | pending_at_start = len(pending) |
|---|
| 1394 | datafile = open(pending_tmp,"w") |
|---|
| 1395 | pending_csv_writer = csv.DictWriter(datafile, |
|---|
| 1396 | csv_keys, |
|---|
| 1397 | extrasaction='ignore') |
|---|
| 1398 | pending_csv_writer.writerow(headline_mapping) |
|---|
| 1399 | datafile.close() |
|---|
| 1400 | # |
|---|
| 1401 | # imported |
|---|
| 1402 | # |
|---|
| 1403 | if not os.path.exists(imported_path): |
|---|
| 1404 | datafile = open(imported_path,"w") |
|---|
| 1405 | imported_csv_writer = csv.DictWriter(datafile, |
|---|
| 1406 | csv_keys, |
|---|
| 1407 | extrasaction='ignore') |
|---|
| 1408 | imported_csv_writer.writerow(headline_mapping) |
|---|
| 1409 | datafile.close() |
|---|
| 1410 | start = True |
|---|
| 1411 | tr_count = 0 |
|---|
| 1412 | total = 0 |
|---|
| 1413 | total_added_to_imported = 0 |
|---|
| 1414 | total_pending = 0 |
|---|
| 1415 | if pending_only: |
|---|
| 1416 | import_source_path = pending_path |
|---|
| [2193] | 1417 | else: |
|---|
| [3172] | 1418 | import_source_path = "%s/import/%s.csv" % (i_home,filename) |
|---|
| 1419 | if not os.path.exists(import_source_path): |
|---|
| 1420 | em = 'No %(import_source_fn)s' % vars() |
|---|
| 1421 | return em |
|---|
| 1422 | import_source_fn = os.path.split(import_source_path)[1] |
|---|
| 1423 | info['imported_from'] = import_source_fn |
|---|
| 1424 | attrs = csv.reader(open(import_source_path,"rb")).next() |
|---|
| 1425 | import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore') |
|---|
| 1426 | or k.strip() in info.keys())] |
|---|
| 1427 | diff2schema = set(import_keys).difference(set(importer.schema.keys())) |
|---|
| 1428 | diff2layout = set(import_keys).difference(set(importer.layout.keys())) |
|---|
| 1429 | if diff2schema and diff2schema != set(['id',]): |
|---|
| 1430 | msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema) |
|---|
| 1431 | return msg |
|---|
| 1432 | if mode in importer.required_modes: |
|---|
| 1433 | required_keys = [importer.layout.getIdUnprefixed(id) |
|---|
| 1434 | for id,widget in importer.layout.objectItems() |
|---|
| 1435 | if widget.is_required] |
|---|
| 1436 | if not set(required_keys).issubset(set(import_keys)): |
|---|
| 1437 | diff2import = set(required_keys).difference(set(import_keys)) |
|---|
| 1438 | msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import) |
|---|
| 1439 | return msg |
|---|
| 1440 | # |
|---|
| 1441 | # start importing |
|---|
| 1442 | # |
|---|
| [2193] | 1443 | try: |
|---|
| [3172] | 1444 | reader = csv.DictReader(open(import_source_path,"rb")) |
|---|
| 1445 | except: |
|---|
| 1446 | msg = 'Error reading %s.csv' % filename |
|---|
| 1447 | logger.error(msg) |
|---|
| 1448 | return msg |
|---|
| 1449 | items = [item for item in reader] |
|---|
| 1450 | total_to_import = len(items) |
|---|
| [3181] | 1451 | tti_float = float(total_to_import) |
|---|
| [3172] | 1452 | if pending_only: |
|---|
| 1453 | pending_at_start = total_to_import |
|---|
| 1454 | count = 0 |
|---|
| 1455 | imported = [] |
|---|
| 1456 | old_commit_count = 0 |
|---|
| 1457 | with_error_count = imported_count = 0 |
|---|
| 1458 | already_in = 0 |
|---|
| 1459 | for record in items: |
|---|
| 1460 | item = {} |
|---|
| 1461 | for k,v in record.items(): |
|---|
| 1462 | if v: |
|---|
| 1463 | item[k.strip()] = v.strip() |
|---|
| 1464 | count += 1 |
|---|
| 1465 | #item = dict((k.strip(),l.strip()) for (k,l) in item.items()) |
|---|
| 1466 | if start: |
|---|
| 1467 | start = False |
|---|
| 1468 | adapters = [MappingStorageAdapter(importer.schema, item)] |
|---|
| 1469 | logger.info('%(member)s starts import from %(import_source_fn)s in %(mode)s mode' % vars()) |
|---|
| 1470 | #logger.info('%s starts import from %s in %s mode with schema and layout %s' % (member,filename,mode,iname)) |
|---|
| 1471 | dm = DataModel(item, adapters,context=self) |
|---|
| 1472 | ds = DataStructure(data=item,datamodel=dm) |
|---|
| 1473 | error_string = "" |
|---|
| 1474 | total += 1 |
|---|
| [3176] | 1475 | error_count = 0 |
|---|
| [3172] | 1476 | for k in import_keys: |
|---|
| 1477 | if k not in item.keys() or k not in importer.validators.keys(): |
|---|
| 1478 | continue |
|---|
| 1479 | if not importer.validators[k](ds,mode=mode): |
|---|
| 1480 | if error_count: |
|---|
| 1481 | error_string += ' ++ ' |
|---|
| 1482 | error_string += "%s: %s" % (k,self.translation_service(ds.getError(k), |
|---|
| 1483 | ds.getErrorMapping(k))) |
|---|
| 1484 | error_count += 1 |
|---|
| 1485 | if error_string: |
|---|
| 1486 | error = error_string |
|---|
| 1487 | id = '' |
|---|
| 1488 | mapping = item |
|---|
| [2699] | 1489 | else: |
|---|
| [3172] | 1490 | temp_item = item.copy() |
|---|
| 1491 | temp_item.update(dm) |
|---|
| 1492 | results = importer.import_method(temp_item) |
|---|
| 1493 | id = results[0] |
|---|
| 1494 | error = results[1] |
|---|
| 1495 | mapping = results[2] |
|---|
| [3181] | 1496 | data_string = ", ".join("%s: %s" % (k,v) for k,v in mapping.items()) |
|---|
| [3172] | 1497 | info['error'] = error |
|---|
| 1498 | mapping.update(info) |
|---|
| [3178] | 1499 | log_list = [] |
|---|
| [3172] | 1500 | if error: |
|---|
| 1501 | digest = makeDigest(mapping,data_keys) |
|---|
| 1502 | with_error_count += 1 |
|---|
| [3178] | 1503 | #percent_finished = (with_error_count + imported_count)/tti_float*100 |
|---|
| [3172] | 1504 | if digest not in pending_digests: |
|---|
| 1505 | pending_digests += digest, |
|---|
| 1506 | pending.append(mapping) |
|---|
| 1507 | if not pending_only: |
|---|
| [3181] | 1508 | log_list += "record added to %(pending_fn)s, %(data_string)s" % vars(), |
|---|
| [3178] | 1509 | #logger.info("%(percent_finished)6.3f %% done added to %(pending_fn)s %(data_string)s" % vars()) |
|---|
| [3172] | 1510 | else: |
|---|
| 1511 | already_in += 1 |
|---|
| 1512 | #logger.info("already in %(pending_fn)s %(data_string)s" % vars()) |
|---|
| 1513 | pass |
|---|
| 1514 | else: |
|---|
| 1515 | imported_count += 1 |
|---|
| 1516 | imported += mapping, |
|---|
| [3181] | 1517 | log_list += "record imported and added to %(imported_fn)s, %(data_string)s" % vars(), |
|---|
| [3178] | 1518 | #percent_finished = (with_error_count + imported_count)/tti_float*100 |
|---|
| 1519 | #logger.info("%(percent_finished)6.3f %% done imported and added to %(imported_fn)s %(data_string)s" % vars()) |
|---|
| 1520 | if log_list: |
|---|
| 1521 | time_till_now = time.time() - elapse |
|---|
| [3172] | 1522 | percent_finished = (with_error_count + imported_count)/tti_float*100 |
|---|
| [3181] | 1523 | log_list.insert(0,("%(percent_finished)6.3f %% done in %(time_till_now)3.2fs," % vars()),) |
|---|
| [3178] | 1524 | logger.info(' '.join(log_list)) |
|---|
| [3172] | 1525 | finished = count > total_to_import - 1 |
|---|
| 1526 | must_commit = False |
|---|
| 1527 | if len(imported) != old_commit_count: |
|---|
| [3180] | 1528 | if not len(imported) % commit_after: |
|---|
| [3172] | 1529 | must_commit = True |
|---|
| 1530 | old_commit_count = len(imported) |
|---|
| [3181] | 1531 | |
|---|
| [3172] | 1532 | if must_commit or finished: |
|---|
| 1533 | if len(imported): |
|---|
| 1534 | transaction.commit() |
|---|
| 1535 | datafile = open(imported_path,"a") |
|---|
| 1536 | writer = csv.DictWriter(datafile, |
|---|
| 1537 | csv_keys, |
|---|
| 1538 | extrasaction='ignore') |
|---|
| 1539 | writer.writerows(imported) |
|---|
| 1540 | datafile.close() |
|---|
| 1541 | total_added_to_imported += len(imported) |
|---|
| 1542 | imported = [] |
|---|
| 1543 | if len(pending) > 0: |
|---|
| 1544 | datafile = open(pending_tmp,"a") |
|---|
| 1545 | writer = csv.DictWriter(datafile, |
|---|
| 1546 | csv_keys, |
|---|
| 1547 | extrasaction='ignore') |
|---|
| 1548 | writer.writerows(pending) |
|---|
| 1549 | datafile.close() |
|---|
| 1550 | total_pending += len(pending) |
|---|
| 1551 | #total_added_to_pending += len(pending) |
|---|
| 1552 | pending = [] |
|---|
| 1553 | if not finished: |
|---|
| 1554 | msg = '%(commit_after)d imports committed of total %(total_added_to_imported)d\n' % vars() |
|---|
| 1555 | logger.info(msg) |
|---|
| 1556 | elapse = time.time() - elapse |
|---|
| 1557 | copy2(pending_tmp,pending_path) |
|---|
| [3181] | 1558 | msg = "finished importing in %(mode)s mode from %(import_source_fn)s in %(elapse)6.2f seconds, " % vars() |
|---|
| 1559 | msg += "%(count)d records totally read, %(total_added_to_imported)d added to %(imported_fn)s, " % vars() |
|---|
| [3172] | 1560 | if pending_only: |
|---|
| 1561 | removed_pending = pending_at_start - total_pending |
|---|
| [3181] | 1562 | msg += "%(removed_pending)d removed from %(pending_fn)s, " % vars() |
|---|
| [2699] | 1563 | else: |
|---|
| [3181] | 1564 | added_pending = total_pending - pending_at_start |
|---|
| 1565 | msg += "%(added_pending)d added to %(pending_fn)s, %(already_in)s already in %(pending_fn)s" % vars() |
|---|
| 1566 | #msg += "%(total_pending)d totally written" % vars() # this line does not make any sense |
|---|
| [3172] | 1567 | logger.info(msg) |
|---|
| [3178] | 1568 | os.remove(pending_tmp) |
|---|
| [3172] | 1569 | return msg |
|---|
| [2514] | 1570 | ###) |
|---|
| 1571 | |
|---|
| [3172] | 1572 | def d1402_importData(self,filename,name,edit=False,bypass_queue_catalog=False): ###( |
|---|
| [1730] | 1573 | """load data from CSV values""" |
|---|
| 1574 | import transaction |
|---|
| 1575 | import random |
|---|
| [3133] | 1576 | member = self.portal_membership.getAuthenticatedMember() |
|---|
| 1577 | logger = logging.getLogger('WAeUPTool.importData') |
|---|
| 1578 | current = DateTime.DateTime().strftime("%d-%m-%y_%H_%M_%S") |
|---|
| [3143] | 1579 | import_date = DateTime.DateTime().strftime("%d/%m/%y %H:%M:%S") |
|---|
| [3133] | 1580 | students_folder = self.portal_url.getPortalObject().campus.students |
|---|
| 1581 | start = True |
|---|
| 1582 | tr_count = 0 |
|---|
| 1583 | total_imported = 0 |
|---|
| 1584 | total_not_imported = 0 |
|---|
| 1585 | total = 0 |
|---|
| 1586 | pending_only = False |
|---|
| 1587 | pend_str = '_pending' |
|---|
| 1588 | if name.endswith(pend_str): |
|---|
| 1589 | pending_only = True |
|---|
| 1590 | name = name[:-len(pend_str)] |
|---|
| 1591 | iname = "import_%s" % name |
|---|
| 1592 | if name in ('application','course_result',): |
|---|
| 1593 | commit_after = 2000 |
|---|
| 1594 | else: |
|---|
| 1595 | commit_after = 100 |
|---|
| 1596 | stool = getToolByName(self, 'portal_schemas') |
|---|
| 1597 | ltool = getToolByName(self, 'portal_layouts') |
|---|
| 1598 | schema = stool._getOb(iname) |
|---|
| 1599 | if schema is None: |
|---|
| 1600 | em = 'No such schema %s' % iname |
|---|
| 1601 | logger.error('No such schema %s' % iname) |
|---|
| 1602 | return em |
|---|
| 1603 | layout = ltool._getOb(iname) |
|---|
| 1604 | if layout is None: |
|---|
| 1605 | em = 'No such layout %s' % iname |
|---|
| 1606 | logger.error(em) |
|---|
| 1607 | return em |
|---|
| 1608 | validators = {} |
|---|
| 1609 | for widget in layout.keys(): |
|---|
| 1610 | validators[widget] = layout[widget].validate |
|---|
| 1611 | mode = "create" |
|---|
| 1612 | if edit: |
|---|
| 1613 | if filename.endswith('_toDelete'): |
|---|
| 1614 | mode = "delete" |
|---|
| 1615 | else: |
|---|
| 1616 | mode = "edit" |
|---|
| 1617 | importer_name = "mass_%(mode)s_%(name)s" % vars() |
|---|
| 1618 | importer = getattr(self, '%s' % importer_name,None) |
|---|
| 1619 | if importer is None: |
|---|
| 1620 | em = 'No importer function %s' % importer_name |
|---|
| 1621 | logger.error(em) |
|---|
| 1622 | return em |
|---|
| [3139] | 1623 | pending_fn = "%s/import/%ss_pending.csv" % (i_home,name) |
|---|
| [3143] | 1624 | pending_imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current) |
|---|
| [3133] | 1625 | if pending_only: |
|---|
| 1626 | import_source_fn = pending_fn |
|---|
| [3139] | 1627 | imported_fn = "%s/import/%ss_pending_imported%s.csv" % (i_home,name,current) |
|---|
| 1628 | not_imported_fn = "%s/import/%ss_pending_not_imported%s.csv" % (i_home,name,current) |
|---|
| [3140] | 1629 | if not os.path.exists(pending_fn): |
|---|
| 1630 | em = 'No %s' % os.path.split(pending_fn) |
|---|
| 1631 | return em |
|---|
| [3133] | 1632 | else: |
|---|
| 1633 | import_source_fn = "%s/import/%s.csv" % (i_home,filename) |
|---|
| 1634 | imported_fn = "%s/import/%s_imported%s.csv" % (i_home,filename,current) |
|---|
| 1635 | not_imported_fn = "%s/import/%s_not_imported%s.csv" % (i_home,filename,current) |
|---|
| [3162] | 1636 | if not os.path.exists(import_source_fn): |
|---|
| 1637 | em = 'No %s' % os.path.split(import_soure_fn) |
|---|
| 1638 | return em |
|---|
| 1639 | attrs = csv.reader(open(import_source_fn,"rb")).next() |
|---|
| [3139] | 1640 | import_keys = [k.strip() for k in attrs if not (k.strip().startswith('ignore') |
|---|
| [3133] | 1641 | or k.strip() == 'Error')] |
|---|
| 1642 | diff2schema = set(import_keys).difference(set(schema.keys())) |
|---|
| 1643 | diff2layout = set(import_keys).difference(set(layout.keys())) |
|---|
| 1644 | if diff2schema and diff2schema != set(['id',]): |
|---|
| 1645 | msg = 'not ignorable key(s): "%s" found in heading' % ", ".join(diff2schema) |
|---|
| 1646 | return msg |
|---|
| [3143] | 1647 | if mode in ("create","edit"): |
|---|
| [3133] | 1648 | required_keys = [layout.getIdUnprefixed(id) |
|---|
| 1649 | for id,widget in layout.objectItems() |
|---|
| 1650 | if widget.is_required] |
|---|
| 1651 | if not set(required_keys).issubset(set(import_keys)): |
|---|
| 1652 | diff2import = set(required_keys).difference(set(import_keys)) |
|---|
| 1653 | msg = 'required key(s): "%s" not found in heading' % ", ".join(diff2import) |
|---|
| 1654 | return msg |
|---|
| 1655 | # |
|---|
| 1656 | # not_imported |
|---|
| 1657 | # |
|---|
| 1658 | not_imported_keys = import_keys + ["Error",] |
|---|
| [3155] | 1659 | not_imported_file = open(not_imported_fn,"w") |
|---|
| [3133] | 1660 | not_imported_csv_writer = csv.DictWriter(not_imported_file, |
|---|
| 1661 | not_imported_keys, |
|---|
| 1662 | extrasaction='ignore') |
|---|
| 1663 | mapping = dict((k,k) for k in not_imported_keys) |
|---|
| 1664 | not_imported_csv_writer.writerow(mapping) |
|---|
| 1665 | not_imported = [] |
|---|
| 1666 | # |
|---|
| 1667 | # imported |
|---|
| 1668 | # |
|---|
| 1669 | imported_keys = import_keys[:] |
|---|
| 1670 | if 'id' not in imported_keys: |
|---|
| 1671 | imported_keys.insert(0,"id") |
|---|
| [3155] | 1672 | imported_file = open(imported_fn,"w") |
|---|
| [3133] | 1673 | imported_csv_writer = csv.DictWriter(imported_file, |
|---|
| 1674 | imported_keys, |
|---|
| 1675 | extrasaction='ignore') |
|---|
| 1676 | mapping = dict((k,k) for k in imported_keys) |
|---|
| 1677 | imported_csv_writer.writerow(mapping) |
|---|
| 1678 | imported = [] |
|---|
| 1679 | # |
|---|
| 1680 | # pending |
|---|
| 1681 | # |
|---|
| 1682 | pending_records = {} |
|---|
| 1683 | imports_pending = 0 |
|---|
| [3143] | 1684 | total_pending_imported = 0 |
|---|
| [3133] | 1685 | total_pending = 0 |
|---|
| 1686 | format = ','.join(['"%%(%s)s"' % fn for fn in import_keys]) |
|---|
| 1687 | format_error = format + ',"%(Error)s"' |
|---|
| 1688 | #format = '"%(id)s",'+ format |
|---|
| 1689 | imported = [] |
|---|
| 1690 | if name in ('verdict','course_result',): |
|---|
| [3143] | 1691 | #pending_keys = imported_keys[:] |
|---|
| 1692 | sname = "%s_pending" % iname |
|---|
| 1693 | pending_schema = stool._getOb(sname) |
|---|
| 1694 | if schema is None: |
|---|
| 1695 | em = 'No such pending_schema %s' % sname |
|---|
| 1696 | logger.error('No such schema %s' % sname) |
|---|
| 1697 | return em |
|---|
| 1698 | pending_keys = pending_schema.keys() |
|---|
| [3133] | 1699 | pending_keys += "Error", |
|---|
| [3143] | 1700 | if 'id' not in pending_keys: |
|---|
| 1701 | pending_keys.insert(0,'id') |
|---|
| [3133] | 1702 | pending_records = {} |
|---|
| 1703 | if os.path.exists(pending_fn): |
|---|
| 1704 | pending_imports_file = open(pending_fn) |
|---|
| 1705 | pending_imports = csv.DictReader(pending_imports_file, |
|---|
| 1706 | fieldnames=pending_keys) |
|---|
| 1707 | if pending_imports: |
|---|
| 1708 | pending_records = dict((row['id'],row) for row in pending_imports |
|---|
| 1709 | if row['id'] != 'id') |
|---|
| 1710 | pending_imports_file.close() |
|---|
| 1711 | new_imports_pending = 0 |
|---|
| 1712 | pending_imported = [] |
|---|
| [3143] | 1713 | if pending_only: |
|---|
| 1714 | pending_imported_ids = [] |
|---|
| 1715 | pending_imported = [] |
|---|
| [3155] | 1716 | for record in pending_records.values(): |
|---|
| 1717 | item = {} |
|---|
| 1718 | for k,v in record.items(): |
|---|
| 1719 | if v: |
|---|
| 1720 | item[k] = v |
|---|
| [3143] | 1721 | results = importer(item) |
|---|
| 1722 | id = results[0] |
|---|
| 1723 | error = results[1] |
|---|
| 1724 | is_pending = len(results) == 3 |
|---|
| 1725 | if is_pending: |
|---|
| 1726 | continue |
|---|
| [3155] | 1727 | del item['Error'] |
|---|
| 1728 | msg = ";".join(["%s : %s" % (k,v) for k,v in item.items()]) |
|---|
| [3133] | 1729 | logger.info("imported from %(pending_fn)s %(msg)s" % vars()) |
|---|
| [3143] | 1730 | pending_imported.append(item) |
|---|
| 1731 | pending_imported_ids += id, |
|---|
| 1732 | total_pending_imported = len(pending_imported_ids) |
|---|
| [3155] | 1733 | logger.info("imported %d pending imports" % len(pending_imported_ids)) |
|---|
| [3143] | 1734 | for id in pending_imported_ids: |
|---|
| [3161] | 1735 | if id: |
|---|
| 1736 | del pending_records[id] |
|---|
| 1737 | else: |
|---|
| 1738 | logger.info("tried to delete record with empty id") |
|---|
| [3133] | 1739 | pending_imports_file = open(pending_fn,'w') |
|---|
| 1740 | pending_csv_writer = csv.DictWriter(pending_imports_file, |
|---|
| 1741 | pending_keys, |
|---|
| 1742 | extrasaction='ignore') |
|---|
| 1743 | mapping = dict((k,k) for k in pending_keys) |
|---|
| 1744 | pending_csv_writer.writerow(mapping) |
|---|
| [3143] | 1745 | if len(pending_imported) > 0: |
|---|
| 1746 | pending_imported_file = open(pending_imported_fn,'w') |
|---|
| 1747 | pending_imported_csv_writer = csv.DictWriter(pending_imported_file, |
|---|
| 1748 | pending_keys, |
|---|
| 1749 | extrasaction='ignore') |
|---|
| 1750 | pending_imported_csv_writer.writerow(dict((k,k) for k in imported_keys)) |
|---|
| 1751 | pending_imported_csv_writer.writerows(pending_imported) |
|---|
| [3133] | 1752 | # |
|---|
| 1753 | if pending_only: |
|---|
| 1754 | items = [] |
|---|
| 1755 | else: |
|---|
| 1756 | try: |
|---|
| 1757 | items = csv.DictReader(open(import_source_fn,"rb")) |
|---|
| 1758 | except: |
|---|
| 1759 | msg = 'Error reading %s.csv' % filename |
|---|
| 1760 | logger.error(msg) |
|---|
| 1761 | return msg |
|---|
| 1762 | not_imported = [] |
|---|
| [3147] | 1763 | pending_keys = pending_records.keys()[:] |
|---|
| [3133] | 1764 | for item in items: |
|---|
| 1765 | item = dict((k.strip(),l.strip()) for (k,l) in item.items()) |
|---|
| 1766 | if start: |
|---|
| 1767 | start = False |
|---|
| 1768 | adapters = [MappingStorageAdapter(schema, item)] |
|---|
| 1769 | logger.info('%s starts import from %s.csv in %s mode with schema and layout %s' % (member,filename,mode,iname)) |
|---|
| 1770 | dm = DataModel(item, adapters,context=self) |
|---|
| 1771 | ds = DataStructure(data=item,datamodel=dm) |
|---|
| 1772 | error_string = "" |
|---|
| 1773 | total += 1 |
|---|
| 1774 | error_count = 0 |
|---|
| 1775 | for k in import_keys: |
|---|
| 1776 | if not validators[k](ds,mode=mode): |
|---|
| 1777 | if error_count: |
|---|
| 1778 | error_string += ' ++ ' |
|---|
| 1779 | error_string += "%s: %s" % (k,self.translation_service(ds.getError(k), |
|---|
| 1780 | ds.getErrorMapping(k))) |
|---|
| 1781 | error_count += 1 |
|---|
| 1782 | if error_string: |
|---|
| 1783 | item['Error'] = error_string |
|---|
| 1784 | not_imported.append(item) |
|---|
| 1785 | total_not_imported += 1 |
|---|
| 1786 | continue |
|---|
| 1787 | temp_item = item.copy() |
|---|
| 1788 | temp_item.update(dm) |
|---|
| [3143] | 1789 | #id,error = importer(temp_item) |
|---|
| 1790 | results = importer(temp_item) |
|---|
| 1791 | id = results[0] |
|---|
| 1792 | error = results[1] |
|---|
| 1793 | is_pending = len(results) == 3 |
|---|
| 1794 | if is_pending: |
|---|
| 1795 | temp_item = results[2] |
|---|
| 1796 | temp_item['Error'] = error |
|---|
| 1797 | msg = format_error % temp_item |
|---|
| [3147] | 1798 | #if id not in pending_records.keys(): |
|---|
| 1799 | if id not in pending_keys: |
|---|
| [3143] | 1800 | temp_item['id'] = id |
|---|
| 1801 | pending_records[id] = temp_item |
|---|
| 1802 | logger.info("%(id)s added to pending %(msg)s" % vars()) |
|---|
| [3147] | 1803 | pending_keys += id, |
|---|
| [3143] | 1804 | new_imports_pending += 1 |
|---|
| 1805 | else: |
|---|
| 1806 | logger.info("%(id)s already in pending %(msg)s" % vars()) |
|---|
| 1807 | continue |
|---|
| 1808 | elif error: |
|---|
| [3133] | 1809 | item['Error'] = error |
|---|
| [3143] | 1810 | not_imported.append(item) |
|---|
| 1811 | total_not_imported += 1 |
|---|
| 1812 | continue |
|---|
| [3133] | 1813 | item = temp_item |
|---|
| 1814 | item['id'] = id |
|---|
| 1815 | imported.append(item) |
|---|
| 1816 | tr_count += 1 |
|---|
| 1817 | total_imported += 1 |
|---|
| 1818 | msg = format % item |
|---|
| 1819 | logger.info("%(total_imported)d of %(total)d imported in %(mode)s mode, %(msg)s" % vars()) |
|---|
| 1820 | if total and not total % commit_after: |
|---|
| 1821 | transaction.commit() |
|---|
| 1822 | if len(imported) > 0: |
|---|
| 1823 | imported_csv_writer.writerows(imported) |
|---|
| 1824 | imported = [] |
|---|
| 1825 | if len(not_imported) > 0: |
|---|
| 1826 | not_imported_csv_writer.writerows(not_imported) |
|---|
| 1827 | not_imported = [] |
|---|
| 1828 | if len(pending_records) > 0: |
|---|
| 1829 | pending_csv_writer.writerows(pending_records.values()) |
|---|
| 1830 | total_pending += len(pending_records) |
|---|
| 1831 | pending_records = {} |
|---|
| 1832 | msg = '%d transactions committed\n' % (tr_count) |
|---|
| 1833 | regs = [] |
|---|
| 1834 | logger.info(msg) |
|---|
| 1835 | tr_count = 0 |
|---|
| 1836 | if len(imported) > 0: |
|---|
| 1837 | imported_csv_writer.writerows(imported) |
|---|
| 1838 | if len(not_imported) > 0: |
|---|
| 1839 | not_imported_csv_writer.writerows(not_imported) |
|---|
| 1840 | if len(pending_records) > 0: |
|---|
| 1841 | pending_csv_writer.writerows(pending_records.values()) |
|---|
| 1842 | total_pending += len(pending_records) |
|---|
| [3140] | 1843 | import_source_fn = os.path.split(import_source_fn)[1] |
|---|
| [3141] | 1844 | pending_fn = os.path.split(pending_fn)[1] |
|---|
| [3133] | 1845 | msg = "Finished importing in %(mode)s mode from %(import_source_fn)s: " % vars() |
|---|
| [3160] | 1846 | msg += "%(total_imported)d imported, %(total_not_imported)d not imported, " % vars() |
|---|
| [3133] | 1847 | if total_pending: |
|---|
| [3145] | 1848 | if pending_only: |
|---|
| 1849 | msg += "%(new_imports_pending)d pending added, %(total_pending_imported)d pending imported " % vars() |
|---|
| 1850 | else: |
|---|
| 1851 | msg += "%(new_imports_pending)d pending added " % vars() |
|---|
| [3160] | 1852 | msg += "(of total %(total)d), " % vars() |
|---|
| [3143] | 1853 | msg += "%(total_pending)d total pending in %(pending_fn)s " % vars() |
|---|
| [3133] | 1854 | logger.info(msg) |
|---|
| 1855 | return msg |
|---|
| 1856 | ###) |
|---|
| [1775] | 1857 | |
|---|
| [2292] | 1858 | security.declareProtected(ModifyPortalContent,"moveImagesToFS")###( |
|---|
| 1859 | def moveImagesToFS(self,student_id="O738726"): |
|---|
| 1860 | "move the images to the filesystem" |
|---|
| [2675] | 1861 | images_dir = getImagesDir(student_id) |
|---|
| 1862 | #images_dir = os.path.join("%s" % images_base,student_id) |
|---|
| [2292] | 1863 | student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
|---|
| 1864 | stool = getToolByName(self, 'portal_schemas') |
|---|
| 1865 | schemas = ['student_application', |
|---|
| 1866 | 'student_clearance', |
|---|
| 1867 | ] |
|---|
| 1868 | created = False |
|---|
| 1869 | for schema_id in schemas: |
|---|
| 1870 | schema = stool._getOb(schema_id) |
|---|
| 1871 | object = getattr(student_folder,schema_id[len('student_'):],None) |
|---|
| 1872 | if object is None: |
|---|
| 1873 | continue |
|---|
| 1874 | doc = object.getContent() |
|---|
| 1875 | for key in schema.keys(): |
|---|
| 1876 | if schema[key].meta_type != "CPS Image Field": |
|---|
| 1877 | continue |
|---|
| 1878 | #import pdb;pdb.set_trace() |
|---|
| [2351] | 1879 | image = getattr(doc,key,None) |
|---|
| 1880 | if not image or not hasattr(image,"data"): |
|---|
| [2292] | 1881 | continue |
|---|
| 1882 | if not created: |
|---|
| 1883 | if not os.path.exists(images_dir): |
|---|
| 1884 | os.mkdir(images_dir) |
|---|
| 1885 | created = True |
|---|
| 1886 | filename = os.path.join(images_dir,"%(key)s_%(student_id)s.jpg" % vars()) |
|---|
| 1887 | open(filename,"wb").write(str(image.data)) |
|---|
| [2500] | 1888 | ###) |
|---|
| [2292] | 1889 | |
|---|
| [2300] | 1890 | security.declareProtected(ModifyPortalContent,"movePassportToFS")###( |
|---|
| 1891 | def movePassportToFS(self,student_id="O738726"): |
|---|
| 1892 | "move the passports to the filesystem" |
|---|
| 1893 | images_dir = os.path.join("%s" % i_home,'passports') |
|---|
| 1894 | student_folder = getattr(self.portal_url.getPortalObject().campus.students,student_id) |
|---|
| 1895 | stool = getToolByName(self, 'portal_schemas') |
|---|
| 1896 | schemas = ['student_application', |
|---|
| 1897 | #'student_clearance', |
|---|
| 1898 | ] |
|---|
| 1899 | created = False |
|---|
| 1900 | for schema_id in schemas: |
|---|
| 1901 | schema = stool._getOb(schema_id) |
|---|
| 1902 | object = getattr(student_folder,schema_id[len('student_'):],None) |
|---|
| 1903 | if object is None: |
|---|
| 1904 | continue |
|---|
| 1905 | doc = object.getContent() |
|---|
| 1906 | for key in schema.keys(): |
|---|
| 1907 | if schema[key].meta_type != "CPS Image Field": |
|---|
| 1908 | continue |
|---|
| 1909 | #import pdb;pdb.set_trace() |
|---|
| 1910 | image = getattr(doc,key) |
|---|
| 1911 | if not hasattr(image,"data"): |
|---|
| 1912 | continue |
|---|
| 1913 | if not created: |
|---|
| 1914 | if not os.path.exists(images_dir): |
|---|
| 1915 | os.mkdir(images_dir) |
|---|
| 1916 | created = True |
|---|
| 1917 | filename = os.path.join(images_dir,"%(student_id)s.jpg" % vars()) |
|---|
| 1918 | open(filename,"wb").write(str(image.data)) |
|---|
| [2500] | 1919 | ###) |
|---|
| [2300] | 1920 | |
|---|
| [828] | 1921 | InitializeClass(WAeUPTool) |
|---|
