[6821] | 1 | """Batch processing components for academics objects. |
---|
| 2 | |
---|
| 3 | Batch processors eat CSV files to add, update or remove large numbers |
---|
| 4 | of certain kinds of objects at once. |
---|
| 5 | |
---|
| 6 | Here we define the processors for academics specific objects like |
---|
| 7 | faculties, departments and the like. |
---|
| 8 | """ |
---|
| 9 | import grok |
---|
[6849] | 10 | import csv |
---|
| 11 | import copy |
---|
[6821] | 12 | from zope.interface import Interface |
---|
[6825] | 13 | from zope.schema import getFields |
---|
| 14 | from zope.component import queryUtility |
---|
| 15 | from zope.catalog.interfaces import ICatalog |
---|
[6849] | 16 | from waeup.sirp.interfaces import ( |
---|
| 17 | IBatchProcessor, FatalCSVError, IObjectConverter) |
---|
[6825] | 18 | from waeup.sirp.students.interfaces import ( |
---|
[6849] | 19 | IStudent, IStudentStudyCourse, IStudentStudyCourseImport, |
---|
| 20 | IStudentUpdateByRegNo, IStudentUpdateByMatricNo) |
---|
[6821] | 21 | from waeup.sirp.utils.batching import BatchProcessor |
---|
| 22 | |
---|
| 23 | class StudentProcessor(BatchProcessor): |
---|
| 24 | """A batch processor for IStudent objects. |
---|
| 25 | """ |
---|
| 26 | grok.implements(IBatchProcessor) |
---|
| 27 | grok.provides(IBatchProcessor) |
---|
| 28 | grok.context(Interface) |
---|
| 29 | util_name = 'studentimporter' |
---|
| 30 | grok.name(util_name) |
---|
| 31 | |
---|
| 32 | name = u'Student Importer' |
---|
| 33 | iface = IStudent |
---|
| 34 | |
---|
[6849] | 35 | location_fields = [] |
---|
[6821] | 36 | factory_name = 'waeup.Student' |
---|
| 37 | |
---|
[6841] | 38 | mode = None |
---|
| 39 | |
---|
[6821] | 40 | @property |
---|
[6849] | 41 | def available_fields(self): |
---|
| 42 | result = [] |
---|
| 43 | return sorted(list(set( |
---|
| 44 | ['student_id','reg_number','matric_number'] + getFields( |
---|
| 45 | self.iface).keys()))) |
---|
[6821] | 46 | |
---|
[6849] | 47 | def checkHeaders(self, headerfields, mode='create'): |
---|
[6854] | 48 | if not 'reg_number' in headerfields and not 'student_id' \ |
---|
| 49 | in headerfields and not 'matric_number' in headerfields: |
---|
[6849] | 50 | raise FatalCSVError( |
---|
[6854] | 51 | "Need at least columns student_id or reg_number " + |
---|
| 52 | "or matric_number for import!") |
---|
[6849] | 53 | if mode == 'create': |
---|
| 54 | for field in self.required_fields: |
---|
| 55 | if not field in headerfields: |
---|
| 56 | raise FatalCSVError( |
---|
| 57 | "Need at least columns %s for import!" % |
---|
| 58 | ', '.join(["'%s'" % x for x in self.required_fields])) |
---|
| 59 | # Check for fields to be ignored... |
---|
| 60 | not_ignored_fields = [x for x in headerfields |
---|
| 61 | if not x.startswith('--')] |
---|
| 62 | if len(set(not_ignored_fields)) < len(not_ignored_fields): |
---|
| 63 | raise FatalCSVError( |
---|
| 64 | "Double headers: each column name may only appear once.") |
---|
| 65 | return True |
---|
| 66 | |
---|
[6821] | 67 | def parentsExist(self, row, site): |
---|
| 68 | return 'students' in site.keys() |
---|
| 69 | |
---|
[6849] | 70 | def getLocator(self, row): |
---|
| 71 | if 'student_id' in row.keys() and row['student_id']: |
---|
| 72 | return 'student_id' |
---|
| 73 | elif 'reg_number' in row.keys() and row['reg_number']: |
---|
| 74 | return 'reg_number' |
---|
| 75 | elif 'matric_number' in row.keys() and row['matric_number']: |
---|
| 76 | return 'matric_number' |
---|
| 77 | else: |
---|
| 78 | return None |
---|
| 79 | |
---|
[6821] | 80 | # The entry never exists in create mode. |
---|
| 81 | def entryExists(self, row, site): |
---|
[6846] | 82 | if not 'students' in site.keys(): |
---|
[6849] | 83 | return None |
---|
| 84 | if self.getLocator(row) == 'student_id': |
---|
[6846] | 85 | if row['student_id'] in site['students']: |
---|
| 86 | student = site['students'][row['student_id']] |
---|
| 87 | return student |
---|
[6849] | 88 | elif self.getLocator(row) == 'reg_number': |
---|
[6846] | 89 | reg_number = row['reg_number'] |
---|
| 90 | cat = queryUtility(ICatalog, name='students_catalog') |
---|
| 91 | results = list( |
---|
| 92 | cat.searchResults(reg_number=(reg_number, reg_number))) |
---|
| 93 | if results: |
---|
| 94 | return results[0] |
---|
[6849] | 95 | elif self.getLocator(row) == 'matric_number': |
---|
[6846] | 96 | matric_number = row['matric_number'] |
---|
| 97 | cat = queryUtility(ICatalog, name='students_catalog') |
---|
| 98 | results = list( |
---|
| 99 | cat.searchResults(matric_number=(matric_number, matric_number))) |
---|
| 100 | if results: |
---|
| 101 | return results[0] |
---|
[6849] | 102 | return None |
---|
[6821] | 103 | |
---|
| 104 | def getParent(self, row, site): |
---|
| 105 | return site['students'] |
---|
| 106 | |
---|
| 107 | def getEntry(self, row, site): |
---|
[6846] | 108 | return self.entryExists(row, site) |
---|
[6821] | 109 | |
---|
| 110 | def addEntry(self, obj, row, site): |
---|
| 111 | parent = self.getParent(row, site) |
---|
| 112 | parent.addStudent(obj) |
---|
| 113 | return |
---|
| 114 | |
---|
| 115 | def delEntry(self, row, site): |
---|
[6846] | 116 | student = self.entryExists(row, site) |
---|
| 117 | if student: |
---|
| 118 | parent = self.getParent(row, site) |
---|
| 119 | del parent[student.student_id] |
---|
[6821] | 120 | pass |
---|
[6825] | 121 | |
---|
[6849] | 122 | def getMapping(self, path, headerfields, mode): |
---|
| 123 | """Get a mapping from CSV file headerfields to actually used fieldnames. |
---|
| 124 | """ |
---|
| 125 | result = dict() |
---|
| 126 | reader = csv.reader(open(path, 'rb')) |
---|
| 127 | raw_header = reader.next() |
---|
| 128 | for num, field in enumerate(headerfields): |
---|
[6854] | 129 | if field not in [ |
---|
| 130 | 'student_id', 'reg_number', 'matric_number'] and mode == 'remove': |
---|
[6849] | 131 | continue |
---|
| 132 | if field == u'--IGNORE--': |
---|
| 133 | # Skip ignored columns in failed and finished data files. |
---|
| 134 | continue |
---|
| 135 | result[raw_header[num]] = field |
---|
| 136 | return result |
---|
| 137 | |
---|
| 138 | def checkConversion(self, row, mode='create'): |
---|
| 139 | """Validates all values in row. |
---|
| 140 | """ |
---|
| 141 | if mode in ['update', 'remove']: |
---|
| 142 | if self.getLocator(row) == 'reg_number': |
---|
| 143 | iface = IStudentUpdateByRegNo |
---|
| 144 | elif self.getLocator(row) == 'matric_number': |
---|
| 145 | iface = IStudentUpdateByMatricNo |
---|
| 146 | else: |
---|
| 147 | iface = self.iface |
---|
| 148 | converter = IObjectConverter(iface) |
---|
| 149 | errs, inv_errs, conv_dict = converter.fromStringDict( |
---|
| 150 | row, self.factory_name) |
---|
| 151 | return errs, inv_errs, conv_dict |
---|
| 152 | |
---|
[6825] | 153 | class StudentStudyCourseProcessor(BatchProcessor): |
---|
| 154 | """A batch processor for IStudentStudyCourse objects. |
---|
| 155 | """ |
---|
| 156 | grok.implements(IBatchProcessor) |
---|
| 157 | grok.provides(IBatchProcessor) |
---|
| 158 | grok.context(Interface) |
---|
[6837] | 159 | util_name = 'studycourseupdater' |
---|
[6825] | 160 | grok.name(util_name) |
---|
| 161 | |
---|
[6837] | 162 | name = u'StudentStudyCourse Importer (update only)' |
---|
[6825] | 163 | iface = IStudentStudyCourseImport |
---|
| 164 | factory_name = 'waeup.StudentStudyCourse' |
---|
| 165 | |
---|
[6849] | 166 | location_fields = [] |
---|
| 167 | |
---|
[6841] | 168 | mode = None |
---|
| 169 | |
---|
[6825] | 170 | @property |
---|
| 171 | def available_fields(self): |
---|
| 172 | result = [] |
---|
| 173 | return sorted(list(set( |
---|
[6843] | 174 | ['student_id','reg_number','matric_number'] + getFields( |
---|
| 175 | self.iface).keys()))) |
---|
[6825] | 176 | |
---|
[6837] | 177 | def checkHeaders(self, headerfields, mode='ignore'): |
---|
[6854] | 178 | if not 'reg_number' in headerfields and not 'student_id' \ |
---|
| 179 | in headerfields and not 'matric_number' in headerfields: |
---|
[6825] | 180 | raise FatalCSVError( |
---|
[6854] | 181 | "Need at least columns student_id " + |
---|
| 182 | "or reg_number or matric_number for import!") |
---|
[6834] | 183 | # Check for fields to be ignored... |
---|
[6825] | 184 | not_ignored_fields = [x for x in headerfields |
---|
| 185 | if not x.startswith('--')] |
---|
| 186 | if len(set(not_ignored_fields)) < len(not_ignored_fields): |
---|
| 187 | raise FatalCSVError( |
---|
| 188 | "Double headers: each column name may only appear once.") |
---|
| 189 | return True |
---|
| 190 | |
---|
| 191 | def parentsExist(self, row, site): |
---|
[6846] | 192 | if not 'students' in site.keys(): |
---|
[6849] | 193 | return None |
---|
[6846] | 194 | if 'student_id' in row.keys() and row['student_id']: |
---|
[6825] | 195 | if row['student_id'] in site['students']: |
---|
| 196 | student = site['students'][row['student_id']] |
---|
| 197 | return student |
---|
[6843] | 198 | elif 'reg_number' in row.keys() and row['reg_number']: |
---|
[6825] | 199 | reg_number = row['reg_number'] |
---|
[6849] | 200 | #import pdb; pdb.set_trace() |
---|
[6825] | 201 | cat = queryUtility(ICatalog, name='students_catalog') |
---|
| 202 | results = list( |
---|
| 203 | cat.searchResults(reg_number=(reg_number, reg_number))) |
---|
| 204 | if results: |
---|
| 205 | return results[0] |
---|
[6843] | 206 | elif 'matric_number' in row.keys() and row['matric_number']: |
---|
| 207 | matric_number = row['matric_number'] |
---|
| 208 | cat = queryUtility(ICatalog, name='students_catalog') |
---|
| 209 | results = list( |
---|
| 210 | cat.searchResults(matric_number=(matric_number, matric_number))) |
---|
| 211 | if results: |
---|
| 212 | return results[0] |
---|
[6849] | 213 | return None |
---|
[6825] | 214 | |
---|
| 215 | def entryExists(self, row, site): |
---|
| 216 | student = self.parentsExist(row, site) |
---|
| 217 | if not student: |
---|
[6849] | 218 | return None |
---|
[6825] | 219 | if 'studycourse' in student: |
---|
| 220 | return student |
---|
[6849] | 221 | return None |
---|
[6825] | 222 | |
---|
| 223 | def getEntry(self, row, site): |
---|
| 224 | student = self.entryExists(row, site) |
---|
| 225 | if not student: |
---|
| 226 | return None |
---|
| 227 | return student.get('studycourse') |
---|