Changeset 7548


Ignore:
Timestamp:
1 Feb 2012, 11:19:56 (15 years ago)
Author:
Henrik Bettermann
Message:

Implement course ticket importer.

Add checkConversion tests.

Location:
main/waeup.sirp/trunk/src/waeup/sirp
Files:
1 added
4 edited

Legend:

Unmodified
Added
Removed
  • main/waeup.sirp/trunk/src/waeup/sirp/browser/batchprocessing.txt

    r7459 r7548  
    9595    ['Applicant Importer',
    9696     'Applicants Container Importer',
     97     'CourseTicket Importer',
    9798     'Student Importer',
    9899     'StudentStudyCourse Importer (update only)',
     100     'StudentStudyLevel Importer',
    99101     'CertificateCourse Importer',
    100102     'Certificate Importer',
  • main/waeup.sirp/trunk/src/waeup/sirp/students/batching.py

    r7536 r7548  
    2828from zope.interface import Interface
    2929from zope.schema import getFields
    30 from zope.component import queryUtility
     30from zope.component import queryUtility, getUtility
    3131from zope.event import notify
    3232from zope.catalog.interfaces import ICatalog
    … …  
    3838    IStudent, IStudentStudyCourse,
    3939    IStudentUpdateByRegNo, IStudentUpdateByMatricNo,
    40     IStudentStudyLevel)
     40    IStudentStudyLevel, ICourseTicket)
    4141from waeup.sirp.students.workflow import  IMPORTABLE_STATES
    4242from waeup.sirp.utils.batching import BatchProcessor
    … …  
    251251        elif 'reg_number' in row.keys() and row['reg_number']:
    252252            reg_number = row['reg_number']
    253             #import pdb; pdb.set_trace()
    254253            cat = queryUtility(ICatalog, name='students_catalog')
    255254            results = list(
    … …  
    296295            row, self.factory_name)
    297296        # We have to check if current_level is in range of certificate.
    298         # This is not done by the converter.
     297        # This is not done by the converter. This kind of conversion
     298        # checking does only work if a combination of certificate and
     299        # current_level is provided.
    299300        if conv_dict.has_key('certificate'):
    300301          certificate = conv_dict['certificate']
    … …  
    355356        elif 'reg_number' in row.keys() and row['reg_number']:
    356357            reg_number = row['reg_number']
    357             #import pdb; pdb.set_trace()
    358358            cat = queryUtility(ICatalog, name='students_catalog')
    359359            results = list(
    … …  
    395395            row, self.factory_name)
    396396        # We have to check if level is a valid integer.
    397         # This is not by the converter.
     397        # This is not done by the converter.
    398398        try:
    399399            level = int(row['level'])
    … …  
    403403            errs.append(('level','no integer'))
    404404        return errs, inv_errs, conv_dict
     405
     406class CourseTicketProcessor(BatchProcessor):
     407    """A batch processor for ICourseTicket objects.
     408    """
     409    grok.implements(IBatchProcessor)
     410    grok.provides(IBatchProcessor)
     411    grok.context(Interface)
     412    util_name = 'courseticketimporter'
     413    grok.name(util_name)
     414
     415    name = u'CourseTicket Importer'
     416    iface = ICourseTicket
     417    factory_name = 'waeup.CourseTicket'
     418
     419    location_fields = []
     420
     421    mode = None
     422
     423    @property
     424    def available_fields(self):
     425        return sorted(list(set(
     426            ['student_id','reg_number','matric_number','level','code'] + getFields(
     427                self.iface).keys())))
     428
     429    def checkHeaders(self, headerfields, mode='ignore'):
     430        if not 'reg_number' in headerfields and not 'student_id' \
     431            in headerfields and not 'matric_number' in headerfields:
     432            raise FatalCSVError(
     433                "Need at least columns student_id " +
     434                "or reg_number or matric_number for import!")
     435        if not 'level' in headerfields:
     436            raise FatalCSVError(
     437                "Need level for import!")
     438        if not 'code' in headerfields:
     439            raise FatalCSVError(
     440                "Need code for import!")
     441        # Check for fields to be ignored...
     442        not_ignored_fields = [x for x in headerfields
     443                              if not x.startswith('--')]
     444        if len(set(not_ignored_fields)) < len(not_ignored_fields):
     445            raise FatalCSVError(
     446                "Double headers: each column name may only appear once.")
     447        return True
     448
     449    def getParent(self, row, site):
     450        if not 'students' in site.keys():
     451            return None
     452        if 'student_id' in row.keys() and row['student_id']:
     453            if row['student_id'] in site['students']:
     454                student = site['students'][row['student_id']]
     455                return student['studycourse'].get(row['level'])
     456        elif 'reg_number' in row.keys() and row['reg_number']:
     457            reg_number = row['reg_number']
     458            #import pdb; pdb.set_trace()
     459            cat = queryUtility(ICatalog, name='students_catalog')
     460            results = list(
     461                cat.searchResults(reg_number=(reg_number, reg_number)))
     462            if results:
     463                return results[0]['studycourse'].get(row['level'])
     464        elif 'matric_number' in row.keys() and row['matric_number']:
     465            matric_number = row['matric_number']
     466            cat = queryUtility(ICatalog, name='students_catalog')
     467            results = list(
     468                cat.searchResults(matric_number=(matric_number, matric_number)))
     469            if results:
     470                return results[0]['studycourse'].get(row['level'])
     471        return None
     472
     473    def parentsExist(self, row, site):
     474        return self.getParent(row, site) is not None
     475
     476    def entryExists(self, row, site):
     477        return self.getEntry(row, site) is not None
     478
     479    def getEntry(self, row, site):
     480        level = self.getParent(row, site)
     481        if level is None:
     482            return None
     483        return level.get(row['code'])
     484
     485    def addEntry(self, obj, row, site):
     486        parent = self.getParent(row, site)
     487        catalog = getUtility(ICatalog, name='courses_catalog')
     488        entries = list(catalog.searchResults(code=(row['code'],row['code'])))
     489        obj.fcode = entries[0].__parent__.__parent__.__parent__.code
     490        obj.dcode = entries[0].__parent__.__parent__.code
     491        obj.title = entries[0].title
     492        obj.credits = entries[0].credits
     493        obj.passmark = entries[0].passmark
     494        obj.semester = entries[0].semester
     495        parent[row['code']] = obj
     496        return
     497
     498    def checkConversion(self, row, mode='ignore'):
     499        """Validates all values in row.
     500        """
     501        converter = IObjectConverter(self.iface)
     502        errs, inv_errs, conv_dict =  converter.fromStringDict(
     503            row, self.factory_name)
     504        # We have to check if course really exists.
     505        # This is not done by the converter.
     506        catalog = getUtility(ICatalog, name='courses_catalog')
     507        entries = catalog.searchResults(code=(row['code'],row['code']))
     508        if len(entries) == 0:
     509            errs.append(('code','non-existent'))
     510            return errs, inv_errs, conv_dict
     511        return errs, inv_errs, conv_dict
  • main/waeup.sirp/trunk/src/waeup/sirp/students/studylevel.py

    r7536 r7548  
    103103
    104104CourseTicket = attrs_to_fields(CourseTicket)
     105
     106class CourseTicketFactory(grok.GlobalUtility):
     107    """A factory for student study levels.
     108    """
     109    grok.implements(IFactory)
     110    grok.name(u'waeup.CourseTicket')
     111    title = u"Create a new course ticket.",
     112    description = u"This factory instantiates new course ticket instances."
     113
     114    def __call__(self, *args, **kw):
     115        return CourseTicket()
     116
     117    def getInterfaces(self):
     118        return implementedBy(CourseTicket)
  • main/waeup.sirp/trunk/src/waeup/sirp/students/tests/test_batching.py

    r7536 r7548  
    3232from waeup.sirp.students.batching import (
    3333    StudentProcessor, StudentStudyCourseProcessor,
    34     StudentStudyLevelProcessor)
     34    StudentStudyLevelProcessor, CourseTicketProcessor)
    3535from waeup.sirp.students.student import Student
    3636from waeup.sirp.testing import FunctionalLayer, FunctionalTestCase
    … …  
    7777
    7878STUDYLEVEL_HEADER_FIELDS = STUDYLEVEL_SAMPLE_DATA.split(
     79    '\n')[0].split(',')
     80
     81COURSETICKET_SAMPLE_DATA = open(
     82    os.path.join(os.path.dirname(__file__), 'sample_courseticket_data.csv'),
     83    'rb').read()
     84
     85COURSETICKET_HEADER_FIELDS = COURSETICKET_SAMPLE_DATA.split(
    7986    '\n')[0].split(',')
    8087
    … …  
    161168        assert len(self.app['students'].keys()) == 2
    162169
     170    def test_checkConversion(self):
     171        errs, inv_errs, conv_dict = self.importer.checkConversion(
     172            dict(reg_number='1', reg_state='admitted'))
     173        self.assertEqual(len(errs),0)
     174        errs, inv_errs, conv_dict = self.importer.checkConversion(
     175            dict(reg_number='1', reg_state=''))
     176        self.assertEqual(len(errs),1)
     177        self.assertTrue(('reg_state', 'no value provided') in errs)
     178        errs, inv_errs, conv_dict = self.importer.checkConversion(
     179            dict(reg_number='1', reg_state='nonsense'))
     180        self.assertEqual(len(errs),1)
     181        self.assertTrue(('reg_state', 'not allowed') in errs)
     182
    163183    def test_delEntry(self):
    164184        assert self.student.student_id in self.app['students'].keys()
    … …  
    283303        self.assertEqual(student.reg_number,'1')
    284304
     305    def test_checkConversion(self):
     306        errs, inv_errs, conv_dict = self.importer.checkConversion(
     307            dict(reg_number='1', certificate='CERT1', current_level='200'))
     308        self.assertEqual(len(errs),0)
     309        errs, inv_errs, conv_dict = self.importer.checkConversion(
     310            dict(reg_number='1', certificate='CERT999'))
     311        self.assertEqual(len(errs),1)
     312        self.assertTrue(('certificate', u'Invalid value') in errs)
     313        errs, inv_errs, conv_dict = self.importer.checkConversion(
     314            dict(reg_number='1', certificate='CERT1', current_level='100'))
     315        self.assertEqual(len(errs),1)
     316        self.assertTrue(('current_level','not in range') in errs)
     317        # If we import only current_level, no conversion checking is done.
     318        errs, inv_errs, conv_dict = self.importer.checkConversion(
     319            dict(reg_number='1', current_level='100'))
     320        self.assertEqual(len(errs),0)
     321
    285322    def test_import(self):
    286323        num, num_warns, fin_file, fail_file = self.importer.doImport(
    … …  
    350387            IBatchProcessor, StudentStudyLevelProcessor) is True
    351388
     389    def test_checkConversion(self):
     390        errs, inv_errs, conv_dict = self.importer.checkConversion(
     391            dict(reg_number='1', level='220'))
     392        self.assertEqual(len(errs),0)
     393        errs, inv_errs, conv_dict = self.importer.checkConversion(
     394            dict(reg_number='1', level='900'))
     395        self.assertEqual(len(errs),1)
     396        self.assertTrue(('level','no valid integer') in errs)
     397        errs, inv_errs, conv_dict = self.importer.checkConversion(
     398            dict(reg_number='1', level='xyz'))
     399        self.assertEqual(len(errs),1)
     400        self.assertTrue(('level','no integer') in errs)
     401
    352402    def test_import(self):
    353403        num, num_warns, fin_file, fail_file = self.importer.doImport(
    … …  
    365415       
    366416
     417class CourseTicketImporterTest(FunctionalTestCase):
     418
     419    layer = FunctionalLayer
     420
     421    def setUp(self):
     422        super(CourseTicketImporterTest, self).setUp()
     423        self.dc_root = tempfile.mkdtemp()
     424        self.workdir = tempfile.mkdtemp()
     425        app = University()
     426        app['datacenter'].setStoragePath(self.dc_root)
     427        self.getRootFolder()['app'] = app
     428        self.app = self.getRootFolder()['app']
     429        setSite(app)
     430
     431        # Import students with subobjects
     432        student_file = os.path.join(self.workdir, 'sample_student_data.csv')
     433        open(student_file, 'wb').write(STUDENT_SAMPLE_DATA)
     434        num, num_warns, fin_file, fail_file = StudentProcessor().doImport(
     435            student_file, STUDENT_HEADER_FIELDS)
     436        shutil.rmtree(os.path.dirname(fin_file))
     437
     438        # Populate university
     439        self.certificate = createObject('waeup.Certificate')
     440        self.certificate.code = 'CERT1'
     441        self.certificate.application_category = 'basic'
     442        self.certificate.start_level = 200
     443        self.certificate.end_level = 500
     444        self.app['faculties']['fac1'] = Faculty()
     445        self.app['faculties']['fac1']['dep1'] = Department()
     446        self.app['faculties']['fac1']['dep1'].certificates.addCertificate(
     447            self.certificate)
     448        self.course = createObject('waeup.Course')
     449        self.course.code = 'COURSE1'
     450        self.course.semester = 1
     451        self.course.credits = 10
     452        self.course.passmark = 40
     453        self.app['faculties']['fac1']['dep1'].courses.addCourse(
     454            self.course)
     455        self.app['faculties']['fac1']['dep1'].certificates['CERT1'].addCourseRef(
     456            self.course, level=100)
     457
     458        # Update study courses
     459        studycourse_file = os.path.join(
     460            self.workdir, 'sample_studycourse_data.csv')
     461        open(studycourse_file, 'wb').write(STUDYCOURSE_SAMPLE_DATA)
     462        importer = StudentStudyCourseProcessor()
     463        num, num_warns, fin_file, fail_file = importer.doImport(
     464            studycourse_file, STUDYCOURSE_HEADER_FIELDS,'update')
     465        shutil.rmtree(os.path.dirname(fin_file))
     466
     467        # Import study levels
     468        importer = StudentStudyLevelProcessor()
     469        studylevel_file = os.path.join(
     470            self.workdir, 'sample_studylevel_data.csv')
     471        open(studylevel_file, 'wb').write(STUDYLEVEL_SAMPLE_DATA)
     472        num, num_warns, fin_file, fail_file = importer.doImport(
     473            studylevel_file, STUDYLEVEL_HEADER_FIELDS,'create')
     474        shutil.rmtree(os.path.dirname(fin_file))
     475
     476        self.importer = CourseTicketProcessor()
     477        self.csv_file = os.path.join(
     478            self.workdir, 'sample_courseticket_data.csv')
     479        open(self.csv_file, 'wb').write(COURSETICKET_SAMPLE_DATA)
     480
     481    def tearDown(self):
     482        super(CourseTicketImporterTest, self).tearDown()
     483        shutil.rmtree(self.workdir)
     484        shutil.rmtree(self.dc_root)
     485        clearSite()
     486        return
     487
     488    def test_interface(self):
     489        # Make sure we fulfill the interface contracts.
     490        assert verifyObject(IBatchProcessor, self.importer) is True
     491        assert verifyClass(
     492            IBatchProcessor, CourseTicketProcessor) is True
     493
     494    def test_checkConversion(self):
     495        errs, inv_errs, conv_dict = self.importer.checkConversion(
     496            dict(reg_number='1', code='COURSE1', level='220'))
     497        self.assertEqual(len(errs),0)
     498        errs, inv_errs, conv_dict = self.importer.checkConversion(
     499            dict(reg_number='1', code='COURSE2', level='220'))
     500        self.assertEqual(len(errs),1)
     501        self.assertTrue(('code','non-existent') in errs)
     502
     503    def test_import(self):
     504
     505        num, num_warns, fin_file, fail_file = self.importer.doImport(
     506            self.csv_file, COURSETICKET_HEADER_FIELDS,'create')
     507
     508        self.assertEqual(num_warns,2)
     509        assert self.importer.entryExists(
     510            dict(reg_number='1', level='100', code='COURSE1'), self.app) is True
     511        courseticket = self.importer.getEntry(
     512            dict(reg_number='1', level='100', code='COURSE1'), self.app)
     513        self.assertEqual(courseticket.__parent__.__parent__.certificate.code, u'CERT1')
     514        self.assertEqual(courseticket.score, 1)
     515        self.assertEqual(courseticket.core_or_elective, True)
     516        self.assertEqual(courseticket.fcode, 'NA')
     517        self.assertEqual(courseticket.dcode, 'NA')
     518        self.assertEqual(courseticket.code, 'COURSE1')
     519        self.assertEqual(courseticket.title, 'Unnamed Course')
     520        self.assertEqual(courseticket.credits, 10)
     521        self.assertEqual(courseticket.passmark, 40)
     522        self.assertEqual(courseticket.semester, 1)
     523        #import pdb; pdb.set_trace()
     524        shutil.rmtree(os.path.dirname(fin_file))
     525
     526
    367527def test_suite():
    368528    suite = unittest.TestSuite()
    369529    for testcase in [
    370530        StudentImporterTest,StudentStudyCourseImporterTest,
    371         StudentStudyLevelImporterTest,]:
     531        StudentStudyLevelImporterTest,CourseTicketImporterTest,]:
    372532        suite.addTest(unittest.TestLoader().loadTestsFromTestCase(
    373533                testcase
Note: See TracChangeset for help on using the changeset viewer.