source: main/waeup.kofa/trunk/src/waeup/kofa/students/export.py @ 15543

## $Id: export.py 15297 2019-01-16 12:20:18Z henrik $
##
## Copyright (C) 2011 Uli Fouquet & Henrik Bettermann
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##
"""Exporters for student related stuff.
"""
import os
import grok
from datetime import datetime, timedelta
from zope.component import getUtility
from waeup.kofa.interfaces import (
    IExtFileStore, IFileStoreNameChooser, IKofaUtils)
from waeup.kofa.interfaces import MessageFactory as _
from waeup.kofa.students.catalog import StudentsQuery, CourseTicketsQuery
from waeup.kofa.students.interfaces import (
    IStudent, IStudentStudyCourse, IStudentStudyLevel, ICourseTicket,
    IStudentOnlinePayment, ICSVStudentExporter, IBedTicket)
from waeup.kofa.students.vocabularies import study_levels
from waeup.kofa.utils.batching import ExporterBase
from waeup.kofa.utils.helpers import iface_names, to_timezone


def get_students(site, stud_filter=StudentsQuery()):
    """Get all students registered in catalog in `site`.
    """
    return stud_filter.query()

def get_studycourses(students):
    """Get studycourses of `students`.
    """
    return [x.get('studycourse', None) for x in students
            if x is not None]

def get_levels(students):
    """Get all studylevels of `students`.
    """
    levels = []
    for course in get_studycourses(students):
        for level in course.values():
            levels.append(level)
    return levels

def get_tickets(students, **kw):
    """Get course tickets of `students`.
    If code is passed through, filter course tickets
    which belong to this course code and meet level=level
    and level_session=level_session.
    If not, but ct_level and ct_session
    are passed through, filter course tickets
    which meet level==ct_level and level_session==ct_session.
    """
    tickets = []
    code = kw.get('code', None)
    level = kw.get('level', None)
    session = kw.get('session', None)
    ct_level = kw.get('ct_level', None)
    ct_session = kw.get('ct_session', None)
    if code is None:
        for level_obj in get_levels(students):
            for ticket in level_obj.values():
                if ct_level not in ('all', None):
                    if level_obj.level in (10, 999, None)  \
                        and int(ct_level) != level_obj.level:
                        continue
                    if level_obj.level not in range(
                        int(ct_level), int(ct_level)+100, 10):
                        continue
                if ct_session not in ('all', None) and \
                    int(ct_session) != level_obj.level_session:
                    continue
                tickets.append(ticket)
    else:
        for level_obj in get_levels(students):
            for ticket in level_obj.values():
                if ticket.code != code:
                    continue
                if level not in ('all', None):
                    if level_obj.level in (10, 999, None)  \
                        and int(level) != level_obj.level:
                        continue
                    if level_obj.level not in range(
                        int(level), int(level)+100, 10):
                        continue
                if session not in ('all', None) and \
                    int(session) != level_obj.level_session:
                    continue
                tickets.append(ticket)
    return tickets

def get_tickets_for_lecturer(students, **kw):
    """Get course tickets of `students`.
    Filter course tickets which belong to this course code and
    which are editable by lecturers.
    """
    tickets = []
    code = kw.get('code', None)
    level_session = kw.get('session', None)
    level = kw.get('level', None)
    for level_obj in get_levels(students):
        for ticket in level_obj.values():
            if ticket.code != code:
                continue
            if not ticket.editable_by_lecturer:
                continue
            if level not in ('all', None):
                if level_obj.level in (10, 999, None)  \
                    and int(level) != level_obj.level:
                    continue
                if level_obj.level not in range(int(level), int(level)+100, 10):
                    continue
            if level_session not in ('all', None) and \
                int(level_session) != level_obj.level_session:
                continue
            tickets.append(ticket)
    return tickets

def get_payments(students, p_states=None, **kw):
    """Get all payments of `students` within given payment_date period.
    """
    date_format = '%d/%m/%Y'
    payments = []
    p_start = kw.get('payments_start')
    p_end = kw.get('payments_end')
    paycat = kw.get('paycat')
    paysession = kw.get('paysession')
    for student in students:
        for payment in student.get('payments', {}).values():
            if p_start and p_end:
                if not payment.payment_date:
                    continue
                payments_start = datetime.strptime(p_start, date_format)
                payments_end = datetime.strptime(p_end, date_format)
                tz = getUtility(IKofaUtils).tzinfo
                payments_start = tz.localize(payments_start)
                payments_end = tz.localize(payments_end) + timedelta(days=1)
                payment_date = to_timezone(payment.payment_date, tz)
                if payment_date < payments_start or payment_date > payments_end:
                    continue
            if p_states and not payment.p_state in p_states:
                continue
            if paycat not in ('all', None) and payment.p_category != paycat:
                continue
            if paysession not in ('all', None) \
                and payment.p_session != int(paysession):
                continue
            payments.append(payment)
    return payments

def get_bedtickets(students):
    """Get all bedtickets of `students`.
    """
    tickets = []
    for student in students:
        for ticket in student.get('accommodation', {}).values():
            tickets.append(ticket)
    return tickets

class StudentExporterBase(ExporterBase):
    """Exporter for students or related objects.
    This is a baseclass.
    """
    grok.baseclass()
    grok.implements(ICSVStudentExporter)
    grok.provides(ICSVStudentExporter)

    def filter_func(self, x, **kw):
        return x

    def get_filtered(self, site, **kw):
        """Get students from a catalog filtered by keywords.
        students_catalog is the default catalog. The keys must be valid
        catalog index names.
        Returns a simple empty list, a list with `Student`
        objects or a catalog result set with `Student`
        objects.

        .. seealso:: `waeup.kofa.students.catalog.StudentsCatalog`

        """
        # Pass only given keywords to create FilteredCatalogQuery objects.
        # This way we avoid
        # trouble with `None` value ambivalences and queries are also
        # faster (normally less indexes to ask). Drawback is, that
        # developers must look into catalog to see what keywords are
        # valid.
        if kw.get('catalog', None) == 'coursetickets':
            coursetickets = CourseTicketsQuery(**kw).query()
            students = []
            for ticket in coursetickets:
                students.append(ticket.student)
            return list(set(students))
        # Payments can be filtered by payment date and payment category.
        # These parameters are not keys of the catalog and must thus be
        # removed from kw.
        try:
            del kw['payments_start']
            del kw['payments_end']
            del kw['paycat']
            del kw['paysession']
        except KeyError:
            pass
        # Coursetickets can be filtered by level and session.
        # These parameters are not keys of the catalog and must thus be
        # removed from kw.
        try:
            del kw['ct_level']
            del kw['ct_session']
        except KeyError:
            pass
        query = StudentsQuery(**kw)
        return query.query()

    def get_selected(self, site, selected):
        """Get set of selected students.
        Returns a simple empty list or a list with `Student`
        objects.
        """
        students = []
        students_container = site.get('students', {})
        for id in selected:
            student = students_container.get(id, None)
            if student is None:
                # try matric number
                result = list(StudentsQuery(matric_number=id).query())
                if result:
                    student = result[0]
                else:
                    continue
            students.append(student)
        return students

    def export(self, values, filepath=None):
        """Export `values`, an iterable, as CSV file.
        If `filepath` is ``None``, a raw string with CSV data is returned.
        """
        writer, outfile = self.get_csv_writer(filepath)
        for value in values:
            self.write_item(value, writer)
        return self.close_outfile(filepath, outfile)

    def export_all(self, site, filepath=None):
        """Export students into filepath as CSV data.
        If `filepath` is ``None``, a raw string with CSV data is returned.
        """
        return self.export(self.filter_func(get_students(site)), filepath)

    def export_student(self, student, filepath=None):
        return self.export(self.filter_func([student]), filepath=filepath)

    def export_filtered(self, site, filepath=None, **kw):
        """Export items denoted by `kw`.
        If `filepath` is ``None``, a raw string with CSV data should
        be returned.
        """
        data = self.get_filtered(site, **kw)
        return self.export(self.filter_func(data, **kw), filepath=filepath)

    def export_selected(self,site, filepath=None, **kw):
        """Export data for selected set of students.
        """
        selected = kw.get('selected', [])
        data = self.get_selected(site, selected)
        return self.export(self.filter_func(data, **kw), filepath=filepath)


class StudentExporter(grok.GlobalUtility, StudentExporterBase):
    """The Student Exporter first filters the set of students by searching the
    students catalog. Then it exports student base data of this set of students.
    """
    grok.name('students')

    fields = tuple(sorted(iface_names(IStudent))) + (
        'password', 'state', 'history', 'certcode', 'is_postgrad',
        'current_level', 'current_session')
    title = _(u'Students')

    def mangle_value(self, value, name, context=None):
        """The mangler prepares the history messages and adds a hash symbol at
        the end of the phone number to avoid annoying automatic number
        transformation by Excel or Calc."""
        if name == 'history':
            value = value.messages
        if 'phone' in name and value is not None:
            # Append hash '#' to phone numbers to circumvent
            # unwanted excel automatic
            value = str('%s#' % value)
        return super(
            StudentExporter, self).mangle_value(
            value, name, context=context)


class StudentStudyCourseExporter(grok.GlobalUtility, StudentExporterBase):
    """The Student Study Course Exporter first filters the set of students
    by searching the students catalog. Then it exports the data of the current
    study course container of each student from this set. It does
    not export their content.
    """
    grok.name('studentstudycourses')

    fields = tuple(sorted(iface_names(IStudentStudyCourse))) + ('student_id',)
    title = _(u'Student Study Courses')

    def filter_func(self, x, **kw):
        return get_studycourses(x)

    def mangle_value(self, value, name, context=None):
        """The mangler determines the certificate code and the student id.
        """
        if name == 'certificate' and value is not None:
            # XXX: hopefully cert codes are unique site-wide
            value = value.code
        if name == 'student_id' and context is not None:
            student = context.student
            value = getattr(student, name, None)
        return super(
            StudentStudyCourseExporter, self).mangle_value(
            value, name, context=context)


class StudentStudyLevelExporter(grok.GlobalUtility, StudentExporterBase):
    """The Student Study Level Exporter first filters the set of students
    by searching the students catalog. Then it exports the data of the student's
    study level container data but not their content (course tickets).
    The exporter iterates over all objects in the students' ``studycourse``
    containers.
    """
    grok.name('studentstudylevels')

    fields = tuple(sorted(iface_names(
        IStudentStudyLevel))) + (
        'student_id', 'number_of_tickets','certcode')
    title = _(u'Student Study Levels')

    def filter_func(self, x, **kw):
        return get_levels(x)

    def mangle_value(self, value, name, context=None):
        """The mangler determines the student id, nothing else.
        """
        if name == 'student_id' and context is not None:
            student = context.student
            value = getattr(student, name, None)
        return super(
            StudentStudyLevelExporter, self).mangle_value(
            value, name, context=context)

class CourseTicketExporter(grok.GlobalUtility, StudentExporterBase):
    """The Course Ticket Exporter exports course tickets. Usually,
    the exporter first filters the set of students by searching the
    students catalog. Then it collects and iterates over all ``studylevel``
    containers of the filtered student set and finally
    iterates over all items inside these containers.

    If the course code is passed through, the exporter uses a different
    catalog. It searches for students in the course tickets catalog and
    exports those course tickets which belong to the given course code and
    also meet level and session passed through at the same time.
    This happens if the exporter is called at course level in the academic
    section.
    """
    grok.name('coursetickets')

    fields = tuple(sorted(iface_names(ICourseTicket) +
        ['level', 'code', 'level_session'])) + ('student_id',
        'certcode', 'display_fullname')
    title = _(u'Course Tickets')

    def filter_func(self, x, **kw):
        return get_tickets(x, **kw)

    def mangle_value(self, value, name, context=None):
        """The mangler determines the student's id and fullname.
        """
        if context is not None:
            student = context.student
            if name in ('student_id', 'display_fullname') and student is not None:
                value = getattr(student, name, None)
        return super(
            CourseTicketExporter, self).mangle_value(
            value, name, context=context)

class DataForLecturerExporter(grok.GlobalUtility, StudentExporterBase):
    """The Data for Lecturer Exporter searches for students in the course
    tickets catalog and exports those course tickets which belong to the
    given course code, meet level and session passed through at the
    same time, and which are editable by lecturers. This exporter can only
    be called at course level in the academic section.
    """
    grok.name('lecturer')

    fields = ('matric_number', 'student_id', 'display_fullname',
              'level', 'code', 'level_session', 'score')

    title = _(u'Data for Lecturer')

    def filter_func(self, x, **kw):
        return get_tickets_for_lecturer(x, **kw)

    def mangle_value(self, value, name, context=None):
        """The mangler determines the student's id and fullname.
        """
        if context is not None:
            student = context.student
            if name in ('matric_number',
                        'reg_number',
                        'student_id',
                        'display_fullname',) and student is not None:
                value = getattr(student, name, None)
        return super(
            DataForLecturerExporter, self).mangle_value(
            value, name, context=context)

class StudentPaymentExporter(grok.GlobalUtility, StudentExporterBase):
    """The Student Payment Exporter first filters the set of students
    by searching the students catalog. Then it exports student payment
    tickets by iterating over the items of the student's ``payments``
    container. If the payment period is given, only tickets, which were
    paid in payment period, are considered for export.
    """
    grok.name('studentpayments')

    fields = tuple(
        sorted(iface_names(
            IStudentOnlinePayment, exclude_attribs=False,
            omit=['display_item', 'certificate', 'student']))) + (
            'student_id','state','current_session')
    title = _(u'Student Payments')

    def filter_func(self, x, **kw):
        return get_payments(x, **kw)

    def mangle_value(self, value, name, context=None):
        """The mangler determines the student's id, registration
        state and current session.
        """
        if context is not None:
            student = context.student
            if name in ['student_id','state',
                        'current_session'] and student is not None:
                value = getattr(student, name, None)
        return super(
            StudentPaymentExporter, self).mangle_value(
            value, name, context=context)

class StudentUnpaidPaymentExporter(StudentPaymentExporter):
    """The Student Unpaid Payment Exporter works just like the
    Student Payments Exporter but it exports only unpaid tickets.
    This exporter is designed for finding and finally purging outdated
    payment ticket.
    """
    grok.name('studentunpaidpayments')

    title = _(u'Student Unpaid Payments')

    def filter_func(self, x, **kw):
        return get_payments(x, p_states=('unpaid',) , **kw)

class DataForBursaryExporter(StudentPaymentExporter):
    """The Data for Bursary Exporter works just like the Student Payment
    Exporter but it exports much more information about the student. It combines
    payment and student data in one table in order to spare postprocessing of 
    two seperate export files. The exporter is primarily used by bursary
    officers who have exclusively access to this exporter. The exporter
    exports ``paid`` and ``waived`` payment tickets.
    """
    grok.name('bursary')

    def filter_func(self, x, **kw):
        return get_payments(x, p_states=('paid', 'waived'), **kw)

    fields = tuple(
        sorted(iface_names(
            IStudentOnlinePayment, exclude_attribs=False,
            omit=['display_item', 'certificate', 'student']))) + (
            'student_id','matric_number','reg_number',
            'firstname', 'middlename', 'lastname',
            'state','current_session',
            'entry_session', 'entry_mode',
            'faccode', 'depcode','certcode')
    title = _(u'Payment Data for Bursary')

    def mangle_value(self, value, name, context=None):
        """The mangler fetches the student data.
        """
        if context is not None:
            student = context.student
            if name in [
                'student_id','matric_number', 'reg_number',
                'firstname', 'middlename', 'lastname',
                'state', 'current_session',
                'entry_session', 'entry_mode',
                'faccode', 'depcode', 'certcode'] and student is not None:
                value = getattr(student, name, None)
        return super(
            StudentPaymentExporter, self).mangle_value(
            value, name, context=context)

class AccommodationPaymentsExporter(DataForBursaryExporter):
    """The Accommodation Payments Exporter works like the Data for Bursary
    Exporter above. The exporter exports ``paid`` and ``waived`` payment
    tickets with category ``bed_allocation`` or ``hostel_maintenance``.
    The exporter is primarily used by accommodation officers who have
    exclusively access to this exporter.
    """
    grok.name('accommodationpayments')

    def filter_func(self, x, **kw):
        kw['paycat'] = 'bed_allocation'
        payments = get_payments(x, p_states=('paid', 'waived'), **kw)
        kw['paycat'] = 'hostel_maintenance'
        payments += get_payments(x, p_states=('paid', 'waived'), **kw)
        return payments

    title = _(u'Accommodation Payments')

class BedTicketExporter(grok.GlobalUtility, StudentExporterBase):
    """The Bed Ticket Exporter first filters the set of students
    by searching the students catalog. Then it exports bed
    tickets by iterating over the items of the student's ``accommodation``
    container.
    """
    grok.name('bedtickets')

    fields = tuple(
        sorted(iface_names(
            IBedTicket, exclude_attribs=False,
            omit=['display_coordinates', 'maint_payment_made']))) + (
            'student_id', 'actual_bed_type')
    title = _(u'Bed Tickets')

    def filter_func(self, x, **kw):
        return get_bedtickets(x)

    def mangle_value(self, value, name, context=None):
        """The mangler determines the student id and the type of the bed
        which has been booked in the ticket.
        """
        if context is not None:
            student = context.student
            if name in ['student_id'] and student is not None:
                value = getattr(student, name, None)
        if name == 'bed' and value is not None:
            value = getattr(value, 'bed_id', None)
        if name == 'actual_bed_type':
            value = getattr(getattr(context, 'bed', None), 'bed_type', None)
        return super(
            BedTicketExporter, self).mangle_value(
            value, name, context=context)

class SchoolFeePaymentsOverviewExporter(StudentExporter):
    """The School Fee Payments Overview Exporter first filters the set of students
    by searching the students catalog. Then it exports some student base data
    together with the total school fee amount paid in each year over a
    predefined year range (current year - 9, ... , current year + 1).
    """
    grok.name('sfpaymentsoverview')

    curr_year = datetime.now().year
    year_range = range(curr_year - 11, curr_year + 1)
    year_range_tuple = tuple([str(year) for year in year_range])
    fields = ('student_id', 'matric_number', 'display_fullname',
        'state', 'certcode', 'faccode', 'depcode', 'is_postgrad',
        'current_level', 'current_session', 'current_mode',
        'entry_session', 'reg_number'
        ) + year_range_tuple
    title = _(u'Student School Fee Payments Overview')

    def mangle_value(self, value, name, context=None):
        """The mangler summarizes the school fee amounts made per year. It
        iterates over all paid school fee payment tickets and
        adds together the amounts paid in a year. Waived payments
        are marked ``waived``.
        """
        if name in self.year_range_tuple and context is not None:
            value = 0
            for ticket in context['payments'].values():
                if ticket.p_category == 'schoolfee' and \
                    ticket.p_session == int(name):
                    if ticket.p_state == 'waived':
                        value = 'waived'
                        break
                    if ticket.p_state == 'paid':
                        try:
                            value += ticket.amount_auth
                        except TypeError:
                            pass
            if value == 0:
                value = ''
            elif isinstance(value, float):
                value = round(value, 2)
        return super(
            StudentExporter, self).mangle_value(
            value, name, context=context)

class SessionPaymentsOverviewExporter(StudentExporter):
    """The Session Payments Overview Exporter first filters the set of students
    by searching the students catalog. Then it exports some student base data
    together with the total amount paid in predefined payment categories
    over the previous three session (referring to current academic session).
    Sample output:

    header: ``...schoolfee13,schoolfee14,schoolfee15,gown13,gown14,gown15...``

    data: ``...2000.0,,3000.0,,,1000.0,...``

    This csv data string means that the student paid 2000.0 school fee in 2013
    and 3000.0 in 2015. S/He furthermore paid 1000.0 for gown rental in 2015.
    """
    grok.name('sessionpaymentsoverview')

    paycats = ('schoolfee', 'clearance', 'gown', 'transcript')
    regular_fields = ('student_id', 'matric_number', 'display_fullname',
        'state', 'certcode', 'faccode', 'depcode', 'is_postgrad',
        'current_level', 'current_session', 'current_mode',
        'entry_session', 'reg_number'
        )
    title = _(u'Session Payments Overview')

    @property
    def paycatyears(self):
        cas = grok.getSite()['configuration'].current_academic_session
        paycatyears = []
        if cas:
            year_range = range(cas - 2, cas+1)
            year_range_tuple = tuple([str(year)[2:] for year in year_range])
            paycatyears = [
                cat+year for cat in self.paycats for year in year_range_tuple]
        return paycatyears

    @property
    def fields(self):
        return self.regular_fields + tuple(self.paycatyears)

    def mangle_value(self, value, name, context=None):
        """
        """
        amounts = dict()
        for catyear in self.paycatyears:
            amounts[catyear] = 0.0
        if name[:-2] in self.paycats and context is not None:
            for ticket in context['payments'].values():
                if ticket.p_category == name[:-2]:
                    if ticket.p_state in ('waived', 'paid'):
                        if str(ticket.p_session)[2:] == name[-2:]:
                            amounts[name] += ticket.amount_auth
            if amounts[name] == 0.0:
                value = ''
            elif isinstance(amounts[name], float):
                value = round(amounts[name], 2)
        return super(
            StudentExporter, self).mangle_value(
            value, name, context=context)

class StudentStudyLevelsOverviewExporter(StudentExporter):
    """The Student Study Levels Overview Exporter first filters the set of
    students by searching the students catalog. Then it exports some student
    base data together with the session key of registered levels.
    Sample output:

    header: ``...100,110,120,200,210,220,300...``

    data: ``...2010,,,2011,2012,,2013...``

    This csv data string means that level 100 was registered in session 
    2010/2011, level 200 in session 2011/2012, level 210 (200 on 1st probation)
    in session 2012/2013 and level 300 in session 2013/2014.
    """
    grok.name('studylevelsoverview')

    avail_levels = tuple([str(x) for x in study_levels(None)])

    fields = ('student_id', ) + (
        'state', 'certcode', 'faccode', 'depcode', 'is_postgrad',
        'entry_session', 'current_level', 'current_session',
        ) + avail_levels
    title = _(u'Student Study Levels Overview')

    def mangle_value(self, value, name, context=None):
        """The mangler checks if a given level has been registered. It returns
        the ``level_session`` attribute of the student study level object
        if the named level exists.
        """
        if name in self.avail_levels and context is not None:
            value = ''
            for level in context['studycourse'].values():
                if level.level == int(name):
                    value = '%s' % level.level_session
                    break
        return super(
            StudentExporter, self).mangle_value(
            value, name, context=context)

class ComboCardDataExporter(grok.GlobalUtility, StudentExporterBase):
    """Like all other exporters the Combo Card Data Exporter first filters the
    set of students by searching the students catalog. Then it exports some
    student base data which are neccessary to print for the Interswitch combo 
    card (identity card for students). The output contains a ``passport_path``
    column which contains the filesystem path of the passport image file.
    If no path is given, no passport image file exists.
    """
    grok.name('combocard')

    fields = ('display_fullname',
              'student_id','matric_number',
              'certificate', 'faculty', 'department', 'passport_path')
    title = _(u'Combo Card Data')

    def mangle_value(self, value, name, context=None):
        """The mangler determines the titles of faculty, department
        and certificate. It also computes the path of passport image file
        stored in the filesystem.
        """
        certificate = context['studycourse'].certificate
        if name == 'certificate' and certificate is not None:
            value = certificate.title
        if name == 'department' and certificate is not None:
            value = certificate.__parent__.__parent__.longtitle
        if name == 'faculty' and certificate is not None:
            value = certificate.__parent__.__parent__.__parent__.longtitle
        if name == 'passport_path' and certificate is not None:
            file_id = IFileStoreNameChooser(context).chooseName(
                attr='passport.jpg')
            os_path = getUtility(IExtFileStore)._pathFromFileID(file_id)
            if not os.path.exists(os_path):
                value = None
            else:
                value = '/'.join(os_path.split('/')[-4:])
        return super(
            ComboCardDataExporter, self).mangle_value(
            value, name, context=context)