Changeset 8403 for main


Ignore:
Timestamp:
9 May 2012, 16:43:10 (13 years ago)
Author:
uli
Message:

Backup student data when a student is deleted (blocking of student_id yet missing).

Location:
main/waeup.kofa/trunk/src/waeup/kofa/students
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • main/waeup.kofa/trunk/src/waeup/kofa/students/student.py

    r8375 r8403  
    2020"""
    2121import os
     22import shutil
    2223import grok
    2324from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
     
    2627from zope.interface import implementedBy
    2728from zope.securitypolicy.interfaces import IPrincipalRoleManager
     29
    2830from waeup.kofa.image import KofaImageFile
    2931from waeup.kofa.imagestorage import DefaultFileStoreHandler
    3032from waeup.kofa.interfaces import (
    3133    IObjectHistory, IUserAccount, IFileStoreNameChooser, IFileStoreHandler,
    32     IKofaUtils, CLEARANCE, registration_states_vocab)
     34    IKofaUtils, CLEARANCE, registration_states_vocab, IExtFileStore,
     35    ICSVExporter)
    3336from waeup.kofa.students.accommodation import StudentAccommodation
     37from waeup.kofa.students.export import EXPORTER_NAMES
    3438from waeup.kofa.students.interfaces import IStudent, IStudentNavigation
    3539from waeup.kofa.students.payments import StudentPaymentsContainer
    36 from waeup.kofa.students.studycourse import StudentStudyCourse
    3740from waeup.kofa.students.utils import generate_student_id
    38 from waeup.kofa.utils.helpers import attrs_to_fields
    39 
     41from waeup.kofa.utils.helpers import attrs_to_fields, now, copy_filesystem_tree
    4042
    4143class Student(grok.Container):
     
    177179    return
    178180
     181def move_student_files(student, del_dir):
     182    """Move files belonging to `student` to `del_dir`.
     183
     184    `del_dir` is expected to be the path to the site-wide directory
     185    for storing backup data.
     186
     187    The current files of the student are removed after backup.
     188
     189    If the student has no associated files stored, nothing is done.
     190    """
     191    stud_id = student.student_id
     192
     193    src = getUtility(IExtFileStore).root
     194    src = os.path.join(src, 'students', stud_id[0], stud_id)
     195
     196    dst = os.path.join(del_dir, 'media', 'students', stud_id[0], stud_id)
     197
     198    if not os.path.isdir(src):
     199        # Do not copy if no files were stored.
     200        return
     201    if not os.path.exists(dst):
     202        os.makedirs(dst, 0755)
     203    copy_filesystem_tree(src, dst)
     204    shutil.rmtree(src)
     205    return
     206
     207def update_student_deletion_csvs(student, del_dir):
     208    """Update deletion CSV files with data from student.
     209
     210    `del_dir` is expected to be the path to the site-wide directory
     211    for storing backup data.
     212
     213    Each exporter available for students (and their many subobjects)
     214    is called in order to export CSV data of the given student to csv
     215    files in the site-wide backup directory for object data (see
     216    DataCenter).
     217
     218    Each exported row is appended a column giving the deletion date
     219    (column `del_date`) as a UTC timestamp.
     220    """
     221    for name in EXPORTER_NAMES:
     222        exporter = getUtility(ICSVExporter, name=name)
     223        csv_data = exporter.export([student])
     224        csv_data = csv_data.split('\r\n')
     225
     226        # append a deletion timestamp on each data row
     227        timestamp = str(now().replace(microsecond=0)) # store UTC timestamp
     228        for num, row in enumerate(csv_data[1:-1]):
     229            csv_data[num+1] = csv_data[num+1] + ',' + timestamp
     230        csv_path = os.path.join(del_dir, '%s.csv' % name)
     231
     232        # write data to CSV file
     233        if not os.path.exists(csv_path):
     234            # create new CSV file (including header line)
     235            csv_data[0] = csv_data[0] + ',del_date'
     236            open(csv_path, 'wb').write('\r\n'.join(csv_data))
     237        else:
     238            # append existing CSV file (omitting headerline)
     239            open(csv_path, 'a').write('\r\n'.join(csv_data[1:]))
     240    return
     241
    179242@grok.subscribe(IStudent, grok.IObjectRemovedEvent)
    180243def handle_student_removed(student, event):
    181     """If a student is removed a message is logged.
     244    """If a student is removed a message is logged and data is put
     245       into a backup location.
     246
     247    The data of the removed student is appended to CSV files in local
     248    datacenter and any existing external files (passport images, etc.)
     249    are copied over to this location as well.
     250
     251    Documents in the file storage refering to the given student are
     252    removed afterwards (if they exist). Please make no assumptions
     253    about how the deletion takes place. Files might be deleted
     254    individually (leaving the students file directory intact) or the
     255    whole student directory might be deleted completely.
     256
     257    All CSV rows created/appended contain a timestamp with the
     258    datetime of removal in an additional `del_date` column.
     259
     260    XXX: blocking of used student_ids yet not implemented.
    182261    """
    183262    comment = 'Student record removed'
    184263    target = student.student_id
    185264    try:
    186         grok.getSite()['students'].logger.info('%s - %s' % (
     265        site = grok.getSite()
     266        site['students'].logger.info('%s - %s' % (
    187267            target, comment))
    188268    except KeyError:
     
    190270        # a students subcontainer
    191271        return
     272
     273    del_dir = site['datacenter'].deleted_path
     274
     275    # save files of the student
     276    move_student_files(student, del_dir)
     277
     278    # update CSV files
     279    update_student_deletion_csvs(student, del_dir)
    192280    return
    193281
  • main/waeup.kofa/trunk/src/waeup/kofa/students/tests/test_student.py

    r8194 r8403  
    1818"""Tests for students and related.
    1919"""
     20import os
     21import re
     22from cStringIO import StringIO
    2023from datetime import tzinfo
     24from zope.component import getUtility
    2125from zope.component.interfaces import IFactory
    2226from zope.interface import verify
    23 from waeup.kofa.students.student import Student, StudentFactory
     27from waeup.kofa.interfaces import IExtFileStore, IFileStoreNameChooser
     28from waeup.kofa.students.export import EXPORTER_NAMES
     29from waeup.kofa.students.student import (
     30    Student, StudentFactory, handle_student_removed)
    2431from waeup.kofa.students.studycourse import StudentStudyCourse
    2532from waeup.kofa.students.studylevel import StudentStudyLevel, CourseTicket
    2633from waeup.kofa.students.payments import StudentPaymentsContainer
    2734from waeup.kofa.students.accommodation import StudentAccommodation, BedTicket
    28 from waeup.kofa.applicants.interfaces import IApplicantBaseData
    2935from waeup.kofa.students.interfaces import (
    30     IStudent, IStudentStudyCourse, IStudentPaymentsContainer, IStudentAccommodation,
    31     IStudentStudyLevel, ICourseTicket, IBedTicket)
     36    IStudent, IStudentStudyCourse, IStudentPaymentsContainer,
     37    IStudentAccommodation, IStudentStudyLevel, ICourseTicket, IBedTicket)
     38from waeup.kofa.students.tests.test_batching import StudentImportExportSetup
    3239from waeup.kofa.testing import FunctionalLayer, FunctionalTestCase
    3340from waeup.kofa.university.department import Department
     
    8390        return
    8491
     92class StudentRemovalTests(StudentImportExportSetup):
     93    # Test handle_student_removed
     94    #
     95    # This is a complex action updating several CSV files and moving
     96    # stored files to a backup location.
     97    #
     98    # These tests make no assumptions about the CSV files except that
     99    # they contain a deletion timestamp at end of each data row
     100
     101    layer = FunctionalLayer
     102
     103    def setUp(self):
     104        super(StudentRemovalTests, self).setUp()
     105        self.setup_for_export()
     106        return
     107
     108    def create_passport_img(self, student):
     109        # create some faked passport file for `student`
     110        storage = getUtility(IExtFileStore)
     111        file_id = IFileStoreNameChooser(student).chooseName(
     112            attr='passport.jpg')
     113        storage.createFile(file_id, StringIO('I am a fake image.'))
     114
     115    def test_backup_single_student_data(self):
     116        # when a single student is removed, the data is backed up.
     117        self.setup_student(self.student)
     118        # Add a fake image
     119        self.create_passport_img(self.student)
     120        handle_student_removed(self.student, None)
     121        del_dir = self.app['datacenter'].deleted_path
     122        del_img_path = os.path.join(
     123            del_dir, 'media', 'students', 'A', 'A111111',
     124            'passport_A111111.jpg')
     125
     126        # The image was copied over
     127        self.assertTrue(os.path.isfile(del_img_path))
     128        self.assertEqual(
     129            open(del_img_path, 'rb').read(),
     130            'I am a fake image.')
     131
     132        # The student data were put into CSV files
     133        for name in EXPORTER_NAMES:
     134            csv_path = os.path.join(del_dir, '%s.csv' % name)
     135            self.assertTrue(os.path.isfile(csv_path))
     136            contents = open(csv_path, 'rb').read().split('\r\n')
     137            # We expect 3 lines output including a linebreak at end of file.
     138            self.assertEqual(len(contents), 3)
     139        return
     140
     141    def test_backup_append_csv(self):
     142        # when several students are removed, existing CSVs are appended
     143        self.setup_student(self.student)
     144        # Add a fake image
     145        self.create_passport_img(self.student)
     146        del_dir = self.app['datacenter'].deleted_path
     147        # put fake data into students.csv with trailing linebreak
     148        students_csv = os.path.join(del_dir, 'students.csv')
     149        open(students_csv, 'wb').write('line1\r\nline2\r\n')
     150        handle_student_removed(self.student, None)
     151        contents = open(students_csv, 'rb').read().split('\r\n')
     152        # there should be 4 lines in result csv (including trailing linebreak)
     153        self.assertEqual(len(contents), 4)
     154        return
     155
     156    def test_old_files_removed(self):
     157        # make sure old files are not accessible any more
     158        self.setup_student(self.student)
     159        # Add a fake image
     160        self.create_passport_img(self.student)
     161        # make sure we can access the image before removal
     162        file_store = getUtility(IExtFileStore)
     163        image = file_store.getFileByContext(self.student, attr='passport.jpg')
     164        self.assertTrue(image is not None)
     165
     166        # remove image (hopefully)
     167        handle_student_removed(self.student, None)
     168
     169        # the is not accessible anymore
     170        image = file_store.getFileByContext(self.student, attr='passport.jpg')
     171        self.assertEqual(image, None)
     172        return
     173
     174    def test_csv_file_entries_have_timestamp(self):
     175        # each row in written csv files has a ``del_date`` column to
     176        # tell when the associated student was deleted
     177        self.setup_student(self.student)
     178        del_dir = self.app['datacenter'].deleted_path
     179        students_csv = os.path.join(del_dir, 'students.csv')
     180        handle_student_removed(self.student, None)
     181        contents = open(students_csv, 'rb').read().split('\r\n')
     182        # the CSV header ends with a ``del_date`` column
     183        self.assertTrue(contents[0].endswith(',del_date'))
     184        # each line ends with an UTC timestamp
     185        timestamp = contents[1][-23:]
     186        self.assertTrue(re.match(
     187            '^\d\d-\d\d-\d\d \d\d:\d\d:\d\d\+00:00$', timestamp))
     188        return
     189
     190
    85191class StudentFactoryTest(FunctionalTestCase):
    86192
Note: See TracChangeset for help on using the changeset viewer.