Changeset 8403 for main/waeup.kofa/trunk

Timestamp:

9 May 2012, 16:43:10 (13 years ago)

Author:

uli

Message:

Backup student data when a student is deleted (blocking of student_id yet missing).

Location:

main/waeup.kofa/trunk/src/waeup/kofa/students

Files:

• student.py (modified) (4 diffs)
• tests/test_student.py (modified) (2 diffs)

main/waeup.kofa/trunk/src/waeup/kofa/students/student.py

@@ -20 +20 @@
 """
 import os
+import shutil
 import grok
 from hurry.workflow.interfaces import IWorkflowState, IWorkflowInfo
@@ -26 +27 @@
 from zope.interface import implementedBy
 from zope.securitypolicy.interfaces import IPrincipalRoleManager
+
 from waeup.kofa.image import KofaImageFile
 from waeup.kofa.imagestorage import DefaultFileStoreHandler
 from waeup.kofa.interfaces import (
     IObjectHistory, IUserAccount, IFileStoreNameChooser, IFileStoreHandler,
-    IKofaUtils, CLEARANCE, registration_states_vocab)
+    IKofaUtils, CLEARANCE, registration_states_vocab, IExtFileStore,
+    ICSVExporter)
 from waeup.kofa.students.accommodation import StudentAccommodation
+from waeup.kofa.students.export import EXPORTER_NAMES
 from waeup.kofa.students.interfaces import IStudent, IStudentNavigation
 from waeup.kofa.students.payments import StudentPaymentsContainer
-from waeup.kofa.students.studycourse import StudentStudyCourse
 from waeup.kofa.students.utils import generate_student_id
-from waeup.kofa.utils.helpers import attrs_to_fields
-
+from waeup.kofa.utils.helpers import attrs_to_fields, now, copy_filesystem_tree
 
 class Student(grok.Container):
@@ -177 +179 @@
     return
 
+def move_student_files(student, del_dir):
+    """Move files belonging to `student` to `del_dir`.
+
+    `del_dir` is expected to be the path to the site-wide directory
+    for storing backup data.
+
+    The current files of the student are removed after backup.
+
+    If the student has no associated files stored, nothing is done.
+    """
+    stud_id = student.student_id
+
+    src = getUtility(IExtFileStore).root
+    src = os.path.join(src, 'students', stud_id[0], stud_id)
+
+    dst = os.path.join(del_dir, 'media', 'students', stud_id[0], stud_id)
+
+    if not os.path.isdir(src):
+        # Do not copy if no files were stored.
+        return
+    if not os.path.exists(dst):
+        os.makedirs(dst, 0755)
+    copy_filesystem_tree(src, dst)
+    shutil.rmtree(src)
+    return
+
+def update_student_deletion_csvs(student, del_dir):
+    """Update deletion CSV files with data from student.
+
+    `del_dir` is expected to be the path to the site-wide directory
+    for storing backup data.
+
+    Each exporter available for students (and their many subobjects)
+    is called in order to export CSV data of the given student to csv
+    files in the site-wide backup directory for object data (see
+    DataCenter).
+
+    Each exported row is appended a column giving the deletion date
+    (column `del_date`) as a UTC timestamp.
+    """
+    for name in EXPORTER_NAMES:
+        exporter = getUtility(ICSVExporter, name=name)
+        csv_data = exporter.export([student])
+        csv_data = csv_data.split('\r\n')
+
+        # append a deletion timestamp on each data row
+        timestamp = str(now().replace(microsecond=0)) # store UTC timestamp
+        for num, row in enumerate(csv_data[1:-1]):
+            csv_data[num+1] = csv_data[num+1] + ',' + timestamp
+        csv_path = os.path.join(del_dir, '%s.csv' % name)
+
+        # write data to CSV file
+        if not os.path.exists(csv_path):
+            # create new CSV file (including header line)
+            csv_data[0] = csv_data[0] + ',del_date'
+            open(csv_path, 'wb').write('\r\n'.join(csv_data))
+        else:
+            # append existing CSV file (omitting headerline)
+            open(csv_path, 'a').write('\r\n'.join(csv_data[1:]))
+    return
+
 @grok.subscribe(IStudent, grok.IObjectRemovedEvent)
 def handle_student_removed(student, event):
-    """If a student is removed a message is logged.
+    """If a student is removed a message is logged and data is put
+       into a backup location.
+
+    The data of the removed student is appended to CSV files in local
+    datacenter and any existing external files (passport images, etc.)
+    are copied over to this location as well.
+
+    Documents in the file storage refering to the given student are
+    removed afterwards (if they exist). Please make no assumptions
+    about how the deletion takes place. Files might be deleted
+    individually (leaving the students file directory intact) or the
+    whole student directory might be deleted completely.
+
+    All CSV rows created/appended contain a timestamp with the
+    datetime of removal in an additional `del_date` column.
+
+    XXX: blocking of used student_ids yet not implemented.
     """
     comment = 'Student record removed'
     target = student.student_id
     try:
-        grok.getSite()['students'].logger.info('%s - %s' % (
+        site = grok.getSite()
+        site['students'].logger.info('%s - %s' % (
             target, comment))
     except KeyError:
@@ -190 +270 @@
         # a students subcontainer
         return
+
+    del_dir = site['datacenter'].deleted_path
+
+    # save files of the student
+    move_student_files(student, del_dir)
+
+    # update CSV files
+    update_student_deletion_csvs(student, del_dir)
     return
 

main/waeup.kofa/trunk/src/waeup/kofa/students/tests/test_student.py

@@ -18 +18 @@
 """Tests for students and related.
 """
+import os
+import re
+from cStringIO import StringIO
 from datetime import tzinfo
+from zope.component import getUtility
 from zope.component.interfaces import IFactory
 from zope.interface import verify
-from waeup.kofa.students.student import Student, StudentFactory
+from waeup.kofa.interfaces import IExtFileStore, IFileStoreNameChooser
+from waeup.kofa.students.export import EXPORTER_NAMES
+from waeup.kofa.students.student import (
+    Student, StudentFactory, handle_student_removed)
 from waeup.kofa.students.studycourse import StudentStudyCourse
 from waeup.kofa.students.studylevel import StudentStudyLevel, CourseTicket
 from waeup.kofa.students.payments import StudentPaymentsContainer
 from waeup.kofa.students.accommodation import StudentAccommodation, BedTicket
-from waeup.kofa.applicants.interfaces import IApplicantBaseData
 from waeup.kofa.students.interfaces import (
-    IStudent, IStudentStudyCourse, IStudentPaymentsContainer, IStudentAccommodation,
-    IStudentStudyLevel, ICourseTicket, IBedTicket)
+    IStudent, IStudentStudyCourse, IStudentPaymentsContainer,
+    IStudentAccommodation, IStudentStudyLevel, ICourseTicket, IBedTicket)
+from waeup.kofa.students.tests.test_batching import StudentImportExportSetup
 from waeup.kofa.testing import FunctionalLayer, FunctionalTestCase
 from waeup.kofa.university.department import Department
@@ -83 +90 @@
         return
 
+class StudentRemovalTests(StudentImportExportSetup):
+    # Test handle_student_removed
+    #
+    # This is a complex action updating several CSV files and moving
+    # stored files to a backup location.
+    #
+    # These tests make no assumptions about the CSV files except that
+    # they contain a deletion timestamp at end of each data row
+
+    layer = FunctionalLayer
+
+    def setUp(self):
+        super(StudentRemovalTests, self).setUp()
+        self.setup_for_export()
+        return
+
+    def create_passport_img(self, student):
+        # create some faked passport file for `student`
+        storage = getUtility(IExtFileStore)
+        file_id = IFileStoreNameChooser(student).chooseName(
+            attr='passport.jpg')
+        storage.createFile(file_id, StringIO('I am a fake image.'))
+
+    def test_backup_single_student_data(self):
+        # when a single student is removed, the data is backed up.
+        self.setup_student(self.student)
+        # Add a fake image
+        self.create_passport_img(self.student)
+        handle_student_removed(self.student, None)
+        del_dir = self.app['datacenter'].deleted_path
+        del_img_path = os.path.join(
+            del_dir, 'media', 'students', 'A', 'A111111',
+            'passport_A111111.jpg')
+
+        # The image was copied over
+        self.assertTrue(os.path.isfile(del_img_path))
+        self.assertEqual(
+            open(del_img_path, 'rb').read(),
+            'I am a fake image.')
+
+        # The student data were put into CSV files
+        for name in EXPORTER_NAMES:
+            csv_path = os.path.join(del_dir, '%s.csv' % name)
+            self.assertTrue(os.path.isfile(csv_path))
+            contents = open(csv_path, 'rb').read().split('\r\n')
+            # We expect 3 lines output including a linebreak at end of file.
+            self.assertEqual(len(contents), 3)
+        return
+
+    def test_backup_append_csv(self):
+        # when several students are removed, existing CSVs are appended
+        self.setup_student(self.student)
+        # Add a fake image
+        self.create_passport_img(self.student)
+        del_dir = self.app['datacenter'].deleted_path
+        # put fake data into students.csv with trailing linebreak
+        students_csv = os.path.join(del_dir, 'students.csv')
+        open(students_csv, 'wb').write('line1\r\nline2\r\n')
+        handle_student_removed(self.student, None)
+        contents = open(students_csv, 'rb').read().split('\r\n')
+        # there should be 4 lines in result csv (including trailing linebreak)
+        self.assertEqual(len(contents), 4)
+        return
+
+    def test_old_files_removed(self):
+        # make sure old files are not accessible any more
+        self.setup_student(self.student)
+        # Add a fake image
+        self.create_passport_img(self.student)
+        # make sure we can access the image before removal
+        file_store = getUtility(IExtFileStore)
+        image = file_store.getFileByContext(self.student, attr='passport.jpg')
+        self.assertTrue(image is not None)
+
+        # remove image (hopefully)
+        handle_student_removed(self.student, None)
+
+        # the is not accessible anymore
+        image = file_store.getFileByContext(self.student, attr='passport.jpg')
+        self.assertEqual(image, None)
+        return
+
+    def test_csv_file_entries_have_timestamp(self):
+        # each row in written csv files has a ``del_date`` column to
+        # tell when the associated student was deleted
+        self.setup_student(self.student)
+        del_dir = self.app['datacenter'].deleted_path
+        students_csv = os.path.join(del_dir, 'students.csv')
+        handle_student_removed(self.student, None)
+        contents = open(students_csv, 'rb').read().split('\r\n')
+        # the CSV header ends with a ``del_date`` column
+        self.assertTrue(contents[0].endswith(',del_date'))
+        # each line ends with an UTC timestamp
+        timestamp = contents[1][-23:]
+        self.assertTrue(re.match(
+            '^\d\d-\d\d-\d\d \d\d:\d\d:\d\d\+00:00$', timestamp))
+        return
+
+
 class StudentFactoryTest(FunctionalTestCase):