Changeset 1941

Timestamp:

21 Jun 2007, 14:36:15 (17 years ago)

Author:

joachim

Message:

faster version of exportStudents

File:

• WAeUP_SRP/trunk/Students.py (modified) (2 diffs)

WAeUP_SRP/trunk/Students.py

@@ -942 +942 @@
                                 'courses_validated'))
         students = aq_students(query)
-        logger.info('%s starts exportStudents, %s student records to export' % (member,len(students)))
-        chunk = 100
+        nr2export = len(students)
+        logger.info('%s starts exportStudents, %s student records to export' % (member,nr2export))
+        chunk = 2000
         total = 0
+        start = DateTime.DateTime().timeTime()
+        start_chunk = DateTime.DateTime().timeTime()
+        ## alternative method slightly slower
+        # student_recs = {}
+        # for s in students:
+        #      student_recs[s.id] = s
+        # catalog_recs = {}
+        # brains = self.portal_catalog(portal_type = 'Student')
+        # for cat_rec in brains:
+        #     sid = cat_rec.getId
+        #     catalog_recs[sid] = cat_rec
+        # #import pdb;pdb.set_trace()
+        # start = DateTime.DateTime().timeTime()
+        # start_chunk = DateTime.DateTime().timeTime()
+        # for student in students:
+        #     if student.id not in student_recs.keys():
+        #         continue
+        #     not_all = False
+        #     d = self.getFormattedStudentEntry(student_recs[student.id])
+        #     student_obj = catalog_recs[student.id].getObject()
         for student in students:
             not_all = False
             d = self.getFormattedStudentEntry(student)
+            student_obj = getattr(students_folder,student.id)
             for k in toexport.keys()[1:]:
                 try:
-                    object = getattr(getattr(students_folder,student.id),k)
+                    object = getattr(student_obj,k)
                     object_doc = object.getContent()
                 except:
@@ -967 +989 @@
                 open(export_file,"a").write('\n'.join(lines) +'\n')
                 anz = len(lines)
-                logger.info("wrote %(anz)d to %(export_file)s" % vars())
+                logger.info("wrote %(anz)d  total written %(total)d" % vars())
+                end_chunk = DateTime.DateTime().timeTime()
+                duration = end_chunk-start_chunk
+                per_record = duration/anz
+                till_now = end_chunk - start
+                avarage_per_record = till_now/total
+                estimated_end = DateTime.DateTime(start + avarage_per_record * nr2export)
+                estimated_end = estimated_end.strftime("%H:%M:%S")
+                logger.info('%(duration)4.1f, %(per_record)4.3f,end %(estimated_end)s' % vars())
+                start_chunk = DateTime.DateTime().timeTime()
                 lines = []
+        end = DateTime.DateTime().timeTime()
+        logger.info('total time %6.2f m' % ((end-start)/60))
         filename, extension = os.path.splitext(export_file)
-        os.system('zip %s %s' % (filename,export_file))
-        os.system('rm %s' % (export_file))
-        msg = "finished wrote %(total)d records to %(filename)s.zip" % vars()
+        from subprocess import call
+        msg = "wrote %(total)d records to %(export_file)s" % vars()
+        try:
+            retcode = call('gzip %s' % (export_file),shell=True)
+            if retcode == 0:
+                msg = "wrote %(total)d records to %(export_file)s.gz" % vars()
+        except OSError, e:
+            retcode = -99
+            logger.info("zip failed with %s" % e)
         logger.info(msg)
         args = {'portal_status_message': msg}
-        #url = self.REQUEST.get('URL1') + '?' + urlencode(args)
-        url = self.REQUEST.get('URL1')
+        #url = self.REQUEST.get('URL1') + '?' + urlencode(args) 
+        url = self.REQUEST.get('URL2')
         return self.REQUEST.RESPONSE.redirect(url)
     ###)