Batch processing via browser **************************** Preliminaries: We define a function that looks up a form with several submit buttons for the one with a given value (this functionality is missing in zope.testbrowser): >>> def lookup_submit_value(name, value, browser): ... """Find a button with a certain value.""" ... for num in range(0, 100): ... try: ... button = browser.getControl(name=name, index=num) ... if button.value.endswith(value): ... return button ... except IndexError: ... break ... return None Create a site: >>> from waeup.kofa.app import University >>> getRootFolder()['app'] = University() Create a datacenter storage path: >>> import os >>> import tempfile >>> dc_path = tempfile.mkdtemp() Log in: >>> from zope.testbrowser.testing import Browser >>> browser = Browser() >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw') >>> browser.handleErrors = False Set datacenter path and deselect moving old data: >>> browser.open('http://localhost/app') >>> browser.getLink('Data Center').click() >>> browser.getLink('Edit settings').click() >>> browser.getControl(name='newpath').value = dc_path >>> browser.getControl(name='move').value = False >>> browser.getControl(name='save').click() Set non-usable datacenter path: >>> browser.getLink('Edit settings').click() >>> browser.getControl(name='newpath').value = '/' >>> browser.getControl(name='save').click() >>> 'Given storage path cannot be used.' in browser.contents True >>> browser.getControl('Cancel').click() Batch processing faculties ========================== Go to datacenter page: >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for faculties: >>> open('faculties.csv', 'wb').write( ... """code,title,title_prefix ... FAC1,Faculty 1,faculty ... FAC2,Faculty 2,institute ... FAC3,Faculty 3,school ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('faculties.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'faculties.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'faculties_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.displayOptions ['AccessCodeBatch Processor', 'AccessCode Processor', 'Applicant Processor', 'Applicants Container Processor', 'CertificateCourse Processor', 'Certificate Processor', 'Course Processor', 'CourseTicket Processor', 'Department Processor', 'Faculty Processor', 'Hostel Processor', 'Student Payment Processor', 'Student Processor', 'StudentStudyCourse Processor (update only)', 'StudentStudyLevel Processor', 'User Processor', 'Verdict Processor (update only)'] >>> importerselect.getControl('Faculty Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.options ['create', 'update', 'remove'] >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.getControl('View processing log').click() >>> print browser.contents Logfile datacenter.log... >>> browser.getControl(name='query').value = "zope.mgr" >>> browser.getControl('Search').click() >>> 'zope.mgr: Batch processing finished: OK' in browser.contents True >>> 'Mode: create' in browser.contents True >>> browser.getControl(name='query').value = "finished" >>> browser.getControl('Search').click() >>> 'zope.mgr: Batch processing finished: OK' in browser.contents True >>> 'Mode: create' in browser.contents False Batch processing departments ============================ >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for departments: >>> open('departments.csv', 'wb').write( ... """code,faculty_code,title,title_prefix ... DEP1,FAC1,Department 1,department ... DEP2,FAC2,Department 2,centre ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('departments.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'departments.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'departments_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Department Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for courses: >>> open('courses.csv', 'wb').write( ... """code,faculty_code,department_code,title,level,passmark,credits,semester ... CRS1,FAC1,DEP1,Course 1,100,40,2,1 ... CRS2,FAC1,DEP1,Course 2,100,40,2,2 ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('courses.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'courses.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'courses_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Course Processor', index=1).selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificates: >>> open('certificates.csv', 'wb').write( ... """code,faculty_code,department_code,title,study_mode,start_level,end_level,application_category ... CERT1,FAC1,DEP1,Certificate 1,pg_ft,999,999,basic ... CERT2,FAC1,DEP1,Certificate 2,ug_ft,200,300,cest ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('certificates.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'certificates.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'certificates_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Certificate Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificate courses: >>> open('mycertcourses.csv', 'wb').write( ... """course,faculty_code,department_code,certificate_code,level,mandatory ... CRS1,FAC1,DEP1,CERT1,100,True ... CRS2,FAC1,DEP1,CERT1,100,True ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('mycertcourses.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'mycertcourses.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'mycertcourses_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('CertificateCourse Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for users: >>> open('users.csv', 'wb').write( ... """name,title,public_name,email,phone,roles ... uli,Uli Fouquet,Chief Developer,uli@abc.de,+49-234-567,[] ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']" ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('users.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'users.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'users_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('User Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> henrik = getRootFolder()['app']['users']['henrik'] >>> henrik.roles ['waeup.PortalManager', 'waeup.ImportManager'] Pending files ============= When an error occurs during an import, two files are generated: a CSV file with finished files and a CSV file with pending data. Both are stored in the appropriate subdirectories in datacenter. We try to create faculties, from which one already exists. Go to datacenter page: >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for faculties: >>> open('newfaculties.csv', 'wb').write( ... """code,title,title_prefix ... FAC1,Faculty 1,faculty ... FAC4,Faculty 4,school ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('newfaculties.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'newfaculties.csv') >>> browser.getControl(name='SUBMIT').click() Since we now have a user with waeup.ImportManager role, an email has been sent: >>> print browser.contents >> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'newfaculties_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Faculty Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines As there should be no problem with the headers, we can immediately perfom the import: >>> browser.getControl('Perform import').click() One line could not be imported: >>> print browser.contents >> pending_file = dc_path + '/newfaculties_zope.mgr.create.pending.csv' >>> print open(pending_file).read() title_prefix,code,title,--ERRORS-- faculty,FAC1,Faculty 1,This object already exists. Skipping. >>> finished_file = dc_path + '/finished/newfaculties_zope.mgr.create.finished.csv' >>> print open(finished_file).read() title_prefix,code,title school,FAC4,Faculty 4 The finished-file contains the dataset we could import, while the pending file contains the dataset that failed, appended by an error message. Fixing the pending file ----------------------- We 'edit' the pending file (setting code to ``FAC5`` and title appropriately, and removing the --ERROR-- column) and finish the import this way: >>> open(dc_path + '/newfaculties_zope.mgr.create.pending.csv', 'wb').write( ... """title_prefix,--IGNORE--,code,title ... faculty,FAC5,Faculty 5 ... """) Step 1: start batch processing: >>> browser.open('http://localhost/app/datacenter') >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'newfaculties_zope.mgr.create.pending.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Faculty Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3/4: Fix headerlines and import: As there should be no problem with the headers, we can immediately perfom the import: >>> browser.getControl('Perform import').click() This time everything should work: >>> print browser.contents >> sorted(os.listdir(dc_path)) ['deleted', 'finished', 'logs', 'unfinished'] >>> os.listdir(dc_path + '/unfinished') [] >>> sorted(os.listdir(dc_path + '/finished')) ['certificates_zope.mgr.create.finished.csv', ..., 'users_zope.mgr.create.finished.csv', 'users_zope.mgr.csv'] Processed (finished) Files ========================== >>> browser.open('http://localhost/app/datacenter/processed') >>> '' in browser.contents True Log Files ========= >>> browser.open('http://localhost/app/datacenter/logs') >>> 'datacenter.log' in browser.contents True >>> browser.getControl('Show', index=0).click() >>> browser.getControl('Back', index=0).click() >>> browser.getControl('Back to Data Center').click() >>> 'Storage path:' in browser.contents True Clean up: >>> import shutil >>> shutil.rmtree(dc_path)