Batch processing via browser **************************** Preliminaries: We define a function that looks up a form with several submit buttons for the one with a given value (this functionality is missing in zope.testbrowser): >>> def lookup_submit_value(name, value, browser): ... """Find a button with a certain value.""" ... for num in range(0, 100): ... try: ... button = browser.getControl(name=name, index=num) ... if button.value.endswith(value): ... return button ... except IndexError: ... break ... return None Create a site: >>> from waeup.kofa.app import University >>> getRootFolder()['app'] = University() Create a datacenter storage path: >>> import os >>> import tempfile >>> dc_path = tempfile.mkdtemp() Log in: >>> from zope.testbrowser.testing import Browser >>> browser = Browser() >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw') >>> browser.handleErrors = False Set datacenter path and deselect moving old data: >>> browser.open('http://localhost/app') >>> browser.getLink('Data Center').click() >>> browser.getLink('Edit settings').click() >>> browser.getControl(name='newpath').value = dc_path >>> browser.getControl(name='move').value = False >>> browser.getControl(name='save').click() Set non-usable datacenter path: >>> browser.getLink('Edit settings').click() >>> browser.getControl(name='newpath').value = '/' >>> browser.getControl(name='save').click() >>> 'Given storage path cannot be used.' in browser.contents True >>> browser.getControl('Cancel').click() Batch processing faculties ========================== Go to datacenter page: >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for faculties: >>> open('faculties.csv', 'wb').write( ... """code,title,title_prefix ... FAC1,Faculty 1,faculty ... FAC2,Faculty 2,institute ... FAC3,Faculty 3,school ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('faculties.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'faculties.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'faculties_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.displayOptions ['Applicant Processor', 'Applicants Container Processor', 'CertificateCourse Processor', 'Certificate Processor', 'Course Processor', 'CourseTicket Processor', 'Department Processor', 'Faculty Processor', 'Student Payment Processor', 'Student Processor', 'StudentStudyCourse Processor (update only)', 'StudentStudyLevel Processor', 'User Processor', 'Verdict Processor (update only)'] >>> importerselect.getControl('Faculty Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.options ['create', 'update', 'remove'] >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.getControl('View processing log').click() >>> print browser.contents Logfile datacenter.log... >>> browser.getControl(name='query').value = "zope.mgr" >>> browser.getControl('Search').click() >>> 'zope.mgr: Batch processing finished: OK' in browser.contents True >>> 'Mode: create' in browser.contents True >>> browser.getControl(name='query').value = "finished" >>> browser.getControl('Search').click() >>> 'zope.mgr: Batch processing finished: OK' in browser.contents True >>> 'Mode: create' in browser.contents False Batch processing departments ============================ >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for departments: >>> open('departments.csv', 'wb').write( ... """code,faculty_code,title,title_prefix ... DEP1,FAC1,Department 1,department ... DEP2,FAC2,Department 2,centre ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('departments.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'departments.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'departments_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Department Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for courses: >>> open('courses.csv', 'wb').write( ... """code,faculty_code,department_code,title,level,passmark,credits,semester ... CRS1,FAC1,DEP1,Course 1,100,40,2,1 ... CRS2,FAC1,DEP1,Course 2,100,40,2,2 ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('courses.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'courses.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'courses_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Course Processor', index=1).selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificates: >>> open('certificates.csv', 'wb').write( ... """code,faculty_code,department_code,title,study_mode,start_level,end_level,application_category ... CERT1,FAC1,DEP1,Certificate 1,pg_ft,999,999,basic ... CERT2,FAC1,DEP1,Certificate 2,ug_ft,200,300,cest ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('certificates.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'certificates.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'certificates_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Certificate Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificate courses: >>> open('mycertcourses.csv', 'wb').write( ... """course,faculty_code,department_code,certificate_code,level,mandatory ... CRS1,FAC1,DEP1,CERT1,100,True ... CRS2,FAC1,DEP1,CERT1,100,True ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('mycertcourses.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'mycertcourses.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'mycertcourses_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('CertificateCourse Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificate courses: >>> open('users.csv', 'wb').write( ... """name,title,public_name,email,phone,roles ... uli,Uli Fouquet,Chief Developer,uli@abc.de,+49-234-567 ... henrik, Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,['waeup.PortalManager'] ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('users.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'users.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'users_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('User Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import').disabled False >>> browser.getControl('Perform import').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for faculties: >>> open('newfaculties.csv', 'wb').write( ... """code,title,title_prefix ... FAC1,Faculty 1,faculty ... FAC4,Faculty 4,school ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload data').click() >>> filecontents = cStringIO.StringIO( ... open('newfaculties.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'newfaculties.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'newfaculties_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Faculty Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3: Fix headerlines As there should be no problem with the headers, we can immediately perfom the import: >>> browser.getControl('Perform import').click() One line could not be imported: >>> print browser.contents >> pending_file = dc_path + '/newfaculties_zope.mgr.create.pending.csv' >>> print open(pending_file).read() title_prefix,code,title,--ERRORS-- faculty,FAC1,Faculty 1,This object already exists. Skipping. >>> finished_file = dc_path + '/finished/newfaculties_zope.mgr.create.finished.csv' >>> print open(finished_file).read() title_prefix,code,title school,FAC4,Faculty 4 The finished-file contains the dataset we could import, while the pending file contains the dataset that failed, appended by an error message. Fixing the pending file ----------------------- We 'edit' the pending file (setting code to ``FAC5`` and title appropriately, and removing the --ERROR-- column) and finish the import this way: >>> open(dc_path + '/newfaculties_zope.mgr.create.pending.csv', 'wb').write( ... """title_prefix,--IGNORE--,code,title ... faculty,FAC5,Faculty 5 ... """) Step 1: start batch processing: >>> browser.open('http://localhost/app/datacenter') >>> browser.getLink('Process data').click() >>> button = lookup_submit_value( ... 'select', 'newfaculties_zope.mgr.create.pending.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Faculty Processor').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3').click() Step 3/4: Fix headerlines and import: As there should be no problem with the headers, we can immediately perfom the import: >>> browser.getControl('Perform import').click() This time everything should work: >>> print browser.contents >> sorted(os.listdir(dc_path)) ['deleted', 'finished', 'logs', 'unfinished'] >>> os.listdir(dc_path + '/unfinished') [] >>> sorted(os.listdir(dc_path + '/finished')) ['certificates_zope.mgr.create.finished.csv', ..., 'users_zope.mgr.create.finished.csv', 'users_zope.mgr.csv'] Processed (finished) Files ========================== >>> browser.open('http://localhost/app/datacenter/processed') >>> '' in browser.contents True Log Files ========= >>> browser.open('http://localhost/app/datacenter/logs') >>> 'datacenter.log' in browser.contents True >>> browser.getControl('Show', index=0).click() >>> browser.getControl('Back', index=0).click() >>> browser.getControl('Back to Data Center').click() >>> 'Storage path:' in browser.contents True Clean up: >>> import shutil >>> shutil.rmtree(dc_path)