Batch processing via browser **************************** :test-layer: functional Preliminaries: We define a function that looks up a form with several submit buttons for the one with a given value (this functionality is missing in zope.testbrowser): >>> def lookup_submit_value(name, value, browser): ... """Find a button with a certain value.""" ... for num in range(0, 100): ... try: ... button = browser.getControl(name=name, index=num) ... if button.value.endswith(value): ... return button ... except IndexError: ... break ... return None Create a site: >>> from waeup.sirp.app import University >>> getRootFolder()['app'] = University() Create a datacenter storage path: >>> import os >>> import tempfile >>> dc_path = tempfile.mkdtemp() Log in: >>> from zope.testbrowser.testing import Browser >>> browser = Browser() >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw') >>> browser.handleErrors = False Set datacenter path and deselect moving old data: >>> browser.open('http://localhost/app') >>> browser.getLink('Data Center').click() >>> browser.getLink('Edit settings').click() >>> browser.getControl(name='newpath').value = dc_path >>> browser.getControl(name='move').value = False >>> browser.getControl(name='save').click() Batch processing faculties ========================== Go to datacenter page: >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for faculties: >>> open('faculties.csv', 'wb').write( ... """code,review_state,title,title_prefix ... FAC1,static,Faculty 1,faculty ... FAC2,static,Faculty 2,institute ... FAC3,static,Faculty 3,school_of ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload CSV file').click() >>> filecontents = cStringIO.StringIO( ... open('faculties.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'faculties.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'faculties_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.displayOptions ['JAMB Data Importer', 'CertificateCourse importer', 'Certificate importer', 'Course importer', 'Department importer', 'Faculty importer'] >>> importerselect.getControl('Faculty importer').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.options ['create', 'update', 'remove'] >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import...').disabled False >>> browser.getControl('Perform import...').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.getControl('View processing log').click() >>> print browser.contents ... ... - INFO - -------------------- ... - INFO - zope.mgr: Batch processing finished: OK ... - INFO - zope.mgr: Source: /.../faculties_zope.mgr.csv ... - INFO - zope.mgr: Mode: create ... - INFO - zope.mgr: User: zope.mgr ... - INFO - zope.mgr: Processing time: ... s (... s/item) ... - INFO - zope.mgr: Processed: 3 lines (3 successful/ 0 failed) ... - INFO - -------------------- .... Batch processing departments ============================ >>> browser.open('http://localhost/app/datacenter') Prepare a CSV file for departments: >>> open('departments.csv', 'wb').write( ... """code,faculty_code,review_state,title,title_prefix ... DEP1,FAC1,static,Department 1,department ... DEP2,FAC2,static,Department 2,centre ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload CSV file').click() >>> filecontents = cStringIO.StringIO( ... open('departments.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'departments.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'departments_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Department importer').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import...').disabled False >>> browser.getControl('Perform import...').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for courses: >>> open('courses.csv', 'wb').write( ... """code,faculty_code,department_code,title,level,passmark,credits,semester ... CRS1,FAC1,DEP1,Course 1,100,40,2,1 ... CRS2,FAC1,DEP1,Course 2,100,40,2,2 ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload CSV file').click() >>> filecontents = cStringIO.StringIO( ... open('courses.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'courses.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'courses_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Course importer', index=1).selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import...').disabled False >>> browser.getControl('Perform import...').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificates: >>> open('certificates.csv', 'wb').write( ... """code,faculty_code,department_code,title,study_mode,start_level,end_level,application_category ... CERT1,FAC1,DEP1,Certificate 1,ug_ft,100,300,basic ... CERT2,FAC1,DEP1,Certificate 2,ug_ft,200,300,cest ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload CSV file').click() >>> filecontents = cStringIO.StringIO( ... open('certificates.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'certificates.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'certificates_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Certificate importer').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import...').disabled False >>> browser.getControl('Perform import...').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for certificate courses: >>> open('mycertcourses.csv', 'wb').write( ... """course,faculty_code,department_code,certificate_code,level,core_or_elective ... CRS1,FAC1,DEP1,CERT1,100,True ... CRS2,FAC1,DEP1,CERT1,100,True ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload CSV file').click() >>> filecontents = cStringIO.StringIO( ... open('mycertcourses.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'mycertcourses.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'mycertcourses_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('CertificateCourse importer').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3: Fix headerlines We get informed that there are no problems with the current header: >>> print browser.contents >> browser.getControl('Perform import...').disabled False >>> browser.getControl('Perform import...').click() Step 4: See import results The import was successful: >>> print browser.contents >> browser.open('http://localhost/app/datacenter') Prepare a CSV file for faculties: >>> open('newfaculties.csv', 'wb').write( ... """code,review_state,title,title_prefix ... FAC1,static,Faculty 1,faculty ... FAC4,static,Faculty 4,school_of ... """) Upload the file: >>> import cStringIO >>> browser.getLink('Upload CSV file').click() >>> filecontents = cStringIO.StringIO( ... open('newfaculties.csv', 'rb').read()) >>> filewidget = browser.getControl(name='uploadfile:file') >>> filewidget.add_file(filecontents, 'text/plain', 'newfaculties.csv') >>> browser.getControl(name='SUBMIT').click() Step 1: start batch processing: >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'newfaculties_zope.mgr.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Faculty importer').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3: Fix headerlines As there should be no problem with the headers, we can immediately perfom the import: >>> browser.getControl('Perform import...').click() One line could not be imported: >>> print browser.contents >> pending_file = dc_path + '/newfaculties_zope.mgr.create.pending.csv' >>> print open(pending_file).read() title_prefix,--IGNORE--,code,title,--ERRORS-- faculty,static,FAC1,Faculty 1,This object already exists in the same container. Skipping. >>> finished_file = dc_path + '/finished/newfaculties_zope.mgr.create.finished.csv' >>> print open(finished_file).read() title_prefix,--IGNORE--,code,title school_of,static,FAC4,Faculty 4 The finished-file contains the dataset we could import, while the pending file contains the dataset that failed, appended by an error message. Fixing the pending file ----------------------- We 'edit' the pending file (setting code to ``FAC5`` and title appropriately, and removing the --ERROR-- column) and finish the import this way: >>> open(dc_path + '/newfaculties_zope.mgr.create.pending.csv', 'wb').write( ... """title_prefix,--IGNORE--,code,title ... faculty,static,FAC5,Faculty 5 ... """) Step 1: start batch processing: >>> browser.open('http://localhost/app/datacenter') >>> browser.getLink('Batch processing').click() >>> button = lookup_submit_value( ... 'select', 'newfaculties_zope.mgr.create.pending.csv', browser) >>> button.click() Step 2: select a processor and mode: >>> importerselect = browser.getControl(name='importer') >>> importerselect.getControl('Faculty importer').selected = True >>> modeselect = browser.getControl(name='mode') >>> modeselect.getControl(value='create').selected = True >>> browser.getControl('Proceed to step 3...').click() Step 3/4: Fix headerlines and import: As there should be no problem with the headers, we can immediately perfom the import: >>> browser.getControl('Perform import...').click() This time everything should work: >>> print browser.contents >> sorted(os.listdir(dc_path)) ['finished', 'logs', 'unfinished'] >>> os.listdir(dc_path + '/unfinished') [] >>> sorted(os.listdir(dc_path + '/finished')) ['certificates_zope.mgr.create.finished.csv', ..., 'newfaculties_zope.mgr.create.finished.csv', 'newfaculties_zope.mgr.csv'] Clean up: >>> import shutil >>> shutil.rmtree(dc_path)