source: main/waeup.ikoba/trunk/src/waeup/ikoba/doctests/batchprocessing_browser.txt @ 14195

Last change on this file since 14195 was 13806, checked in by Henrik Bettermann, 8 years ago

Add portal maintenance mode.

See r13394, r13396, r13468.
File size: 9.3 KB
RevLine 
[12993]1Batch Processing via Browser
[4857]2****************************
3
[4869]4Preliminaries:
5
6We define a function that looks up a form with several submit buttons
7for the one with a given value (this functionality is missing in
8zope.testbrowser):
9
10    >>> def lookup_submit_value(name, value, browser):
11    ...   """Find a button with a certain value."""
12    ...   for num in range(0, 100):
13    ...     try:
14    ...       button = browser.getControl(name=name, index=num)
[4899]15    ...       if button.value.endswith(value):
[4869]16    ...         return button
17    ...     except IndexError:
18    ...       break
19    ...   return None
20
[4857]21Create a site:
22
[11954]23    >>> from waeup.ikoba.app import Company
24    >>> getRootFolder()['app'] = Company()
[9312]25    >>> from zope.component.hooks import setSite
26    >>> setSite(getRootFolder()['app'])
[4857]27
28Create a datacenter storage path:
29
30    >>> import os
31    >>> import tempfile
32    >>> dc_path = tempfile.mkdtemp()
33
34Log in:
35
36    >>> from zope.testbrowser.testing import Browser
37    >>> browser = Browser()
38    >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw')
39    >>> browser.handleErrors = False
40
41Set datacenter path and deselect moving old data:
42
43    >>> browser.open('http://localhost/app')
44    >>> browser.getLink('Data Center').click()
45    >>> browser.getLink('Edit settings').click()
46    >>> browser.getControl(name='newpath').value = dc_path
47    >>> browser.getControl(name='move').value = False
48    >>> browser.getControl(name='save').click()
49
[6611]50Set non-usable datacenter path:
51
52    >>> browser.getLink('Edit settings').click()
53    >>> browser.getControl(name='newpath').value = '/'
54    >>> browser.getControl(name='save').click()
[12193]55    >>> 'Given storage path cannot be used' in browser.contents
[6611]56    True
[11254]57    >>> browser.getControl('Back to Data Center').click()
[6611]58
59
[8973]60Batch processing users
61======================
[4857]62
[8973]63    >>> browser.open('http://localhost/app/datacenter')
64
[9203]65Prepare a CSV file for users:
[8973]66
67    >>> open('users.csv', 'wb').write(
[8976]68    ... """name,title,public_name,email,phone,roles
[9310]69    ... uli,Uli Fouquet,Chief Developer,uli@abc.de,+49-234-567,[]
70    ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']"
[8973]71    ... """)
72
73Upload the file:
74
75    >>> import cStringIO
[9024]76    >>> browser.getLink('Upload data').click()
[8973]77    >>> filecontents = cStringIO.StringIO(
78    ...   open('users.csv', 'rb').read())
79    >>> filewidget = browser.getControl(name='uploadfile:file')
80    >>> filewidget.add_file(filecontents, 'text/plain', 'users.csv')
81    >>> browser.getControl(name='SUBMIT').click()
82
83Step 1: start batch processing:
84
[9024]85    >>> browser.getLink('Process data').click()
[13806]86    >>> browser.getLink('Switch maintenance mode').click()
[8973]87    >>> button = lookup_submit_value(
88    ...   'select', 'users_zope.mgr.csv', browser)
89    >>> button.click()
90
91Step 2: select a processor and mode:
92
93    >>> importerselect = browser.getControl(name='importer')
94    >>> importerselect.getControl('User Processor').selected = True
95    >>> modeselect = browser.getControl(name='mode')
96    >>> modeselect.getControl(value='create').selected = True
97    >>> browser.getControl('Proceed to step 3').click()
98
99Step 3: Fix headerlines
100
101We get informed that there are no problems with the current header:
102
103    >>> print browser.contents
104    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
105    ...
106    Header fields OK
107    ...
108
109The submit button is enabled:
110
111    >>> browser.getControl('Perform import').disabled
112    False
113
114    >>> browser.getControl('Perform import').click()
115
116Step 4: See import results
117
118The import was successful:
119
120    >>> print browser.contents
121    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
122    ...Successfully processed 2 rows...
123    ...Batch processing finished...
124    ...File:...users_zope.mgr.csv...
125
[9310]126User henrik has got the global roles:
127
128   >>> henrik = getRootFolder()['app']['users']['henrik']
129   >>> henrik.roles
[11947]130   ['waeup.ProductsOfficer', 'waeup.PortalManager', 'waeup.ImportManager']
[9310]131
[4899]132Pending files
133=============
134
135When an error occurs during an import, two files are generated: a CSV
136file with finished files and a CSV file with pending data. Both are
137stored in the appropriate subdirectories in datacenter. We try to
138create faculties, from which one already exists.
139
140Go to datacenter page:
141
142    >>> browser.open('http://localhost/app/datacenter')
143
[11947]144Prepare a CSV file for users:
[4899]145
[11947]146    >>> open('newusers.csv', 'wb').write(
147    ... """name,title,public_name,email,phone,roles
148    ... isouaba,Issoufou Abba Moussa,Chief Developer,isouaba@abc.de,+49-234-567,[]
149    ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']"
[12189]150    ... anne,Anne Palina,,anne@abc.de,+49-234-567,"['waeup.Nonsense']"
[4899]151    ... """)
152
153Upload the file:
154
155    >>> import cStringIO
[9024]156    >>> browser.getLink('Upload data').click()
[4899]157    >>> filecontents = cStringIO.StringIO(
[11947]158    ...   open('newusers.csv', 'rb').read())
[4899]159    >>> filewidget = browser.getControl(name='uploadfile:file')
[11947]160    >>> filewidget.add_file(filecontents, 'text/plain', 'newusers.csv')
[4899]161    >>> browser.getControl(name='SUBMIT').click()
162
[9310]163Since we now have a user with waeup.ImportManager role, an email has been sent:
164    >>> print browser.contents
165    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
166    ...
167    ...All import managers have been notified by email...
168    ...
169
[4899]170Step 1: start batch processing:
171
[9024]172    >>> browser.getLink('Process data').click()
[13806]173    >>> browser.getLink('Switch maintenance mode').click()
[4899]174    >>> button = lookup_submit_value(
[11947]175    ...   'select', 'newusers_zope.mgr.csv', browser)
[4899]176    >>> button.click()
177
178Step 2: select a processor and mode:
179
180    >>> importerselect = browser.getControl(name='importer')
[11947]181    >>> importerselect.getControl('User Processor').selected = True
[4899]182    >>> modeselect = browser.getControl(name='mode')
183    >>> modeselect.getControl(value='create').selected = True
[7705]184    >>> browser.getControl('Proceed to step 3').click()
[4899]185
186Step 3: Fix headerlines
187
188As there should be no problem with the headers, we can immediately
189perfom the import:
190
[7705]191    >>> browser.getControl('Perform import').click()
[4899]192
[12189]193Two lines could not be imported:
[4899]194
195    >>> print browser.contents
196    <!DOCTYPE html PUBLIC...
197    ...
[12189]198    ...Processing of 2 rows failed...
[4899]199    ...Successfully processed 1 rows...
200    ...
201
202Now there are two files as a result in datacenter storage's root and
203``finished`` dirs:
204
[11947]205    >>> pending_file = dc_path + '/newusers_zope.mgr.create.pending.csv'
[4899]206    >>> print open(pending_file).read()
[11947]207    name,roles,title,public_name,phone,email,--ERRORS--
208    henrik,"['waeup.PortalManager', 'waeup.ImportManager']",Henrik Bettermann,Admin,+49-234-567,henrik@abc.de,This object already exists. Skipping.
[12986]209    anne,['waeup.Nonsense'],Anne Palina,<IGNORE>,+49-234-567,anne@abc.de,roles: invalid role
[4899]210
[11947]211    >>> finished_file = dc_path + '/finished/newusers_zope.mgr.create.finished.csv'
[4899]212    >>> print open(finished_file).read()
[11947]213    name,roles,title,public_name,phone,email
[12986]214    isouaba,<IGNORE>,Issoufou Abba Moussa,Chief Developer,+49-234-567,isouaba@abc.de
[4899]215
216The finished-file contains the dataset we could import, while the
217pending file contains the dataset that failed, appended by an error
218message.
219
220
[4981]221Fixing the pending file
222-----------------------
223
[12189]224We 'edit' the pending file by removing anne and replacing henrik by gbenga
225and finish the import this way:
[4981]226
[11947]227    >>> open(dc_path + '/newusers_zope.mgr.create.pending.csv', 'wb').write(
228    ... """name,title,public_name,email,phone,roles
229    ... gbenga,Jason Bamidele,Chief Developer,gbenga@abc.de,+49-234-567,[]
[4981]230    ... """)
231
232Step 1: start batch processing:
233
234    >>> browser.open('http://localhost/app/datacenter')
[9024]235    >>> browser.getLink('Process data').click()
[13806]236    >>> browser.getLink('Switch maintenance mode').click()
[4981]237    >>> button = lookup_submit_value(
[11947]238    ...   'select', 'newusers_zope.mgr.create.pending.csv', browser)
[4981]239    >>> button.click()
240
241Step 2: select a processor and mode:
242
243    >>> importerselect = browser.getControl(name='importer')
[11947]244    >>> importerselect.getControl('User Processor').selected = True
[4981]245    >>> modeselect = browser.getControl(name='mode')
246    >>> modeselect.getControl(value='create').selected = True
[7705]247    >>> browser.getControl('Proceed to step 3').click()
[4981]248
249Step 3/4: Fix headerlines and import:
250
251As there should be no problem with the headers, we can immediately
252perfom the import:
253
[7705]254    >>> browser.getControl('Perform import').click()
[4981]255
256This time everything should work:
257
258    >>> print browser.contents
259    <!DOCTYPE html PUBLIC...
260    ...
261    ...Successfully processed 1 rows...
262    ...
263
264    >>> sorted(os.listdir(dc_path))
[8372]265    ['deleted', 'finished', 'logs', 'unfinished']
[4981]266
267    >>> os.listdir(dc_path + '/unfinished')
268    []
269
270    >>> sorted(os.listdir(dc_path + '/finished'))
[11947]271    ['newusers_zope.mgr.create.finished.csv', 'newusers_zope.mgr.csv', 'users_zope.mgr.create.finished.csv', 'users_zope.mgr.csv']
[4981]272
[9023]273Processed (finished) Files
274==========================
[4981]275
[9023]276    >>> browser.open('http://localhost/app/datacenter/processed')
[11947]277    >>> 'download?filename=finished/users_zope.mgr.create.finished.csv' in browser.contents
[9023]278    True
279
[6608]280Log Files
281=========
282
283    >>> browser.open('http://localhost/app/datacenter/logs')
284    >>> 'datacenter.log' in browser.contents
285    True
[6754]286    >>> browser.getControl('Show', index=0).click()
[6611]287    >>> browser.getControl('Back', index=0).click()
288    >>> browser.getControl('Back to Data Center').click()
289    >>> 'Storage path:' in browser.contents
290    True
[6608]291
292
[4857]293Clean up:
294
295    >>> import shutil
[6734]296    >>> shutil.rmtree(dc_path)
Note: See TracBrowser for help on using the repository browser.