source: main/waeup.ikoba/trunk/src/waeup/ikoba/doctests/batchprocessing_browser.txt @ 13100

Last change on this file since 13100 was 12993, checked in by Henrik Bettermann, 9 years ago

First adjustments for the upcoming Ikoba User Handbook.
File size: 9.2 KB
RevLine 
[12993]1Batch Processing via Browser
[4857]2****************************
3
[4869]4Preliminaries:
5
6We define a function that looks up a form with several submit buttons
7for the one with a given value (this functionality is missing in
8zope.testbrowser):
9
10    >>> def lookup_submit_value(name, value, browser):
11    ...   """Find a button with a certain value."""
12    ...   for num in range(0, 100):
13    ...     try:
14    ...       button = browser.getControl(name=name, index=num)
[4899]15    ...       if button.value.endswith(value):
[4869]16    ...         return button
17    ...     except IndexError:
18    ...       break
19    ...   return None
20
[4857]21Create a site:
22
[11954]23    >>> from waeup.ikoba.app import Company
24    >>> getRootFolder()['app'] = Company()
[9312]25    >>> from zope.component.hooks import setSite
26    >>> setSite(getRootFolder()['app'])
[4857]27
28Create a datacenter storage path:
29
30    >>> import os
31    >>> import tempfile
32    >>> dc_path = tempfile.mkdtemp()
33
34Log in:
35
36    >>> from zope.testbrowser.testing import Browser
37    >>> browser = Browser()
38    >>> browser.addHeader('Authorization', 'Basic mgr:mgrpw')
39    >>> browser.handleErrors = False
40
41Set datacenter path and deselect moving old data:
42
43    >>> browser.open('http://localhost/app')
44    >>> browser.getLink('Data Center').click()
45    >>> browser.getLink('Edit settings').click()
46    >>> browser.getControl(name='newpath').value = dc_path
47    >>> browser.getControl(name='move').value = False
48    >>> browser.getControl(name='save').click()
49
[6611]50Set non-usable datacenter path:
51
52    >>> browser.getLink('Edit settings').click()
53    >>> browser.getControl(name='newpath').value = '/'
54    >>> browser.getControl(name='save').click()
[12193]55    >>> 'Given storage path cannot be used' in browser.contents
[6611]56    True
[11254]57    >>> browser.getControl('Back to Data Center').click()
[6611]58
59
[8973]60Batch processing users
61======================
[4857]62
[8973]63    >>> browser.open('http://localhost/app/datacenter')
64
[9203]65Prepare a CSV file for users:
[8973]66
67    >>> open('users.csv', 'wb').write(
[8976]68    ... """name,title,public_name,email,phone,roles
[9310]69    ... uli,Uli Fouquet,Chief Developer,uli@abc.de,+49-234-567,[]
70    ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']"
[8973]71    ... """)
72
73Upload the file:
74
75    >>> import cStringIO
[9024]76    >>> browser.getLink('Upload data').click()
[8973]77    >>> filecontents = cStringIO.StringIO(
78    ...   open('users.csv', 'rb').read())
79    >>> filewidget = browser.getControl(name='uploadfile:file')
80    >>> filewidget.add_file(filecontents, 'text/plain', 'users.csv')
81    >>> browser.getControl(name='SUBMIT').click()
82
83Step 1: start batch processing:
84
[9024]85    >>> browser.getLink('Process data').click()
[8973]86    >>> button = lookup_submit_value(
87    ...   'select', 'users_zope.mgr.csv', browser)
88    >>> button.click()
89
90Step 2: select a processor and mode:
91
92    >>> importerselect = browser.getControl(name='importer')
93    >>> importerselect.getControl('User Processor').selected = True
94    >>> modeselect = browser.getControl(name='mode')
95    >>> modeselect.getControl(value='create').selected = True
96    >>> browser.getControl('Proceed to step 3').click()
97
98Step 3: Fix headerlines
99
100We get informed that there are no problems with the current header:
101
102    >>> print browser.contents
103    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
104    ...
105    Header fields OK
106    ...
107
108The submit button is enabled:
109
110    >>> browser.getControl('Perform import').disabled
111    False
112
113    >>> browser.getControl('Perform import').click()
114
115Step 4: See import results
116
117The import was successful:
118
119    >>> print browser.contents
120    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
121    ...Successfully processed 2 rows...
122    ...Batch processing finished...
123    ...File:...users_zope.mgr.csv...
124
[9310]125User henrik has got the global roles:
126
127   >>> henrik = getRootFolder()['app']['users']['henrik']
128   >>> henrik.roles
[11947]129   ['waeup.ProductsOfficer', 'waeup.PortalManager', 'waeup.ImportManager']
[9310]130
[4899]131Pending files
132=============
133
134When an error occurs during an import, two files are generated: a CSV
135file with finished files and a CSV file with pending data. Both are
136stored in the appropriate subdirectories in datacenter. We try to
137create faculties, from which one already exists.
138
139Go to datacenter page:
140
141    >>> browser.open('http://localhost/app/datacenter')
142
[11947]143Prepare a CSV file for users:
[4899]144
[11947]145    >>> open('newusers.csv', 'wb').write(
146    ... """name,title,public_name,email,phone,roles
147    ... isouaba,Issoufou Abba Moussa,Chief Developer,isouaba@abc.de,+49-234-567,[]
148    ... henrik,Henrik Bettermann,Admin,henrik@abc.de,+49-234-567,"['waeup.PortalManager', 'waeup.ImportManager']"
[12189]149    ... anne,Anne Palina,,anne@abc.de,+49-234-567,"['waeup.Nonsense']"
[4899]150    ... """)
151
152Upload the file:
153
154    >>> import cStringIO
[9024]155    >>> browser.getLink('Upload data').click()
[4899]156    >>> filecontents = cStringIO.StringIO(
[11947]157    ...   open('newusers.csv', 'rb').read())
[4899]158    >>> filewidget = browser.getControl(name='uploadfile:file')
[11947]159    >>> filewidget.add_file(filecontents, 'text/plain', 'newusers.csv')
[4899]160    >>> browser.getControl(name='SUBMIT').click()
161
[9310]162Since we now have a user with waeup.ImportManager role, an email has been sent:
163    >>> print browser.contents
164    <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
165    ...
166    ...All import managers have been notified by email...
167    ...
168
[4899]169Step 1: start batch processing:
170
[9024]171    >>> browser.getLink('Process data').click()
[4899]172    >>> button = lookup_submit_value(
[11947]173    ...   'select', 'newusers_zope.mgr.csv', browser)
[4899]174    >>> button.click()
175
176Step 2: select a processor and mode:
177
178    >>> importerselect = browser.getControl(name='importer')
[11947]179    >>> importerselect.getControl('User Processor').selected = True
[4899]180    >>> modeselect = browser.getControl(name='mode')
181    >>> modeselect.getControl(value='create').selected = True
[7705]182    >>> browser.getControl('Proceed to step 3').click()
[4899]183
184Step 3: Fix headerlines
185
186As there should be no problem with the headers, we can immediately
187perfom the import:
188
[7705]189    >>> browser.getControl('Perform import').click()
[4899]190
[12189]191Two lines could not be imported:
[4899]192
193    >>> print browser.contents
194    <!DOCTYPE html PUBLIC...
195    ...
[12189]196    ...Processing of 2 rows failed...
[4899]197    ...Successfully processed 1 rows...
198    ...
199
200Now there are two files as a result in datacenter storage's root and
201``finished`` dirs:
202
[11947]203    >>> pending_file = dc_path + '/newusers_zope.mgr.create.pending.csv'
[4899]204    >>> print open(pending_file).read()
[11947]205    name,roles,title,public_name,phone,email,--ERRORS--
206    henrik,"['waeup.PortalManager', 'waeup.ImportManager']",Henrik Bettermann,Admin,+49-234-567,henrik@abc.de,This object already exists. Skipping.
[12986]207    anne,['waeup.Nonsense'],Anne Palina,<IGNORE>,+49-234-567,anne@abc.de,roles: invalid role
[4899]208
[11947]209    >>> finished_file = dc_path + '/finished/newusers_zope.mgr.create.finished.csv'
[4899]210    >>> print open(finished_file).read()
[11947]211    name,roles,title,public_name,phone,email
[12986]212    isouaba,<IGNORE>,Issoufou Abba Moussa,Chief Developer,+49-234-567,isouaba@abc.de
[4899]213
214The finished-file contains the dataset we could import, while the
215pending file contains the dataset that failed, appended by an error
216message.
217
218
[4981]219Fixing the pending file
220-----------------------
221
[12189]222We 'edit' the pending file by removing anne and replacing henrik by gbenga
223and finish the import this way:
[4981]224
[11947]225    >>> open(dc_path + '/newusers_zope.mgr.create.pending.csv', 'wb').write(
226    ... """name,title,public_name,email,phone,roles
227    ... gbenga,Jason Bamidele,Chief Developer,gbenga@abc.de,+49-234-567,[]
[4981]228    ... """)
229
230Step 1: start batch processing:
231
232    >>> browser.open('http://localhost/app/datacenter')
[9024]233    >>> browser.getLink('Process data').click()
[4981]234    >>> button = lookup_submit_value(
[11947]235    ...   'select', 'newusers_zope.mgr.create.pending.csv', browser)
[4981]236    >>> button.click()
237
238Step 2: select a processor and mode:
239
240    >>> importerselect = browser.getControl(name='importer')
[11947]241    >>> importerselect.getControl('User Processor').selected = True
[4981]242    >>> modeselect = browser.getControl(name='mode')
243    >>> modeselect.getControl(value='create').selected = True
[7705]244    >>> browser.getControl('Proceed to step 3').click()
[4981]245
246Step 3/4: Fix headerlines and import:
247
248As there should be no problem with the headers, we can immediately
249perfom the import:
250
[7705]251    >>> browser.getControl('Perform import').click()
[4981]252
253This time everything should work:
254
255    >>> print browser.contents
256    <!DOCTYPE html PUBLIC...
257    ...
258    ...Successfully processed 1 rows...
259    ...
260
261    >>> sorted(os.listdir(dc_path))
[8372]262    ['deleted', 'finished', 'logs', 'unfinished']
[4981]263
264    >>> os.listdir(dc_path + '/unfinished')
265    []
266
267    >>> sorted(os.listdir(dc_path + '/finished'))
[11947]268    ['newusers_zope.mgr.create.finished.csv', 'newusers_zope.mgr.csv', 'users_zope.mgr.create.finished.csv', 'users_zope.mgr.csv']
[4981]269
[9023]270Processed (finished) Files
271==========================
[4981]272
[9023]273    >>> browser.open('http://localhost/app/datacenter/processed')
[11947]274    >>> 'download?filename=finished/users_zope.mgr.create.finished.csv' in browser.contents
[9023]275    True
276
[6608]277Log Files
278=========
279
280    >>> browser.open('http://localhost/app/datacenter/logs')
281    >>> 'datacenter.log' in browser.contents
282    True
[6754]283    >>> browser.getControl('Show', index=0).click()
[6611]284    >>> browser.getControl('Back', index=0).click()
285    >>> browser.getControl('Back to Data Center').click()
286    >>> 'Storage path:' in browser.contents
287    True
[6608]288
289
[4857]290Clean up:
291
292    >>> import shutil
[6734]293    >>> shutil.rmtree(dc_path)
Note: See TracBrowser for help on using the repository browser.