Changeset 2259 for WAeUP_SRP


Ignore:
Timestamp:
19 Sep 2007, 15:23:58 (17 years ago)
Author:
joachim
Message:

new feature bypass catalog_queue

Location:
WAeUP_SRP/base
Files:
1 added
4 edited

Legend:

Unmodified
Added
Removed
  • WAeUP_SRP/base/PatchQueueCatalogProcessQueue.py

    r2235 r2259  
    11import sets
    22import logging
     3from types import StringType
     4from AccessControl.Permissions \
     5import manage_zcatalog_entries, view_management_screens
     6from AccessControl import ModuleSecurityInfo, ClassSecurityInfo
     7from AccessControl.SecurityInfo import ClassSecurityInformation
     8from Products.PageTemplates.PageTemplateFile import PageTemplateFile
    39from Products.QueueCatalog.CatalogEventQueue import CatalogEventQueue, EVENT_TYPES, ADDED_EVENTS
    410from Products.QueueCatalog.CatalogEventQueue import ADDED, CHANGED, CHANGED_ADDED, REMOVED
    511from Products.QueueCatalog.CatalogEventQueue import SAFE_POLICY, ALTERNATIVE_POLICY
    612from Products.QueueCatalog.QueueCatalog import cataloged
     13from Products.QueueCatalog.QueueCatalog import QueueCatalog
    714from ZODB.POSException import ConflictError
    815from ZEO.Exceptions import ClientDisconnected
    916logger = logging.getLogger('event.QueueCatalog')
     17security = ClassSecurityInformation()
    1018
    1119def __init__(self,
    1220             buckets=1009,
    1321             conflict_policy=SAFE_POLICY,
     22             bypass = False,
    1423             location = "portal_catalog_real"):
    1524    self._buckets = buckets
    1625    self._conflict_policy = conflict_policy
    1726    self._clearQueues()
     27    self._bypass = bypass
    1828    self._location = location
    19    
     29
     30security.declareProtected(view_management_screens, 'getBypassQueue')
     31def getBypassQueue(self):
     32    "get _by_pass"
     33    if not hasattr(self,"_bypass"):
     34        self._bypass = False
     35    return self._bypass
     36
     37security.declareProtected(view_management_screens, 'setBypassQueue')
     38def setBypassQueue(self, bypass=False):
     39    "set _bypass"
     40    self._bypass = bypass
     41
     42def catalog_object(self, obj, uid=None, idxs=None, update_metadata=1):
     43    # update_metadata=0 is ignored if the queued catalog is set to
     44    # update metadata during queue processing, rather than immediately
     45
     46    # similarly, limiting the idxs only limits the immediate indexes.  If
     47    # any work needs to be done in the queue processing, it will all be
     48    # done: we have not implemented partial indexing during queue
     49    # processing.  The only way to avoid any of it is to avoid all of it
     50    # (i.e., update metadata immediately and don't have any indexes to
     51    # update on the queued side).
     52
     53    # Make sure the current context is allowed to do this:
     54    catalog_object = self.getZCatalog('catalog_object')
     55
     56    if uid is None:
     57        uid = '/'.join(obj.getPhysicalPath())
     58    elif not isinstance(uid, StringType):
     59        uid = '/'.join(uid)
     60
     61    catalog = self.getZCatalog()
     62    cat_indexes = sets.Set(catalog.indexes())
     63    immediate_indexes = sets.Set(self._immediate_indexes)
     64    cat_indexes -= immediate_indexes
     65
     66    # The ZCatalog API doesn't allow us to distinguish between
     67    # adds and updates, so we have to try to figure this out
     68    # ourselves.
     69
     70    # There's a risk of a race here. What if there is a previously
     71    # unprocessed add event? If so, then this should be a changed
     72    # event. If we undo this transaction later, we'll generate a
     73    # remove event, when we should generate an add changed event.
     74    # To avoid this, we need to make sure we see consistent values
     75    # of the event queue. We also need to avoid resolving
     76    # (non-undo) conflicts of add events. This will slow things
     77    # down a bit, but adds should be relatively infrequent.
     78    bypass_queue = self.getBypassQueue()   
     79    # Now, try to decide if the catalog has the uid (path).
     80    already_cataloged = cataloged(catalog, uid)
     81    if not already_cataloged:
     82        # Looks like we should add, but maybe there's already a
     83        # pending add event. We'd better check the event queue:
     84        already_cataloged = (
     85            self._queues[hash(uid) % self._buckets].getEvent(uid) in
     86            ADDED_EVENTS)
     87    if idxs and already_cataloged:
     88        # if not already_cataloged, we index the whole thing
     89        idxs = sets.Set(idxs)
     90        immediate_indexes.intersection_update(idxs)
     91        cat_indexes.intersection_update(idxs)
     92
     93    immediate_metadata = self.getImmediateMetadataUpdate()
     94    if cat_indexes or update_metadata and not immediate_metadata:
     95        if bypass_queue:
     96            catalog_object(obj, uid, cat_indexes)
     97        else:
     98            import pdb;pdb.set_trace()
     99            self._update(uid, already_cataloged and CHANGED or ADDED)
     100        # Update some of the indexes immediately.
     101    elif immediate_indexes:
     102        # Update some of the indexes immediately.
     103        catalog.catalog_object(
     104            obj, uid, immediate_indexes,
     105            update_metadata=update_metadata and immediate_metadata)
     106    elif update_metadata and immediate_metadata:
     107        # if it is added, no point in doing the metadata, and it will be
     108        # done in the queue process anyway
     109        catalog._catalog.updateMetadata(obj, uid)
     110
    20111def _process_queue(self, queue, limit):
    21112    """Process a single queue"""
     
    65156
    66157    return count
    67 from Products.QueueCatalog.QueueCatalog import QueueCatalog
     158def manage_edit(self, title='', location='', immediate_indexes=(),
     159                immediate_removal=0, bucket_count=0, immediate_metadata=0,
     160                all_indexes=0, conflict_policy=SAFE_POLICY, RESPONSE=None,
     161               enable_bypass=0):
     162    """ Edit the instance """
     163    self.title = title
     164    self.setLocation(location or None)
     165    self.setImmediateIndexes(immediate_indexes)
     166    self.setImmediateRemoval(immediate_removal)
     167    self.setImmediateMetadataUpdate(immediate_metadata)
     168    self.setProcessAllIndexes(all_indexes)
     169    self.setConflictPolicy(conflict_policy)
     170    self.setBypassQueue(enable_bypass)
     171    if bucket_count:
     172        bucket_count = int(bucket_count)
     173        if bucket_count != self.getBucketCount():
     174            self.setBucketCount(bucket_count)
     175
     176    if RESPONSE is not None:
     177        RESPONSE.redirect('%s/manage_editForm?manage_tabs_message='
     178                            'Properties+changed' % self.absolute_url())
     179
    68180QueueCatalog.__init__ = __init__
     181QueueCatalog.getBypassQueue = getBypassQueue
     182QueueCatalog.getBypassQueue__roles__ = ['Manager', 'Owner',]
     183QueueCatalog.setBypassQueue = setBypassQueue
     184QueueCatalog.setBypassQueue__roles__ = ['Manager', 'Owner',]
     185QueueCatalog.catalog_object = catalog_object
    69186QueueCatalog._process_queue = _process_queue
     187QueueCatalog.manage_editForm = PageTemplateFile('zmi/qc_edit', globals())
     188QueueCatalog.manage_edit = manage_edit
    70189
  • WAeUP_SRP/base/WAeUPTool.py

    r2237 r2259  
    9393        log = open(path,"w")
    9494        return log
    95 
    96 ###)
     95    ###)
     96
     97    security.declareProtected(ModifyPortalContent,'bypassQueueCatalog') ###(
     98    def bypassQueueCatalog(self,enable=True):
     99        """bypass the QueueCatalog by setting all indexes to process imediate,
     100        if enable is True (default) the old settings are restored
     101        """
     102
     103    ###)
    97104
    98105    security.declareProtected(ModifyPortalContent,'measureOaT') ###(
     
    11791186
    11801187    security.declareProtected(ModifyPortalContent,"importData")###(
    1181     def importData(self,filename,name,edit=False):
     1188    def importData(self,filename,name,edit=False,bypass_queue_catalog=False):
    11821189        """load data from CSV values"""
    11831190        import transaction
     
    12501257                format_error = format + ',"%(Error)s"'
    12511258                format = '"%(id)s",'+ format
     1259                by_pass_queue = getattr(self.portal_catalog,"setBypassQueue",None)
     1260                if by_pass_queue is not None:
     1261                    bypass_queue(bypass_queue_catalog)
     1262                           
    12521263            dm = DataModel(item, adapters,context=self)
    12531264            ds = DataStructure(data=item,datamodel=dm)
     
    12931304                                                '\n'.join(not_imported))
    12941305        em = "Imported: %d, not imported: %d of total %d" % (total_imported,total_not_imported,total)
    1295         logger.info(em)
    12961306        return em
    12971307    ###)
  • WAeUP_SRP/base/skins/waeup_default/import.py

    r1774 r2259  
    55return html renderer + psm
    66"""
     7try:
     8    from Products.zdb import set_trace
     9except:
     10    def set_trace():
     11        pass
    712import DateTime
    813from urllib import urlencode
     
    1116
    1217validate = REQUEST.has_key("import")
    13 
     18enable = REQUEST.has_key("enable_bypass")
     19disable = REQUEST.has_key("disable_bypass")
     20#set_trace()
    1421lt = context.portal_layouts
    1522#pr = context.portal_registration
     
    2330                      commit=False,
    2431                      )
    25 if psm == 'invalid':
     32if enable:
     33    context.portal_catalog.setBypassQueue(1)
     34    psm = "QueueCatalog Bypass enabled"
     35    #return REQUEST.RESPONSE.redirect("%s" % context.absolute_url())
     36elif disable:
     37    context.portal_catalog.setBypassQueue(0)
     38    psm = "QueueCatalog Bypass disabled"
     39elif psm == 'invalid':
    2640    return context.import_form(rendered = res,
    2741                               psm = "Please correct your input!",
     
    3448                              )
    3549elif psm == 'valid':
    36     pass
    37 psm = context.waeup_tool.importData(ds.get('filename'),
     50    psm = context.waeup_tool.importData(ds.get('filename'),
    3851                                    ds.get('name'),
    3952                                    ds.get('import_or_edit')
  • WAeUP_SRP/base/skins/waeup_default/import_form.pt

    r1783 r2259  
    2121                 tal:attributes="value button"
    2222                 tal:condition="not:creation" />
     23          <span tal:define="bypass here/portal_catalog/getBypassQueue|nothing">
     24            <input type="submit" class="standalone"
     25                   name="enable_bypass" value="bypass QueueCatalog"
     26                   tal:condition="not:bypass" />
     27            <input type="submit" class="standalone"
     28                   name="disable_bypass" value="use QueueCatalog"
     29                   tal:condition="bypass" />
     30          </span>
    2331        </form>
    2432      </metal:block>
Note: See TracChangeset for help on using the changeset viewer.