Changeset 15739 for main/waeup.kofa/trunk/src/waeup/kofa/utils

Timestamp:

2 Nov 2019, 01:16:36 (5 years ago)

Author:

uli

Message:

Add helpers to reindex catalogs.

File:

• main/waeup.kofa/trunk/src/waeup/kofa/utils/helpers.py (modified) (3 diffs)

main/waeup.kofa/trunk/src/waeup/kofa/utils/helpers.py

@@ -35 +35 @@
 from zope.interface import implementedBy
 from zope.interface.interface import Method, Attribute
+from zope.intid.interfaces import IIntIds
 from zope.schema import getFieldNames
 from zope.schema.fieldproperty import FieldProperty
@@ -41 +42 @@
 from zope.pluggableauth.interfaces import IAuthenticatorPlugin
 from zope.formlib.widget import renderElement
+
 
 BUFSIZE = 8 * 1024
@@ -918 +920 @@
     parser.feed(html_code)
     return parser.form_vars
+
+
+def get_catalog_docids(cat):
+    """Get all docids for a given catalog `cat`.
+
+    Catalogs store the ids of objects they index. Get all of these object ids.
+    This function works at least for catalogs that provide field- and text
+    indexes only.
+    """
+    result = []
+    for index in cat.values():
+        try:
+            # FieldIndexes
+            result.extend(list(index._rev_index.keys()))
+        except AttributeError:
+            # TextIndexes
+            result.extend(list(index.index._docwords.keys()))
+    return set(result)
+
+
+def reindex_cat(cat):
+    """Reindex all objects stored in a catalog `cat`.
+
+    Regular catalogs try to reindex all stored object ids of a ZODB when asked
+    to reindex all contents. That can be overkill. This function reindexes only
+    those objects, that were already stored in a catalog. It was tested for
+    catalogs with at least 650000 objects.
+
+    Please note, that reindexing catalgos, can take a considerable amount of
+    time. 100.000 objects took about 12 minutes to reindex on a 16 GB machine.
+    """
+    d1 = datetime.datetime.now()
+    print("Collecting doc ids...")
+    uidutil = getUtility(IIntIds, context=cat)
+    uids = get_catalog_docids(cat)
+    print("Found %s entries..." % len(uids))
+    for docid in uids:
+        ob = uidutil.getObject(docid)
+        cat.index_doc(docid, ob)
+    d2 = datetime.datetime.now()
+    print("Finished. %s" % (d2 - d1))