Merge pull request #1877 from UlrichB22/add_whoosh_idx

Add 3rd minimal whoosh index for performance

Author: RogerHaase

src/moin/app.py

@@ -169,16 +169,8 @@ class ItemNameConverter(PathConverter):
     clock.stop("create_app flask-cache")
     # init storage
     clock.start("create_app init backends")
-    try:
-        init_backends(app)
-    except EmptyIndexError:
-        # create-instance has no index at start and index-* subcommands check the index individually
-        if info_name not in ["create-instance", "build-instance"] and not info_name.startswith("index-"):
-            clock.stop("create_app init backends")
-            clock.stop("create_app total")
-            logging.error("Error: Wiki index not found. Try 'moin help' or 'moin --help' to get further information.")
-            raise SystemExit(1)
-        logging.debug("Wiki index not found.")
+    # start init_backends
+    _init_backends(app, info_name, clock)
     clock.stop("create_app init backends")
     clock.start("create_app flask-babel")
     i18n_init(app)
@@ -212,6 +204,32 @@ def destroy_app(app):
     deinit_backends(app)
 
 
+def _init_backends(app, info_name, clock):
+    """
+    initialize the backends with exception handling
+    """
+    try:
+        init_backends(app)
+    except EmptyIndexError:
+        # create-instance has no index at start and index-* subcommands check the index individually
+        if info_name not in ["create-instance", "build-instance"] and not info_name.startswith("index-"):
+            missing_indexes = app.storage.missing_index_check()
+            if missing_indexes == "all":
+                logging.error(
+                    "Error: all wiki indexes missing. Try 'moin help' or 'moin --help' to get further information."
+                )
+            elif missing_indexes == "'latest_meta'":  # TODO: remove this check after 6-12 month
+                logging.error(
+                    "Error: Wiki index 'latest_meta' missing. Please see https://github.com/moinwiki/moin/pull/1877"
+                )
+            else:
+                logging.error(f"Error: Wiki index {missing_indexes} missing, please check.")
+            clock.stop("create_app init backends")
+            clock.stop("create_app total")
+            raise SystemExit(1)
+        logging.debug("Wiki index not found.")
+
+
 def init_backends(app, create_backend=False):
     """
     initialize the backends

src/moin/apps/frontend/views.py

@@ -585,6 +585,8 @@ def wrapper(item_name, rev):
             abort(404, item_name)
         if add_trail:
             flaskg.user.add_trail(item_name, aliases=item.meta.revision.fqnames)
+        """if view has been called with default rev=CURRENT we can avoid an index query in flash_if_item_deleted"""
+        item.is_current = rev == CURRENT
         return wrapped(item)
 
     return wrapper
@@ -602,6 +604,7 @@ def flash_if_item_deleted(item_name, rev_id, itemrev):
     Show flash info message if target item is deleted, show another message if revision is deleted.
     Return True if item is deleted or this revision is deleted.
     """
+    rev_id = CURRENT if getattr(itemrev, "is_current", False) else rev_id
     if not rev_id == CURRENT:
         ret = False
         current_item = Item.create(item_name, rev_id=CURRENT)
@@ -1503,13 +1506,13 @@ def name_initial(files, uppercase=False, lowercase=False):
     # request.args is a MultiDict instance, which degenerates into a normal
     # single-valued dict on most occasions (making the first value the *only*
     # value for a specific key) unless explicitly told to expose multiple
-    # values, eg. calling items with multi=True. See Werkzeug documentation for
+    # values, e.g. calling items with multi=True. See Werkzeug documentation for
     # more.
 
     form = IndexForm.from_flat(request.args.items(multi=True))
     selected_groups = form["contenttype"].value
     startswith = request.values.get("startswith")
-    dirs, files = item.get_index(startswith, selected_groups)
+    dirs, files = item.get_index(startswith, selected_groups, short=True)
     dirs_fullname = [x.fullname for x in dirs]
     initials = request.values.get("initials")
     if initials:

src/moin/cli/_tests/__init__.py

@@ -16,6 +16,7 @@
 
 from moin._tests import get_dirs
 from moin import log
+from moin.constants.keys import ALL_REVS, LATEST_META
 
 logging = log.getLogger(__name__)
 
@@ -131,7 +132,9 @@ def read_index_dump(out: str, latest=False):
             if item:
                 yield item
                 item = {}
-            if latest and "all_revs" in line:
+            if latest and ALL_REVS in line:
+                break
+            if LATEST_META in line:
                 break
             continue
         space_index = line.index(" ")

src/moin/cli/maint/index.py

@@ -13,7 +13,7 @@
 from flask.cli import FlaskGroup
 
 from moin.app import create_app, init_backends
-from moin.constants.keys import LATEST_REVS, ALL_REVS
+from moin.constants.keys import LATEST_REVS, ALL_REVS, LATEST_META
 from moin.utils.filesys import wiki_index_exists
 
 
@@ -134,7 +134,7 @@ def IndexDump(tmp, truncate):
         logging.error(ERR_NO_INDEX)
         raise SystemExit(1)
     logging.info("Index dump started")
-    for idx_name in [LATEST_REVS, ALL_REVS]:
+    for idx_name in [LATEST_REVS, ALL_REVS, LATEST_META]:
         print(f" {'-' * 10} {idx_name} {'-' * 60}")
         for kvs in app.storage.dump(tmp=tmp, idx_name=idx_name):
             for k, v in kvs:

src/moin/constants/keys.py

@@ -170,6 +170,7 @@
 # index names
 LATEST_REVS = "latest_revs"
 ALL_REVS = "all_revs"
+LATEST_META = "latest_meta"
 
 # values for ACTION key
 ACTION_SAVE = "SAVE"

src/moin/items/__init__.py

@@ -83,6 +83,7 @@
     TAGS,
     TEMPLATE,
     LATEST_REVS,
+    LATEST_META,
     EDIT_ROWS,
     FQNAMES,
     USERGROUP,
@@ -1186,7 +1187,6 @@ def handle_variables(self, data, meta):
         @rtype: string
         @return: new text of wikipage, variables replaced
         """
-        logging.debug(f"handle_variable data: {data!r}")
         if self.contenttype not in CONTENTTYPE_VARIABLES:
             return data
         if "@" not in data:
@@ -1196,6 +1196,7 @@ def handle_variables(self, data, meta):
         if TEMPLATE in meta["tags"]:
             return data
 
+        logging.debug(f"handle_variable data: {data!r}")  # log only if necessary
         item_name = request.path.split("/", 2)[-1]
         signature = flaskg.user.name0 if flaskg.user.valid else request.remote_addr
 
@@ -1356,7 +1357,7 @@ def build_index_query(self, startswith=None, selected_groups=None, isglobalindex
 
         return query
 
-    def get_index(self, startswith=None, selected_groups=None, regex=None):
+    def get_index(self, startswith=None, selected_groups=None, regex=None, short=False):
         """
         Get index enties for descendents of the matching items
 
@@ -1371,11 +1372,12 @@ def get_index(self, startswith=None, selected_groups=None, regex=None):
              - one for "dirs" (direct descendents that also contain descendents)
         """
         fqname = self.fqname
+        idx_name = LATEST_META if short else LATEST_REVS
         isglobalindex = not fqname.value or fqname.value == NAMESPACE_ALL
         query = self.build_index_query(startswith, selected_groups, isglobalindex)
         if not fqname.value.startswith(NAMESPACE_ALL + "/") and fqname.value != NAMESPACE_ALL:
             query = Term(NAMESPACE, fqname.namespace) & query
-        revs = flaskg.storage.search_meta(query, idx_name=LATEST_REVS, sortedby=NAME_EXACT, limit=None, regex=regex)
+        revs = flaskg.storage.search_meta(query, idx_name=idx_name, sortedby=NAME_EXACT, limit=None, regex=regex)
         return self.make_flat_index(revs, isglobalindex)
 
 

src/moin/macros/ItemList.py

@@ -131,7 +131,7 @@ def macro(self, content, arguments, page_url, alternative):
             item = ""
         # verify item exists and current user has read permission
         elif item != "":
-            if not flaskg.storage.get_item(**(split_fqname(item).query)):
+            if not flaskg.storage.get_item(short=True, **(split_fqname(item).query)):
                 err_msg = _("Item does not exist or read access blocked by ACLs: {0}").format(item)
                 return fail_message(err_msg, alternative)
 

src/moin/macros/_base.py

@@ -49,7 +49,7 @@ def get_item_names(name="", startswith="", kind="files", skiptag="", tag="", reg
         item = Item.create(name)
     except AccessDenied:
         abort(403)
-    dirs, files = item.get_index(startswith, regex=regex)
+    dirs, files = item.get_index(startswith, regex=regex, short=True)
     item_names = []
     if not kind or kind == "files" or kind == "both":
         for item in files: