start project

backend/apps/search/indexing.py

@@ -30,7 +30,7 @@ from datetime import UTC, datetime
 from typing import Any
 
 from django.conf import settings
-from elasticsearch import Elasticsearch
+from elasticsearch import BadRequestError, Elasticsearch
 from elasticsearch.helpers import bulk
 
 from apps.profiles.models import Profile
@@ -227,8 +227,33 @@ def _default_queryset():  # noqa: ANN202
     )
 
 
+# Creating an index waits for the master to ack and the shard to be allocated,
+# which can take far longer than the per-request timeout used for searches.
+INDEX_CREATE_TIMEOUT = 120
+
+
 def create_index(client: Elasticsearch, name: str) -> None:
-    client.indices.create(index=name, settings=INDEX_SETTINGS, mappings=INDEX_MAPPINGS)
+    """
+    Create the physical index, tolerating a retried request.
+
+    ``indices.create`` is not idempotent, but the client runs with
+    ``retry_on_timeout``: when the cluster acks the creation more slowly than
+    the request timeout, the retry comes back as
+    ``resource_already_exists_exception`` and the rebuild fails even though the
+    index is there.  The name carries a timestamp generated moments earlier, so
+    an index already under that name can only be our own first attempt.
+    """
+    try:
+        client.options(request_timeout=INDEX_CREATE_TIMEOUT).indices.create(
+            index=name, settings=INDEX_SETTINGS, mappings=INDEX_MAPPINGS
+        )
+    except BadRequestError as exc:
+        if exc.error != "resource_already_exists_exception":
+            raise
+        logger.warning(
+            "index_create_retried",
+            extra={"event": "index_create_retried", "index": name},
+        )
 
 
 def rebuild_index(client: Elasticsearch | None = None) -> dict[str, Any]:
@@ -245,13 +270,19 @@ def rebuild_index(client: Elasticsearch | None = None) -> dict[str, Any]:
     create_index(client, new_index)
     logger.info("index_created", extra={"event": "index_created", "index": new_index})
 
-    succeeded, errors = bulk(
-        client,
-        ({"_index": new_index, **doc} for doc in iter_documents()),
-        raise_on_error=False,
-        stats_only=False,
-    )
-    client.indices.refresh(index=new_index)
+    # Anything that goes wrong from here on leaves an index nothing points at;
+    # drop it rather than accumulating one orphan per failed attempt.
+    try:
+        succeeded, errors = bulk(
+            client,
+            ({"_index": new_index, **doc} for doc in iter_documents()),
+            raise_on_error=False,
+            stats_only=False,
+        )
+        client.indices.refresh(index=new_index)
+    except Exception:
+        client.indices.delete(index=new_index, ignore_unavailable=True)
+        raise
 
     previous = _current_indices(client, alias)
     actions: list[dict[str, Any]] = [{"add": {"index": new_index, "alias": alias}}]