start project

backend/apps/search/backends/elastic.py

@@ -80,6 +80,18 @@ FILTER_FIELDS = {
     "companies": "company.keyword",
 }
 
+#: Maps a facet (as returned to the frontend) to the request attribute that
+#: selects values *within that same facet*.  Used to exclude a facet's own
+#: filter from its own aggregation - see ``_build_aggregations``.
+FACET_TO_FILTER_ATTR = {
+    "skills": "skills",
+    "job_title_role": "roles",
+    "seniority": "seniority",
+    "country": "countries",
+    "industry": "industries",
+    "company": "companies",
+}
+
 
 def build_query(request: SearchRequest) -> dict[str, Any]:
     """Translate a search request into an Elasticsearch request body."""
@@ -118,20 +130,26 @@ def build_query(request: SearchRequest) -> dict[str, Any]:
     else:
         must.append({"match_all": {}})
 
+    active_filters = _active_filters(request)
+    range_filter = _range_filter(request)
+
     body: dict[str, Any] = {
-        "query": {
-            "bool": {
-                "must": must,
-                "should": should,
-                "filter": _build_filters(request),
-            }
-        },
+        "query": {"bool": {"must": must, "should": should}},
         "from": request.offset,
         "size": request.page_size,
         "track_total_hits": True,
         "sort": _build_sort(request),
     }
 
+    # Filters live in ``post_filter``, not ``query.bool.filter``: post_filter
+    # narrows the hits and the total count *after* aggregations are computed,
+    # which is what lets a multi-select facet stay populated with its own
+    # sibling values - see ``_build_aggregations`` for the other half of the
+    # trick (each facet additionally excludes its own filter).
+    all_filters = [*active_filters.values(), *([range_filter] if range_filter else [])]
+    if all_filters:
+        body["post_filter"] = {"bool": {"filter": all_filters}}
+
     if request.q:
         body["highlight"] = {
             "pre_tags": ["<mark>"],
@@ -140,33 +158,35 @@ def build_query(request: SearchRequest) -> dict[str, Any]:
         }
 
     if request.include_facets:
-        body["aggs"] = _build_aggregations()
+        body["aggs"] = _build_aggregations(active_filters, range_filter)
 
     return body
 
 
-def _build_filters(request: SearchRequest) -> list[dict[str, Any]]:
-    filters: list[dict[str, Any]] = []
-
+def _active_filters(request: SearchRequest) -> dict[str, dict[str, Any]]:
+    """One ``terms`` clause per attribute that actually has a value selected."""
+    filters: dict[str, dict[str, Any]] = {}
     for attribute, field_name in FILTER_FIELDS.items():
         values = getattr(request, attribute)
         if values:
             # ``terms`` is OR within one filter; separate filters are ANDed,
             # which matches how users read the UI: "python OR django" skills,
             # AND located in Germany.
-            filters.append({"terms": {field_name: list(values)}})
-
-    if request.min_experience is not None or request.max_experience is not None:
-        bounds: dict[str, int] = {}
-        if request.min_experience is not None:
-            bounds["gte"] = request.min_experience
-        if request.max_experience is not None:
-            bounds["lte"] = request.max_experience
-        filters.append({"range": {"years_experience": bounds}})
-
+            filters[attribute] = {"terms": {field_name: list(values)}}
     return filters
 
 
+def _range_filter(request: SearchRequest) -> dict[str, Any] | None:
+    if request.min_experience is None and request.max_experience is None:
+        return None
+    bounds: dict[str, int] = {}
+    if request.min_experience is not None:
+        bounds["gte"] = request.min_experience
+    if request.max_experience is not None:
+        bounds["lte"] = request.max_experience
+    return {"range": {"years_experience": bounds}}
+
+
 def _build_sort(request: SearchRequest) -> list[Any]:
     if request.sort == SORT_NAME:
         return [{"full_name.keyword": "asc"}, {"id": "asc"}]
@@ -177,12 +197,35 @@ def _build_sort(request: SearchRequest) -> list[Any]:
     return ["_score", {"id": "asc"}]  # deterministic tie-break
 
 
-def _build_aggregations() -> dict[str, Any]:
+def _build_aggregations(
+    active_filters: dict[str, dict[str, Any]], range_filter: dict[str, Any] | None
+) -> dict[str, Any]:
+    """
+    One ``terms`` aggregation per facet, each scoped by every *other* active
+    filter - the standard multi-select faceting pattern.  Without this, a
+    facet's own filter (applied globally through ``post_filter``) would
+    already have narrowed the document set before the aggregation runs, so
+    ticking "Engineering" under Job family would make every other job family
+    option vanish from that same list instead of just narrowing the results.
+    The experience range applies to every facet unconditionally: it is not a
+    facet with its own checkboxes, so it is never excluded.
+    """
     size = settings.SEARCH["FACET_SIZE"]
-    return {
-        name: {"terms": {"field": field_name, "size": size, "order": {"_count": "desc"}}}
-        for name, field_name in FACET_FIELDS.items()
-    }
+    aggs: dict[str, Any] = {}
+    for name, field_name in FACET_FIELDS.items():
+        own_attribute = FACET_TO_FILTER_ATTR[name]
+        other_filters = [
+            clause for attribute, clause in active_filters.items() if attribute != own_attribute
+        ]
+        if range_filter is not None:
+            other_filters.append(range_filter)
+
+        terms_agg = {"terms": {"field": field_name, "size": size, "order": {"_count": "desc"}}}
+        if other_filters:
+            aggs[name] = {"filter": {"bool": {"filter": other_filters}}, "aggs": {name: terms_agg}}
+        else:
+            aggs[name] = terms_agg
+    return aggs
 
 
 class ElasticsearchBackend(SearchBackend):
@@ -269,11 +312,16 @@ class ElasticsearchBackend(SearchBackend):
 
     @staticmethod
     def _to_facets(aggregations: dict[str, Any]) -> dict[str, list[FacetBucket]]:
-        return {
-            name: [
+        facets: dict[str, list[FacetBucket]] = {}
+        for name, payload in aggregations.items():
+            # A facet scoped by other filters is wrapped in an outer `filter`
+            # aggregation (see ``_build_aggregations``); unwrap it to reach the
+            # `terms` buckets.  When no other filter was active it is the bare
+            # `terms` aggregation already.
+            terms_payload = payload.get(name, payload)
+            facets[name] = [
                 FacetBucket(key=str(bucket["key"]), count=bucket["doc_count"])
-                for bucket in payload.get("buckets", [])
+                for bucket in terms_payload.get("buckets", [])
                 if bucket["key"] != ""
             ]
-            for name, payload in aggregations.items()
-        }
+        return facets

backend/apps/search/backends/postgres.py

@@ -42,14 +42,26 @@ _TOKEN_RE = re.compile(r"[\w']+", re.UNICODE)
 _FACET_LIMIT = 25
 _MIN_TRIGRAM_SIMILARITY = 0.3
 
+#: Maps a facet (as returned to the frontend) to the request attribute that
+#: selects values *within that same facet*.  Used to exclude a facet's own
+#: filter from its own count - see ``_facets``.
+_FACET_TO_FILTER_ATTR = {
+    "skills": "skills",
+    "job_title_role": "roles",
+    "seniority": "seniority",
+    "country": "countries",
+    "industry": "industries",
+    "company": "companies",
+}
+
 
 class PostgresBackend(SearchBackend):
     name = "postgres"
 
     def search(self, request: SearchRequest) -> SearchResult:
         started = time.perf_counter()
-        queryset = self._apply_filters(Profile.objects.all(), request)
-        queryset = self._apply_keyword(queryset, request.q)
+        keyword_queryset = self._apply_keyword(Profile.objects.all(), request.q)
+        queryset = self._apply_filters(keyword_queryset, request)
         total = queryset.count()
 
         page = (
@@ -60,7 +72,7 @@ class PostgresBackend(SearchBackend):
             ]
         )
         hits = [self._to_hit(profile, request.q) for profile in page]
-        facets = self._facets(queryset) if request.include_facets else {}
+        facets = self._facets(keyword_queryset, request) if request.include_facets else {}
 
         return SearchResult(
             total=total,
@@ -87,23 +99,35 @@ class PostgresBackend(SearchBackend):
 
     # -- Query construction -------------------------------------------------
     @staticmethod
-    def _apply_filters(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
-        if request.skills:
+    def _apply_filters(
+        queryset: QuerySet[Profile], request: SearchRequest, *, exclude: str | None = None
+    ) -> QuerySet[Profile]:
+        """
+        Apply every active filter except ``exclude``.
+
+        ``exclude`` is how a facet's own count query avoids narrowing itself:
+        pass e.g. ``exclude="roles"`` when computing the job-family facet so
+        ticking "Engineering" does not make "Human resources" disappear from
+        that same list.  The main hits queryset calls this with no exclusion.
+        """
+        if request.skills and exclude != "skills":
             # Every selected skill widens the result set (OR), mirroring the
             # ``terms`` filter used by the Elasticsearch backend.
             queryset = queryset.filter(
                 skills__slug__in=[Skill.make_slug(s) for s in request.skills]
             )
-        if request.roles:
+        if request.roles and exclude != "roles":
             queryset = queryset.filter(job_title_role__in=request.roles)
-        if request.seniority:
+        if request.seniority and exclude != "seniority":
             queryset = queryset.filter(seniority_levels__overlap=list(request.seniority))
-        if request.countries:
+        if request.countries and exclude != "countries":
             queryset = queryset.filter(location_country__in=request.countries)
-        if request.industries:
+        if request.industries and exclude != "industries":
             queryset = queryset.filter(industry__in=request.industries)
-        if request.companies:
+        if request.companies and exclude != "companies":
             queryset = queryset.filter(current_company__name__in=request.companies)
+        # The experience range is not a facet with its own checkboxes, so it
+        # always applies - it is never the thing being excluded.
         if request.min_experience is not None:
             queryset = queryset.filter(years_experience__gte=request.min_experience)
         if request.max_experience is not None:
@@ -152,53 +176,65 @@ class PostgresBackend(SearchBackend):
         return queryset.order_by("-rank", "full_name", "id")
 
     # -- Facets -------------------------------------------------------------
-    def _facets(self, queryset: QuerySet[Profile]) -> dict[str, list[FacetBucket]]:
+    def _facets(
+        self, keyword_queryset: QuerySet[Profile], request: SearchRequest
+    ) -> dict[str, list[FacetBucket]]:
+        """
+        One count per facet, each computed on a queryset that excludes that
+        facet's *own* filter.
+
+        This is what keeps a multi-select facet fully populated once one of
+        its values is ticked: the job-family facet must still list "Human
+        resources" after "Engineering" is selected, while every *other* facet
+        (country, seniority, ...) does narrow to the engineering-only result
+        set.  Six independent queries instead of one is a deliberate
+        trade-off - see the module docstring on why this backend does not aim
+        to match Elasticsearch's scale, only its contract.
+        """
+        return {
+            name: self._facet_counts(
+                name, self._apply_filters(keyword_queryset, request, exclude=attribute)
+            )
+            for name, attribute in _FACET_TO_FILTER_ATTR.items()
+        }
+
+    @staticmethod
+    def _facet_counts(facet_name: str, queryset: QuerySet[Profile]) -> list[FacetBucket]:
         ids = list(queryset.values_list("id", flat=True))
         if not ids:
-            return {
-                name: []
-                for name in (
-                    "skills",
-                    "job_title_role",
-                    "seniority",
-                    "country",
-                    "industry",
-                    "company",
+            return []
+
+        if facet_name == "skills":
+            counter = Counter(
+                Skill.objects.filter(profile_skills__profile_id__in=ids).values_list(
+                    "name", flat=True
                 )
-            }
+            )
+        elif facet_name == "seniority":
+            counter = Counter()
+            for levels in Profile.objects.filter(id__in=ids).values_list(
+                "seniority_levels", flat=True
+            ):
+                for level in levels or []:
+                    counter[level] += 1
+        else:
+            column = {
+                "job_title_role": "job_title_role",
+                "country": "location_country",
+                "industry": "industry",
+                "company": "current_company__name",
+            }[facet_name]
+            counter = Counter(
+                value or "" for value in Profile.objects.filter(id__in=ids).values_list(
+                    column, flat=True
+                )
+            )
 
-        skills = Skill.objects.filter(profile_skills__profile_id__in=ids).values_list(
-            "name", flat=True
-        )
-        rows = Profile.objects.filter(id__in=ids).values(
-            "job_title_role", "location_country", "industry", "current_company__name",
-            "seniority_levels",
-        )  # fmt: skip
-
-        counters: dict[str, Counter] = {
-            "skills": Counter(skills),
-            "job_title_role": Counter(),
-            "seniority": Counter(),
-            "country": Counter(),
-            "industry": Counter(),
-            "company": Counter(),
-        }
-        for row in rows:
-            counters["job_title_role"][row["job_title_role"]] += 1
-            counters["country"][row["location_country"]] += 1
-            counters["industry"][row["industry"]] += 1
-            counters["company"][row["current_company__name"] or ""] += 1
-            for level in row["seniority_levels"] or []:
-                counters["seniority"][level] += 1
-
-        return {
-            name: [
-                FacetBucket(key=key, count=count)
-                for key, count in counter.most_common(_FACET_LIMIT)
-                if key
-            ]
-            for name, counter in counters.items()
-        }
+        return [
+            FacetBucket(key=key, count=count)
+            for key, count in counter.most_common(_FACET_LIMIT)
+            if key
+        ]
 
     # -- Result mapping -----------------------------------------------------
     @classmethod