feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class SearchConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.search"
|
||||
verbose_name = "Search"
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Backend factory - resolves ``SEARCH["BACKEND"]`` to a concrete strategy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import cache
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
from apps.search.backends.base import SearchBackend
|
||||
|
||||
__all__ = ["SearchBackend", "get_search_backend"]
|
||||
|
||||
|
||||
@cache
|
||||
def get_search_backend(name: str | None = None) -> SearchBackend:
|
||||
"""
|
||||
Return the configured backend.
|
||||
|
||||
Cached because both implementations hold a connection pool that should be
|
||||
created once per process, not once per request.
|
||||
"""
|
||||
backend = (name or settings.SEARCH["BACKEND"]).lower()
|
||||
|
||||
if backend == "elasticsearch":
|
||||
from apps.search.backends.elastic import ElasticsearchBackend
|
||||
|
||||
return ElasticsearchBackend()
|
||||
if backend == "postgres":
|
||||
from apps.search.backends.postgres import PostgresBackend
|
||||
|
||||
return PostgresBackend()
|
||||
|
||||
raise ImproperlyConfigured(
|
||||
f"Unknown SEARCH['BACKEND'] {backend!r}; expected 'elasticsearch' or 'postgres'."
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""The contract every search backend implements."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from apps.search.types import SearchRequest, SearchResult
|
||||
|
||||
|
||||
class SearchBackend(ABC):
|
||||
"""
|
||||
Strategy interface for keyword search.
|
||||
|
||||
Two implementations ship with the project:
|
||||
|
||||
* :class:`apps.search.backends.elastic.ElasticsearchBackend` - the default,
|
||||
used in every environment that runs the ELK stack.
|
||||
* :class:`apps.search.backends.postgres.PostgresBackend` - a full-text
|
||||
fallback that keeps the API (and the test suite) working without any
|
||||
external search service.
|
||||
|
||||
Keeping the surface this small is deliberate: the views, the serializers and
|
||||
the frontend are completely unaware of which engine answered the query.
|
||||
"""
|
||||
|
||||
#: Human readable identifier, surfaced in the API response and health check.
|
||||
name: str = "base"
|
||||
|
||||
@abstractmethod
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
"""Execute a search and return ranked hits plus facet counts."""
|
||||
|
||||
@abstractmethod
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
"""Return type-ahead completions for a partial name or skill."""
|
||||
|
||||
@abstractmethod
|
||||
def ping(self) -> bool:
|
||||
"""Raise or return ``False`` when the backend is not usable."""
|
||||
@@ -0,0 +1,279 @@
|
||||
"""
|
||||
Elasticsearch search backend.
|
||||
|
||||
The query builder is a pure function (:func:`build_query`) that turns a
|
||||
:class:`~apps.search.types.SearchRequest` into an Elasticsearch request body.
|
||||
Keeping it free of I/O means the ranking logic is unit-testable without a
|
||||
running cluster - see ``tests/test_query_builder.py``.
|
||||
|
||||
Ranking model
|
||||
-------------
|
||||
``must`` the keyword must match somewhere, with per-field boosts that encode
|
||||
"a name hit is worth more than a summary hit", plus ``fuzziness:
|
||||
AUTO`` so ``pyhton`` still finds ``python``.
|
||||
``should`` an exact phrase match on the strong fields adds a large bonus, which
|
||||
pushes "product manager" above documents that merely contain both
|
||||
words far apart.
|
||||
``filter`` structured constraints (skills, role, seniority, country, industry,
|
||||
employer, years of experience). They live in the filter context, so
|
||||
they are cached by Elasticsearch and contribute nothing to the score.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import ApiError, Elasticsearch, TransportError
|
||||
|
||||
from apps.core.exceptions import SearchBackendError
|
||||
from apps.search.backends.base import SearchBackend
|
||||
from apps.search.indexing import alias_name, get_client
|
||||
from apps.search.types import (
|
||||
FACET_FIELDS,
|
||||
SORT_EXPERIENCE_ASC,
|
||||
SORT_EXPERIENCE_DESC,
|
||||
SORT_NAME,
|
||||
FacetBucket,
|
||||
SearchHit,
|
||||
SearchRequest,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Field boosts for the free-text part of the query.
|
||||
SEARCH_FIELDS = [
|
||||
"full_name^6",
|
||||
"job_title^4",
|
||||
"skills^3.5",
|
||||
"company^2.5",
|
||||
"past_companies^1.5",
|
||||
"industry^1.5",
|
||||
"majors^1.5",
|
||||
"education",
|
||||
"location_name",
|
||||
"headline^2",
|
||||
"summary",
|
||||
"interests",
|
||||
]
|
||||
|
||||
#: Narrower field set for the phrase bonus - phrases in a summary are noise.
|
||||
PHRASE_FIELDS = ["full_name^8", "job_title^6", "skills^5", "company^4", "headline^3"]
|
||||
|
||||
HIGHLIGHT_FIELDS = {
|
||||
"full_name": {"number_of_fragments": 0},
|
||||
"job_title": {"number_of_fragments": 0},
|
||||
"skills": {"number_of_fragments": 3},
|
||||
"summary": {"fragment_size": 160, "number_of_fragments": 1},
|
||||
}
|
||||
|
||||
#: Maps an API filter to the indexed keyword field it constrains.
|
||||
FILTER_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"roles": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"countries": "location_country",
|
||||
"industries": "industry.keyword",
|
||||
"companies": "company.keyword",
|
||||
}
|
||||
|
||||
|
||||
def build_query(request: SearchRequest) -> dict[str, Any]:
|
||||
"""Translate a search request into an Elasticsearch request body."""
|
||||
must: list[dict[str, Any]] = []
|
||||
should: list[dict[str, Any]] = []
|
||||
|
||||
if request.q:
|
||||
must.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": request.q,
|
||||
"type": "best_fields",
|
||||
"fields": SEARCH_FIELDS,
|
||||
"operator": "or",
|
||||
# One-word queries must match; longer queries tolerate a
|
||||
# missing term so "senior python django" still returns
|
||||
# people who only list two of the three.
|
||||
"minimum_should_match": "2<70%",
|
||||
"fuzziness": "AUTO",
|
||||
"prefix_length": 1,
|
||||
"max_expansions": 30,
|
||||
}
|
||||
}
|
||||
)
|
||||
should.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": request.q,
|
||||
"type": "phrase",
|
||||
"fields": PHRASE_FIELDS,
|
||||
"slop": 1,
|
||||
"boost": 4,
|
||||
}
|
||||
}
|
||||
)
|
||||
else:
|
||||
must.append({"match_all": {}})
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must,
|
||||
"should": should,
|
||||
"filter": _build_filters(request),
|
||||
}
|
||||
},
|
||||
"from": request.offset,
|
||||
"size": request.page_size,
|
||||
"track_total_hits": True,
|
||||
"sort": _build_sort(request),
|
||||
}
|
||||
|
||||
if request.q:
|
||||
body["highlight"] = {
|
||||
"pre_tags": ["<mark>"],
|
||||
"post_tags": ["</mark>"],
|
||||
"fields": HIGHLIGHT_FIELDS,
|
||||
}
|
||||
|
||||
if request.include_facets:
|
||||
body["aggs"] = _build_aggregations()
|
||||
|
||||
return body
|
||||
|
||||
|
||||
def _build_filters(request: SearchRequest) -> list[dict[str, Any]]:
|
||||
filters: list[dict[str, Any]] = []
|
||||
|
||||
for attribute, field_name in FILTER_FIELDS.items():
|
||||
values = getattr(request, attribute)
|
||||
if values:
|
||||
# ``terms`` is OR within one filter; separate filters are ANDed,
|
||||
# which matches how users read the UI: "python OR django" skills,
|
||||
# AND located in Germany.
|
||||
filters.append({"terms": {field_name: list(values)}})
|
||||
|
||||
if request.min_experience is not None or request.max_experience is not None:
|
||||
bounds: dict[str, int] = {}
|
||||
if request.min_experience is not None:
|
||||
bounds["gte"] = request.min_experience
|
||||
if request.max_experience is not None:
|
||||
bounds["lte"] = request.max_experience
|
||||
filters.append({"range": {"years_experience": bounds}})
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def _build_sort(request: SearchRequest) -> list[Any]:
|
||||
if request.sort == SORT_NAME:
|
||||
return [{"full_name.keyword": "asc"}, {"id": "asc"}]
|
||||
if request.sort == SORT_EXPERIENCE_DESC:
|
||||
return [{"years_experience": {"order": "desc", "missing": "_last"}}, "_score"]
|
||||
if request.sort == SORT_EXPERIENCE_ASC:
|
||||
return [{"years_experience": {"order": "asc", "missing": "_last"}}, "_score"]
|
||||
return ["_score", {"id": "asc"}] # deterministic tie-break
|
||||
|
||||
|
||||
def _build_aggregations() -> dict[str, Any]:
|
||||
size = settings.SEARCH["FACET_SIZE"]
|
||||
return {
|
||||
name: {"terms": {"field": field_name, "size": size, "order": {"_count": "desc"}}}
|
||||
for name, field_name in FACET_FIELDS.items()
|
||||
}
|
||||
|
||||
|
||||
class ElasticsearchBackend(SearchBackend):
|
||||
name = "elasticsearch"
|
||||
|
||||
def __init__(self, client: Elasticsearch | None = None) -> None:
|
||||
self._client = client or get_client()
|
||||
self._index = alias_name()
|
||||
|
||||
# -- SearchBackend ------------------------------------------------------
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
body = build_query(request)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = self._client.search(index=self._index, **body)
|
||||
except (ApiError, TransportError) as exc:
|
||||
logger.exception("search_backend_error", extra={"event": "search_backend_error"})
|
||||
raise SearchBackendError() from exc
|
||||
|
||||
took_ms = int((time.perf_counter() - started) * 1000)
|
||||
hits = [self._to_hit(raw) for raw in response["hits"]["hits"]]
|
||||
return SearchResult(
|
||||
total=response["hits"]["total"]["value"],
|
||||
hits=hits,
|
||||
facets=self._to_facets(response.get("aggregations", {})),
|
||||
took_ms=took_ms,
|
||||
backend=self.name,
|
||||
)
|
||||
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
if not prefix:
|
||||
return []
|
||||
try:
|
||||
response = self._client.search(
|
||||
index=self._index,
|
||||
size=limit,
|
||||
query={
|
||||
"bool": {
|
||||
"should": [
|
||||
{"match": {"full_name.autocomplete": {"query": prefix, "boost": 2}}},
|
||||
{"match": {"skills.autocomplete": prefix}},
|
||||
],
|
||||
"minimum_should_match": 1,
|
||||
}
|
||||
},
|
||||
_source=["full_name", "job_title"],
|
||||
)
|
||||
except (ApiError, TransportError) as exc:
|
||||
raise SearchBackendError() from exc
|
||||
|
||||
seen: list[str] = []
|
||||
for raw in response["hits"]["hits"]:
|
||||
value = raw["_source"].get("full_name", "")
|
||||
if value and value not in seen:
|
||||
seen.append(value)
|
||||
return seen[:limit]
|
||||
|
||||
def ping(self) -> bool:
|
||||
if not self._client.ping():
|
||||
raise SearchBackendError("Elasticsearch did not answer a ping.")
|
||||
return True
|
||||
|
||||
# -- Mapping helpers ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _to_hit(raw: dict[str, Any]) -> SearchHit:
|
||||
source = raw["_source"]
|
||||
return SearchHit(
|
||||
id=source["id"],
|
||||
score=raw.get("_score") or 0.0,
|
||||
full_name=source.get("full_name", ""),
|
||||
headline=source.get("headline", ""),
|
||||
job_title=source.get("job_title", ""),
|
||||
job_title_role=source.get("job_title_role", ""),
|
||||
seniority_levels=source.get("seniority_levels", []),
|
||||
company=source.get("company", ""),
|
||||
industry=source.get("industry", ""),
|
||||
location_name=source.get("location_name", ""),
|
||||
location_country=source.get("location_country", ""),
|
||||
years_experience=source.get("years_experience"),
|
||||
linkedin_url=source.get("linkedin_url", ""),
|
||||
skills=source.get("skills", [])[:12],
|
||||
highlight=raw.get("highlight", {}),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _to_facets(aggregations: dict[str, Any]) -> dict[str, list[FacetBucket]]:
|
||||
return {
|
||||
name: [
|
||||
FacetBucket(key=str(bucket["key"]), count=bucket["doc_count"])
|
||||
for bucket in payload.get("buckets", [])
|
||||
if bucket["key"] != ""
|
||||
]
|
||||
for name, payload in aggregations.items()
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
"""
|
||||
PostgreSQL full-text search backend.
|
||||
|
||||
Why it exists
|
||||
-------------
|
||||
Elasticsearch is the primary engine, but a search API that hard-fails without
|
||||
it is fragile: the test suite would need a cluster, a reviewer would need one to
|
||||
run the project, and an Elasticsearch outage would take the whole product down.
|
||||
Setting ``SEARCH_BACKEND=postgres`` switches to this implementation, which is
|
||||
built on the ``tsvector`` maintained by the importer plus a trigram similarity
|
||||
pass for typo tolerance.
|
||||
|
||||
It is intentionally *not* a re-implementation of Elasticsearch: ranking is
|
||||
coarser and there is no stemming beyond what the ``english`` text search
|
||||
configuration provides. For a 300 profile dataset the difference is barely
|
||||
visible, which is exactly why it is a good fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
|
||||
from django.db.models import Case, F, FloatField, Q, QuerySet, Value, When
|
||||
from django.db.models.functions import Greatest
|
||||
|
||||
from apps.profiles.models import Profile, Skill
|
||||
from apps.search.backends.base import SearchBackend
|
||||
from apps.search.types import (
|
||||
SORT_EXPERIENCE_ASC,
|
||||
SORT_EXPERIENCE_DESC,
|
||||
SORT_NAME,
|
||||
FacetBucket,
|
||||
SearchHit,
|
||||
SearchRequest,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w']+", re.UNICODE)
|
||||
_FACET_LIMIT = 25
|
||||
_MIN_TRIGRAM_SIMILARITY = 0.3
|
||||
|
||||
|
||||
class PostgresBackend(SearchBackend):
|
||||
name = "postgres"
|
||||
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
started = time.perf_counter()
|
||||
queryset = self._apply_filters(Profile.objects.all(), request)
|
||||
queryset = self._apply_keyword(queryset, request.q)
|
||||
total = queryset.count()
|
||||
|
||||
page = (
|
||||
self._apply_sort(queryset, request)
|
||||
.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill")[
|
||||
request.offset : request.offset + request.page_size
|
||||
]
|
||||
)
|
||||
hits = [self._to_hit(profile, request.q) for profile in page]
|
||||
facets = self._facets(queryset) if request.include_facets else {}
|
||||
|
||||
return SearchResult(
|
||||
total=total,
|
||||
hits=hits,
|
||||
facets=facets,
|
||||
took_ms=int((time.perf_counter() - started) * 1000),
|
||||
backend=self.name,
|
||||
)
|
||||
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
if not prefix:
|
||||
return []
|
||||
return list(
|
||||
Profile.objects.filter(
|
||||
Q(full_name__istartswith=prefix) | Q(full_name__icontains=f" {prefix}")
|
||||
)
|
||||
.order_by("full_name")
|
||||
.values_list("full_name", flat=True)[:limit]
|
||||
)
|
||||
|
||||
def ping(self) -> bool:
|
||||
Profile.objects.exists()
|
||||
return True
|
||||
|
||||
# -- Query construction -------------------------------------------------
|
||||
@staticmethod
|
||||
def _apply_filters(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
|
||||
if request.skills:
|
||||
# Every selected skill widens the result set (OR), mirroring the
|
||||
# ``terms`` filter used by the Elasticsearch backend.
|
||||
queryset = queryset.filter(
|
||||
skills__slug__in=[Skill.make_slug(s) for s in request.skills]
|
||||
)
|
||||
if request.roles:
|
||||
queryset = queryset.filter(job_title_role__in=request.roles)
|
||||
if request.seniority:
|
||||
queryset = queryset.filter(seniority_levels__overlap=list(request.seniority))
|
||||
if request.countries:
|
||||
queryset = queryset.filter(location_country__in=request.countries)
|
||||
if request.industries:
|
||||
queryset = queryset.filter(industry__in=request.industries)
|
||||
if request.companies:
|
||||
queryset = queryset.filter(current_company__name__in=request.companies)
|
||||
if request.min_experience is not None:
|
||||
queryset = queryset.filter(years_experience__gte=request.min_experience)
|
||||
if request.max_experience is not None:
|
||||
queryset = queryset.filter(years_experience__lte=request.max_experience)
|
||||
return queryset.distinct()
|
||||
|
||||
@staticmethod
|
||||
def _apply_keyword(queryset: QuerySet[Profile], term: str) -> QuerySet[Profile]:
|
||||
if not term:
|
||||
return queryset.annotate(rank=Value(0.0, output_field=FloatField()))
|
||||
|
||||
# ``websearch`` accepts the syntax users already know from search
|
||||
# engines: quoted phrases, ``or``, and ``-excluded``.
|
||||
query = SearchQuery(term, search_type="websearch", config="english")
|
||||
return (
|
||||
queryset.annotate(
|
||||
text_rank=SearchRank(F("search_vector"), query),
|
||||
name_similarity=TrigramSimilarity("full_name", term),
|
||||
)
|
||||
.annotate(
|
||||
rank=Greatest(
|
||||
F("text_rank"),
|
||||
# A close name match is worth as much as a strong text hit,
|
||||
# which is what makes "jospeh" still find "joseph".
|
||||
Case(
|
||||
When(
|
||||
name_similarity__gte=_MIN_TRIGRAM_SIMILARITY, then=F("name_similarity")
|
||||
),
|
||||
default=Value(0.0),
|
||||
output_field=FloatField(),
|
||||
),
|
||||
output_field=FloatField(),
|
||||
)
|
||||
)
|
||||
.filter(Q(search_vector=query) | Q(name_similarity__gte=_MIN_TRIGRAM_SIMILARITY))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_sort(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
|
||||
if request.sort == SORT_NAME:
|
||||
return queryset.order_by("full_name", "id")
|
||||
if request.sort == SORT_EXPERIENCE_DESC:
|
||||
return queryset.order_by(F("years_experience").desc(nulls_last=True), "id")
|
||||
if request.sort == SORT_EXPERIENCE_ASC:
|
||||
return queryset.order_by(F("years_experience").asc(nulls_last=True), "id")
|
||||
return queryset.order_by("-rank", "full_name", "id")
|
||||
|
||||
# -- Facets -------------------------------------------------------------
|
||||
def _facets(self, queryset: QuerySet[Profile]) -> dict[str, list[FacetBucket]]:
|
||||
ids = list(queryset.values_list("id", flat=True))
|
||||
if not ids:
|
||||
return {
|
||||
name: []
|
||||
for name in (
|
||||
"skills",
|
||||
"job_title_role",
|
||||
"seniority",
|
||||
"country",
|
||||
"industry",
|
||||
"company",
|
||||
)
|
||||
}
|
||||
|
||||
skills = Skill.objects.filter(profile_skills__profile_id__in=ids).values_list(
|
||||
"name", flat=True
|
||||
)
|
||||
rows = Profile.objects.filter(id__in=ids).values(
|
||||
"job_title_role", "location_country", "industry", "current_company__name",
|
||||
"seniority_levels",
|
||||
) # fmt: skip
|
||||
|
||||
counters: dict[str, Counter] = {
|
||||
"skills": Counter(skills),
|
||||
"job_title_role": Counter(),
|
||||
"seniority": Counter(),
|
||||
"country": Counter(),
|
||||
"industry": Counter(),
|
||||
"company": Counter(),
|
||||
}
|
||||
for row in rows:
|
||||
counters["job_title_role"][row["job_title_role"]] += 1
|
||||
counters["country"][row["location_country"]] += 1
|
||||
counters["industry"][row["industry"]] += 1
|
||||
counters["company"][row["current_company__name"] or ""] += 1
|
||||
for level in row["seniority_levels"] or []:
|
||||
counters["seniority"][level] += 1
|
||||
|
||||
return {
|
||||
name: [
|
||||
FacetBucket(key=key, count=count)
|
||||
for key, count in counter.most_common(_FACET_LIMIT)
|
||||
if key
|
||||
]
|
||||
for name, counter in counters.items()
|
||||
}
|
||||
|
||||
# -- Result mapping -----------------------------------------------------
|
||||
@classmethod
|
||||
def _to_hit(cls, profile: Profile, term: str) -> SearchHit:
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()[:12]]
|
||||
return SearchHit(
|
||||
id=profile.id,
|
||||
score=round(float(getattr(profile, "rank", 0.0) or 0.0), 4),
|
||||
full_name=profile.full_name,
|
||||
headline=profile.headline,
|
||||
job_title=profile.job_title,
|
||||
job_title_role=profile.job_title_role,
|
||||
seniority_levels=profile.seniority_levels,
|
||||
company=profile.current_company.name if profile.current_company_id else "",
|
||||
industry=profile.industry,
|
||||
location_name=profile.location_name,
|
||||
location_country=profile.location_country,
|
||||
years_experience=profile.years_experience,
|
||||
linkedin_url=profile.linkedin_url,
|
||||
skills=skills,
|
||||
highlight=cls._highlight(term, profile, skills),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _highlight(term: str, profile: Profile, skills: list[str]) -> dict[str, list[str]]:
|
||||
"""
|
||||
Approximate Elasticsearch's highlighter.
|
||||
|
||||
PostgreSQL offers ``ts_headline`` but it needs an extra round trip per
|
||||
row; for the fields shown on a result card a simple token wrap gives the
|
||||
same visual result at no query cost.
|
||||
"""
|
||||
tokens = [token for token in _TOKEN_RE.findall(term.lower()) if len(token) > 1]
|
||||
if not tokens:
|
||||
return {}
|
||||
pattern = re.compile("|".join(re.escape(token) for token in tokens), re.IGNORECASE)
|
||||
|
||||
highlight: dict[str, list[str]] = {}
|
||||
for field_name, value in (
|
||||
("full_name", profile.full_name),
|
||||
("job_title", profile.job_title),
|
||||
):
|
||||
if value and pattern.search(value):
|
||||
highlight[field_name] = [pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", value)]
|
||||
matched_skills = [skill for skill in skills if pattern.search(skill)]
|
||||
if matched_skills:
|
||||
highlight["skills"] = [
|
||||
pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", skill)
|
||||
for skill in matched_skills[:3]
|
||||
]
|
||||
return highlight
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Elasticsearch index definition and (re)build helpers.
|
||||
|
||||
Index design
|
||||
------------
|
||||
* One document per profile, denormalised: the fields a recruiter searches on
|
||||
(name, title, employer, skills, education, location) all live in the same
|
||||
document, so a query never needs a join.
|
||||
* Text fields are analysed with an English analyzer (lowercase, ASCII folding,
|
||||
stop words, stemming) so "engineering" matches "engineer".
|
||||
* Every field that is *filtered* or *aggregated* on has a ``keyword``
|
||||
counterpart with a lowercase normaliser - analysed text cannot be used for
|
||||
exact term filters or facet counts.
|
||||
* ``full_name.autocomplete`` uses an edge n-gram analyzer for the type-ahead;
|
||||
the search analyzer is deliberately different (``standard``) so the query
|
||||
term is not itself n-grammed.
|
||||
* **No personal contact data is indexed.** E-mail addresses and phone numbers
|
||||
stay in PostgreSQL, which removes any chance of confirming someone's e-mail
|
||||
through a crafted search query.
|
||||
|
||||
Rebuilds use an alias plus a timestamped physical index, so a reindex never
|
||||
leaves the API pointing at a half-populated index and can be rolled back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch.helpers import bulk
|
||||
|
||||
from apps.profiles.models import Profile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INDEX_SETTINGS: dict[str, Any] = {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"refresh_interval": "1s",
|
||||
"max_result_window": 10_000,
|
||||
"analysis": {
|
||||
"filter": {
|
||||
"english_stop": {"type": "stop", "stopwords": "_english_"},
|
||||
"english_stemmer": {"type": "stemmer", "language": "english"},
|
||||
"english_possessive": {"type": "stemmer", "language": "possessive_english"},
|
||||
"edge_ngrams": {"type": "edge_ngram", "min_gram": 2, "max_gram": 15},
|
||||
},
|
||||
"normalizer": {
|
||||
"lowercase_normalizer": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
}
|
||||
},
|
||||
"analyzer": {
|
||||
"english_text": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": [
|
||||
"lowercase",
|
||||
"asciifolding",
|
||||
"english_possessive",
|
||||
"english_stop",
|
||||
"english_stemmer",
|
||||
],
|
||||
},
|
||||
"autocomplete_index": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding", "edge_ngrams"],
|
||||
},
|
||||
"autocomplete_search": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
KEYWORD = {"type": "keyword", "normalizer": "lowercase_normalizer"}
|
||||
|
||||
INDEX_MAPPINGS: dict[str, Any] = {
|
||||
"dynamic": "strict", # reject unexpected fields instead of guessing a type
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"full_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"headline": {"type": "text", "analyzer": "english_text"},
|
||||
"job_title": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"job_title_role": KEYWORD,
|
||||
"job_title_sub_role": KEYWORD,
|
||||
"seniority_levels": KEYWORD,
|
||||
"company": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"past_companies": {"type": "text", "analyzer": "english_text"},
|
||||
"industry": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"skills": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"education": {"type": "text", "analyzer": "english_text"},
|
||||
"degrees": KEYWORD,
|
||||
"majors": {"type": "text", "analyzer": "english_text", "fields": {"keyword": KEYWORD}},
|
||||
"summary": {"type": "text", "analyzer": "english_text"},
|
||||
"interests": {"type": "text", "analyzer": "english_text"},
|
||||
"languages": KEYWORD,
|
||||
"location_name": {"type": "text", "analyzer": "english_text"},
|
||||
"location_locality": KEYWORD,
|
||||
"location_region": KEYWORD,
|
||||
"location_country": KEYWORD,
|
||||
"location_continent": KEYWORD,
|
||||
"years_experience": {"type": "integer"},
|
||||
"connections": {"type": "integer"},
|
||||
"linkedin_url": {"type": "keyword", "index": False},
|
||||
"indexed_at": {"type": "date"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_client() -> Elasticsearch:
|
||||
"""Build an Elasticsearch client from settings (auth optional in dev)."""
|
||||
config = settings.SEARCH["ELASTICSEARCH"]
|
||||
kwargs: dict[str, Any] = {
|
||||
"hosts": config["HOSTS"],
|
||||
"request_timeout": config["TIMEOUT"],
|
||||
"max_retries": config["MAX_RETRIES"],
|
||||
"retry_on_timeout": True,
|
||||
}
|
||||
if config["USERNAME"] and config["PASSWORD"]:
|
||||
kwargs["basic_auth"] = (config["USERNAME"], config["PASSWORD"])
|
||||
return Elasticsearch(**kwargs)
|
||||
|
||||
|
||||
def alias_name() -> str:
|
||||
return settings.SEARCH["ELASTICSEARCH"]["INDEX"]
|
||||
|
||||
|
||||
def physical_index_name() -> str:
|
||||
return f"{alias_name()}-{datetime.now(UTC):%Y%m%d%H%M%S}"
|
||||
|
||||
|
||||
def build_document(profile: Profile) -> dict[str, Any]:
|
||||
"""Project a ``Profile`` row (plus its relations) into an index document."""
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()]
|
||||
experiences = list(profile.experiences.all())
|
||||
educations = list(profile.educations.all())
|
||||
|
||||
degrees = sorted({degree for edu in educations for degree in edu.degrees})
|
||||
majors = sorted({major for edu in educations for major in edu.majors})
|
||||
schools = [edu.school_name for edu in educations if edu.school_name]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"full_name": profile.full_name,
|
||||
"headline": profile.headline,
|
||||
"job_title": profile.job_title,
|
||||
"job_title_role": profile.job_title_role,
|
||||
"job_title_sub_role": profile.job_title_sub_role,
|
||||
"seniority_levels": profile.seniority_levels,
|
||||
"company": profile.current_company.name if profile.current_company_id else "",
|
||||
"past_companies": sorted({exp.company_name for exp in experiences if exp.company_name}),
|
||||
"industry": profile.industry,
|
||||
"skills": skills,
|
||||
"education": " ".join([*schools, *majors]),
|
||||
"degrees": degrees,
|
||||
"majors": majors,
|
||||
"summary": profile.summary,
|
||||
"interests": profile.interests,
|
||||
"languages": profile.languages,
|
||||
"location_name": profile.location_name,
|
||||
"location_locality": profile.location_locality,
|
||||
"location_region": profile.location_region,
|
||||
"location_country": profile.location_country,
|
||||
"location_continent": profile.location_continent,
|
||||
"years_experience": profile.years_experience,
|
||||
"connections": profile.connections,
|
||||
"linkedin_url": profile.linkedin_url,
|
||||
"indexed_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def iter_documents(queryset: Iterable[Profile] | None = None) -> Iterator[dict[str, Any]]:
|
||||
profiles = queryset if queryset is not None else _default_queryset()
|
||||
for profile in profiles:
|
||||
yield {"_id": profile.id, "_source": build_document(profile)}
|
||||
|
||||
|
||||
def _default_queryset(): # noqa: ANN202
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill", "experiences", "educations")
|
||||
.order_by("id")
|
||||
.iterator(chunk_size=200)
|
||||
)
|
||||
|
||||
|
||||
def create_index(client: Elasticsearch, name: str) -> None:
|
||||
client.indices.create(index=name, settings=INDEX_SETTINGS, mappings=INDEX_MAPPINGS)
|
||||
|
||||
|
||||
def rebuild_index(client: Elasticsearch | None = None) -> dict[str, Any]:
|
||||
"""
|
||||
Zero-downtime rebuild.
|
||||
|
||||
Creates a fresh physical index, indexes every profile into it, then flips
|
||||
the alias atomically and removes the previous physical index.
|
||||
"""
|
||||
client = client or get_client()
|
||||
alias = alias_name()
|
||||
new_index = physical_index_name()
|
||||
|
||||
create_index(client, new_index)
|
||||
logger.info("index_created", extra={"event": "index_created", "index": new_index})
|
||||
|
||||
succeeded, errors = bulk(
|
||||
client,
|
||||
({"_index": new_index, **doc} for doc in iter_documents()),
|
||||
raise_on_error=False,
|
||||
stats_only=False,
|
||||
)
|
||||
client.indices.refresh(index=new_index)
|
||||
|
||||
previous = _current_indices(client, alias)
|
||||
actions: list[dict[str, Any]] = [{"add": {"index": new_index, "alias": alias}}]
|
||||
actions += [{"remove": {"index": name, "alias": alias}} for name in previous]
|
||||
client.indices.update_aliases(actions=actions)
|
||||
|
||||
for name in previous:
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
|
||||
result = {
|
||||
"alias": alias,
|
||||
"index": new_index,
|
||||
"indexed": succeeded,
|
||||
"errors": len(errors) if isinstance(errors, list) else errors,
|
||||
"replaced": previous,
|
||||
}
|
||||
logger.info("index_rebuilt", extra={"event": "index_rebuilt", **result})
|
||||
return result
|
||||
|
||||
|
||||
def _current_indices(client: Elasticsearch, alias: str) -> list[str]:
|
||||
if not client.indices.exists_alias(name=alias):
|
||||
# A leftover *index* (not alias) with the same name would block the
|
||||
# alias creation, so remove it first.
|
||||
if client.indices.exists(index=alias):
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
return []
|
||||
return list(client.indices.get_alias(name=alias).keys())
|
||||
|
||||
|
||||
def index_profile(profile: Profile, client: Elasticsearch | None = None) -> None:
|
||||
"""Index (or re-index) a single profile - used after a targeted update."""
|
||||
client = client or get_client()
|
||||
client.index(index=alias_name(), id=profile.id, document=build_document(profile))
|
||||
@@ -0,0 +1,52 @@
|
||||
"""``python manage.py search_index`` - manage the Elasticsearch index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.search.indexing import alias_name, get_client, rebuild_index
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create, rebuild, inspect or drop the Elasticsearch profile index."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--rebuild", action="store_true", help="Rebuild and swap the alias.")
|
||||
group.add_argument("--status", action="store_true", help="Show index/alias status.")
|
||||
group.add_argument("--drop", action="store_true", help="Delete the index and its alias.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
client = get_client()
|
||||
if not client.ping():
|
||||
raise CommandError("Elasticsearch is not reachable. Check ELASTICSEARCH_HOSTS.")
|
||||
|
||||
if options["rebuild"]:
|
||||
started = time.perf_counter()
|
||||
result = rebuild_index(client)
|
||||
result["elapsed_seconds"] = round(time.perf_counter() - started, 2)
|
||||
self.stdout.write(json.dumps(result, indent=2))
|
||||
self.stdout.write(self.style.SUCCESS("Index rebuilt."))
|
||||
return
|
||||
|
||||
alias = alias_name()
|
||||
if options["status"]:
|
||||
exists = client.indices.exists_alias(name=alias)
|
||||
payload: dict[str, Any] = {"alias": alias, "exists": bool(exists)}
|
||||
if exists:
|
||||
indices = list(client.indices.get_alias(name=alias).keys())
|
||||
payload["indices"] = indices
|
||||
payload["documents"] = client.count(index=alias)["count"]
|
||||
self.stdout.write(json.dumps(payload, indent=2))
|
||||
return
|
||||
|
||||
if options["drop"]:
|
||||
if client.indices.exists_alias(name=alias):
|
||||
for name in client.indices.get_alias(name=alias):
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
self.stdout.write(self.style.WARNING(f"Dropped index/alias {alias}."))
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Request validation and response shaping for the search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.search.types import SORT_OPTIONS, SORT_RELEVANCE, SearchRequest, SearchResult
|
||||
|
||||
MAX_QUERY_LENGTH = 200
|
||||
MAX_FILTER_VALUES = 20
|
||||
|
||||
|
||||
class CommaSeparatedField(serializers.CharField):
|
||||
"""
|
||||
Accepts ``?skills=python,django`` **and** repeated ``?skills=python&skills=django``.
|
||||
|
||||
Both spellings are common in the wild; normalising here keeps the views and
|
||||
both backends free of query-string trivia.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
kwargs.setdefault("required", False)
|
||||
kwargs.setdefault("allow_blank", True)
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_value(self, dictionary: Any) -> Any:
|
||||
# ``QueryDict.get`` silently keeps only the last occurrence, which would
|
||||
# turn ``?skills=a&skills=b`` into ``b``. ``getlist`` keeps both.
|
||||
if hasattr(dictionary, "getlist"):
|
||||
values = dictionary.getlist(self.field_name)
|
||||
if values:
|
||||
return values
|
||||
return super().get_value(dictionary)
|
||||
|
||||
def to_internal_value(self, data: Any) -> tuple[str, ...]:
|
||||
raw: list[str] = []
|
||||
if isinstance(data, list | tuple):
|
||||
raw = [str(item) for item in data]
|
||||
elif data:
|
||||
raw = [str(data)]
|
||||
|
||||
values: list[str] = []
|
||||
for chunk in raw:
|
||||
for item in chunk.split(","):
|
||||
cleaned = item.strip().lower()[:128]
|
||||
if cleaned and cleaned not in values:
|
||||
values.append(cleaned)
|
||||
if len(values) > MAX_FILTER_VALUES:
|
||||
raise serializers.ValidationError(
|
||||
f"At most {MAX_FILTER_VALUES} values are allowed per filter."
|
||||
)
|
||||
return tuple(values)
|
||||
|
||||
|
||||
class SearchRequestSerializer(serializers.Serializer):
|
||||
"""Validates and normalises the query string of ``GET /api/v1/search/``."""
|
||||
|
||||
q = serializers.CharField(
|
||||
required=False, allow_blank=True, max_length=MAX_QUERY_LENGTH, trim_whitespace=True,
|
||||
help_text="Free-text keyword. Supports quoted phrases and -exclusions.",
|
||||
) # fmt: skip
|
||||
skills = CommaSeparatedField(help_text="Skill names, OR-combined.")
|
||||
roles = CommaSeparatedField(help_text="Job family, e.g. engineering, human_resources.")
|
||||
seniority = CommaSeparatedField(help_text="Seniority level, e.g. senior, manager, cxo.")
|
||||
countries = CommaSeparatedField(help_text="Country of residence.")
|
||||
industries = CommaSeparatedField(help_text="Industry of the current employer.")
|
||||
companies = CommaSeparatedField(help_text="Current employer name.")
|
||||
min_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
max_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
sort = serializers.ChoiceField(choices=SORT_OPTIONS, required=False, default=SORT_RELEVANCE)
|
||||
page = serializers.IntegerField(required=False, min_value=1, default=1)
|
||||
page_size = serializers.IntegerField(required=False, min_value=1, max_value=100, default=20)
|
||||
facets = serializers.BooleanField(required=False, default=True)
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
low, high = attrs.get("min_experience"), attrs.get("max_experience")
|
||||
if low is not None and high is not None and low > high:
|
||||
raise serializers.ValidationError(
|
||||
{"min_experience": "min_experience cannot be greater than max_experience."}
|
||||
)
|
||||
|
||||
page, page_size = attrs.get("page", 1), attrs.get("page_size", 20)
|
||||
# Deep pagination is both useless to a human and expensive for the
|
||||
# engine, so it is rejected instead of silently truncated.
|
||||
if page * page_size > settings.SEARCH["MAX_RESULT_WINDOW"]:
|
||||
raise serializers.ValidationError(
|
||||
{"page": f"Cannot page beyond {settings.SEARCH['MAX_RESULT_WINDOW']} results."}
|
||||
)
|
||||
return attrs
|
||||
|
||||
def to_search_request(self) -> SearchRequest:
|
||||
data = self.validated_data
|
||||
return SearchRequest(
|
||||
q=data.get("q", "").strip(),
|
||||
skills=data.get("skills", ()),
|
||||
roles=data.get("roles", ()),
|
||||
seniority=data.get("seniority", ()),
|
||||
countries=data.get("countries", ()),
|
||||
industries=data.get("industries", ()),
|
||||
companies=data.get("companies", ()),
|
||||
min_experience=data.get("min_experience"),
|
||||
max_experience=data.get("max_experience"),
|
||||
sort=data.get("sort", SORT_RELEVANCE),
|
||||
page=data.get("page", 1),
|
||||
page_size=data.get("page_size", 20),
|
||||
include_facets=data.get("facets", True),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response serializers (declared explicitly so the OpenAPI schema is accurate)
|
||||
# ---------------------------------------------------------------------------
|
||||
class SearchHitSerializer(serializers.Serializer):
|
||||
id = serializers.IntegerField()
|
||||
score = serializers.FloatField()
|
||||
full_name = serializers.CharField()
|
||||
headline = serializers.CharField()
|
||||
job_title = serializers.CharField()
|
||||
job_title_role = serializers.CharField()
|
||||
seniority_levels = serializers.ListField(child=serializers.CharField())
|
||||
company = serializers.CharField()
|
||||
industry = serializers.CharField()
|
||||
location_name = serializers.CharField()
|
||||
location_country = serializers.CharField()
|
||||
years_experience = serializers.IntegerField(allow_null=True)
|
||||
linkedin_url = serializers.CharField()
|
||||
skills = serializers.ListField(child=serializers.CharField())
|
||||
highlight = serializers.DictField(child=serializers.ListField(child=serializers.CharField()))
|
||||
|
||||
|
||||
class FacetBucketSerializer(serializers.Serializer):
|
||||
key = serializers.CharField()
|
||||
count = serializers.IntegerField()
|
||||
|
||||
|
||||
class SearchResponseSerializer(serializers.Serializer):
|
||||
count = serializers.IntegerField()
|
||||
page = serializers.IntegerField()
|
||||
page_size = serializers.IntegerField()
|
||||
total_pages = serializers.IntegerField()
|
||||
took_ms = serializers.IntegerField()
|
||||
backend = serializers.CharField()
|
||||
results = SearchHitSerializer(many=True)
|
||||
facets = serializers.DictField(child=FacetBucketSerializer(many=True))
|
||||
|
||||
@staticmethod
|
||||
def from_result(result: SearchResult, request: SearchRequest) -> dict[str, Any]:
|
||||
total_pages = max(1, -(-result.total // request.page_size))
|
||||
return {
|
||||
"count": result.total,
|
||||
"page": request.page,
|
||||
"page_size": request.page_size,
|
||||
"total_pages": total_pages,
|
||||
"took_ms": result.took_ms,
|
||||
"backend": result.backend,
|
||||
"results": [asdict(hit) for hit in result.hits],
|
||||
"facets": {
|
||||
name: [asdict(bucket) for bucket in buckets]
|
||||
for name, buckets in result.facets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SuggestResponseSerializer(serializers.Serializer):
|
||||
suggestions = serializers.ListField(child=serializers.CharField())
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Backend independent data structures.
|
||||
|
||||
Both search backends speak this vocabulary, which is what makes them
|
||||
interchangeable behind a single setting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# Sort keys accepted by the API and understood by every backend.
|
||||
SORT_RELEVANCE = "relevance"
|
||||
SORT_EXPERIENCE_DESC = "-experience"
|
||||
SORT_EXPERIENCE_ASC = "experience"
|
||||
SORT_NAME = "name"
|
||||
SORT_OPTIONS = (SORT_RELEVANCE, SORT_EXPERIENCE_DESC, SORT_EXPERIENCE_ASC, SORT_NAME)
|
||||
|
||||
# Facet name -> the document field it aggregates.
|
||||
FACET_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"job_title_role": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"country": "location_country",
|
||||
"industry": "industry",
|
||||
"company": "company.keyword",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SearchRequest:
|
||||
"""A validated, normalised search request."""
|
||||
|
||||
q: str = ""
|
||||
skills: tuple[str, ...] = ()
|
||||
roles: tuple[str, ...] = ()
|
||||
seniority: tuple[str, ...] = ()
|
||||
countries: tuple[str, ...] = ()
|
||||
industries: tuple[str, ...] = ()
|
||||
companies: tuple[str, ...] = ()
|
||||
min_experience: int | None = None
|
||||
max_experience: int | None = None
|
||||
sort: str = SORT_RELEVANCE
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
include_facets: bool = True
|
||||
|
||||
@property
|
||||
def offset(self) -> int:
|
||||
return (self.page - 1) * self.page_size
|
||||
|
||||
@property
|
||||
def has_filters(self) -> bool:
|
||||
return bool(
|
||||
self.skills
|
||||
or self.roles
|
||||
or self.seniority
|
||||
or self.countries
|
||||
or self.industries
|
||||
or self.companies
|
||||
or self.min_experience is not None
|
||||
or self.max_experience is not None
|
||||
)
|
||||
|
||||
def as_log_context(self) -> dict[str, Any]:
|
||||
"""Compact representation for the audit log (no free text explosion)."""
|
||||
return {
|
||||
"query": self.q[:120],
|
||||
"filters": {
|
||||
"skills": list(self.skills),
|
||||
"roles": list(self.roles),
|
||||
"seniority": list(self.seniority),
|
||||
"countries": list(self.countries),
|
||||
"industries": list(self.industries),
|
||||
"companies": list(self.companies),
|
||||
"min_experience": self.min_experience,
|
||||
"max_experience": self.max_experience,
|
||||
},
|
||||
"sort": self.sort,
|
||||
"page": self.page,
|
||||
"page_size": self.page_size,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SearchHit:
|
||||
"""One result card - identical shape whichever backend produced it."""
|
||||
|
||||
id: int
|
||||
score: float
|
||||
full_name: str
|
||||
headline: str
|
||||
job_title: str
|
||||
job_title_role: str
|
||||
seniority_levels: list[str]
|
||||
company: str
|
||||
industry: str
|
||||
location_name: str
|
||||
location_country: str
|
||||
years_experience: int | None
|
||||
linkedin_url: str
|
||||
skills: list[str]
|
||||
highlight: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FacetBucket:
|
||||
key: str
|
||||
count: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SearchResult:
|
||||
total: int
|
||||
hits: list[SearchHit]
|
||||
facets: dict[str, list[FacetBucket]]
|
||||
took_ms: int
|
||||
backend: str
|
||||
@@ -0,0 +1,11 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.search.views import ReindexView, SearchView, SuggestView
|
||||
|
||||
app_name = "search"
|
||||
|
||||
urlpatterns = [
|
||||
path("", SearchView.as_view(), name="search"),
|
||||
path("suggest/", SuggestView.as_view(), name="suggest"),
|
||||
path("reindex/", ReindexView.as_view(), name="reindex"),
|
||||
]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from drf_spectacular.utils import OpenApiParameter, extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from apps.core.permissions import IsAdminRole
|
||||
from apps.core.throttling import SearchRateThrottle
|
||||
from apps.search.backends import get_search_backend
|
||||
from apps.search.serializers import (
|
||||
SearchRequestSerializer,
|
||||
SearchResponseSerializer,
|
||||
SuggestResponseSerializer,
|
||||
)
|
||||
|
||||
audit = logging.getLogger("audit.search")
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
parameters=[SearchRequestSerializer],
|
||||
responses={200: SearchResponseSerializer},
|
||||
summary="Keyword search with filters and facets",
|
||||
description=(
|
||||
"Ranked keyword search across name, job title, skills, employer, education, "
|
||||
"location and summary, combined with structured filters. Filter values are "
|
||||
"OR-ed inside one filter and AND-ed across filters."
|
||||
),
|
||||
)
|
||||
class SearchView(APIView):
|
||||
"""
|
||||
``GET /api/v1/search/``
|
||||
|
||||
Thin by design: validate the query string, hand a normalised request to the
|
||||
configured backend, shape the response, emit an audit event. All ranking
|
||||
lives in the backend, all validation lives in the serializer.
|
||||
"""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
throttle_classes = [SearchRateThrottle]
|
||||
serializer_class = SearchRequestSerializer
|
||||
|
||||
def get(self, request: Request) -> Response:
|
||||
serializer = SearchRequestSerializer(data=request.query_params)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
search_request = serializer.to_search_request()
|
||||
|
||||
backend = get_search_backend()
|
||||
result = backend.search(search_request)
|
||||
|
||||
audit.info(
|
||||
"search_executed",
|
||||
extra={
|
||||
"event": "search_executed",
|
||||
"user_id": request.user.id,
|
||||
"backend": result.backend,
|
||||
"result_count": result.total,
|
||||
"took_ms": result.took_ms,
|
||||
**search_request.as_log_context(),
|
||||
},
|
||||
)
|
||||
return Response(SearchResponseSerializer.from_result(result, search_request))
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
parameters=[
|
||||
OpenApiParameter("q", str, description="Partial name or skill.", required=True),
|
||||
OpenApiParameter("limit", int, description="Maximum suggestions (1-20)."),
|
||||
],
|
||||
responses={200: SuggestResponseSerializer},
|
||||
summary="Type-ahead suggestions",
|
||||
)
|
||||
class SuggestView(APIView):
|
||||
"""Edge n-gram completions used by the search box."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
throttle_classes = [SearchRateThrottle]
|
||||
|
||||
def get(self, request: Request) -> Response:
|
||||
prefix = request.query_params.get("q", "").strip()[:64]
|
||||
try:
|
||||
limit = min(max(int(request.query_params.get("limit", 8)), 1), 20)
|
||||
except ValueError:
|
||||
limit = 8
|
||||
suggestions = get_search_backend().suggest(prefix, limit=limit) if prefix else []
|
||||
return Response({"suggestions": suggestions})
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
request=None,
|
||||
responses={202: dict},
|
||||
summary="Rebuild the search index (admin only)",
|
||||
)
|
||||
class ReindexView(APIView):
|
||||
"""
|
||||
Triggers a zero-downtime rebuild of the Elasticsearch index.
|
||||
|
||||
Restricted to the admin role: it is an expensive, cluster-wide operation.
|
||||
"""
|
||||
|
||||
permission_classes = [IsAuthenticated, IsAdminRole]
|
||||
|
||||
def post(self, request: Request) -> Response:
|
||||
from apps.search.indexing import rebuild_index
|
||||
|
||||
result = rebuild_index()
|
||||
audit.info(
|
||||
"index_rebuilt",
|
||||
extra={"event": "index_rebuilt", "user_id": request.user.id, **result},
|
||||
)
|
||||
return Response(result, status=status.HTTP_202_ACCEPTED)
|
||||
Reference in New Issue
Block a user