feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Elasticsearch index definition and (re)build helpers.
|
||||
|
||||
Index design
|
||||
------------
|
||||
* One document per profile, denormalised: the fields a recruiter searches on
|
||||
(name, title, employer, skills, education, location) all live in the same
|
||||
document, so a query never needs a join.
|
||||
* Text fields are analysed with an English analyzer (lowercase, ASCII folding,
|
||||
stop words, stemming) so "engineering" matches "engineer".
|
||||
* Every field that is *filtered* or *aggregated* on has a ``keyword``
|
||||
counterpart with a lowercase normaliser - analysed text cannot be used for
|
||||
exact term filters or facet counts.
|
||||
* ``full_name.autocomplete`` uses an edge n-gram analyzer for the type-ahead;
|
||||
the search analyzer is deliberately different (``standard``) so the query
|
||||
term is not itself n-grammed.
|
||||
* **No personal contact data is indexed.** E-mail addresses and phone numbers
|
||||
stay in PostgreSQL, which removes any chance of confirming someone's e-mail
|
||||
through a crafted search query.
|
||||
|
||||
Rebuilds use an alias plus a timestamped physical index, so a reindex never
|
||||
leaves the API pointing at a half-populated index and can be rolled back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch.helpers import bulk
|
||||
|
||||
from apps.profiles.models import Profile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INDEX_SETTINGS: dict[str, Any] = {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"refresh_interval": "1s",
|
||||
"max_result_window": 10_000,
|
||||
"analysis": {
|
||||
"filter": {
|
||||
"english_stop": {"type": "stop", "stopwords": "_english_"},
|
||||
"english_stemmer": {"type": "stemmer", "language": "english"},
|
||||
"english_possessive": {"type": "stemmer", "language": "possessive_english"},
|
||||
"edge_ngrams": {"type": "edge_ngram", "min_gram": 2, "max_gram": 15},
|
||||
},
|
||||
"normalizer": {
|
||||
"lowercase_normalizer": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
}
|
||||
},
|
||||
"analyzer": {
|
||||
"english_text": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": [
|
||||
"lowercase",
|
||||
"asciifolding",
|
||||
"english_possessive",
|
||||
"english_stop",
|
||||
"english_stemmer",
|
||||
],
|
||||
},
|
||||
"autocomplete_index": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding", "edge_ngrams"],
|
||||
},
|
||||
"autocomplete_search": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
KEYWORD = {"type": "keyword", "normalizer": "lowercase_normalizer"}
|
||||
|
||||
INDEX_MAPPINGS: dict[str, Any] = {
|
||||
"dynamic": "strict", # reject unexpected fields instead of guessing a type
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"full_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"headline": {"type": "text", "analyzer": "english_text"},
|
||||
"job_title": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"job_title_role": KEYWORD,
|
||||
"job_title_sub_role": KEYWORD,
|
||||
"seniority_levels": KEYWORD,
|
||||
"company": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"past_companies": {"type": "text", "analyzer": "english_text"},
|
||||
"industry": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"skills": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"education": {"type": "text", "analyzer": "english_text"},
|
||||
"degrees": KEYWORD,
|
||||
"majors": {"type": "text", "analyzer": "english_text", "fields": {"keyword": KEYWORD}},
|
||||
"summary": {"type": "text", "analyzer": "english_text"},
|
||||
"interests": {"type": "text", "analyzer": "english_text"},
|
||||
"languages": KEYWORD,
|
||||
"location_name": {"type": "text", "analyzer": "english_text"},
|
||||
"location_locality": KEYWORD,
|
||||
"location_region": KEYWORD,
|
||||
"location_country": KEYWORD,
|
||||
"location_continent": KEYWORD,
|
||||
"years_experience": {"type": "integer"},
|
||||
"connections": {"type": "integer"},
|
||||
"linkedin_url": {"type": "keyword", "index": False},
|
||||
"indexed_at": {"type": "date"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_client() -> Elasticsearch:
|
||||
"""Build an Elasticsearch client from settings (auth optional in dev)."""
|
||||
config = settings.SEARCH["ELASTICSEARCH"]
|
||||
kwargs: dict[str, Any] = {
|
||||
"hosts": config["HOSTS"],
|
||||
"request_timeout": config["TIMEOUT"],
|
||||
"max_retries": config["MAX_RETRIES"],
|
||||
"retry_on_timeout": True,
|
||||
}
|
||||
if config["USERNAME"] and config["PASSWORD"]:
|
||||
kwargs["basic_auth"] = (config["USERNAME"], config["PASSWORD"])
|
||||
return Elasticsearch(**kwargs)
|
||||
|
||||
|
||||
def alias_name() -> str:
|
||||
return settings.SEARCH["ELASTICSEARCH"]["INDEX"]
|
||||
|
||||
|
||||
def physical_index_name() -> str:
|
||||
return f"{alias_name()}-{datetime.now(UTC):%Y%m%d%H%M%S}"
|
||||
|
||||
|
||||
def build_document(profile: Profile) -> dict[str, Any]:
|
||||
"""Project a ``Profile`` row (plus its relations) into an index document."""
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()]
|
||||
experiences = list(profile.experiences.all())
|
||||
educations = list(profile.educations.all())
|
||||
|
||||
degrees = sorted({degree for edu in educations for degree in edu.degrees})
|
||||
majors = sorted({major for edu in educations for major in edu.majors})
|
||||
schools = [edu.school_name for edu in educations if edu.school_name]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"full_name": profile.full_name,
|
||||
"headline": profile.headline,
|
||||
"job_title": profile.job_title,
|
||||
"job_title_role": profile.job_title_role,
|
||||
"job_title_sub_role": profile.job_title_sub_role,
|
||||
"seniority_levels": profile.seniority_levels,
|
||||
"company": profile.current_company.name if profile.current_company_id else "",
|
||||
"past_companies": sorted({exp.company_name for exp in experiences if exp.company_name}),
|
||||
"industry": profile.industry,
|
||||
"skills": skills,
|
||||
"education": " ".join([*schools, *majors]),
|
||||
"degrees": degrees,
|
||||
"majors": majors,
|
||||
"summary": profile.summary,
|
||||
"interests": profile.interests,
|
||||
"languages": profile.languages,
|
||||
"location_name": profile.location_name,
|
||||
"location_locality": profile.location_locality,
|
||||
"location_region": profile.location_region,
|
||||
"location_country": profile.location_country,
|
||||
"location_continent": profile.location_continent,
|
||||
"years_experience": profile.years_experience,
|
||||
"connections": profile.connections,
|
||||
"linkedin_url": profile.linkedin_url,
|
||||
"indexed_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def iter_documents(queryset: Iterable[Profile] | None = None) -> Iterator[dict[str, Any]]:
|
||||
profiles = queryset if queryset is not None else _default_queryset()
|
||||
for profile in profiles:
|
||||
yield {"_id": profile.id, "_source": build_document(profile)}
|
||||
|
||||
|
||||
def _default_queryset(): # noqa: ANN202
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill", "experiences", "educations")
|
||||
.order_by("id")
|
||||
.iterator(chunk_size=200)
|
||||
)
|
||||
|
||||
|
||||
def create_index(client: Elasticsearch, name: str) -> None:
|
||||
client.indices.create(index=name, settings=INDEX_SETTINGS, mappings=INDEX_MAPPINGS)
|
||||
|
||||
|
||||
def rebuild_index(client: Elasticsearch | None = None) -> dict[str, Any]:
|
||||
"""
|
||||
Zero-downtime rebuild.
|
||||
|
||||
Creates a fresh physical index, indexes every profile into it, then flips
|
||||
the alias atomically and removes the previous physical index.
|
||||
"""
|
||||
client = client or get_client()
|
||||
alias = alias_name()
|
||||
new_index = physical_index_name()
|
||||
|
||||
create_index(client, new_index)
|
||||
logger.info("index_created", extra={"event": "index_created", "index": new_index})
|
||||
|
||||
succeeded, errors = bulk(
|
||||
client,
|
||||
({"_index": new_index, **doc} for doc in iter_documents()),
|
||||
raise_on_error=False,
|
||||
stats_only=False,
|
||||
)
|
||||
client.indices.refresh(index=new_index)
|
||||
|
||||
previous = _current_indices(client, alias)
|
||||
actions: list[dict[str, Any]] = [{"add": {"index": new_index, "alias": alias}}]
|
||||
actions += [{"remove": {"index": name, "alias": alias}} for name in previous]
|
||||
client.indices.update_aliases(actions=actions)
|
||||
|
||||
for name in previous:
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
|
||||
result = {
|
||||
"alias": alias,
|
||||
"index": new_index,
|
||||
"indexed": succeeded,
|
||||
"errors": len(errors) if isinstance(errors, list) else errors,
|
||||
"replaced": previous,
|
||||
}
|
||||
logger.info("index_rebuilt", extra={"event": "index_rebuilt", **result})
|
||||
return result
|
||||
|
||||
|
||||
def _current_indices(client: Elasticsearch, alias: str) -> list[str]:
|
||||
if not client.indices.exists_alias(name=alias):
|
||||
# A leftover *index* (not alias) with the same name would block the
|
||||
# alias creation, so remove it first.
|
||||
if client.indices.exists(index=alias):
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
return []
|
||||
return list(client.indices.get_alias(name=alias).keys())
|
||||
|
||||
|
||||
def index_profile(profile: Profile, client: Elasticsearch | None = None) -> None:
|
||||
"""Index (or re-index) a single profile - used after a targeted update."""
|
||||
client = client or get_client()
|
||||
client.index(index=alias_name(), id=profile.id, document=build_document(profile))
|
||||
Reference in New Issue
Block a user