feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,169 @@
|
||||
"""Request validation and response shaping for the search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.search.types import SORT_OPTIONS, SORT_RELEVANCE, SearchRequest, SearchResult
|
||||
|
||||
MAX_QUERY_LENGTH = 200
|
||||
MAX_FILTER_VALUES = 20
|
||||
|
||||
|
||||
class CommaSeparatedField(serializers.CharField):
|
||||
"""
|
||||
Accepts ``?skills=python,django`` **and** repeated ``?skills=python&skills=django``.
|
||||
|
||||
Both spellings are common in the wild; normalising here keeps the views and
|
||||
both backends free of query-string trivia.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
kwargs.setdefault("required", False)
|
||||
kwargs.setdefault("allow_blank", True)
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_value(self, dictionary: Any) -> Any:
|
||||
# ``QueryDict.get`` silently keeps only the last occurrence, which would
|
||||
# turn ``?skills=a&skills=b`` into ``b``. ``getlist`` keeps both.
|
||||
if hasattr(dictionary, "getlist"):
|
||||
values = dictionary.getlist(self.field_name)
|
||||
if values:
|
||||
return values
|
||||
return super().get_value(dictionary)
|
||||
|
||||
def to_internal_value(self, data: Any) -> tuple[str, ...]:
|
||||
raw: list[str] = []
|
||||
if isinstance(data, list | tuple):
|
||||
raw = [str(item) for item in data]
|
||||
elif data:
|
||||
raw = [str(data)]
|
||||
|
||||
values: list[str] = []
|
||||
for chunk in raw:
|
||||
for item in chunk.split(","):
|
||||
cleaned = item.strip().lower()[:128]
|
||||
if cleaned and cleaned not in values:
|
||||
values.append(cleaned)
|
||||
if len(values) > MAX_FILTER_VALUES:
|
||||
raise serializers.ValidationError(
|
||||
f"At most {MAX_FILTER_VALUES} values are allowed per filter."
|
||||
)
|
||||
return tuple(values)
|
||||
|
||||
|
||||
class SearchRequestSerializer(serializers.Serializer):
|
||||
"""Validates and normalises the query string of ``GET /api/v1/search/``."""
|
||||
|
||||
q = serializers.CharField(
|
||||
required=False, allow_blank=True, max_length=MAX_QUERY_LENGTH, trim_whitespace=True,
|
||||
help_text="Free-text keyword. Supports quoted phrases and -exclusions.",
|
||||
) # fmt: skip
|
||||
skills = CommaSeparatedField(help_text="Skill names, OR-combined.")
|
||||
roles = CommaSeparatedField(help_text="Job family, e.g. engineering, human_resources.")
|
||||
seniority = CommaSeparatedField(help_text="Seniority level, e.g. senior, manager, cxo.")
|
||||
countries = CommaSeparatedField(help_text="Country of residence.")
|
||||
industries = CommaSeparatedField(help_text="Industry of the current employer.")
|
||||
companies = CommaSeparatedField(help_text="Current employer name.")
|
||||
min_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
max_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
sort = serializers.ChoiceField(choices=SORT_OPTIONS, required=False, default=SORT_RELEVANCE)
|
||||
page = serializers.IntegerField(required=False, min_value=1, default=1)
|
||||
page_size = serializers.IntegerField(required=False, min_value=1, max_value=100, default=20)
|
||||
facets = serializers.BooleanField(required=False, default=True)
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
low, high = attrs.get("min_experience"), attrs.get("max_experience")
|
||||
if low is not None and high is not None and low > high:
|
||||
raise serializers.ValidationError(
|
||||
{"min_experience": "min_experience cannot be greater than max_experience."}
|
||||
)
|
||||
|
||||
page, page_size = attrs.get("page", 1), attrs.get("page_size", 20)
|
||||
# Deep pagination is both useless to a human and expensive for the
|
||||
# engine, so it is rejected instead of silently truncated.
|
||||
if page * page_size > settings.SEARCH["MAX_RESULT_WINDOW"]:
|
||||
raise serializers.ValidationError(
|
||||
{"page": f"Cannot page beyond {settings.SEARCH['MAX_RESULT_WINDOW']} results."}
|
||||
)
|
||||
return attrs
|
||||
|
||||
def to_search_request(self) -> SearchRequest:
|
||||
data = self.validated_data
|
||||
return SearchRequest(
|
||||
q=data.get("q", "").strip(),
|
||||
skills=data.get("skills", ()),
|
||||
roles=data.get("roles", ()),
|
||||
seniority=data.get("seniority", ()),
|
||||
countries=data.get("countries", ()),
|
||||
industries=data.get("industries", ()),
|
||||
companies=data.get("companies", ()),
|
||||
min_experience=data.get("min_experience"),
|
||||
max_experience=data.get("max_experience"),
|
||||
sort=data.get("sort", SORT_RELEVANCE),
|
||||
page=data.get("page", 1),
|
||||
page_size=data.get("page_size", 20),
|
||||
include_facets=data.get("facets", True),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response serializers (declared explicitly so the OpenAPI schema is accurate)
|
||||
# ---------------------------------------------------------------------------
|
||||
class SearchHitSerializer(serializers.Serializer):
|
||||
id = serializers.IntegerField()
|
||||
score = serializers.FloatField()
|
||||
full_name = serializers.CharField()
|
||||
headline = serializers.CharField()
|
||||
job_title = serializers.CharField()
|
||||
job_title_role = serializers.CharField()
|
||||
seniority_levels = serializers.ListField(child=serializers.CharField())
|
||||
company = serializers.CharField()
|
||||
industry = serializers.CharField()
|
||||
location_name = serializers.CharField()
|
||||
location_country = serializers.CharField()
|
||||
years_experience = serializers.IntegerField(allow_null=True)
|
||||
linkedin_url = serializers.CharField()
|
||||
skills = serializers.ListField(child=serializers.CharField())
|
||||
highlight = serializers.DictField(child=serializers.ListField(child=serializers.CharField()))
|
||||
|
||||
|
||||
class FacetBucketSerializer(serializers.Serializer):
|
||||
key = serializers.CharField()
|
||||
count = serializers.IntegerField()
|
||||
|
||||
|
||||
class SearchResponseSerializer(serializers.Serializer):
|
||||
count = serializers.IntegerField()
|
||||
page = serializers.IntegerField()
|
||||
page_size = serializers.IntegerField()
|
||||
total_pages = serializers.IntegerField()
|
||||
took_ms = serializers.IntegerField()
|
||||
backend = serializers.CharField()
|
||||
results = SearchHitSerializer(many=True)
|
||||
facets = serializers.DictField(child=FacetBucketSerializer(many=True))
|
||||
|
||||
@staticmethod
|
||||
def from_result(result: SearchResult, request: SearchRequest) -> dict[str, Any]:
|
||||
total_pages = max(1, -(-result.total // request.page_size))
|
||||
return {
|
||||
"count": result.total,
|
||||
"page": request.page,
|
||||
"page_size": request.page_size,
|
||||
"total_pages": total_pages,
|
||||
"took_ms": result.took_ms,
|
||||
"backend": result.backend,
|
||||
"results": [asdict(hit) for hit in result.hits],
|
||||
"facets": {
|
||||
name: [asdict(bucket) for bucket in buckets]
|
||||
for name, buckets in result.facets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SuggestResponseSerializer(serializers.Serializer):
|
||||
suggestions = serializers.ListField(child=serializers.CharField())
|
||||
Reference in New Issue
Block a user