feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)

Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
2026-09-03 08:13:10 +00:00
commit a464e550c9
133 changed files with 17019 additions and 0 deletions
@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class CoreConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "apps.core"
+    verbose_name = "Core"
@@ -0,0 +1,101 @@
+"""
+A single, predictable error envelope for the whole API.
+
+Every failure - validation, permission, throttling, or an unexpected crash -
+comes back in the same shape:
+
+    {"error": {"code": "validation_error", "message": "...", "details": {...},
+               "request_id": "..."}}
+
+Unhandled exceptions are logged with their stack trace but never leak internals
+to the client.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from django.core.exceptions import PermissionDenied
+from django.http import Http404
+from rest_framework import exceptions, status
+from rest_framework.response import Response
+from rest_framework.views import exception_handler as drf_exception_handler
+
+from apps.core.logging import get_request_id
+
+logger = logging.getLogger("apps.errors")
+
+_CODE_BY_STATUS = {
+    status.HTTP_400_BAD_REQUEST: "validation_error",
+    status.HTTP_401_UNAUTHORIZED: "authentication_failed",
+    status.HTTP_403_FORBIDDEN: "permission_denied",
+    status.HTTP_404_NOT_FOUND: "not_found",
+    status.HTTP_405_METHOD_NOT_ALLOWED: "method_not_allowed",
+    status.HTTP_429_TOO_MANY_REQUESTS: "throttled",
+    status.HTTP_503_SERVICE_UNAVAILABLE: "service_unavailable",
+}
+
+
+class ServiceUnavailable(exceptions.APIException):
+    status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    default_detail = "A downstream dependency is unavailable. Please retry shortly."
+    default_code = "service_unavailable"
+
+
+class SearchBackendError(ServiceUnavailable):
+    default_detail = "The search backend is currently unavailable."
+    default_code = "search_unavailable"
+
+
+def api_exception_handler(exc: Exception, context: dict[str, Any]) -> Response | None:
+    response = drf_exception_handler(exc, context)
+
+    if response is None:
+        # Anything DRF does not know about is a bug on our side.
+        if isinstance(exc, Http404):
+            response = Response(status=status.HTTP_404_NOT_FOUND)
+        elif isinstance(exc, PermissionDenied):
+            response = Response(status=status.HTTP_403_FORBIDDEN)
+        else:
+            logger.exception(
+                "unhandled_exception",
+                extra={"event": "unhandled_exception", "view": str(context.get("view"))},
+            )
+            return Response(
+                _envelope("internal_error", "An unexpected error occurred.", None),
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            )
+
+    detail = getattr(exc, "detail", None)
+    if isinstance(exc, exceptions.ValidationError):
+        # DRF labels every field error "invalid"; one stable code per failure
+        # mode is far easier for a client to branch on.
+        code = "validation_error"
+    else:
+        code = getattr(exc, "default_code", None) or _CODE_BY_STATUS.get(
+            response.status_code, "error"
+        )
+
+    if isinstance(detail, dict):
+        message = "The request payload failed validation."
+        details: Any = detail
+    elif isinstance(detail, list):
+        message = str(detail[0]) if detail else "Request failed."
+        details = detail
+    elif detail is not None:
+        message = str(detail)
+        details = None
+    else:
+        message = response.status_text
+        details = None
+
+    response.data = _envelope(code, message, details)
+    return response
+
+
+def _envelope(code: str, message: str, details: Any) -> dict[str, Any]:
+    error: dict[str, Any] = {"code": code, "message": message, "request_id": get_request_id()}
+    if details:
+        error["details"] = details
+    return {"error": error}
@@ -0,0 +1,90 @@
+"""
+Structured logging primitives.
+
+Every log record is emitted as a single JSON object so Logstash can ingest it
+without grok patterns, and every record carries the request id of the HTTP
+request that produced it, which makes a single user action traceable end to end
+in Kibana.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import json
+import logging
+import traceback
+from contextvars import ContextVar
+from typing import Any
+
+# Propagates the current request id to any code running in the same context,
+# including background threads started per request.
+_request_id: ContextVar[str] = ContextVar("request_id", default="-")
+
+# Attributes that ``logging`` puts on every record; anything else the caller
+# passed through ``extra=`` is treated as structured context and serialised.
+_STANDARD_ATTRS = frozenset(
+    {
+        "args", "asctime", "created", "exc_info", "exc_text", "filename",
+        "funcName", "levelname", "levelno", "lineno", "module", "msecs",
+        "message", "msg", "name", "pathname", "process", "processName",
+        "relativeCreated", "stack_info", "thread", "threadName", "taskName",
+        "request_id",
+    }
+)  # fmt: skip
+
+
+def set_request_id(value: str) -> None:
+    _request_id.set(value)
+
+
+def get_request_id() -> str:
+    return _request_id.get()
+
+
+class RequestIDFilter(logging.Filter):
+    """Attaches the ambient request id to every record."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.request_id = getattr(record, "request_id", None) or get_request_id()
+        return True
+
+
+class JSONFormatter(logging.Formatter):
+    """Renders a ``LogRecord`` as one line of JSON."""
+
+    def __init__(self, service: str = "app", environment: str = "development") -> None:
+        super().__init__()
+        self.service = service
+        self.environment = environment
+
+    def format(self, record: logging.LogRecord) -> str:
+        payload: dict[str, Any] = {
+            "@timestamp": dt.datetime.fromtimestamp(record.created, tz=dt.UTC).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "service": self.service,
+            "environment": self.environment,
+            "request_id": getattr(record, "request_id", "-"),
+        }
+
+        if record.exc_info:
+            exc_type, exc_value, exc_tb = record.exc_info
+            payload["error"] = {
+                "type": getattr(exc_type, "__name__", str(exc_type)),
+                "message": str(exc_value),
+                "stack": "".join(traceback.format_exception(exc_type, exc_value, exc_tb))[:8000],
+            }
+
+        for key, value in record.__dict__.items():
+            if key not in _STANDARD_ATTRS and not key.startswith("_"):
+                payload[key] = _coerce(value)
+
+        return json.dumps(payload, ensure_ascii=False, default=str)
+
+
+def _coerce(value: Any) -> Any:
+    """Keep the payload JSON serialisable without blowing up on odd types."""
+    if isinstance(value, str | int | float | bool | type(None) | list | dict):
+        return value
+    return str(value)
@@ -0,0 +1,87 @@
+"""Request scoped middleware: correlation ids and structured access logs."""
+
+from __future__ import annotations
+
+import logging
+import time
+import uuid
+from collections.abc import Callable
+
+from django.http import HttpRequest, HttpResponse
+
+from apps.core.logging import set_request_id
+
+logger = logging.getLogger("apps.access")
+
+REQUEST_ID_HEADER = "HTTP_X_REQUEST_ID"
+RESPONSE_HEADER = "X-Request-ID"
+_MAX_REQUEST_ID_LEN = 64
+
+
+class RequestIDMiddleware:
+    """
+    Reads ``X-Request-ID`` from the incoming request or generates one.
+
+    The value is echoed back on the response and injected into every log record
+    produced while handling the request, so a browser network tab entry can be
+    pasted straight into Kibana.
+    """
+
+    def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
+        self.get_response = get_response
+
+    def __call__(self, request: HttpRequest) -> HttpResponse:
+        incoming = request.META.get(REQUEST_ID_HEADER, "")
+        # Never trust client input verbatim: keep it short and alphanumeric.
+        request_id = (
+            incoming[:_MAX_REQUEST_ID_LEN]
+            if incoming and incoming.replace("-", "").replace("_", "").isalnum()
+            else uuid.uuid4().hex
+        )
+        request.request_id = request_id  # type: ignore[attr-defined]
+        set_request_id(request_id)
+
+        response = self.get_response(request)
+        response[RESPONSE_HEADER] = request_id
+        return response
+
+
+class AccessLogMiddleware:
+    """Emits one structured log line per request with timing and identity."""
+
+    def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
+        self.get_response = get_response
+
+    def __call__(self, request: HttpRequest) -> HttpResponse:
+        started = time.perf_counter()
+        response = self.get_response(request)
+        duration_ms = round((time.perf_counter() - started) * 1000, 2)
+
+        user = getattr(request, "user", None)
+        logger.info(
+            "http_request",
+            extra={
+                "event": "http_request",
+                "http_method": request.method,
+                "http_path": request.path,
+                "http_status": response.status_code,
+                "duration_ms": duration_ms,
+                "client_ip": _client_ip(request),
+                "user_agent": request.META.get("HTTP_USER_AGENT", "")[:200],
+                "user_id": getattr(user, "id", None) if user and user.is_authenticated else None,
+            },
+        )
+        return response
+
+
+def _client_ip(request: HttpRequest) -> str:
+    """
+    Resolve the client IP.
+
+    ``X-Forwarded-For`` is only consulted because the API always sits behind a
+    reverse proxy we control; the left-most entry is the original client.
+    """
+    forwarded = request.META.get("HTTP_X_FORWARDED_FOR")
+    if forwarded:
+        return forwarded.split(",")[0].strip()[:45]
+    return request.META.get("REMOTE_ADDR", "")[:45]
@@ -0,0 +1,32 @@
+"""Shared pagination classes."""
+
+from __future__ import annotations
+
+from collections import OrderedDict
+from typing import Any
+
+from rest_framework import pagination
+from rest_framework.response import Response
+
+
+class PageNumberPagination(pagination.PageNumberPagination):
+    """Page/page_size pagination with a hard upper bound on page size."""
+
+    page_size = 20
+    page_size_query_param = "page_size"
+    max_page_size = 100
+
+    def get_paginated_response(self, data: Any) -> Response:
+        return Response(
+            OrderedDict(
+                [
+                    ("count", self.page.paginator.count),
+                    ("page", self.page.number),
+                    ("page_size", self.get_page_size(self.request)),
+                    ("total_pages", self.page.paginator.num_pages),
+                    ("next", self.get_next_link()),
+                    ("previous", self.get_previous_link()),
+                    ("results", data),
+                ]
+            )
+        )
@@ -0,0 +1,20 @@
+"""Reusable permission classes."""
+
+from __future__ import annotations
+
+from rest_framework.permissions import SAFE_METHODS, BasePermission
+
+
+class IsAdminRole(BasePermission):
+    """Only users with the ADMIN role (or Django superusers)."""
+
+    message = "This action requires the admin role."
+
+    def has_permission(self, request, view) -> bool:  # noqa: ANN001, ARG002
+        user = request.user
+        return bool(user and user.is_authenticated and user.is_admin_role)
+
+
+class ReadOnly(BasePermission):
+    def has_permission(self, request, view) -> bool:  # noqa: ANN001, ARG002
+        return request.method in SAFE_METHODS
@@ -0,0 +1,25 @@
+"""Scoped throttles used by the search and auth endpoints."""
+
+from __future__ import annotations
+
+from rest_framework.throttling import ScopedRateThrottle, SimpleRateThrottle
+
+
+class SearchRateThrottle(ScopedRateThrottle):
+    """Rate limit for the (relatively expensive) search endpoints."""
+
+    scope = "search"
+
+
+class LoginRateThrottle(SimpleRateThrottle):
+    """
+    Brute-force protection for credential endpoints.
+
+    Keyed on the client IP rather than the user, because the attacker controls
+    the username field.
+    """
+
+    scope = "login"
+
+    def get_cache_key(self, request, view) -> str | None:  # noqa: ANN001, ARG002
+        return self.cache_format % {"scope": self.scope, "ident": self.get_ident(request)}
@@ -0,0 +1,10 @@
+from django.urls import path
+
+from apps.core.views import LivenessView, ReadinessView
+
+app_name = "core"
+
+urlpatterns = [
+    path("live/", LivenessView.as_view(), name="liveness"),
+    path("ready/", ReadinessView.as_view(), name="readiness"),
+]
@@ -0,0 +1,74 @@
+"""Operational endpoints used by Docker/Kubernetes probes and by humans."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from django.conf import settings
+from django.db import connection
+from drf_spectacular.utils import extend_schema
+from rest_framework import status
+from rest_framework.permissions import AllowAny
+from rest_framework.request import Request
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from apps.search.backends import get_search_backend
+
+
+class LivenessView(APIView):
+    """Cheap check: the process is up and can answer HTTP."""
+
+    permission_classes = [AllowAny]
+    authentication_classes: list = []
+    throttle_classes: list = []
+
+    @extend_schema(tags=["health"], responses={200: dict})
+    def get(self, request: Request) -> Response:  # noqa: ARG002
+        return Response({"status": "ok", "service": settings.SERVICE_NAME})
+
+
+class ReadinessView(APIView):
+    """
+    Deep check: verifies every dependency the API needs to serve traffic.
+
+    Returns 503 when any dependency is down so an orchestrator stops routing
+    traffic to this instance.
+    """
+
+    permission_classes = [AllowAny]
+    authentication_classes: list = []
+    throttle_classes: list = []
+
+    @extend_schema(tags=["health"], responses={200: dict, 503: dict})
+    def get(self, request: Request) -> Response:  # noqa: ARG002
+        checks: dict[str, Any] = {
+            "database": _check(_database),
+            "search": _check(_search),
+        }
+        healthy = all(c["healthy"] for c in checks.values())
+        return Response(
+            {"status": "ok" if healthy else "degraded", "checks": checks},
+            status=status.HTTP_200_OK if healthy else status.HTTP_503_SERVICE_UNAVAILABLE,
+        )
+
+
+def _check(fn) -> dict[str, Any]:  # noqa: ANN001
+    try:
+        detail = fn()
+    except Exception as exc:  # noqa: BLE001 - health checks must never raise
+        return {"healthy": False, "detail": str(exc)[:200]}
+    return {"healthy": True, "detail": detail}
+
+
+def _database() -> str:
+    with connection.cursor() as cursor:
+        cursor.execute("SELECT 1")
+        cursor.fetchone()
+    return "reachable"
+
+
+def _search() -> str:
+    backend = get_search_backend()
+    backend.ping()
+    return f"{backend.name} reachable"