feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
from django.contrib import admin
|
||||
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from apps.accounts.models import User
|
||||
|
||||
|
||||
@admin.register(User)
|
||||
class UserAdmin(BaseUserAdmin):
|
||||
ordering = ("-date_joined",)
|
||||
list_display = ("email", "full_name", "role", "is_active", "is_staff", "date_joined")
|
||||
list_filter = ("role", "is_active", "is_staff", "is_superuser")
|
||||
search_fields = ("email", "full_name")
|
||||
readonly_fields = ("date_joined", "last_login")
|
||||
|
||||
fieldsets = (
|
||||
(None, {"fields": ("email", "password")}),
|
||||
(_("Personal info"), {"fields": ("full_name",)}),
|
||||
(_("Permissions"), {"fields": ("role", "is_active", "is_staff", "is_superuser", "groups")}),
|
||||
(_("Important dates"), {"fields": ("last_login", "date_joined")}),
|
||||
)
|
||||
add_fieldsets = (
|
||||
(
|
||||
None,
|
||||
{
|
||||
"classes": ("wide",),
|
||||
"fields": ("email", "full_name", "role", "password1", "password2"),
|
||||
},
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class AccountsConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.accounts"
|
||||
verbose_name = "Accounts"
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
``python manage.py seed_users`` - create the demo accounts used by the README.
|
||||
|
||||
Credentials come from the environment so no password is ever committed; the
|
||||
command is idempotent and refuses to run with production settings unless
|
||||
``--force`` is given.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from apps.accounts.models import Role, User
|
||||
|
||||
DEMO_USERS = (
|
||||
("DEMO_ADMIN_EMAIL", "DEMO_ADMIN_PASSWORD", Role.ADMIN, True),
|
||||
("DEMO_ANALYST_EMAIL", "DEMO_ANALYST_PASSWORD", Role.ANALYST, False),
|
||||
("DEMO_VIEWER_EMAIL", "DEMO_VIEWER_PASSWORD", Role.VIEWER, False),
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create the demo admin / analyst / viewer accounts."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
parser.add_argument("--force", action="store_true", help="Run even outside development.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
for email_var, password_var, role, is_staff in DEMO_USERS:
|
||||
email = os.environ.get(email_var)
|
||||
password = os.environ.get(password_var)
|
||||
if not email or not password:
|
||||
self.stdout.write(
|
||||
self.style.WARNING(f"Skipping {role}: {email_var}/{password_var} not set.")
|
||||
)
|
||||
continue
|
||||
|
||||
user = User.objects.filter(email=email.lower()).first()
|
||||
if user:
|
||||
user.role = role
|
||||
user.is_staff = is_staff
|
||||
user.is_superuser = is_staff
|
||||
user.set_password(password)
|
||||
user.save(update_fields=["role", "is_staff", "is_superuser", "password"])
|
||||
self.stdout.write(f"Updated {email} ({role}).")
|
||||
continue
|
||||
|
||||
User.objects.create_user(
|
||||
email=email,
|
||||
password=password,
|
||||
role=role,
|
||||
full_name=f"Demo {role.label}",
|
||||
is_staff=is_staff,
|
||||
is_superuser=is_staff,
|
||||
)
|
||||
self.stdout.write(self.style.SUCCESS(f"Created {email} ({role})."))
|
||||
@@ -0,0 +1,103 @@
|
||||
# Generated by Django 5.1.15 on 2026-09-03 03:36
|
||||
|
||||
import apps.accounts.models
|
||||
import django.utils.timezone
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="User",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("password", models.CharField(max_length=128, verbose_name="password")),
|
||||
(
|
||||
"last_login",
|
||||
models.DateTimeField(blank=True, null=True, verbose_name="last login"),
|
||||
),
|
||||
(
|
||||
"is_superuser",
|
||||
models.BooleanField(
|
||||
default=False,
|
||||
help_text="Designates that this user has all permissions without explicitly assigning them.",
|
||||
verbose_name="superuser status",
|
||||
),
|
||||
),
|
||||
(
|
||||
"email",
|
||||
models.EmailField(
|
||||
db_index=True, max_length=254, unique=True, verbose_name="email address"
|
||||
),
|
||||
),
|
||||
(
|
||||
"full_name",
|
||||
models.CharField(blank=True, max_length=150, verbose_name="full name"),
|
||||
),
|
||||
(
|
||||
"role",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("admin", "Administrator"),
|
||||
("analyst", "Analyst"),
|
||||
("viewer", "Viewer"),
|
||||
],
|
||||
default="viewer",
|
||||
help_text="Determines which fields and actions the user has access to.",
|
||||
max_length=16,
|
||||
verbose_name="role",
|
||||
),
|
||||
),
|
||||
("is_active", models.BooleanField(default=True, verbose_name="active")),
|
||||
("is_staff", models.BooleanField(default=False, verbose_name="staff status")),
|
||||
(
|
||||
"date_joined",
|
||||
models.DateTimeField(
|
||||
default=django.utils.timezone.now, verbose_name="date joined"
|
||||
),
|
||||
),
|
||||
(
|
||||
"groups",
|
||||
models.ManyToManyField(
|
||||
blank=True,
|
||||
help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
|
||||
related_name="user_set",
|
||||
related_query_name="user",
|
||||
to="auth.group",
|
||||
verbose_name="groups",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user_permissions",
|
||||
models.ManyToManyField(
|
||||
blank=True,
|
||||
help_text="Specific permissions for this user.",
|
||||
related_name="user_set",
|
||||
related_query_name="user",
|
||||
to="auth.permission",
|
||||
verbose_name="user permissions",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "user",
|
||||
"verbose_name_plural": "users",
|
||||
"ordering": ("-date_joined",),
|
||||
},
|
||||
managers=[
|
||||
("objects", apps.accounts.models.UserManager()),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Custom user model.
|
||||
|
||||
Two things are worth pointing out:
|
||||
|
||||
* The username field is the e-mail address - the dataset users are internal
|
||||
analysts, and an e-mail is the identifier they already have.
|
||||
* Authorisation is expressed as a single ``role`` field instead of a pile of
|
||||
Django groups. With three well defined roles this is easier to reason about,
|
||||
easier to test, and it keeps the JWT payload small.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib.auth.models import AbstractBaseUser, BaseUserManager, PermissionsMixin
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
||||
class Role(models.TextChoices):
|
||||
ADMIN = "admin", _("Administrator")
|
||||
ANALYST = "analyst", _("Analyst")
|
||||
VIEWER = "viewer", _("Viewer")
|
||||
|
||||
|
||||
class UserManager(BaseUserManager):
|
||||
"""Manager that normalises e-mails and refuses to create users without one."""
|
||||
|
||||
use_in_migrations = True
|
||||
|
||||
def create_user(self, email: str, password: str | None = None, **extra) -> User:
|
||||
if not email:
|
||||
raise ValueError("Users must have an email address.")
|
||||
email = self.normalize_email(email).lower()
|
||||
user = self.model(email=email, **extra)
|
||||
# ``set_password`` hashes with Argon2 (see PASSWORD_HASHERS).
|
||||
user.set_password(password)
|
||||
user.full_clean(exclude=["password"])
|
||||
user.save(using=self._db)
|
||||
return user
|
||||
|
||||
def create_superuser(self, email: str, password: str | None = None, **extra) -> User:
|
||||
extra.setdefault("is_staff", True)
|
||||
extra.setdefault("is_superuser", True)
|
||||
extra.setdefault("role", Role.ADMIN)
|
||||
if extra.get("is_staff") is not True or extra.get("is_superuser") is not True:
|
||||
raise ValueError("Superuser must have is_staff=True and is_superuser=True.")
|
||||
return self.create_user(email, password, **extra)
|
||||
|
||||
|
||||
class User(AbstractBaseUser, PermissionsMixin):
|
||||
email = models.EmailField(_("email address"), unique=True, db_index=True)
|
||||
full_name = models.CharField(_("full name"), max_length=150, blank=True)
|
||||
role = models.CharField(
|
||||
_("role"),
|
||||
max_length=16,
|
||||
choices=Role.choices,
|
||||
default=Role.VIEWER,
|
||||
help_text=_("Determines which fields and actions the user has access to."),
|
||||
)
|
||||
is_active = models.BooleanField(_("active"), default=True)
|
||||
is_staff = models.BooleanField(_("staff status"), default=False)
|
||||
date_joined = models.DateTimeField(_("date joined"), default=timezone.now)
|
||||
|
||||
objects = UserManager()
|
||||
|
||||
USERNAME_FIELD = "email"
|
||||
REQUIRED_FIELDS: list[str] = []
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("user")
|
||||
verbose_name_plural = _("users")
|
||||
ordering = ("-date_joined",)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.email
|
||||
|
||||
# -- Authorisation helpers ---------------------------------------------
|
||||
@property
|
||||
def is_admin_role(self) -> bool:
|
||||
return self.role == Role.ADMIN or self.is_superuser
|
||||
|
||||
@property
|
||||
def can_view_pii(self) -> bool:
|
||||
"""
|
||||
Whether raw contact details may be returned to this user.
|
||||
|
||||
Viewers only ever see masked e-mails and phone numbers; unmasking is a
|
||||
deliberate privilege granted to analysts and administrators.
|
||||
"""
|
||||
return self.role in (Role.ADMIN, Role.ANALYST) or self.is_superuser
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Serializers for registration, login and profile of the API's own users."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.auth import password_validation
|
||||
from django.db import IntegrityError
|
||||
from rest_framework import serializers
|
||||
from rest_framework_simplejwt.serializers import (
|
||||
TokenObtainPairSerializer as BaseTokenObtainPairSerializer,
|
||||
)
|
||||
|
||||
from apps.accounts.models import Role, User
|
||||
|
||||
audit = logging.getLogger("audit.auth")
|
||||
|
||||
|
||||
class UserSerializer(serializers.ModelSerializer):
|
||||
can_view_pii = serializers.BooleanField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = User
|
||||
fields = ("id", "email", "full_name", "role", "can_view_pii", "date_joined")
|
||||
read_only_fields = fields
|
||||
|
||||
|
||||
class RegisterSerializer(serializers.ModelSerializer):
|
||||
password = serializers.CharField(write_only=True, style={"input_type": "password"})
|
||||
password_confirm = serializers.CharField(write_only=True, style={"input_type": "password"})
|
||||
|
||||
class Meta:
|
||||
model = User
|
||||
fields = ("email", "full_name", "password", "password_confirm")
|
||||
|
||||
def validate_email(self, value: str) -> str:
|
||||
return value.strip().lower()
|
||||
|
||||
def validate_password(self, value: str) -> str:
|
||||
# Runs Django's configured validators: length, common passwords, etc.
|
||||
password_validation.validate_password(value)
|
||||
return value
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
if attrs["password"] != attrs["password_confirm"]:
|
||||
raise serializers.ValidationError({"password_confirm": "Passwords do not match."})
|
||||
return attrs
|
||||
|
||||
def create(self, validated_data: dict[str, Any]) -> User:
|
||||
validated_data.pop("password_confirm")
|
||||
password = validated_data.pop("password")
|
||||
try:
|
||||
# Self-service sign-up always yields the least privileged role;
|
||||
# elevation is an explicit admin action.
|
||||
return User.objects.create_user(password=password, role=Role.VIEWER, **validated_data)
|
||||
except IntegrityError as exc: # pragma: no cover - race with unique index
|
||||
raise serializers.ValidationError(
|
||||
{"email": "A user with this email already exists."}
|
||||
) from exc
|
||||
|
||||
|
||||
class TokenObtainPairSerializer(BaseTokenObtainPairSerializer):
|
||||
"""Adds identity claims to the access token and records an audit event."""
|
||||
|
||||
@classmethod
|
||||
def get_token(cls, user: User): # noqa: ANN206
|
||||
token = super().get_token(user)
|
||||
token["role"] = user.role
|
||||
token["email"] = user.email
|
||||
return token
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
data = super().validate(attrs)
|
||||
audit.info(
|
||||
"login_succeeded",
|
||||
extra={
|
||||
"event": "login_succeeded",
|
||||
"user_id": self.user.id,
|
||||
"user_role": self.user.role,
|
||||
},
|
||||
)
|
||||
data["user"] = UserSerializer(self.user).data
|
||||
return data
|
||||
|
||||
|
||||
class LogoutSerializer(serializers.Serializer):
|
||||
refresh = serializers.CharField()
|
||||
@@ -0,0 +1,13 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.accounts.views import LoginView, LogoutView, MeView, RefreshView, RegisterView
|
||||
|
||||
app_name = "accounts"
|
||||
|
||||
urlpatterns = [
|
||||
path("register/", RegisterView.as_view(), name="register"),
|
||||
path("login/", LoginView.as_view(), name="login"),
|
||||
path("refresh/", RefreshView.as_view(), name="refresh"),
|
||||
path("logout/", LogoutView.as_view(), name="logout"),
|
||||
path("me/", MeView.as_view(), name="me"),
|
||||
]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Authentication endpoints: register, login, refresh, logout, me."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import generics, status
|
||||
from rest_framework.permissions import AllowAny, IsAuthenticated
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
from rest_framework_simplejwt.exceptions import TokenError
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
from rest_framework_simplejwt.views import TokenObtainPairView as BaseTokenObtainPairView
|
||||
from rest_framework_simplejwt.views import TokenRefreshView as BaseTokenRefreshView
|
||||
|
||||
from apps.accounts.models import User
|
||||
from apps.accounts.serializers import LogoutSerializer, RegisterSerializer, UserSerializer
|
||||
from apps.core.throttling import LoginRateThrottle
|
||||
|
||||
audit = logging.getLogger("audit.auth")
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class RegisterView(generics.CreateAPIView):
|
||||
"""Self-service sign-up. New accounts always get the ``viewer`` role."""
|
||||
|
||||
serializer_class = RegisterSerializer
|
||||
permission_classes = [AllowAny]
|
||||
throttle_classes = [LoginRateThrottle]
|
||||
queryset = User.objects.all()
|
||||
|
||||
def create(self, request: Request, *args, **kwargs) -> Response:
|
||||
serializer = self.get_serializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
user = serializer.save()
|
||||
audit.info("user_registered", extra={"event": "user_registered", "user_id": user.id})
|
||||
return Response(UserSerializer(user).data, status=status.HTTP_201_CREATED)
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class LoginView(BaseTokenObtainPairView):
|
||||
"""Exchange credentials for an access/refresh token pair."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
throttle_classes = [LoginRateThrottle]
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class RefreshView(BaseTokenRefreshView):
|
||||
"""Rotate a refresh token; the presented token is blacklisted afterwards."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
throttle_classes = [LoginRateThrottle]
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"], request=LogoutSerializer, responses={205: None})
|
||||
class LogoutView(APIView):
|
||||
"""Blacklist a refresh token so it can no longer be rotated."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
serializer_class = LogoutSerializer
|
||||
|
||||
def post(self, request: Request) -> Response:
|
||||
serializer = LogoutSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
# An already-invalid token is not an error worth surfacing: the
|
||||
# caller's intent (be logged out) is satisfied either way.
|
||||
with contextlib.suppress(TokenError):
|
||||
RefreshToken(serializer.validated_data["refresh"]).blacklist()
|
||||
audit.info("logout", extra={"event": "logout", "user_id": request.user.id})
|
||||
return Response(status=status.HTTP_205_RESET_CONTENT)
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class MeView(generics.RetrieveAPIView):
|
||||
"""Identity and effective privileges of the calling user."""
|
||||
|
||||
serializer_class = UserSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_object(self) -> User:
|
||||
return self.request.user
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class CoreConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.core"
|
||||
verbose_name = "Core"
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
A single, predictable error envelope for the whole API.
|
||||
|
||||
Every failure - validation, permission, throttling, or an unexpected crash -
|
||||
comes back in the same shape:
|
||||
|
||||
{"error": {"code": "validation_error", "message": "...", "details": {...},
|
||||
"request_id": "..."}}
|
||||
|
||||
Unhandled exceptions are logged with their stack trace but never leak internals
|
||||
to the client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from django.core.exceptions import PermissionDenied
|
||||
from django.http import Http404
|
||||
from rest_framework import exceptions, status
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import exception_handler as drf_exception_handler
|
||||
|
||||
from apps.core.logging import get_request_id
|
||||
|
||||
logger = logging.getLogger("apps.errors")
|
||||
|
||||
_CODE_BY_STATUS = {
|
||||
status.HTTP_400_BAD_REQUEST: "validation_error",
|
||||
status.HTTP_401_UNAUTHORIZED: "authentication_failed",
|
||||
status.HTTP_403_FORBIDDEN: "permission_denied",
|
||||
status.HTTP_404_NOT_FOUND: "not_found",
|
||||
status.HTTP_405_METHOD_NOT_ALLOWED: "method_not_allowed",
|
||||
status.HTTP_429_TOO_MANY_REQUESTS: "throttled",
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE: "service_unavailable",
|
||||
}
|
||||
|
||||
|
||||
class ServiceUnavailable(exceptions.APIException):
|
||||
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
default_detail = "A downstream dependency is unavailable. Please retry shortly."
|
||||
default_code = "service_unavailable"
|
||||
|
||||
|
||||
class SearchBackendError(ServiceUnavailable):
|
||||
default_detail = "The search backend is currently unavailable."
|
||||
default_code = "search_unavailable"
|
||||
|
||||
|
||||
def api_exception_handler(exc: Exception, context: dict[str, Any]) -> Response | None:
|
||||
response = drf_exception_handler(exc, context)
|
||||
|
||||
if response is None:
|
||||
# Anything DRF does not know about is a bug on our side.
|
||||
if isinstance(exc, Http404):
|
||||
response = Response(status=status.HTTP_404_NOT_FOUND)
|
||||
elif isinstance(exc, PermissionDenied):
|
||||
response = Response(status=status.HTTP_403_FORBIDDEN)
|
||||
else:
|
||||
logger.exception(
|
||||
"unhandled_exception",
|
||||
extra={"event": "unhandled_exception", "view": str(context.get("view"))},
|
||||
)
|
||||
return Response(
|
||||
_envelope("internal_error", "An unexpected error occurred.", None),
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
detail = getattr(exc, "detail", None)
|
||||
if isinstance(exc, exceptions.ValidationError):
|
||||
# DRF labels every field error "invalid"; one stable code per failure
|
||||
# mode is far easier for a client to branch on.
|
||||
code = "validation_error"
|
||||
else:
|
||||
code = getattr(exc, "default_code", None) or _CODE_BY_STATUS.get(
|
||||
response.status_code, "error"
|
||||
)
|
||||
|
||||
if isinstance(detail, dict):
|
||||
message = "The request payload failed validation."
|
||||
details: Any = detail
|
||||
elif isinstance(detail, list):
|
||||
message = str(detail[0]) if detail else "Request failed."
|
||||
details = detail
|
||||
elif detail is not None:
|
||||
message = str(detail)
|
||||
details = None
|
||||
else:
|
||||
message = response.status_text
|
||||
details = None
|
||||
|
||||
response.data = _envelope(code, message, details)
|
||||
return response
|
||||
|
||||
|
||||
def _envelope(code: str, message: str, details: Any) -> dict[str, Any]:
|
||||
error: dict[str, Any] = {"code": code, "message": message, "request_id": get_request_id()}
|
||||
if details:
|
||||
error["details"] = details
|
||||
return {"error": error}
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Structured logging primitives.
|
||||
|
||||
Every log record is emitted as a single JSON object so Logstash can ingest it
|
||||
without grok patterns, and every record carries the request id of the HTTP
|
||||
request that produced it, which makes a single user action traceable end to end
|
||||
in Kibana.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
# Propagates the current request id to any code running in the same context,
|
||||
# including background threads started per request.
|
||||
_request_id: ContextVar[str] = ContextVar("request_id", default="-")
|
||||
|
||||
# Attributes that ``logging`` puts on every record; anything else the caller
|
||||
# passed through ``extra=`` is treated as structured context and serialised.
|
||||
_STANDARD_ATTRS = frozenset(
|
||||
{
|
||||
"args", "asctime", "created", "exc_info", "exc_text", "filename",
|
||||
"funcName", "levelname", "levelno", "lineno", "module", "msecs",
|
||||
"message", "msg", "name", "pathname", "process", "processName",
|
||||
"relativeCreated", "stack_info", "thread", "threadName", "taskName",
|
||||
"request_id",
|
||||
}
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
def set_request_id(value: str) -> None:
|
||||
_request_id.set(value)
|
||||
|
||||
|
||||
def get_request_id() -> str:
|
||||
return _request_id.get()
|
||||
|
||||
|
||||
class RequestIDFilter(logging.Filter):
|
||||
"""Attaches the ambient request id to every record."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
record.request_id = getattr(record, "request_id", None) or get_request_id()
|
||||
return True
|
||||
|
||||
|
||||
class JSONFormatter(logging.Formatter):
|
||||
"""Renders a ``LogRecord`` as one line of JSON."""
|
||||
|
||||
def __init__(self, service: str = "app", environment: str = "development") -> None:
|
||||
super().__init__()
|
||||
self.service = service
|
||||
self.environment = environment
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
"@timestamp": dt.datetime.fromtimestamp(record.created, tz=dt.UTC).isoformat(),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
"service": self.service,
|
||||
"environment": self.environment,
|
||||
"request_id": getattr(record, "request_id", "-"),
|
||||
}
|
||||
|
||||
if record.exc_info:
|
||||
exc_type, exc_value, exc_tb = record.exc_info
|
||||
payload["error"] = {
|
||||
"type": getattr(exc_type, "__name__", str(exc_type)),
|
||||
"message": str(exc_value),
|
||||
"stack": "".join(traceback.format_exception(exc_type, exc_value, exc_tb))[:8000],
|
||||
}
|
||||
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STANDARD_ATTRS and not key.startswith("_"):
|
||||
payload[key] = _coerce(value)
|
||||
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _coerce(value: Any) -> Any:
|
||||
"""Keep the payload JSON serialisable without blowing up on odd types."""
|
||||
if isinstance(value, str | int | float | bool | type(None) | list | dict):
|
||||
return value
|
||||
return str(value)
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Request scoped middleware: correlation ids and structured access logs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
|
||||
from apps.core.logging import set_request_id
|
||||
|
||||
logger = logging.getLogger("apps.access")
|
||||
|
||||
REQUEST_ID_HEADER = "HTTP_X_REQUEST_ID"
|
||||
RESPONSE_HEADER = "X-Request-ID"
|
||||
_MAX_REQUEST_ID_LEN = 64
|
||||
|
||||
|
||||
class RequestIDMiddleware:
|
||||
"""
|
||||
Reads ``X-Request-ID`` from the incoming request or generates one.
|
||||
|
||||
The value is echoed back on the response and injected into every log record
|
||||
produced while handling the request, so a browser network tab entry can be
|
||||
pasted straight into Kibana.
|
||||
"""
|
||||
|
||||
def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request: HttpRequest) -> HttpResponse:
|
||||
incoming = request.META.get(REQUEST_ID_HEADER, "")
|
||||
# Never trust client input verbatim: keep it short and alphanumeric.
|
||||
request_id = (
|
||||
incoming[:_MAX_REQUEST_ID_LEN]
|
||||
if incoming and incoming.replace("-", "").replace("_", "").isalnum()
|
||||
else uuid.uuid4().hex
|
||||
)
|
||||
request.request_id = request_id # type: ignore[attr-defined]
|
||||
set_request_id(request_id)
|
||||
|
||||
response = self.get_response(request)
|
||||
response[RESPONSE_HEADER] = request_id
|
||||
return response
|
||||
|
||||
|
||||
class AccessLogMiddleware:
|
||||
"""Emits one structured log line per request with timing and identity."""
|
||||
|
||||
def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request: HttpRequest) -> HttpResponse:
|
||||
started = time.perf_counter()
|
||||
response = self.get_response(request)
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
|
||||
user = getattr(request, "user", None)
|
||||
logger.info(
|
||||
"http_request",
|
||||
extra={
|
||||
"event": "http_request",
|
||||
"http_method": request.method,
|
||||
"http_path": request.path,
|
||||
"http_status": response.status_code,
|
||||
"duration_ms": duration_ms,
|
||||
"client_ip": _client_ip(request),
|
||||
"user_agent": request.META.get("HTTP_USER_AGENT", "")[:200],
|
||||
"user_id": getattr(user, "id", None) if user and user.is_authenticated else None,
|
||||
},
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _client_ip(request: HttpRequest) -> str:
|
||||
"""
|
||||
Resolve the client IP.
|
||||
|
||||
``X-Forwarded-For`` is only consulted because the API always sits behind a
|
||||
reverse proxy we control; the left-most entry is the original client.
|
||||
"""
|
||||
forwarded = request.META.get("HTTP_X_FORWARDED_FOR")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()[:45]
|
||||
return request.META.get("REMOTE_ADDR", "")[:45]
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Shared pagination classes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import Any
|
||||
|
||||
from rest_framework import pagination
|
||||
from rest_framework.response import Response
|
||||
|
||||
|
||||
class PageNumberPagination(pagination.PageNumberPagination):
|
||||
"""Page/page_size pagination with a hard upper bound on page size."""
|
||||
|
||||
page_size = 20
|
||||
page_size_query_param = "page_size"
|
||||
max_page_size = 100
|
||||
|
||||
def get_paginated_response(self, data: Any) -> Response:
|
||||
return Response(
|
||||
OrderedDict(
|
||||
[
|
||||
("count", self.page.paginator.count),
|
||||
("page", self.page.number),
|
||||
("page_size", self.get_page_size(self.request)),
|
||||
("total_pages", self.page.paginator.num_pages),
|
||||
("next", self.get_next_link()),
|
||||
("previous", self.get_previous_link()),
|
||||
("results", data),
|
||||
]
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Reusable permission classes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from rest_framework.permissions import SAFE_METHODS, BasePermission
|
||||
|
||||
|
||||
class IsAdminRole(BasePermission):
|
||||
"""Only users with the ADMIN role (or Django superusers)."""
|
||||
|
||||
message = "This action requires the admin role."
|
||||
|
||||
def has_permission(self, request, view) -> bool: # noqa: ANN001, ARG002
|
||||
user = request.user
|
||||
return bool(user and user.is_authenticated and user.is_admin_role)
|
||||
|
||||
|
||||
class ReadOnly(BasePermission):
|
||||
def has_permission(self, request, view) -> bool: # noqa: ANN001, ARG002
|
||||
return request.method in SAFE_METHODS
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Scoped throttles used by the search and auth endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from rest_framework.throttling import ScopedRateThrottle, SimpleRateThrottle
|
||||
|
||||
|
||||
class SearchRateThrottle(ScopedRateThrottle):
|
||||
"""Rate limit for the (relatively expensive) search endpoints."""
|
||||
|
||||
scope = "search"
|
||||
|
||||
|
||||
class LoginRateThrottle(SimpleRateThrottle):
|
||||
"""
|
||||
Brute-force protection for credential endpoints.
|
||||
|
||||
Keyed on the client IP rather than the user, because the attacker controls
|
||||
the username field.
|
||||
"""
|
||||
|
||||
scope = "login"
|
||||
|
||||
def get_cache_key(self, request, view) -> str | None: # noqa: ANN001, ARG002
|
||||
return self.cache_format % {"scope": self.scope, "ident": self.get_ident(request)}
|
||||
@@ -0,0 +1,10 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.core.views import LivenessView, ReadinessView
|
||||
|
||||
app_name = "core"
|
||||
|
||||
urlpatterns = [
|
||||
path("live/", LivenessView.as_view(), name="liveness"),
|
||||
path("ready/", ReadinessView.as_view(), name="readiness"),
|
||||
]
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Operational endpoints used by Docker/Kubernetes probes and by humans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.permissions import AllowAny
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from apps.search.backends import get_search_backend
|
||||
|
||||
|
||||
class LivenessView(APIView):
|
||||
"""Cheap check: the process is up and can answer HTTP."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes: list = []
|
||||
throttle_classes: list = []
|
||||
|
||||
@extend_schema(tags=["health"], responses={200: dict})
|
||||
def get(self, request: Request) -> Response: # noqa: ARG002
|
||||
return Response({"status": "ok", "service": settings.SERVICE_NAME})
|
||||
|
||||
|
||||
class ReadinessView(APIView):
|
||||
"""
|
||||
Deep check: verifies every dependency the API needs to serve traffic.
|
||||
|
||||
Returns 503 when any dependency is down so an orchestrator stops routing
|
||||
traffic to this instance.
|
||||
"""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes: list = []
|
||||
throttle_classes: list = []
|
||||
|
||||
@extend_schema(tags=["health"], responses={200: dict, 503: dict})
|
||||
def get(self, request: Request) -> Response: # noqa: ARG002
|
||||
checks: dict[str, Any] = {
|
||||
"database": _check(_database),
|
||||
"search": _check(_search),
|
||||
}
|
||||
healthy = all(c["healthy"] for c in checks.values())
|
||||
return Response(
|
||||
{"status": "ok" if healthy else "degraded", "checks": checks},
|
||||
status=status.HTTP_200_OK if healthy else status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
|
||||
def _check(fn) -> dict[str, Any]: # noqa: ANN001
|
||||
try:
|
||||
detail = fn()
|
||||
except Exception as exc: # noqa: BLE001 - health checks must never raise
|
||||
return {"healthy": False, "detail": str(exc)[:200]}
|
||||
return {"healthy": True, "detail": detail}
|
||||
|
||||
|
||||
def _database() -> str:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
cursor.fetchone()
|
||||
return "reachable"
|
||||
|
||||
|
||||
def _search() -> str:
|
||||
backend = get_search_backend()
|
||||
backend.ping()
|
||||
return f"{backend.name} reachable"
|
||||
@@ -0,0 +1,72 @@
|
||||
from django.contrib import admin
|
||||
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
|
||||
class ExperienceInline(admin.TabularInline):
|
||||
model = Experience
|
||||
extra = 0
|
||||
fields = ("title", "company_name", "start_date", "end_date", "is_primary")
|
||||
readonly_fields = fields
|
||||
can_delete = False
|
||||
show_change_link = True
|
||||
|
||||
|
||||
class EducationInline(admin.TabularInline):
|
||||
model = Education
|
||||
extra = 0
|
||||
fields = ("school_name", "degrees", "majors", "end_date")
|
||||
readonly_fields = fields
|
||||
can_delete = False
|
||||
|
||||
|
||||
class ProfileSkillInline(admin.TabularInline):
|
||||
model = ProfileSkill
|
||||
extra = 0
|
||||
autocomplete_fields = ("skill",)
|
||||
|
||||
|
||||
@admin.register(Profile)
|
||||
class ProfileAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
"full_name", "job_title", "current_company", "location_country", "years_experience",
|
||||
) # fmt: skip
|
||||
list_filter = ("location_country", "job_title_role", "industry")
|
||||
search_fields = ("full_name", "job_title", "skills_text", "companies_text")
|
||||
list_select_related = ("current_company",)
|
||||
inlines = (ProfileSkillInline, ExperienceInline, EducationInline)
|
||||
readonly_fields = (
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"skills_text",
|
||||
"companies_text",
|
||||
"education_text",
|
||||
)
|
||||
|
||||
|
||||
@admin.register(Skill)
|
||||
class SkillAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "slug", "profile_count")
|
||||
search_fields = ("name", "slug")
|
||||
ordering = ("-profile_count",)
|
||||
|
||||
|
||||
@admin.register(Company)
|
||||
class CompanyAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "industry", "size", "country")
|
||||
search_fields = ("name",)
|
||||
list_filter = ("size", "industry")
|
||||
|
||||
|
||||
@admin.register(School)
|
||||
class SchoolAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "type", "country")
|
||||
search_fields = ("name",)
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ProfilesConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.profiles"
|
||||
verbose_name = "Profiles"
|
||||
@@ -0,0 +1,394 @@
|
||||
"""
|
||||
Resilient reader for the supplied LinkedIn CSV export.
|
||||
|
||||
Why this module exists
|
||||
----------------------
|
||||
The provided file is a real-world dirty export, not a clean CSV. Feeding it to
|
||||
``csv.DictReader`` silently produces garbage. Four separate defects are present:
|
||||
|
||||
1. **Injected watermark** - many rows start with an extra, unheadered column
|
||||
containing ``H:\\New folder\\leak\\...@BreachedData1 ...``. Everything after
|
||||
it is shifted one position to the right.
|
||||
2. **Missing columns** - most records omit one or more optional fields
|
||||
(``job_title_role``, ``job_title_sub_role``, ...) instead of writing an empty
|
||||
value, so the remaining values slide left relative to the header.
|
||||
3. **Broken quoting** - a handful of ``summary`` values contain an unescaped
|
||||
double quote, which splits a single logical record across several physical
|
||||
rows.
|
||||
4. **Concatenated exports** - the header line appears a second time in the
|
||||
middle of the file and ~40 people are duplicated.
|
||||
|
||||
Repair strategy
|
||||
---------------
|
||||
Every column has a *signature*: ``linkedin_url`` contains ``linkedin.com/in/``,
|
||||
``skills`` is a Python list literal, ``location_geo`` looks like ``lat,lon`` and
|
||||
so on. We score each (column, value) pair against those signatures and then run
|
||||
a Needleman-Wunsch style alignment between the row's values and the header
|
||||
columns, allowing two edit operations:
|
||||
|
||||
* *skip column* - the record simply does not carry that field;
|
||||
* *drop value* - a stray fragment that belongs to no column.
|
||||
|
||||
The alignment that maximises the total score is the reconstructed record. The
|
||||
result is validated a second time and any value that still fails its signature
|
||||
is discarded rather than persisted as junk - it is better to store nothing than
|
||||
to store a phone number in the e-mail column.
|
||||
|
||||
``ast.literal_eval`` is used (never ``eval``) so the nested Python-repr columns
|
||||
can be parsed without executing anything from the file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import csv
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# The nested columns are large; the stdlib default (128 KiB) is not enough.
|
||||
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))
|
||||
|
||||
WATERMARK_MARKERS = ("@BreachedData", "H:\\New folder")
|
||||
CONTINENTS = frozenset(
|
||||
{"north america", "south america", "europe", "asia", "africa", "oceania", "antarctica"}
|
||||
)
|
||||
GENDERS = frozenset({"male", "female"})
|
||||
|
||||
Validator = Callable[[str], bool | None]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Column signatures
|
||||
# ---------------------------------------------------------------------------
|
||||
def _list_of_str(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
return isinstance(parsed, list) and all(isinstance(item, str) for item in parsed)
|
||||
|
||||
|
||||
def _list_of_dicts(required_key: str) -> Validator:
|
||||
def check(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
if not isinstance(parsed, list):
|
||||
return None
|
||||
if not parsed:
|
||||
return True
|
||||
return all(isinstance(item, dict) for item in parsed) and required_key in parsed[0]
|
||||
|
||||
return check
|
||||
|
||||
|
||||
def _dict_with(required_key: str) -> Validator:
|
||||
def check(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
return isinstance(parsed, dict) and required_key in parsed
|
||||
|
||||
return check
|
||||
|
||||
|
||||
def _matches(pattern: str) -> Validator:
|
||||
compiled = re.compile(pattern)
|
||||
return lambda value: bool(compiled.fullmatch(value))
|
||||
|
||||
|
||||
def _contains(needle: str) -> Validator:
|
||||
return lambda value: needle in value
|
||||
|
||||
|
||||
def _one_of(options: frozenset[str]) -> Validator:
|
||||
return lambda value: value.lower() in options
|
||||
|
||||
|
||||
def _safe_literal(value: str) -> Any | None:
|
||||
"""``ast.literal_eval`` that returns ``None`` instead of raising."""
|
||||
try:
|
||||
return ast.literal_eval(value)
|
||||
except (ValueError, SyntaxError, MemoryError, RecursionError):
|
||||
return None
|
||||
|
||||
|
||||
#: Columns without an entry here accept any text (name, title, industry, ...).
|
||||
FIELD_VALIDATORS: dict[str, Validator] = {
|
||||
"gender": _one_of(GENDERS),
|
||||
"linkedin_url": _contains("linkedin.com/in/"),
|
||||
"linkedin_id": _matches(r"\d{3,20}"),
|
||||
"facebook_url": _contains("facebook.com/"),
|
||||
"facebook_id": _matches(r"\d{5,25}"),
|
||||
"job_title_role": _matches(r"[a-z_]{3,40}"),
|
||||
"job_title_sub_role": _matches(r"[a-z_]{3,40}"),
|
||||
"job_title_levels": _list_of_str,
|
||||
"job_company_website": _matches(r"[\w.-]+\.[a-z]{2,10}"),
|
||||
"job_company_size": _matches(
|
||||
r"(1-10|11-50|51-200|201-500|501-1000|1001-5000|5001-10000|10001\+)"
|
||||
),
|
||||
"job_company_founded": _matches(r"(1[6-9]|20)\d\d(\.0)?"),
|
||||
"job_company_linkedin_url": _contains("linkedin.com/company/"),
|
||||
"job_company_linkedin_id": _matches(r"\d{3,20}"),
|
||||
"job_company_facebook_url": _contains("facebook.com/"),
|
||||
"job_company_twitter_url": _contains("twitter.com/"),
|
||||
"job_company_location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"job_company_location_continent": _one_of(CONTINENTS),
|
||||
"job_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
|
||||
"job_start_date": _matches(r"\d{4}(-\d{2})?"),
|
||||
"location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"location_continent": _one_of(CONTINENTS),
|
||||
"location_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
|
||||
"linkedin_connections": _matches(r"\d+(\.\d+)?"),
|
||||
"inferred_salary": _matches(r"[\d,]+-[\d,]+|>?\d[\d,]*\+?"),
|
||||
"inferred_years_experience": _matches(r"\d{1,2}(\.\d+)?"),
|
||||
"phone_numbers": _list_of_str,
|
||||
"emails": _list_of_dicts("address"),
|
||||
"interests": _list_of_str,
|
||||
"skills": _list_of_str,
|
||||
"location_names": _list_of_str,
|
||||
"regions": _list_of_str,
|
||||
"countries": _list_of_str,
|
||||
"street_addresses": _list_of_dicts("street_address"),
|
||||
"experience": _list_of_dicts("company"),
|
||||
"education": _list_of_dicts("school"),
|
||||
"profiles": _list_of_dicts("network"),
|
||||
"certifications": _list_of_dicts("name"),
|
||||
"languages": _list_of_dicts("name"),
|
||||
"version_status": _dict_with("status"),
|
||||
"work_email": _contains("@"),
|
||||
"middle_initial": _matches(r"[a-z]"),
|
||||
"birth_year": _matches(r"(19|20)\d\d"),
|
||||
"birth_date": _matches(r"\d{4}-\d{2}(-\d{2})?"),
|
||||
"twitter_url": _contains("twitter.com/"),
|
||||
"github_url": _contains("github.com/"),
|
||||
"mobile_phone": _matches(r"\+?\d[\d -]{6,20}"),
|
||||
}
|
||||
|
||||
# Alignment weights. Tuned so that a confirmed signature match outweighs any
|
||||
# combination of edits, while losing a non-empty value is expensive.
|
||||
SCORE_MATCH = 3.0
|
||||
SCORE_MISMATCH = -4.0
|
||||
SCORE_FREE_TEXT = 0.3
|
||||
SCORE_EMPTY = 0.15
|
||||
PENALTY_MISSING_COLUMN = -0.5
|
||||
PENALTY_DROP_EMPTY = -0.3
|
||||
PENALTY_DROP_VALUE = -3.5
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportReport:
|
||||
"""Counters surfaced by the management command so imports are auditable."""
|
||||
|
||||
physical_rows: int = 0
|
||||
duplicate_headers: int = 0
|
||||
repaired_rows: int = 0
|
||||
orphan_fragments: int = 0
|
||||
duplicate_records: int = 0
|
||||
discarded_values: int = 0
|
||||
records: int = 0
|
||||
issues: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"physical_rows": self.physical_rows,
|
||||
"duplicate_headers": self.duplicate_headers,
|
||||
"repaired_rows": self.repaired_rows,
|
||||
"orphan_fragments": self.orphan_fragments,
|
||||
"duplicate_records": self.duplicate_records,
|
||||
"discarded_values": self.discarded_values,
|
||||
"records": self.records,
|
||||
}
|
||||
|
||||
|
||||
class DatasetReader:
|
||||
"""Yields clean, de-duplicated records from the raw CSV export."""
|
||||
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path)
|
||||
|
||||
def read(self) -> tuple[list[dict[str, str]], ImportReport]:
|
||||
report = ImportReport()
|
||||
with self.path.open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.reader(handle))
|
||||
|
||||
if not rows:
|
||||
return [], report
|
||||
|
||||
header = rows[0]
|
||||
report.physical_rows = len(rows) - 1
|
||||
|
||||
logical_rows = self._reassemble(rows[1:], header, report)
|
||||
|
||||
seen: set[str] = set()
|
||||
records: list[dict[str, str]] = []
|
||||
for values in logical_rows:
|
||||
record = self._align(values, header)
|
||||
self._sanitize(record, report)
|
||||
key = self._identity(record)
|
||||
if key in seen:
|
||||
report.duplicate_records += 1
|
||||
continue
|
||||
seen.add(key)
|
||||
records.append(record)
|
||||
|
||||
report.records = len(records)
|
||||
return records, report
|
||||
|
||||
# -- Step 1: turn physical rows into logical records --------------------
|
||||
def _reassemble(
|
||||
self, rows: list[list[str]], header: list[str], report: ImportReport
|
||||
) -> list[list[str]]:
|
||||
expected = len(header)
|
||||
logical: list[list[str]] = []
|
||||
for raw_row in rows:
|
||||
row = self._strip_watermark(list(raw_row))
|
||||
if not row:
|
||||
continue
|
||||
if self._is_repeated_header(row, header):
|
||||
report.duplicate_headers += 1
|
||||
continue
|
||||
if self._starts_record(row) or not logical:
|
||||
if len(row) != expected:
|
||||
report.repaired_rows += 1
|
||||
logical.append(row)
|
||||
elif len(logical[-1]) < expected - 7:
|
||||
# The previous record is short: this fragment is its tail,
|
||||
# split by a stray quote inside a free-text column.
|
||||
logical[-1].extend(row)
|
||||
report.repaired_rows += 1
|
||||
else:
|
||||
# Cannot be attributed to any record without guessing.
|
||||
report.orphan_fragments += 1
|
||||
return logical
|
||||
|
||||
@staticmethod
|
||||
def _strip_watermark(row: list[str]) -> list[str]:
|
||||
while row and any(marker in row[0] for marker in WATERMARK_MARKERS):
|
||||
row = row[1:]
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def _is_repeated_header(row: list[str], header: list[str]) -> bool:
|
||||
return len(row) > 3 and row[:3] == header[:3]
|
||||
|
||||
@staticmethod
|
||||
def _starts_record(row: list[str]) -> bool:
|
||||
"""
|
||||
A record always begins with ``full_name`` followed by ``first_name``,
|
||||
and the full name starts with the first name. That is a strong,
|
||||
cheap anchor for telling a new record from a continuation fragment.
|
||||
"""
|
||||
if len(row) < 9:
|
||||
return False
|
||||
full_name, first_name = row[0].strip(), row[1].strip()
|
||||
return bool(
|
||||
full_name
|
||||
and first_name
|
||||
and len(full_name) < 80
|
||||
and "@" not in full_name
|
||||
and full_name.startswith(first_name)
|
||||
)
|
||||
|
||||
# -- Step 2: align values to columns ------------------------------------
|
||||
@staticmethod
|
||||
def _score(column: str, value: str) -> float:
|
||||
if value == "":
|
||||
return SCORE_EMPTY
|
||||
validator = FIELD_VALIDATORS.get(column)
|
||||
if validator is None:
|
||||
return SCORE_FREE_TEXT
|
||||
try:
|
||||
verdict = validator(value)
|
||||
except Exception: # noqa: BLE001 - a broken value is just a mismatch
|
||||
verdict = False
|
||||
if verdict is None:
|
||||
return SCORE_MISMATCH
|
||||
return SCORE_MATCH if verdict else SCORE_MISMATCH
|
||||
|
||||
def _align(self, values: list[str], header: list[str]) -> dict[str, str]:
|
||||
"""Needleman-Wunsch alignment of row values against header columns."""
|
||||
n, m = len(values), len(header)
|
||||
neg_inf = float("-inf")
|
||||
best = [[neg_inf] * (m + 1) for _ in range(n + 1)]
|
||||
move = [[0] * (m + 1) for _ in range(n + 1)]
|
||||
best[0][0] = 0.0
|
||||
|
||||
for j in range(1, m + 1):
|
||||
best[0][j] = best[0][j - 1] + PENALTY_MISSING_COLUMN
|
||||
move[0][j] = 2
|
||||
for i in range(1, n + 1):
|
||||
best[i][0] = best[i - 1][0] + self._drop_penalty(values[i - 1])
|
||||
move[i][0] = 3
|
||||
|
||||
for i in range(1, n + 1):
|
||||
value = values[i - 1]
|
||||
drop = self._drop_penalty(value)
|
||||
for j in range(1, m + 1):
|
||||
score = best[i - 1][j - 1] + self._score(header[j - 1], value)
|
||||
choice = 1
|
||||
skip_column = best[i][j - 1] + PENALTY_MISSING_COLUMN
|
||||
if skip_column > score:
|
||||
score, choice = skip_column, 2
|
||||
drop_value = best[i - 1][j] + drop
|
||||
if drop_value > score:
|
||||
score, choice = drop_value, 3
|
||||
best[i][j], move[i][j] = score, choice
|
||||
|
||||
record: dict[str, str] = {}
|
||||
i, j = n, m
|
||||
while i > 0 or j > 0:
|
||||
choice = move[i][j]
|
||||
if choice == 1:
|
||||
record[header[j - 1]] = values[i - 1].strip()
|
||||
i, j = i - 1, j - 1
|
||||
elif choice == 2:
|
||||
record.setdefault(header[j - 1], "")
|
||||
j -= 1
|
||||
else:
|
||||
i -= 1
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def _drop_penalty(value: str) -> float:
|
||||
return PENALTY_DROP_EMPTY if value == "" else PENALTY_DROP_VALUE
|
||||
|
||||
# -- Step 3: final validation ------------------------------------------
|
||||
@staticmethod
|
||||
def _sanitize(record: dict[str, str], report: ImportReport) -> None:
|
||||
for column, value in list(record.items()):
|
||||
if not value:
|
||||
continue
|
||||
validator = FIELD_VALIDATORS.get(column)
|
||||
if validator is None:
|
||||
continue
|
||||
try:
|
||||
verdict = validator(value)
|
||||
except Exception: # noqa: BLE001
|
||||
verdict = False
|
||||
if not verdict:
|
||||
record[column] = ""
|
||||
report.discarded_values += 1
|
||||
|
||||
@staticmethod
|
||||
def _identity(record: dict[str, str]) -> str:
|
||||
return (
|
||||
record.get("linkedin_id") or record.get("linkedin_url") or record.get("full_name") or ""
|
||||
).lower()
|
||||
|
||||
|
||||
def parse_literal(value: str | None, default: Any) -> Any:
|
||||
"""Public helper for the importer: parse a nested column defensively."""
|
||||
if not value:
|
||||
return default
|
||||
parsed = _safe_literal(value)
|
||||
return default if parsed is None else parsed
|
||||
|
||||
|
||||
def read_dataset(path: str | Path) -> Iterator[dict[str, str]]:
|
||||
"""Convenience wrapper for callers that do not need the report."""
|
||||
records, _ = DatasetReader(path).read()
|
||||
yield from records
|
||||
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Maps the repaired CSV records onto the relational model.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* **Idempotent.** The natural key is the LinkedIn id (falling back to the
|
||||
profile URL), so re-running the import updates instead of duplicating.
|
||||
* **Bulk friendly.** Companies, schools and skills are resolved through
|
||||
in-memory caches, and the child rows of a profile are replaced wholesale with
|
||||
``bulk_create`` - 284 profiles import in a couple of seconds.
|
||||
* **Self healing input.** The top level columns of this export are frequently
|
||||
missing, but the same information usually exists inside the nested
|
||||
``experience`` blob. ``_derive_current_position`` falls back to the primary
|
||||
experience entry, which raises coverage of ``job_title_role`` from 108 to 183
|
||||
profiles and of the current employer from 219 to 271.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.postgres.search import SearchVector
|
||||
from django.db import transaction
|
||||
|
||||
from apps.profiles.dataset import DatasetReader, ImportReport, parse_literal
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_YEAR_RE = re.compile(r"(\d{4})")
|
||||
_MAX_SKILLS = 60
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportStats:
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skills: int = 0
|
||||
companies: int = 0
|
||||
schools: int = 0
|
||||
experiences: int = 0
|
||||
educations: int = 0
|
||||
report: ImportReport = field(default_factory=ImportReport)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"profiles_created": self.created,
|
||||
"profiles_updated": self.updated,
|
||||
"skills": self.skills,
|
||||
"companies": self.companies,
|
||||
"schools": self.schools,
|
||||
"experiences": self.experiences,
|
||||
"educations": self.educations,
|
||||
**self.report.as_dict(),
|
||||
}
|
||||
|
||||
|
||||
class ProfileImporter:
|
||||
"""Loads the dataset into PostgreSQL."""
|
||||
|
||||
def __init__(self, path: str, limit: int | None = None) -> None:
|
||||
self.path = path
|
||||
self.limit = limit
|
||||
self._companies: dict[str, Company] = {}
|
||||
self._schools: dict[str, School] = {}
|
||||
self._skills: dict[str, Skill] = {}
|
||||
|
||||
# -- Entry point --------------------------------------------------------
|
||||
def run(self) -> ImportStats:
|
||||
records, report = DatasetReader(self.path).read()
|
||||
if self.limit:
|
||||
records = records[: self.limit]
|
||||
|
||||
stats = ImportStats(report=report)
|
||||
self._warm_caches()
|
||||
|
||||
for record in records:
|
||||
try:
|
||||
with transaction.atomic():
|
||||
self._import_record(record, stats)
|
||||
except Exception: # noqa: BLE001 - one bad row must not abort the run
|
||||
logger.exception(
|
||||
"profile_import_failed",
|
||||
extra={"event": "profile_import_failed", "name": record.get("full_name")},
|
||||
)
|
||||
report.issues.append(record.get("full_name", "<unknown>"))
|
||||
|
||||
self._refresh_skill_counters()
|
||||
self._refresh_search_vectors()
|
||||
|
||||
stats.skills = len(self._skills)
|
||||
stats.companies = len(self._companies)
|
||||
stats.schools = len(self._schools)
|
||||
return stats
|
||||
|
||||
# -- Caches -------------------------------------------------------------
|
||||
def _warm_caches(self) -> None:
|
||||
self._companies = {c.name: c for c in Company.objects.all()}
|
||||
self._schools = {s.name: s for s in School.objects.all()}
|
||||
self._skills = {s.slug: s for s in Skill.objects.all()}
|
||||
|
||||
# -- Per record ---------------------------------------------------------
|
||||
def _import_record(self, record: dict[str, str], stats: ImportStats) -> None:
|
||||
experiences = parse_literal(record.get("experience"), [])
|
||||
educations = parse_literal(record.get("education"), [])
|
||||
primary = self._primary_experience(experiences)
|
||||
|
||||
title, role, sub_role, levels = self._derive_current_position(record, primary)
|
||||
company = self._resolve_current_company(record, primary)
|
||||
|
||||
skills = self._clean_str_list(parse_literal(record.get("skills"), []))[:_MAX_SKILLS]
|
||||
past_companies = [
|
||||
str(item.get("company", {}).get("name") or "")
|
||||
for item in experiences
|
||||
if isinstance(item, dict) and isinstance(item.get("company"), dict)
|
||||
]
|
||||
schools = [
|
||||
str(item.get("school", {}).get("name") or "")
|
||||
for item in educations
|
||||
if isinstance(item, dict) and isinstance(item.get("school"), dict)
|
||||
]
|
||||
majors = [
|
||||
major
|
||||
for item in educations
|
||||
if isinstance(item, dict)
|
||||
for major in self._clean_str_list(item.get("majors") or [])
|
||||
]
|
||||
|
||||
defaults: dict[str, Any] = {
|
||||
"full_name": record.get("full_name", "")[:255],
|
||||
"first_name": record.get("first_name", "")[:128],
|
||||
"last_name": record.get("last_name", "")[:128],
|
||||
"gender": record.get("gender", "")[:16],
|
||||
"linkedin_url": record.get("linkedin_url", "")[:255],
|
||||
"linkedin_username": record.get("linkedin_username", "")[:128],
|
||||
"github_url": record.get("github_url", "")[:255],
|
||||
"twitter_url": record.get("twitter_url", "")[:255],
|
||||
"job_title": title[:255],
|
||||
"job_title_role": role[:64],
|
||||
"job_title_sub_role": sub_role[:64],
|
||||
"seniority_levels": levels,
|
||||
"current_company": company,
|
||||
"industry": (record.get("industry") or "")[:128],
|
||||
"job_summary": record.get("job_summary", ""),
|
||||
"job_start_date": record.get("job_start_date", "")[:10],
|
||||
"location_name": record.get("location_name", "")[:255],
|
||||
"location_locality": record.get("location_locality", "")[:128],
|
||||
"location_region": record.get("location_region", "")[:128],
|
||||
"location_country": record.get("location_country", "")[:128],
|
||||
"location_continent": record.get("location_continent", "")[:64],
|
||||
"summary": record.get("summary", ""),
|
||||
"years_experience": self._to_int(record.get("inferred_years_experience"), 0, 70),
|
||||
"inferred_salary": record.get("inferred_salary", "")[:32],
|
||||
"connections": self._to_int(record.get("linkedin_connections"), 0, 100_000),
|
||||
"interests": self._clean_str_list(parse_literal(record.get("interests"), []))[:40],
|
||||
"languages": self._language_names(parse_literal(record.get("languages"), [])),
|
||||
"emails": self._emails(parse_literal(record.get("emails"), [])),
|
||||
"phone_numbers": self._clean_str_list(parse_literal(record.get("phone_numbers"), []))[
|
||||
:10
|
||||
],
|
||||
"skills_text": " ".join(skills),
|
||||
"companies_text": " ".join(
|
||||
filter(None, {*past_companies, company.name if company else ""})
|
||||
),
|
||||
"education_text": " ".join(filter(None, [*schools, *majors])),
|
||||
}
|
||||
|
||||
external_id = (record.get("linkedin_id") or "")[:64]
|
||||
lookup = (
|
||||
{"external_id": external_id}
|
||||
if external_id
|
||||
else {"linkedin_url": defaults["linkedin_url"]}
|
||||
)
|
||||
profile, created = Profile.objects.update_or_create(**lookup, defaults=defaults)
|
||||
if created:
|
||||
stats.created += 1
|
||||
else:
|
||||
stats.updated += 1
|
||||
|
||||
self._sync_skills(profile, skills)
|
||||
stats.experiences += self._sync_experiences(profile, experiences)
|
||||
stats.educations += self._sync_educations(profile, educations)
|
||||
|
||||
# -- Current position ---------------------------------------------------
|
||||
@staticmethod
|
||||
def _primary_experience(experiences: list[Any]) -> dict[str, Any]:
|
||||
entries = [item for item in experiences if isinstance(item, dict)]
|
||||
for item in entries:
|
||||
if item.get("is_primary"):
|
||||
return item
|
||||
return entries[0] if entries else {}
|
||||
|
||||
def _derive_current_position(
|
||||
self, record: dict[str, str], primary: dict[str, Any]
|
||||
) -> tuple[str, str, str, list[str]]:
|
||||
title_blob = primary.get("title") if isinstance(primary.get("title"), dict) else {}
|
||||
title = record.get("job_title") or str(title_blob.get("name") or "")
|
||||
role = record.get("job_title_role") or str(title_blob.get("role") or "")
|
||||
sub_role = record.get("job_title_sub_role") or str(title_blob.get("sub_role") or "")
|
||||
levels = self._clean_str_list(
|
||||
parse_literal(record.get("job_title_levels"), None) or title_blob.get("levels") or []
|
||||
)
|
||||
return title, role, sub_role, levels[:5]
|
||||
|
||||
def _resolve_current_company(
|
||||
self, record: dict[str, str], primary: dict[str, Any]
|
||||
) -> Company | None:
|
||||
name = (record.get("job_company_name") or "").strip()
|
||||
payload: dict[str, Any] = {
|
||||
"external_id": record.get("job_company_id", ""),
|
||||
"industry": record.get("job_company_industry", ""),
|
||||
"size": record.get("job_company_size", ""),
|
||||
"founded": self._to_int(record.get("job_company_founded"), 1600, 2100),
|
||||
"website": record.get("job_company_website", ""),
|
||||
"linkedin_url": record.get("job_company_linkedin_url", ""),
|
||||
"location_name": record.get("job_company_location_name", ""),
|
||||
"country": record.get("job_company_location_country", ""),
|
||||
}
|
||||
|
||||
if not name and isinstance(primary.get("company"), dict):
|
||||
blob = primary["company"]
|
||||
name = str(blob.get("name") or "").strip()
|
||||
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
|
||||
payload = {
|
||||
"external_id": str(blob.get("id") or ""),
|
||||
"industry": str(blob.get("industry") or ""),
|
||||
"size": str(blob.get("size") or ""),
|
||||
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
|
||||
"website": str(blob.get("website") or ""),
|
||||
"linkedin_url": str(blob.get("linkedin_url") or ""),
|
||||
"location_name": str(location.get("name") or ""),
|
||||
"country": str(location.get("country") or ""),
|
||||
}
|
||||
|
||||
return self._get_or_create_company(name, payload)
|
||||
|
||||
def _get_or_create_company(self, name: str, payload: dict[str, Any]) -> Company | None:
|
||||
name = name.strip()[:255]
|
||||
if not name:
|
||||
return None
|
||||
cached = self._companies.get(name)
|
||||
if cached:
|
||||
return cached
|
||||
company = Company.objects.create(
|
||||
name=name,
|
||||
external_id=str(payload.get("external_id") or "")[:128],
|
||||
industry=str(payload.get("industry") or "")[:128],
|
||||
size=str(payload.get("size") or "")[:32],
|
||||
founded=payload.get("founded"),
|
||||
website=str(payload.get("website") or "")[:255],
|
||||
linkedin_url=str(payload.get("linkedin_url") or "")[:255],
|
||||
location_name=str(payload.get("location_name") or "")[:255],
|
||||
country=str(payload.get("country") or "")[:128],
|
||||
)
|
||||
self._companies[name] = company
|
||||
return company
|
||||
|
||||
def _get_or_create_school(self, blob: dict[str, Any]) -> School | None:
|
||||
name = str(blob.get("name") or "").strip()[:255]
|
||||
if not name:
|
||||
return None
|
||||
cached = self._schools.get(name)
|
||||
if cached:
|
||||
return cached
|
||||
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
|
||||
school = School.objects.create(
|
||||
name=name,
|
||||
type=str(blob.get("type") or "")[:64],
|
||||
website=str(blob.get("website") or "")[:255],
|
||||
linkedin_url=str(blob.get("linkedin_url") or "")[:255],
|
||||
location_name=str(location.get("name") or "")[:255],
|
||||
country=str(location.get("country") or "")[:128],
|
||||
)
|
||||
self._schools[name] = school
|
||||
return school
|
||||
|
||||
# -- Children -----------------------------------------------------------
|
||||
def _sync_skills(self, profile: Profile, skills: list[str]) -> None:
|
||||
ProfileSkill.objects.filter(profile=profile).delete()
|
||||
links: list[ProfileSkill] = []
|
||||
used: set[int] = set()
|
||||
for position, name in enumerate(skills):
|
||||
slug = Skill.make_slug(name)
|
||||
if not slug:
|
||||
continue
|
||||
skill = self._skills.get(slug)
|
||||
if skill is None:
|
||||
skill, _ = Skill.objects.get_or_create(slug=slug, defaults={"name": name[:128]})
|
||||
self._skills[slug] = skill
|
||||
if skill.pk in used:
|
||||
continue
|
||||
used.add(skill.pk)
|
||||
links.append(ProfileSkill(profile=profile, skill=skill, position=position))
|
||||
ProfileSkill.objects.bulk_create(links, ignore_conflicts=True)
|
||||
|
||||
def _sync_experiences(self, profile: Profile, experiences: list[Any]) -> int:
|
||||
Experience.objects.filter(profile=profile).delete()
|
||||
rows: list[Experience] = []
|
||||
for item in experiences:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
blob = item.get("company") if isinstance(item.get("company"), dict) else {}
|
||||
title = item.get("title") if isinstance(item.get("title"), dict) else {}
|
||||
company = self._get_or_create_company(
|
||||
str(blob.get("name") or ""),
|
||||
{
|
||||
"external_id": blob.get("id"),
|
||||
"industry": blob.get("industry"),
|
||||
"size": blob.get("size"),
|
||||
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
|
||||
"website": blob.get("website"),
|
||||
"linkedin_url": blob.get("linkedin_url"),
|
||||
"location_name": (
|
||||
(blob.get("location") or {}).get("name")
|
||||
if isinstance(blob.get("location"), dict)
|
||||
else ""
|
||||
),
|
||||
"country": (
|
||||
(blob.get("location") or {}).get("country")
|
||||
if isinstance(blob.get("location"), dict)
|
||||
else ""
|
||||
),
|
||||
},
|
||||
)
|
||||
rows.append(
|
||||
Experience(
|
||||
profile=profile,
|
||||
company=company,
|
||||
company_name=str(blob.get("name") or "")[:255],
|
||||
title=str(title.get("name") or "")[:255],
|
||||
title_role=str(title.get("role") or "")[:64],
|
||||
title_sub_role=str(title.get("sub_role") or "")[:64],
|
||||
levels=self._clean_str_list(title.get("levels") or [])[:5],
|
||||
start_date=str(item.get("start_date") or "")[:10],
|
||||
end_date=str(item.get("end_date") or "")[:10],
|
||||
start_year=self._year(item.get("start_date")),
|
||||
end_year=self._year(item.get("end_date")),
|
||||
is_primary=bool(item.get("is_primary")),
|
||||
location_name=self._first_location(item.get("location_names")),
|
||||
summary=str(item.get("summary") or ""),
|
||||
)
|
||||
)
|
||||
Experience.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
def _sync_educations(self, profile: Profile, educations: list[Any]) -> int:
|
||||
Education.objects.filter(profile=profile).delete()
|
||||
rows: list[Education] = []
|
||||
for item in educations:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
blob = item.get("school") if isinstance(item.get("school"), dict) else {}
|
||||
rows.append(
|
||||
Education(
|
||||
profile=profile,
|
||||
school=self._get_or_create_school(blob),
|
||||
school_name=str(blob.get("name") or "")[:255],
|
||||
degrees=self._clean_str_list(item.get("degrees") or [])[:5],
|
||||
majors=self._clean_str_list(item.get("majors") or [])[:5],
|
||||
minors=self._clean_str_list(item.get("minors") or [])[:5],
|
||||
gpa=self._to_decimal(item.get("gpa")),
|
||||
start_date=str(item.get("start_date") or "")[:10],
|
||||
end_date=str(item.get("end_date") or "")[:10],
|
||||
end_year=self._year(item.get("end_date")),
|
||||
)
|
||||
)
|
||||
Education.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
# -- Post processing ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _refresh_skill_counters() -> None:
|
||||
"""Single UPDATE ... FROM keeps the facet ordering cheap at query time."""
|
||||
from django.db import connection
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE profiles_skill AS s
|
||||
SET profile_count = COALESCE(counts.total, 0)
|
||||
FROM (
|
||||
SELECT skill_id, COUNT(*) AS total
|
||||
FROM profiles_profileskill
|
||||
GROUP BY skill_id
|
||||
) AS counts
|
||||
WHERE counts.skill_id = s.id
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE profiles_skill SET profile_count = 0
|
||||
WHERE id NOT IN (SELECT skill_id FROM profiles_profileskill)
|
||||
"""
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _refresh_search_vectors() -> None:
|
||||
"""
|
||||
Rebuild the tsvector for every profile in one statement.
|
||||
|
||||
Weights encode the ranking intent: a hit on the person's name beats a
|
||||
hit on their title or skills, which in turn beats a hit in the free-text
|
||||
summary.
|
||||
"""
|
||||
Profile.objects.update(
|
||||
search_vector=(
|
||||
SearchVector("full_name", weight="A", config="english")
|
||||
+ SearchVector("job_title", weight="B", config="english")
|
||||
+ SearchVector("skills_text", weight="B", config="english")
|
||||
+ SearchVector("companies_text", weight="B", config="english")
|
||||
+ SearchVector("industry", weight="C", config="english")
|
||||
+ SearchVector("education_text", weight="C", config="english")
|
||||
+ SearchVector("location_name", weight="C", config="english")
|
||||
+ SearchVector("summary", weight="D", config="english")
|
||||
)
|
||||
)
|
||||
|
||||
# -- Small helpers ------------------------------------------------------
|
||||
@staticmethod
|
||||
def _clean_str_list(values: Any) -> list[str]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for value in values:
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
cleaned = value.strip().lower()[:128]
|
||||
if cleaned and cleaned not in seen:
|
||||
seen.add(cleaned)
|
||||
out.append(cleaned)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _language_names(values: Any) -> list[str]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
return [
|
||||
str(item.get("name")).strip().lower()[:64]
|
||||
for item in values
|
||||
if isinstance(item, dict) and item.get("name")
|
||||
][:10]
|
||||
|
||||
@staticmethod
|
||||
def _emails(values: Any) -> list[dict[str, str]]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"address": str(item.get("address"))[:254],
|
||||
"type": str(item.get("type") or "other")[:32],
|
||||
}
|
||||
for item in values
|
||||
if isinstance(item, dict) and item.get("address")
|
||||
][:10]
|
||||
|
||||
@staticmethod
|
||||
def _first_location(values: Any) -> str:
|
||||
if isinstance(values, list) and values and isinstance(values[0], str):
|
||||
return values[0][:255]
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _to_int(value: str | None, minimum: int, maximum: int) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
number = int(float(str(value).replace(",", "")))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if minimum <= number <= maximum else None
|
||||
|
||||
@staticmethod
|
||||
def _to_decimal(value: Any) -> Any:
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return round(number, 2) if 0 <= number <= 10 else None
|
||||
|
||||
@staticmethod
|
||||
def _year(value: Any) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
match = _YEAR_RE.search(str(value))
|
||||
if not match:
|
||||
return None
|
||||
year = int(match.group(1))
|
||||
return year if 1900 <= year <= 2100 else None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""``python manage.py import_profiles`` - load the CSV export into PostgreSQL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import transaction
|
||||
|
||||
from apps.profiles.importers import ProfileImporter
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Import (or refresh) LinkedIn profiles from the raw CSV export."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
parser.add_argument(
|
||||
"--path",
|
||||
default=settings.DATASET_PATH,
|
||||
help="Path to the CSV export (defaults to DATASET_PATH).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", type=int, default=None, help="Import at most N profiles (smoke tests)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--truncate",
|
||||
action="store_true",
|
||||
help="Delete existing profile data before importing.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reindex",
|
||||
action="store_true",
|
||||
help="Rebuild the Elasticsearch index once the import finishes.",
|
||||
)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
path = options["path"]
|
||||
started = time.perf_counter()
|
||||
|
||||
if options["truncate"]:
|
||||
self.stdout.write("Truncating existing profile data ...")
|
||||
with transaction.atomic():
|
||||
ProfileSkill.objects.all().delete()
|
||||
Experience.objects.all().delete()
|
||||
Education.objects.all().delete()
|
||||
Profile.objects.all().delete()
|
||||
Skill.objects.all().delete()
|
||||
Company.objects.all().delete()
|
||||
School.objects.all().delete()
|
||||
|
||||
self.stdout.write(f"Reading dataset from {path} ...")
|
||||
try:
|
||||
stats = ProfileImporter(path, limit=options["limit"]).run()
|
||||
except FileNotFoundError as exc:
|
||||
raise CommandError(f"Dataset not found: {path}") from exc
|
||||
|
||||
elapsed = round(time.perf_counter() - started, 2)
|
||||
summary = {**stats.as_dict(), "elapsed_seconds": elapsed}
|
||||
self.stdout.write(json.dumps(summary, indent=2))
|
||||
|
||||
if stats.report.issues:
|
||||
self.stdout.write(
|
||||
self.style.WARNING(
|
||||
f"{len(stats.report.issues)} record(s) failed: "
|
||||
f"{', '.join(stats.report.issues[:10])}"
|
||||
)
|
||||
)
|
||||
|
||||
if options["reindex"]:
|
||||
self.stdout.write("Rebuilding the search index ...")
|
||||
from django.core.management import call_command
|
||||
|
||||
call_command("search_index", "--rebuild")
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(f"Import finished in {elapsed}s."))
|
||||
@@ -0,0 +1,416 @@
|
||||
# Generated by Django 5.1.15 on 2026-09-03 03:36
|
||||
|
||||
import django.contrib.postgres.fields
|
||||
import django.contrib.postgres.indexes
|
||||
import django.contrib.postgres.search
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = []
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Company",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("external_id", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("name", models.CharField(max_length=255, unique=True)),
|
||||
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
(
|
||||
"size",
|
||||
models.CharField(
|
||||
blank=True, help_text="Head-count bucket, e.g. 51-200", max_length=32
|
||||
),
|
||||
),
|
||||
("founded", models.PositiveIntegerField(blank=True, null=True)),
|
||||
("website", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("country", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
],
|
||||
options={
|
||||
"verbose_name_plural": "companies",
|
||||
"ordering": ("name",),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="School",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("name", models.CharField(max_length=255, unique=True)),
|
||||
("type", models.CharField(blank=True, max_length=64)),
|
||||
("website", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("country", models.CharField(blank=True, max_length=128)),
|
||||
],
|
||||
options={
|
||||
"ordering": ("name",),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Skill",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("name", models.CharField(max_length=128)),
|
||||
("slug", models.SlugField(max_length=140, unique=True)),
|
||||
(
|
||||
"profile_count",
|
||||
models.PositiveIntegerField(
|
||||
db_index=True,
|
||||
default=0,
|
||||
help_text="Denormalised counter used to rank filter suggestions.",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-profile_count", "name"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Profile",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"external_id",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
max_length=64,
|
||||
),
|
||||
),
|
||||
("full_name", models.CharField(db_index=True, max_length=255)),
|
||||
("first_name", models.CharField(blank=True, max_length=128)),
|
||||
("last_name", models.CharField(blank=True, max_length=128)),
|
||||
(
|
||||
"gender",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
choices=[("male", "Male"), ("female", "Female"), ("", "Unknown")],
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_username", models.CharField(blank=True, max_length=128)),
|
||||
("github_url", models.CharField(blank=True, max_length=255)),
|
||||
("twitter_url", models.CharField(blank=True, max_length=255)),
|
||||
("job_title", models.CharField(blank=True, db_index=True, max_length=255)),
|
||||
(
|
||||
"job_title_role",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="Coarse job family, e.g. engineering.",
|
||||
max_length=64,
|
||||
),
|
||||
),
|
||||
("job_title_sub_role", models.CharField(blank=True, max_length=64)),
|
||||
(
|
||||
"seniority_levels",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("job_summary", models.TextField(blank=True)),
|
||||
("job_start_date", models.CharField(blank=True, max_length=10)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("location_locality", models.CharField(blank=True, max_length=128)),
|
||||
("location_region", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("location_country", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("location_continent", models.CharField(blank=True, max_length=64)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
(
|
||||
"years_experience",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
("inferred_salary", models.CharField(blank=True, max_length=32)),
|
||||
("connections", models.PositiveIntegerField(blank=True, null=True)),
|
||||
(
|
||||
"interests",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"languages",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=64),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("emails", models.JSONField(blank=True, default=list)),
|
||||
(
|
||||
"phone_numbers",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("skills_text", models.TextField(blank=True, editable=False)),
|
||||
("companies_text", models.TextField(blank=True, editable=False)),
|
||||
("education_text", models.TextField(blank=True, editable=False)),
|
||||
(
|
||||
"search_vector",
|
||||
django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
|
||||
),
|
||||
(
|
||||
"current_company",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="current_employees",
|
||||
to="profiles.company",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("full_name", "id"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Education",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("school_name", models.CharField(blank=True, max_length=255)),
|
||||
(
|
||||
"degrees",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"majors",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"minors",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("gpa", models.DecimalField(blank=True, decimal_places=2, max_digits=4, null=True)),
|
||||
("start_date", models.CharField(blank=True, max_length=10)),
|
||||
("end_date", models.CharField(blank=True, max_length=10)),
|
||||
(
|
||||
"end_year",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="educations",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
(
|
||||
"school",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="educations",
|
||||
to="profiles.school",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-end_year", "id"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ProfileSkill",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("position", models.PositiveSmallIntegerField(default=0)),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="profile_skills",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
(
|
||||
"skill",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="profile_skills",
|
||||
to="profiles.skill",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("position",),
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="profile",
|
||||
name="skills",
|
||||
field=models.ManyToManyField(
|
||||
related_name="profiles", through="profiles.ProfileSkill", to="profiles.skill"
|
||||
),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Experience",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("company_name", models.CharField(blank=True, max_length=255)),
|
||||
("title", models.CharField(blank=True, max_length=255)),
|
||||
("title_role", models.CharField(blank=True, max_length=64)),
|
||||
("title_sub_role", models.CharField(blank=True, max_length=64)),
|
||||
(
|
||||
"levels",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("start_date", models.CharField(blank=True, max_length=10)),
|
||||
("end_date", models.CharField(blank=True, max_length=10)),
|
||||
(
|
||||
"start_year",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
("end_year", models.PositiveSmallIntegerField(blank=True, null=True)),
|
||||
("is_primary", models.BooleanField(db_index=True, default=False)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
(
|
||||
"company",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="experiences",
|
||||
to="profiles.company",
|
||||
),
|
||||
),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="experiences",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-is_primary", "-start_year", "id"),
|
||||
"indexes": [
|
||||
models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")
|
||||
],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="profileskill",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("profile", "skill"), name="uniq_profile_skill"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=["search_vector"], name="profile_search_vector_gin"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=["seniority_levels"], name="profile_seniority_gin"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="profile",
|
||||
constraint=models.UniqueConstraint(
|
||||
condition=models.Q(("external_id__gt", "")),
|
||||
fields=("external_id",),
|
||||
name="uniq_profile_external_id",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
Enable the PostgreSQL extensions the search fallback relies on.
|
||||
|
||||
``pg_trgm`` powers ``TrigramSimilarity`` (typo tolerant name matching) and the
|
||||
GIN trigram index below makes ``ILIKE '%term%'`` on skill names fast enough for
|
||||
the type-ahead endpoint.
|
||||
"""
|
||||
|
||||
from django.contrib.postgres.operations import TrigramExtension, UnaccentExtension
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("profiles", "0001_initial")]
|
||||
|
||||
operations = [
|
||||
TrigramExtension(),
|
||||
UnaccentExtension(),
|
||||
migrations.RunSQL(
|
||||
sql=(
|
||||
"CREATE INDEX IF NOT EXISTS profile_full_name_trgm "
|
||||
"ON profiles_profile USING gin (full_name gin_trgm_ops);"
|
||||
),
|
||||
reverse_sql="DROP INDEX IF EXISTS profile_full_name_trgm;",
|
||||
),
|
||||
migrations.RunSQL(
|
||||
sql=(
|
||||
"CREATE INDEX IF NOT EXISTS skill_name_trgm "
|
||||
"ON profiles_skill USING gin (name gin_trgm_ops);"
|
||||
),
|
||||
reverse_sql="DROP INDEX IF EXISTS skill_name_trgm;",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Domain model for the LinkedIn dataset.
|
||||
|
||||
The raw export is one very wide CSV row per person with several columns holding
|
||||
serialised lists of dictionaries. Storing that blob as-is would make filtering
|
||||
and aggregation painful, so it is normalised into five tables:
|
||||
|
||||
Company <--- Experience ---> Profile <--- Education ---> School
|
||||
|
|
||||
+--- ProfileSkill ---> Skill
|
||||
|
||||
PostgreSQL is the source of truth. Elasticsearch holds a denormalised
|
||||
projection of the same data for search (see ``apps.search``); it can always be
|
||||
rebuilt from these tables, which is why the index is disposable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.contrib.postgres.indexes import GinIndex
|
||||
from django.contrib.postgres.search import SearchVectorField
|
||||
from django.db import models
|
||||
from django.utils.text import slugify
|
||||
|
||||
|
||||
class TimeStampedModel(models.Model):
|
||||
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class Company(TimeStampedModel):
|
||||
"""An employer referenced by the current job or by a past experience."""
|
||||
|
||||
external_id = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
size = models.CharField(max_length=32, blank=True, help_text="Head-count bucket, e.g. 51-200")
|
||||
founded = models.PositiveIntegerField(null=True, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
verbose_name_plural = "companies"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class School(TimeStampedModel):
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
type = models.CharField(max_length=64, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class Skill(TimeStampedModel):
|
||||
"""
|
||||
A normalised skill.
|
||||
|
||||
``slug`` is the deduplication key: the raw data contains the same skill with
|
||||
different casing and spacing ("Team Building" / "team building").
|
||||
"""
|
||||
|
||||
name = models.CharField(max_length=128)
|
||||
slug = models.SlugField(max_length=140, unique=True)
|
||||
profile_count = models.PositiveIntegerField(
|
||||
default=0, db_index=True, help_text="Denormalised counter used to rank filter suggestions."
|
||||
)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-profile_count", "name")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
@staticmethod
|
||||
def make_slug(name: str) -> str:
|
||||
return slugify(name)[:140] or name.strip().lower()[:140]
|
||||
|
||||
|
||||
class Profile(TimeStampedModel):
|
||||
"""A single person in the dataset."""
|
||||
|
||||
class Gender(models.TextChoices):
|
||||
MALE = "male", "Male"
|
||||
FEMALE = "female", "Female"
|
||||
UNKNOWN = "", "Unknown"
|
||||
|
||||
# -- Identity ----------------------------------------------------------
|
||||
external_id = models.CharField(
|
||||
max_length=64,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
)
|
||||
full_name = models.CharField(max_length=255, db_index=True)
|
||||
first_name = models.CharField(max_length=128, blank=True)
|
||||
last_name = models.CharField(max_length=128, blank=True)
|
||||
gender = models.CharField(max_length=16, blank=True, choices=Gender.choices)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
linkedin_username = models.CharField(max_length=128, blank=True)
|
||||
github_url = models.CharField(max_length=255, blank=True)
|
||||
twitter_url = models.CharField(max_length=255, blank=True)
|
||||
|
||||
# -- Current position --------------------------------------------------
|
||||
job_title = models.CharField(max_length=255, blank=True, db_index=True)
|
||||
job_title_role = models.CharField(
|
||||
max_length=64, blank=True, db_index=True, help_text="Coarse job family, e.g. engineering."
|
||||
)
|
||||
job_title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
seniority_levels = ArrayField(
|
||||
models.CharField(max_length=32),
|
||||
default=list,
|
||||
blank=True,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
)
|
||||
current_company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="current_employees"
|
||||
)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
job_summary = models.TextField(blank=True)
|
||||
job_start_date = models.CharField(max_length=10, blank=True)
|
||||
|
||||
# -- Location ----------------------------------------------------------
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
location_locality = models.CharField(max_length=128, blank=True)
|
||||
location_region = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_continent = models.CharField(max_length=64, blank=True)
|
||||
|
||||
# -- Narrative & metrics ----------------------------------------------
|
||||
summary = models.TextField(blank=True)
|
||||
years_experience = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
inferred_salary = models.CharField(max_length=32, blank=True)
|
||||
connections = models.PositiveIntegerField(null=True, blank=True)
|
||||
interests = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
languages = ArrayField(models.CharField(max_length=64), default=list, blank=True)
|
||||
|
||||
# -- Contact details (PII: masked unless the caller is privileged) ------
|
||||
emails = models.JSONField(default=list, blank=True)
|
||||
phone_numbers = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
|
||||
skills = models.ManyToManyField(Skill, through="ProfileSkill", related_name="profiles")
|
||||
|
||||
# -- Denormalised text, maintained by the importer ---------------------
|
||||
# PostgreSQL's ``SearchVector`` cannot traverse a many-to-many relation, so
|
||||
# the searchable text of the related rows is flattened here once at import
|
||||
# time. This keeps the tsvector update to a single UPDATE statement and
|
||||
# keeps query time O(index lookup) instead of O(joins).
|
||||
skills_text = models.TextField(blank=True, editable=False)
|
||||
companies_text = models.TextField(blank=True, editable=False)
|
||||
education_text = models.TextField(blank=True, editable=False)
|
||||
|
||||
# Materialised tsvector powering the PostgreSQL search backend.
|
||||
search_vector = SearchVectorField(null=True, editable=False)
|
||||
|
||||
class Meta:
|
||||
ordering = ("full_name", "id")
|
||||
indexes = [
|
||||
GinIndex(fields=["search_vector"], name="profile_search_vector_gin"),
|
||||
GinIndex(fields=["seniority_levels"], name="profile_seniority_gin"),
|
||||
models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["external_id"],
|
||||
condition=models.Q(external_id__gt=""),
|
||||
name="uniq_profile_external_id",
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.full_name
|
||||
|
||||
@property
|
||||
def headline(self) -> str:
|
||||
"""Short "Title at Company" line used in result cards."""
|
||||
company = self.current_company.name if self.current_company_id else ""
|
||||
if self.job_title and company:
|
||||
return f"{self.job_title} at {company}"
|
||||
return self.job_title or company
|
||||
|
||||
|
||||
class ProfileSkill(models.Model):
|
||||
"""Through table - keeps the original ordering of the skill list."""
|
||||
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
skill = models.ForeignKey(Skill, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
position = models.PositiveSmallIntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
ordering = ("position",)
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["profile", "skill"], name="uniq_profile_skill")
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.profile_id}:{self.skill_id}"
|
||||
|
||||
|
||||
class Experience(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="experiences")
|
||||
company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="experiences"
|
||||
)
|
||||
company_name = models.CharField(max_length=255, blank=True)
|
||||
title = models.CharField(max_length=255, blank=True)
|
||||
title_role = models.CharField(max_length=64, blank=True)
|
||||
title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
levels = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
start_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True)
|
||||
is_primary = models.BooleanField(default=False, db_index=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-is_primary", "-start_year", "id")
|
||||
indexes = [models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.title} @ {self.company_name}"
|
||||
|
||||
|
||||
class Education(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="educations")
|
||||
school = models.ForeignKey(
|
||||
School, null=True, blank=True, on_delete=models.SET_NULL, related_name="educations"
|
||||
)
|
||||
school_name = models.CharField(max_length=255, blank=True)
|
||||
degrees = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
majors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
minors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
gpa = models.DecimalField(max_digits=4, decimal_places=2, null=True, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-end_year", "id")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.school_name}"
|
||||
@@ -0,0 +1,46 @@
|
||||
"""
|
||||
Contact-detail masking.
|
||||
|
||||
The dataset contains real personal data (e-mail addresses, mobile numbers,
|
||||
street addresses). Exposing it to every authenticated user would be both a
|
||||
privacy and a compliance problem, so contact details are masked unless the
|
||||
caller holds a role that is explicitly allowed to see them
|
||||
(:pyattr:`apps.accounts.models.User.can_view_pii`).
|
||||
|
||||
Masking happens in the serialisation layer - the raw values never leave the
|
||||
database for an unprivileged caller, and the search index stores them in a
|
||||
non-searchable field so a keyword query can never be used to confirm an e-mail
|
||||
address by brute force.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_EMAIL_RE = re.compile(r"^([^@]+)@(.+)$")
|
||||
|
||||
|
||||
def mask_email(address: str) -> str:
|
||||
"""``joseph.holland@garver.com`` -> ``jo************@garver.com``."""
|
||||
match = _EMAIL_RE.match(address or "")
|
||||
if not match:
|
||||
return "***"
|
||||
local, domain = match.groups()
|
||||
keep = min(2, max(len(local) - 1, 0))
|
||||
return f"{local[:keep]}{'*' * max(len(local) - keep, 1)}@{domain}"
|
||||
|
||||
|
||||
def mask_phone(number: str) -> str:
|
||||
"""``+19402058928`` -> ``+1940*****28``."""
|
||||
digits = (number or "").strip()
|
||||
if len(digits) < 6:
|
||||
return "***"
|
||||
return f"{digits[:5]}{'*' * (len(digits) - 7)}{digits[-2:]}"
|
||||
|
||||
|
||||
def mask_emails(emails: list[dict[str, str]]) -> list[dict[str, str]]:
|
||||
return [{**item, "address": mask_email(item.get("address", ""))} for item in emails or []]
|
||||
|
||||
|
||||
def mask_phones(numbers: list[str]) -> list[str]:
|
||||
return [mask_phone(number) for number in numbers or []]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Read-only serializers for profile payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.profiles.models import Company, Education, Experience, Profile, School, Skill
|
||||
from apps.profiles.privacy import mask_emails, mask_phones
|
||||
|
||||
|
||||
class SkillSerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = Skill
|
||||
fields = ("id", "name", "slug", "profile_count")
|
||||
|
||||
|
||||
class CompanySerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = Company
|
||||
fields = ("id", "name", "industry", "size", "founded", "website", "location_name")
|
||||
|
||||
|
||||
class SchoolSerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = School
|
||||
fields = ("id", "name", "type", "location_name")
|
||||
|
||||
|
||||
class ExperienceSerializer(serializers.ModelSerializer):
|
||||
company = CompanySerializer(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = Experience
|
||||
fields = (
|
||||
"id", "title", "title_role", "title_sub_role", "levels", "company", "company_name",
|
||||
"start_date", "end_date", "start_year", "end_year", "is_primary", "location_name",
|
||||
"summary",
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class EducationSerializer(serializers.ModelSerializer):
|
||||
school = SchoolSerializer(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = Education
|
||||
fields = (
|
||||
"id", "school", "school_name", "degrees", "majors", "minors", "gpa",
|
||||
"start_date", "end_date", "end_year",
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class PIIMixin:
|
||||
"""Masks contact details unless the caller is allowed to see them."""
|
||||
|
||||
def _can_view_pii(self) -> bool:
|
||||
request = self.context.get("request") # type: ignore[attr-defined]
|
||||
user = getattr(request, "user", None)
|
||||
return bool(user and user.is_authenticated and user.can_view_pii)
|
||||
|
||||
def get_emails(self, obj: Profile) -> list[dict[str, str]]:
|
||||
return obj.emails if self._can_view_pii() else mask_emails(obj.emails)
|
||||
|
||||
def get_phone_numbers(self, obj: Profile) -> list[str]:
|
||||
return obj.phone_numbers if self._can_view_pii() else mask_phones(obj.phone_numbers)
|
||||
|
||||
|
||||
class ProfileListSerializer(PIIMixin, serializers.ModelSerializer):
|
||||
"""Compact representation used for result cards."""
|
||||
|
||||
headline = serializers.CharField(read_only=True)
|
||||
company = serializers.CharField(source="current_company.name", default="", read_only=True)
|
||||
skills = serializers.SerializerMethodField()
|
||||
emails = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Profile
|
||||
fields = (
|
||||
"id", "full_name", "headline", "job_title", "job_title_role", "seniority_levels",
|
||||
"company", "industry", "location_name", "location_country", "years_experience",
|
||||
"linkedin_url", "skills", "emails",
|
||||
) # fmt: skip
|
||||
|
||||
def get_skills(self, obj: Profile) -> list[str]:
|
||||
# ``prefetch_related`` in the view keeps this at one extra query total.
|
||||
return [link.skill.name for link in obj.profile_skills.all()[:10]]
|
||||
|
||||
|
||||
class ProfileDetailSerializer(PIIMixin, serializers.ModelSerializer):
|
||||
headline = serializers.CharField(read_only=True)
|
||||
current_company = CompanySerializer(read_only=True)
|
||||
skills = serializers.SerializerMethodField()
|
||||
experiences = ExperienceSerializer(many=True, read_only=True)
|
||||
educations = EducationSerializer(many=True, read_only=True)
|
||||
emails = serializers.SerializerMethodField()
|
||||
phone_numbers = serializers.SerializerMethodField()
|
||||
pii_masked = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Profile
|
||||
fields = (
|
||||
"id", "external_id", "full_name", "first_name", "last_name", "gender", "headline",
|
||||
"job_title", "job_title_role", "job_title_sub_role", "seniority_levels",
|
||||
"current_company", "industry", "job_summary", "job_start_date",
|
||||
"location_name", "location_locality", "location_region", "location_country",
|
||||
"location_continent", "summary", "years_experience", "inferred_salary",
|
||||
"connections", "interests", "languages", "linkedin_url", "linkedin_username",
|
||||
"github_url", "twitter_url", "skills", "experiences", "educations",
|
||||
"emails", "phone_numbers", "pii_masked",
|
||||
) # fmt: skip
|
||||
|
||||
def get_skills(self, obj: Profile) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": link.skill.name, "slug": link.skill.slug} for link in obj.profile_skills.all()
|
||||
]
|
||||
|
||||
def get_pii_masked(self, obj: Profile) -> bool: # noqa: ARG002
|
||||
return not self._can_view_pii()
|
||||
@@ -0,0 +1,11 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.profiles.views import ProfileDetailView, ProfileListView, SkillListView
|
||||
|
||||
app_name = "profiles"
|
||||
|
||||
urlpatterns = [
|
||||
path("", ProfileListView.as_view(), name="list"),
|
||||
path("skills/", SkillListView.as_view(), name="skills"),
|
||||
path("<int:pk>/", ProfileDetailView.as_view(), name="detail"),
|
||||
]
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Profile read endpoints (the search endpoints live in ``apps.search``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.db.models import QuerySet
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import generics
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
|
||||
from apps.profiles.models import Profile, Skill
|
||||
from apps.profiles.serializers import (
|
||||
ProfileDetailSerializer,
|
||||
ProfileListSerializer,
|
||||
SkillSerializer,
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class ProfileListView(generics.ListAPIView):
|
||||
"""
|
||||
Plain, unranked listing.
|
||||
|
||||
Useful for browsing and for verifying the dataset; ranked keyword search is
|
||||
served by ``/api/v1/search/``.
|
||||
"""
|
||||
|
||||
serializer_class = ProfileListSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_queryset(self) -> QuerySet[Profile]:
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill")
|
||||
.order_by("full_name", "id")
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class ProfileDetailView(generics.RetrieveAPIView):
|
||||
"""Full profile including work history, education and (masked) contacts."""
|
||||
|
||||
serializer_class = ProfileDetailSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_queryset(self) -> QuerySet[Profile]:
|
||||
return Profile.objects.select_related("current_company").prefetch_related(
|
||||
"profile_skills__skill",
|
||||
"experiences__company",
|
||||
"educations__school",
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class SkillListView(generics.ListAPIView):
|
||||
"""
|
||||
Skill dictionary ordered by popularity.
|
||||
|
||||
The frontend uses it to power the skill filter's type-ahead without hitting
|
||||
Elasticsearch for every keystroke.
|
||||
"""
|
||||
|
||||
serializer_class = SkillSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
pagination_class = None
|
||||
|
||||
def get_queryset(self) -> QuerySet[Skill]:
|
||||
queryset = Skill.objects.filter(profile_count__gt=0)
|
||||
term = self.request.query_params.get("q", "").strip().lower()
|
||||
if term:
|
||||
queryset = queryset.filter(name__icontains=term[:64])
|
||||
limit = min(int(self.request.query_params.get("limit", 50) or 50), 200)
|
||||
return queryset.order_by("-profile_count", "name")[:limit]
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class SearchConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.search"
|
||||
verbose_name = "Search"
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Backend factory - resolves ``SEARCH["BACKEND"]`` to a concrete strategy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import cache
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
from apps.search.backends.base import SearchBackend
|
||||
|
||||
__all__ = ["SearchBackend", "get_search_backend"]
|
||||
|
||||
|
||||
@cache
|
||||
def get_search_backend(name: str | None = None) -> SearchBackend:
|
||||
"""
|
||||
Return the configured backend.
|
||||
|
||||
Cached because both implementations hold a connection pool that should be
|
||||
created once per process, not once per request.
|
||||
"""
|
||||
backend = (name or settings.SEARCH["BACKEND"]).lower()
|
||||
|
||||
if backend == "elasticsearch":
|
||||
from apps.search.backends.elastic import ElasticsearchBackend
|
||||
|
||||
return ElasticsearchBackend()
|
||||
if backend == "postgres":
|
||||
from apps.search.backends.postgres import PostgresBackend
|
||||
|
||||
return PostgresBackend()
|
||||
|
||||
raise ImproperlyConfigured(
|
||||
f"Unknown SEARCH['BACKEND'] {backend!r}; expected 'elasticsearch' or 'postgres'."
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""The contract every search backend implements."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from apps.search.types import SearchRequest, SearchResult
|
||||
|
||||
|
||||
class SearchBackend(ABC):
|
||||
"""
|
||||
Strategy interface for keyword search.
|
||||
|
||||
Two implementations ship with the project:
|
||||
|
||||
* :class:`apps.search.backends.elastic.ElasticsearchBackend` - the default,
|
||||
used in every environment that runs the ELK stack.
|
||||
* :class:`apps.search.backends.postgres.PostgresBackend` - a full-text
|
||||
fallback that keeps the API (and the test suite) working without any
|
||||
external search service.
|
||||
|
||||
Keeping the surface this small is deliberate: the views, the serializers and
|
||||
the frontend are completely unaware of which engine answered the query.
|
||||
"""
|
||||
|
||||
#: Human readable identifier, surfaced in the API response and health check.
|
||||
name: str = "base"
|
||||
|
||||
@abstractmethod
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
"""Execute a search and return ranked hits plus facet counts."""
|
||||
|
||||
@abstractmethod
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
"""Return type-ahead completions for a partial name or skill."""
|
||||
|
||||
@abstractmethod
|
||||
def ping(self) -> bool:
|
||||
"""Raise or return ``False`` when the backend is not usable."""
|
||||
@@ -0,0 +1,279 @@
|
||||
"""
|
||||
Elasticsearch search backend.
|
||||
|
||||
The query builder is a pure function (:func:`build_query`) that turns a
|
||||
:class:`~apps.search.types.SearchRequest` into an Elasticsearch request body.
|
||||
Keeping it free of I/O means the ranking logic is unit-testable without a
|
||||
running cluster - see ``tests/test_query_builder.py``.
|
||||
|
||||
Ranking model
|
||||
-------------
|
||||
``must`` the keyword must match somewhere, with per-field boosts that encode
|
||||
"a name hit is worth more than a summary hit", plus ``fuzziness:
|
||||
AUTO`` so ``pyhton`` still finds ``python``.
|
||||
``should`` an exact phrase match on the strong fields adds a large bonus, which
|
||||
pushes "product manager" above documents that merely contain both
|
||||
words far apart.
|
||||
``filter`` structured constraints (skills, role, seniority, country, industry,
|
||||
employer, years of experience). They live in the filter context, so
|
||||
they are cached by Elasticsearch and contribute nothing to the score.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import ApiError, Elasticsearch, TransportError
|
||||
|
||||
from apps.core.exceptions import SearchBackendError
|
||||
from apps.search.backends.base import SearchBackend
|
||||
from apps.search.indexing import alias_name, get_client
|
||||
from apps.search.types import (
|
||||
FACET_FIELDS,
|
||||
SORT_EXPERIENCE_ASC,
|
||||
SORT_EXPERIENCE_DESC,
|
||||
SORT_NAME,
|
||||
FacetBucket,
|
||||
SearchHit,
|
||||
SearchRequest,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Field boosts for the free-text part of the query.
|
||||
SEARCH_FIELDS = [
|
||||
"full_name^6",
|
||||
"job_title^4",
|
||||
"skills^3.5",
|
||||
"company^2.5",
|
||||
"past_companies^1.5",
|
||||
"industry^1.5",
|
||||
"majors^1.5",
|
||||
"education",
|
||||
"location_name",
|
||||
"headline^2",
|
||||
"summary",
|
||||
"interests",
|
||||
]
|
||||
|
||||
#: Narrower field set for the phrase bonus - phrases in a summary are noise.
|
||||
PHRASE_FIELDS = ["full_name^8", "job_title^6", "skills^5", "company^4", "headline^3"]
|
||||
|
||||
HIGHLIGHT_FIELDS = {
|
||||
"full_name": {"number_of_fragments": 0},
|
||||
"job_title": {"number_of_fragments": 0},
|
||||
"skills": {"number_of_fragments": 3},
|
||||
"summary": {"fragment_size": 160, "number_of_fragments": 1},
|
||||
}
|
||||
|
||||
#: Maps an API filter to the indexed keyword field it constrains.
|
||||
FILTER_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"roles": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"countries": "location_country",
|
||||
"industries": "industry.keyword",
|
||||
"companies": "company.keyword",
|
||||
}
|
||||
|
||||
|
||||
def build_query(request: SearchRequest) -> dict[str, Any]:
|
||||
"""Translate a search request into an Elasticsearch request body."""
|
||||
must: list[dict[str, Any]] = []
|
||||
should: list[dict[str, Any]] = []
|
||||
|
||||
if request.q:
|
||||
must.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": request.q,
|
||||
"type": "best_fields",
|
||||
"fields": SEARCH_FIELDS,
|
||||
"operator": "or",
|
||||
# One-word queries must match; longer queries tolerate a
|
||||
# missing term so "senior python django" still returns
|
||||
# people who only list two of the three.
|
||||
"minimum_should_match": "2<70%",
|
||||
"fuzziness": "AUTO",
|
||||
"prefix_length": 1,
|
||||
"max_expansions": 30,
|
||||
}
|
||||
}
|
||||
)
|
||||
should.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": request.q,
|
||||
"type": "phrase",
|
||||
"fields": PHRASE_FIELDS,
|
||||
"slop": 1,
|
||||
"boost": 4,
|
||||
}
|
||||
}
|
||||
)
|
||||
else:
|
||||
must.append({"match_all": {}})
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must,
|
||||
"should": should,
|
||||
"filter": _build_filters(request),
|
||||
}
|
||||
},
|
||||
"from": request.offset,
|
||||
"size": request.page_size,
|
||||
"track_total_hits": True,
|
||||
"sort": _build_sort(request),
|
||||
}
|
||||
|
||||
if request.q:
|
||||
body["highlight"] = {
|
||||
"pre_tags": ["<mark>"],
|
||||
"post_tags": ["</mark>"],
|
||||
"fields": HIGHLIGHT_FIELDS,
|
||||
}
|
||||
|
||||
if request.include_facets:
|
||||
body["aggs"] = _build_aggregations()
|
||||
|
||||
return body
|
||||
|
||||
|
||||
def _build_filters(request: SearchRequest) -> list[dict[str, Any]]:
|
||||
filters: list[dict[str, Any]] = []
|
||||
|
||||
for attribute, field_name in FILTER_FIELDS.items():
|
||||
values = getattr(request, attribute)
|
||||
if values:
|
||||
# ``terms`` is OR within one filter; separate filters are ANDed,
|
||||
# which matches how users read the UI: "python OR django" skills,
|
||||
# AND located in Germany.
|
||||
filters.append({"terms": {field_name: list(values)}})
|
||||
|
||||
if request.min_experience is not None or request.max_experience is not None:
|
||||
bounds: dict[str, int] = {}
|
||||
if request.min_experience is not None:
|
||||
bounds["gte"] = request.min_experience
|
||||
if request.max_experience is not None:
|
||||
bounds["lte"] = request.max_experience
|
||||
filters.append({"range": {"years_experience": bounds}})
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def _build_sort(request: SearchRequest) -> list[Any]:
|
||||
if request.sort == SORT_NAME:
|
||||
return [{"full_name.keyword": "asc"}, {"id": "asc"}]
|
||||
if request.sort == SORT_EXPERIENCE_DESC:
|
||||
return [{"years_experience": {"order": "desc", "missing": "_last"}}, "_score"]
|
||||
if request.sort == SORT_EXPERIENCE_ASC:
|
||||
return [{"years_experience": {"order": "asc", "missing": "_last"}}, "_score"]
|
||||
return ["_score", {"id": "asc"}] # deterministic tie-break
|
||||
|
||||
|
||||
def _build_aggregations() -> dict[str, Any]:
|
||||
size = settings.SEARCH["FACET_SIZE"]
|
||||
return {
|
||||
name: {"terms": {"field": field_name, "size": size, "order": {"_count": "desc"}}}
|
||||
for name, field_name in FACET_FIELDS.items()
|
||||
}
|
||||
|
||||
|
||||
class ElasticsearchBackend(SearchBackend):
|
||||
name = "elasticsearch"
|
||||
|
||||
def __init__(self, client: Elasticsearch | None = None) -> None:
|
||||
self._client = client or get_client()
|
||||
self._index = alias_name()
|
||||
|
||||
# -- SearchBackend ------------------------------------------------------
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
body = build_query(request)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = self._client.search(index=self._index, **body)
|
||||
except (ApiError, TransportError) as exc:
|
||||
logger.exception("search_backend_error", extra={"event": "search_backend_error"})
|
||||
raise SearchBackendError() from exc
|
||||
|
||||
took_ms = int((time.perf_counter() - started) * 1000)
|
||||
hits = [self._to_hit(raw) for raw in response["hits"]["hits"]]
|
||||
return SearchResult(
|
||||
total=response["hits"]["total"]["value"],
|
||||
hits=hits,
|
||||
facets=self._to_facets(response.get("aggregations", {})),
|
||||
took_ms=took_ms,
|
||||
backend=self.name,
|
||||
)
|
||||
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
if not prefix:
|
||||
return []
|
||||
try:
|
||||
response = self._client.search(
|
||||
index=self._index,
|
||||
size=limit,
|
||||
query={
|
||||
"bool": {
|
||||
"should": [
|
||||
{"match": {"full_name.autocomplete": {"query": prefix, "boost": 2}}},
|
||||
{"match": {"skills.autocomplete": prefix}},
|
||||
],
|
||||
"minimum_should_match": 1,
|
||||
}
|
||||
},
|
||||
_source=["full_name", "job_title"],
|
||||
)
|
||||
except (ApiError, TransportError) as exc:
|
||||
raise SearchBackendError() from exc
|
||||
|
||||
seen: list[str] = []
|
||||
for raw in response["hits"]["hits"]:
|
||||
value = raw["_source"].get("full_name", "")
|
||||
if value and value not in seen:
|
||||
seen.append(value)
|
||||
return seen[:limit]
|
||||
|
||||
def ping(self) -> bool:
|
||||
if not self._client.ping():
|
||||
raise SearchBackendError("Elasticsearch did not answer a ping.")
|
||||
return True
|
||||
|
||||
# -- Mapping helpers ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _to_hit(raw: dict[str, Any]) -> SearchHit:
|
||||
source = raw["_source"]
|
||||
return SearchHit(
|
||||
id=source["id"],
|
||||
score=raw.get("_score") or 0.0,
|
||||
full_name=source.get("full_name", ""),
|
||||
headline=source.get("headline", ""),
|
||||
job_title=source.get("job_title", ""),
|
||||
job_title_role=source.get("job_title_role", ""),
|
||||
seniority_levels=source.get("seniority_levels", []),
|
||||
company=source.get("company", ""),
|
||||
industry=source.get("industry", ""),
|
||||
location_name=source.get("location_name", ""),
|
||||
location_country=source.get("location_country", ""),
|
||||
years_experience=source.get("years_experience"),
|
||||
linkedin_url=source.get("linkedin_url", ""),
|
||||
skills=source.get("skills", [])[:12],
|
||||
highlight=raw.get("highlight", {}),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _to_facets(aggregations: dict[str, Any]) -> dict[str, list[FacetBucket]]:
|
||||
return {
|
||||
name: [
|
||||
FacetBucket(key=str(bucket["key"]), count=bucket["doc_count"])
|
||||
for bucket in payload.get("buckets", [])
|
||||
if bucket["key"] != ""
|
||||
]
|
||||
for name, payload in aggregations.items()
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
"""
|
||||
PostgreSQL full-text search backend.
|
||||
|
||||
Why it exists
|
||||
-------------
|
||||
Elasticsearch is the primary engine, but a search API that hard-fails without
|
||||
it is fragile: the test suite would need a cluster, a reviewer would need one to
|
||||
run the project, and an Elasticsearch outage would take the whole product down.
|
||||
Setting ``SEARCH_BACKEND=postgres`` switches to this implementation, which is
|
||||
built on the ``tsvector`` maintained by the importer plus a trigram similarity
|
||||
pass for typo tolerance.
|
||||
|
||||
It is intentionally *not* a re-implementation of Elasticsearch: ranking is
|
||||
coarser and there is no stemming beyond what the ``english`` text search
|
||||
configuration provides. For a 300 profile dataset the difference is barely
|
||||
visible, which is exactly why it is a good fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
|
||||
from django.db.models import Case, F, FloatField, Q, QuerySet, Value, When
|
||||
from django.db.models.functions import Greatest
|
||||
|
||||
from apps.profiles.models import Profile, Skill
|
||||
from apps.search.backends.base import SearchBackend
|
||||
from apps.search.types import (
|
||||
SORT_EXPERIENCE_ASC,
|
||||
SORT_EXPERIENCE_DESC,
|
||||
SORT_NAME,
|
||||
FacetBucket,
|
||||
SearchHit,
|
||||
SearchRequest,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w']+", re.UNICODE)
|
||||
_FACET_LIMIT = 25
|
||||
_MIN_TRIGRAM_SIMILARITY = 0.3
|
||||
|
||||
|
||||
class PostgresBackend(SearchBackend):
|
||||
name = "postgres"
|
||||
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
started = time.perf_counter()
|
||||
queryset = self._apply_filters(Profile.objects.all(), request)
|
||||
queryset = self._apply_keyword(queryset, request.q)
|
||||
total = queryset.count()
|
||||
|
||||
page = (
|
||||
self._apply_sort(queryset, request)
|
||||
.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill")[
|
||||
request.offset : request.offset + request.page_size
|
||||
]
|
||||
)
|
||||
hits = [self._to_hit(profile, request.q) for profile in page]
|
||||
facets = self._facets(queryset) if request.include_facets else {}
|
||||
|
||||
return SearchResult(
|
||||
total=total,
|
||||
hits=hits,
|
||||
facets=facets,
|
||||
took_ms=int((time.perf_counter() - started) * 1000),
|
||||
backend=self.name,
|
||||
)
|
||||
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
if not prefix:
|
||||
return []
|
||||
return list(
|
||||
Profile.objects.filter(
|
||||
Q(full_name__istartswith=prefix) | Q(full_name__icontains=f" {prefix}")
|
||||
)
|
||||
.order_by("full_name")
|
||||
.values_list("full_name", flat=True)[:limit]
|
||||
)
|
||||
|
||||
def ping(self) -> bool:
|
||||
Profile.objects.exists()
|
||||
return True
|
||||
|
||||
# -- Query construction -------------------------------------------------
|
||||
@staticmethod
|
||||
def _apply_filters(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
|
||||
if request.skills:
|
||||
# Every selected skill widens the result set (OR), mirroring the
|
||||
# ``terms`` filter used by the Elasticsearch backend.
|
||||
queryset = queryset.filter(
|
||||
skills__slug__in=[Skill.make_slug(s) for s in request.skills]
|
||||
)
|
||||
if request.roles:
|
||||
queryset = queryset.filter(job_title_role__in=request.roles)
|
||||
if request.seniority:
|
||||
queryset = queryset.filter(seniority_levels__overlap=list(request.seniority))
|
||||
if request.countries:
|
||||
queryset = queryset.filter(location_country__in=request.countries)
|
||||
if request.industries:
|
||||
queryset = queryset.filter(industry__in=request.industries)
|
||||
if request.companies:
|
||||
queryset = queryset.filter(current_company__name__in=request.companies)
|
||||
if request.min_experience is not None:
|
||||
queryset = queryset.filter(years_experience__gte=request.min_experience)
|
||||
if request.max_experience is not None:
|
||||
queryset = queryset.filter(years_experience__lte=request.max_experience)
|
||||
return queryset.distinct()
|
||||
|
||||
@staticmethod
|
||||
def _apply_keyword(queryset: QuerySet[Profile], term: str) -> QuerySet[Profile]:
|
||||
if not term:
|
||||
return queryset.annotate(rank=Value(0.0, output_field=FloatField()))
|
||||
|
||||
# ``websearch`` accepts the syntax users already know from search
|
||||
# engines: quoted phrases, ``or``, and ``-excluded``.
|
||||
query = SearchQuery(term, search_type="websearch", config="english")
|
||||
return (
|
||||
queryset.annotate(
|
||||
text_rank=SearchRank(F("search_vector"), query),
|
||||
name_similarity=TrigramSimilarity("full_name", term),
|
||||
)
|
||||
.annotate(
|
||||
rank=Greatest(
|
||||
F("text_rank"),
|
||||
# A close name match is worth as much as a strong text hit,
|
||||
# which is what makes "jospeh" still find "joseph".
|
||||
Case(
|
||||
When(
|
||||
name_similarity__gte=_MIN_TRIGRAM_SIMILARITY, then=F("name_similarity")
|
||||
),
|
||||
default=Value(0.0),
|
||||
output_field=FloatField(),
|
||||
),
|
||||
output_field=FloatField(),
|
||||
)
|
||||
)
|
||||
.filter(Q(search_vector=query) | Q(name_similarity__gte=_MIN_TRIGRAM_SIMILARITY))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_sort(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
|
||||
if request.sort == SORT_NAME:
|
||||
return queryset.order_by("full_name", "id")
|
||||
if request.sort == SORT_EXPERIENCE_DESC:
|
||||
return queryset.order_by(F("years_experience").desc(nulls_last=True), "id")
|
||||
if request.sort == SORT_EXPERIENCE_ASC:
|
||||
return queryset.order_by(F("years_experience").asc(nulls_last=True), "id")
|
||||
return queryset.order_by("-rank", "full_name", "id")
|
||||
|
||||
# -- Facets -------------------------------------------------------------
|
||||
def _facets(self, queryset: QuerySet[Profile]) -> dict[str, list[FacetBucket]]:
|
||||
ids = list(queryset.values_list("id", flat=True))
|
||||
if not ids:
|
||||
return {
|
||||
name: []
|
||||
for name in (
|
||||
"skills",
|
||||
"job_title_role",
|
||||
"seniority",
|
||||
"country",
|
||||
"industry",
|
||||
"company",
|
||||
)
|
||||
}
|
||||
|
||||
skills = Skill.objects.filter(profile_skills__profile_id__in=ids).values_list(
|
||||
"name", flat=True
|
||||
)
|
||||
rows = Profile.objects.filter(id__in=ids).values(
|
||||
"job_title_role", "location_country", "industry", "current_company__name",
|
||||
"seniority_levels",
|
||||
) # fmt: skip
|
||||
|
||||
counters: dict[str, Counter] = {
|
||||
"skills": Counter(skills),
|
||||
"job_title_role": Counter(),
|
||||
"seniority": Counter(),
|
||||
"country": Counter(),
|
||||
"industry": Counter(),
|
||||
"company": Counter(),
|
||||
}
|
||||
for row in rows:
|
||||
counters["job_title_role"][row["job_title_role"]] += 1
|
||||
counters["country"][row["location_country"]] += 1
|
||||
counters["industry"][row["industry"]] += 1
|
||||
counters["company"][row["current_company__name"] or ""] += 1
|
||||
for level in row["seniority_levels"] or []:
|
||||
counters["seniority"][level] += 1
|
||||
|
||||
return {
|
||||
name: [
|
||||
FacetBucket(key=key, count=count)
|
||||
for key, count in counter.most_common(_FACET_LIMIT)
|
||||
if key
|
||||
]
|
||||
for name, counter in counters.items()
|
||||
}
|
||||
|
||||
# -- Result mapping -----------------------------------------------------
|
||||
@classmethod
|
||||
def _to_hit(cls, profile: Profile, term: str) -> SearchHit:
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()[:12]]
|
||||
return SearchHit(
|
||||
id=profile.id,
|
||||
score=round(float(getattr(profile, "rank", 0.0) or 0.0), 4),
|
||||
full_name=profile.full_name,
|
||||
headline=profile.headline,
|
||||
job_title=profile.job_title,
|
||||
job_title_role=profile.job_title_role,
|
||||
seniority_levels=profile.seniority_levels,
|
||||
company=profile.current_company.name if profile.current_company_id else "",
|
||||
industry=profile.industry,
|
||||
location_name=profile.location_name,
|
||||
location_country=profile.location_country,
|
||||
years_experience=profile.years_experience,
|
||||
linkedin_url=profile.linkedin_url,
|
||||
skills=skills,
|
||||
highlight=cls._highlight(term, profile, skills),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _highlight(term: str, profile: Profile, skills: list[str]) -> dict[str, list[str]]:
|
||||
"""
|
||||
Approximate Elasticsearch's highlighter.
|
||||
|
||||
PostgreSQL offers ``ts_headline`` but it needs an extra round trip per
|
||||
row; for the fields shown on a result card a simple token wrap gives the
|
||||
same visual result at no query cost.
|
||||
"""
|
||||
tokens = [token for token in _TOKEN_RE.findall(term.lower()) if len(token) > 1]
|
||||
if not tokens:
|
||||
return {}
|
||||
pattern = re.compile("|".join(re.escape(token) for token in tokens), re.IGNORECASE)
|
||||
|
||||
highlight: dict[str, list[str]] = {}
|
||||
for field_name, value in (
|
||||
("full_name", profile.full_name),
|
||||
("job_title", profile.job_title),
|
||||
):
|
||||
if value and pattern.search(value):
|
||||
highlight[field_name] = [pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", value)]
|
||||
matched_skills = [skill for skill in skills if pattern.search(skill)]
|
||||
if matched_skills:
|
||||
highlight["skills"] = [
|
||||
pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", skill)
|
||||
for skill in matched_skills[:3]
|
||||
]
|
||||
return highlight
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Elasticsearch index definition and (re)build helpers.
|
||||
|
||||
Index design
|
||||
------------
|
||||
* One document per profile, denormalised: the fields a recruiter searches on
|
||||
(name, title, employer, skills, education, location) all live in the same
|
||||
document, so a query never needs a join.
|
||||
* Text fields are analysed with an English analyzer (lowercase, ASCII folding,
|
||||
stop words, stemming) so "engineering" matches "engineer".
|
||||
* Every field that is *filtered* or *aggregated* on has a ``keyword``
|
||||
counterpart with a lowercase normaliser - analysed text cannot be used for
|
||||
exact term filters or facet counts.
|
||||
* ``full_name.autocomplete`` uses an edge n-gram analyzer for the type-ahead;
|
||||
the search analyzer is deliberately different (``standard``) so the query
|
||||
term is not itself n-grammed.
|
||||
* **No personal contact data is indexed.** E-mail addresses and phone numbers
|
||||
stay in PostgreSQL, which removes any chance of confirming someone's e-mail
|
||||
through a crafted search query.
|
||||
|
||||
Rebuilds use an alias plus a timestamped physical index, so a reindex never
|
||||
leaves the API pointing at a half-populated index and can be rolled back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch.helpers import bulk
|
||||
|
||||
from apps.profiles.models import Profile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INDEX_SETTINGS: dict[str, Any] = {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"refresh_interval": "1s",
|
||||
"max_result_window": 10_000,
|
||||
"analysis": {
|
||||
"filter": {
|
||||
"english_stop": {"type": "stop", "stopwords": "_english_"},
|
||||
"english_stemmer": {"type": "stemmer", "language": "english"},
|
||||
"english_possessive": {"type": "stemmer", "language": "possessive_english"},
|
||||
"edge_ngrams": {"type": "edge_ngram", "min_gram": 2, "max_gram": 15},
|
||||
},
|
||||
"normalizer": {
|
||||
"lowercase_normalizer": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
}
|
||||
},
|
||||
"analyzer": {
|
||||
"english_text": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": [
|
||||
"lowercase",
|
||||
"asciifolding",
|
||||
"english_possessive",
|
||||
"english_stop",
|
||||
"english_stemmer",
|
||||
],
|
||||
},
|
||||
"autocomplete_index": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding", "edge_ngrams"],
|
||||
},
|
||||
"autocomplete_search": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
KEYWORD = {"type": "keyword", "normalizer": "lowercase_normalizer"}
|
||||
|
||||
INDEX_MAPPINGS: dict[str, Any] = {
|
||||
"dynamic": "strict", # reject unexpected fields instead of guessing a type
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"full_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"headline": {"type": "text", "analyzer": "english_text"},
|
||||
"job_title": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"job_title_role": KEYWORD,
|
||||
"job_title_sub_role": KEYWORD,
|
||||
"seniority_levels": KEYWORD,
|
||||
"company": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"past_companies": {"type": "text", "analyzer": "english_text"},
|
||||
"industry": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"skills": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"education": {"type": "text", "analyzer": "english_text"},
|
||||
"degrees": KEYWORD,
|
||||
"majors": {"type": "text", "analyzer": "english_text", "fields": {"keyword": KEYWORD}},
|
||||
"summary": {"type": "text", "analyzer": "english_text"},
|
||||
"interests": {"type": "text", "analyzer": "english_text"},
|
||||
"languages": KEYWORD,
|
||||
"location_name": {"type": "text", "analyzer": "english_text"},
|
||||
"location_locality": KEYWORD,
|
||||
"location_region": KEYWORD,
|
||||
"location_country": KEYWORD,
|
||||
"location_continent": KEYWORD,
|
||||
"years_experience": {"type": "integer"},
|
||||
"connections": {"type": "integer"},
|
||||
"linkedin_url": {"type": "keyword", "index": False},
|
||||
"indexed_at": {"type": "date"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_client() -> Elasticsearch:
|
||||
"""Build an Elasticsearch client from settings (auth optional in dev)."""
|
||||
config = settings.SEARCH["ELASTICSEARCH"]
|
||||
kwargs: dict[str, Any] = {
|
||||
"hosts": config["HOSTS"],
|
||||
"request_timeout": config["TIMEOUT"],
|
||||
"max_retries": config["MAX_RETRIES"],
|
||||
"retry_on_timeout": True,
|
||||
}
|
||||
if config["USERNAME"] and config["PASSWORD"]:
|
||||
kwargs["basic_auth"] = (config["USERNAME"], config["PASSWORD"])
|
||||
return Elasticsearch(**kwargs)
|
||||
|
||||
|
||||
def alias_name() -> str:
|
||||
return settings.SEARCH["ELASTICSEARCH"]["INDEX"]
|
||||
|
||||
|
||||
def physical_index_name() -> str:
|
||||
return f"{alias_name()}-{datetime.now(UTC):%Y%m%d%H%M%S}"
|
||||
|
||||
|
||||
def build_document(profile: Profile) -> dict[str, Any]:
|
||||
"""Project a ``Profile`` row (plus its relations) into an index document."""
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()]
|
||||
experiences = list(profile.experiences.all())
|
||||
educations = list(profile.educations.all())
|
||||
|
||||
degrees = sorted({degree for edu in educations for degree in edu.degrees})
|
||||
majors = sorted({major for edu in educations for major in edu.majors})
|
||||
schools = [edu.school_name for edu in educations if edu.school_name]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"full_name": profile.full_name,
|
||||
"headline": profile.headline,
|
||||
"job_title": profile.job_title,
|
||||
"job_title_role": profile.job_title_role,
|
||||
"job_title_sub_role": profile.job_title_sub_role,
|
||||
"seniority_levels": profile.seniority_levels,
|
||||
"company": profile.current_company.name if profile.current_company_id else "",
|
||||
"past_companies": sorted({exp.company_name for exp in experiences if exp.company_name}),
|
||||
"industry": profile.industry,
|
||||
"skills": skills,
|
||||
"education": " ".join([*schools, *majors]),
|
||||
"degrees": degrees,
|
||||
"majors": majors,
|
||||
"summary": profile.summary,
|
||||
"interests": profile.interests,
|
||||
"languages": profile.languages,
|
||||
"location_name": profile.location_name,
|
||||
"location_locality": profile.location_locality,
|
||||
"location_region": profile.location_region,
|
||||
"location_country": profile.location_country,
|
||||
"location_continent": profile.location_continent,
|
||||
"years_experience": profile.years_experience,
|
||||
"connections": profile.connections,
|
||||
"linkedin_url": profile.linkedin_url,
|
||||
"indexed_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def iter_documents(queryset: Iterable[Profile] | None = None) -> Iterator[dict[str, Any]]:
|
||||
profiles = queryset if queryset is not None else _default_queryset()
|
||||
for profile in profiles:
|
||||
yield {"_id": profile.id, "_source": build_document(profile)}
|
||||
|
||||
|
||||
def _default_queryset(): # noqa: ANN202
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill", "experiences", "educations")
|
||||
.order_by("id")
|
||||
.iterator(chunk_size=200)
|
||||
)
|
||||
|
||||
|
||||
def create_index(client: Elasticsearch, name: str) -> None:
|
||||
client.indices.create(index=name, settings=INDEX_SETTINGS, mappings=INDEX_MAPPINGS)
|
||||
|
||||
|
||||
def rebuild_index(client: Elasticsearch | None = None) -> dict[str, Any]:
|
||||
"""
|
||||
Zero-downtime rebuild.
|
||||
|
||||
Creates a fresh physical index, indexes every profile into it, then flips
|
||||
the alias atomically and removes the previous physical index.
|
||||
"""
|
||||
client = client or get_client()
|
||||
alias = alias_name()
|
||||
new_index = physical_index_name()
|
||||
|
||||
create_index(client, new_index)
|
||||
logger.info("index_created", extra={"event": "index_created", "index": new_index})
|
||||
|
||||
succeeded, errors = bulk(
|
||||
client,
|
||||
({"_index": new_index, **doc} for doc in iter_documents()),
|
||||
raise_on_error=False,
|
||||
stats_only=False,
|
||||
)
|
||||
client.indices.refresh(index=new_index)
|
||||
|
||||
previous = _current_indices(client, alias)
|
||||
actions: list[dict[str, Any]] = [{"add": {"index": new_index, "alias": alias}}]
|
||||
actions += [{"remove": {"index": name, "alias": alias}} for name in previous]
|
||||
client.indices.update_aliases(actions=actions)
|
||||
|
||||
for name in previous:
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
|
||||
result = {
|
||||
"alias": alias,
|
||||
"index": new_index,
|
||||
"indexed": succeeded,
|
||||
"errors": len(errors) if isinstance(errors, list) else errors,
|
||||
"replaced": previous,
|
||||
}
|
||||
logger.info("index_rebuilt", extra={"event": "index_rebuilt", **result})
|
||||
return result
|
||||
|
||||
|
||||
def _current_indices(client: Elasticsearch, alias: str) -> list[str]:
|
||||
if not client.indices.exists_alias(name=alias):
|
||||
# A leftover *index* (not alias) with the same name would block the
|
||||
# alias creation, so remove it first.
|
||||
if client.indices.exists(index=alias):
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
return []
|
||||
return list(client.indices.get_alias(name=alias).keys())
|
||||
|
||||
|
||||
def index_profile(profile: Profile, client: Elasticsearch | None = None) -> None:
|
||||
"""Index (or re-index) a single profile - used after a targeted update."""
|
||||
client = client or get_client()
|
||||
client.index(index=alias_name(), id=profile.id, document=build_document(profile))
|
||||
@@ -0,0 +1,52 @@
|
||||
"""``python manage.py search_index`` - manage the Elasticsearch index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.search.indexing import alias_name, get_client, rebuild_index
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create, rebuild, inspect or drop the Elasticsearch profile index."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--rebuild", action="store_true", help="Rebuild and swap the alias.")
|
||||
group.add_argument("--status", action="store_true", help="Show index/alias status.")
|
||||
group.add_argument("--drop", action="store_true", help="Delete the index and its alias.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
client = get_client()
|
||||
if not client.ping():
|
||||
raise CommandError("Elasticsearch is not reachable. Check ELASTICSEARCH_HOSTS.")
|
||||
|
||||
if options["rebuild"]:
|
||||
started = time.perf_counter()
|
||||
result = rebuild_index(client)
|
||||
result["elapsed_seconds"] = round(time.perf_counter() - started, 2)
|
||||
self.stdout.write(json.dumps(result, indent=2))
|
||||
self.stdout.write(self.style.SUCCESS("Index rebuilt."))
|
||||
return
|
||||
|
||||
alias = alias_name()
|
||||
if options["status"]:
|
||||
exists = client.indices.exists_alias(name=alias)
|
||||
payload: dict[str, Any] = {"alias": alias, "exists": bool(exists)}
|
||||
if exists:
|
||||
indices = list(client.indices.get_alias(name=alias).keys())
|
||||
payload["indices"] = indices
|
||||
payload["documents"] = client.count(index=alias)["count"]
|
||||
self.stdout.write(json.dumps(payload, indent=2))
|
||||
return
|
||||
|
||||
if options["drop"]:
|
||||
if client.indices.exists_alias(name=alias):
|
||||
for name in client.indices.get_alias(name=alias):
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
self.stdout.write(self.style.WARNING(f"Dropped index/alias {alias}."))
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Request validation and response shaping for the search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.search.types import SORT_OPTIONS, SORT_RELEVANCE, SearchRequest, SearchResult
|
||||
|
||||
MAX_QUERY_LENGTH = 200
|
||||
MAX_FILTER_VALUES = 20
|
||||
|
||||
|
||||
class CommaSeparatedField(serializers.CharField):
|
||||
"""
|
||||
Accepts ``?skills=python,django`` **and** repeated ``?skills=python&skills=django``.
|
||||
|
||||
Both spellings are common in the wild; normalising here keeps the views and
|
||||
both backends free of query-string trivia.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
kwargs.setdefault("required", False)
|
||||
kwargs.setdefault("allow_blank", True)
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_value(self, dictionary: Any) -> Any:
|
||||
# ``QueryDict.get`` silently keeps only the last occurrence, which would
|
||||
# turn ``?skills=a&skills=b`` into ``b``. ``getlist`` keeps both.
|
||||
if hasattr(dictionary, "getlist"):
|
||||
values = dictionary.getlist(self.field_name)
|
||||
if values:
|
||||
return values
|
||||
return super().get_value(dictionary)
|
||||
|
||||
def to_internal_value(self, data: Any) -> tuple[str, ...]:
|
||||
raw: list[str] = []
|
||||
if isinstance(data, list | tuple):
|
||||
raw = [str(item) for item in data]
|
||||
elif data:
|
||||
raw = [str(data)]
|
||||
|
||||
values: list[str] = []
|
||||
for chunk in raw:
|
||||
for item in chunk.split(","):
|
||||
cleaned = item.strip().lower()[:128]
|
||||
if cleaned and cleaned not in values:
|
||||
values.append(cleaned)
|
||||
if len(values) > MAX_FILTER_VALUES:
|
||||
raise serializers.ValidationError(
|
||||
f"At most {MAX_FILTER_VALUES} values are allowed per filter."
|
||||
)
|
||||
return tuple(values)
|
||||
|
||||
|
||||
class SearchRequestSerializer(serializers.Serializer):
|
||||
"""Validates and normalises the query string of ``GET /api/v1/search/``."""
|
||||
|
||||
q = serializers.CharField(
|
||||
required=False, allow_blank=True, max_length=MAX_QUERY_LENGTH, trim_whitespace=True,
|
||||
help_text="Free-text keyword. Supports quoted phrases and -exclusions.",
|
||||
) # fmt: skip
|
||||
skills = CommaSeparatedField(help_text="Skill names, OR-combined.")
|
||||
roles = CommaSeparatedField(help_text="Job family, e.g. engineering, human_resources.")
|
||||
seniority = CommaSeparatedField(help_text="Seniority level, e.g. senior, manager, cxo.")
|
||||
countries = CommaSeparatedField(help_text="Country of residence.")
|
||||
industries = CommaSeparatedField(help_text="Industry of the current employer.")
|
||||
companies = CommaSeparatedField(help_text="Current employer name.")
|
||||
min_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
max_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
sort = serializers.ChoiceField(choices=SORT_OPTIONS, required=False, default=SORT_RELEVANCE)
|
||||
page = serializers.IntegerField(required=False, min_value=1, default=1)
|
||||
page_size = serializers.IntegerField(required=False, min_value=1, max_value=100, default=20)
|
||||
facets = serializers.BooleanField(required=False, default=True)
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
low, high = attrs.get("min_experience"), attrs.get("max_experience")
|
||||
if low is not None and high is not None and low > high:
|
||||
raise serializers.ValidationError(
|
||||
{"min_experience": "min_experience cannot be greater than max_experience."}
|
||||
)
|
||||
|
||||
page, page_size = attrs.get("page", 1), attrs.get("page_size", 20)
|
||||
# Deep pagination is both useless to a human and expensive for the
|
||||
# engine, so it is rejected instead of silently truncated.
|
||||
if page * page_size > settings.SEARCH["MAX_RESULT_WINDOW"]:
|
||||
raise serializers.ValidationError(
|
||||
{"page": f"Cannot page beyond {settings.SEARCH['MAX_RESULT_WINDOW']} results."}
|
||||
)
|
||||
return attrs
|
||||
|
||||
def to_search_request(self) -> SearchRequest:
|
||||
data = self.validated_data
|
||||
return SearchRequest(
|
||||
q=data.get("q", "").strip(),
|
||||
skills=data.get("skills", ()),
|
||||
roles=data.get("roles", ()),
|
||||
seniority=data.get("seniority", ()),
|
||||
countries=data.get("countries", ()),
|
||||
industries=data.get("industries", ()),
|
||||
companies=data.get("companies", ()),
|
||||
min_experience=data.get("min_experience"),
|
||||
max_experience=data.get("max_experience"),
|
||||
sort=data.get("sort", SORT_RELEVANCE),
|
||||
page=data.get("page", 1),
|
||||
page_size=data.get("page_size", 20),
|
||||
include_facets=data.get("facets", True),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response serializers (declared explicitly so the OpenAPI schema is accurate)
|
||||
# ---------------------------------------------------------------------------
|
||||
class SearchHitSerializer(serializers.Serializer):
|
||||
id = serializers.IntegerField()
|
||||
score = serializers.FloatField()
|
||||
full_name = serializers.CharField()
|
||||
headline = serializers.CharField()
|
||||
job_title = serializers.CharField()
|
||||
job_title_role = serializers.CharField()
|
||||
seniority_levels = serializers.ListField(child=serializers.CharField())
|
||||
company = serializers.CharField()
|
||||
industry = serializers.CharField()
|
||||
location_name = serializers.CharField()
|
||||
location_country = serializers.CharField()
|
||||
years_experience = serializers.IntegerField(allow_null=True)
|
||||
linkedin_url = serializers.CharField()
|
||||
skills = serializers.ListField(child=serializers.CharField())
|
||||
highlight = serializers.DictField(child=serializers.ListField(child=serializers.CharField()))
|
||||
|
||||
|
||||
class FacetBucketSerializer(serializers.Serializer):
|
||||
key = serializers.CharField()
|
||||
count = serializers.IntegerField()
|
||||
|
||||
|
||||
class SearchResponseSerializer(serializers.Serializer):
|
||||
count = serializers.IntegerField()
|
||||
page = serializers.IntegerField()
|
||||
page_size = serializers.IntegerField()
|
||||
total_pages = serializers.IntegerField()
|
||||
took_ms = serializers.IntegerField()
|
||||
backend = serializers.CharField()
|
||||
results = SearchHitSerializer(many=True)
|
||||
facets = serializers.DictField(child=FacetBucketSerializer(many=True))
|
||||
|
||||
@staticmethod
|
||||
def from_result(result: SearchResult, request: SearchRequest) -> dict[str, Any]:
|
||||
total_pages = max(1, -(-result.total // request.page_size))
|
||||
return {
|
||||
"count": result.total,
|
||||
"page": request.page,
|
||||
"page_size": request.page_size,
|
||||
"total_pages": total_pages,
|
||||
"took_ms": result.took_ms,
|
||||
"backend": result.backend,
|
||||
"results": [asdict(hit) for hit in result.hits],
|
||||
"facets": {
|
||||
name: [asdict(bucket) for bucket in buckets]
|
||||
for name, buckets in result.facets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SuggestResponseSerializer(serializers.Serializer):
|
||||
suggestions = serializers.ListField(child=serializers.CharField())
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Backend independent data structures.
|
||||
|
||||
Both search backends speak this vocabulary, which is what makes them
|
||||
interchangeable behind a single setting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# Sort keys accepted by the API and understood by every backend.
|
||||
SORT_RELEVANCE = "relevance"
|
||||
SORT_EXPERIENCE_DESC = "-experience"
|
||||
SORT_EXPERIENCE_ASC = "experience"
|
||||
SORT_NAME = "name"
|
||||
SORT_OPTIONS = (SORT_RELEVANCE, SORT_EXPERIENCE_DESC, SORT_EXPERIENCE_ASC, SORT_NAME)
|
||||
|
||||
# Facet name -> the document field it aggregates.
|
||||
FACET_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"job_title_role": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"country": "location_country",
|
||||
"industry": "industry",
|
||||
"company": "company.keyword",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SearchRequest:
|
||||
"""A validated, normalised search request."""
|
||||
|
||||
q: str = ""
|
||||
skills: tuple[str, ...] = ()
|
||||
roles: tuple[str, ...] = ()
|
||||
seniority: tuple[str, ...] = ()
|
||||
countries: tuple[str, ...] = ()
|
||||
industries: tuple[str, ...] = ()
|
||||
companies: tuple[str, ...] = ()
|
||||
min_experience: int | None = None
|
||||
max_experience: int | None = None
|
||||
sort: str = SORT_RELEVANCE
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
include_facets: bool = True
|
||||
|
||||
@property
|
||||
def offset(self) -> int:
|
||||
return (self.page - 1) * self.page_size
|
||||
|
||||
@property
|
||||
def has_filters(self) -> bool:
|
||||
return bool(
|
||||
self.skills
|
||||
or self.roles
|
||||
or self.seniority
|
||||
or self.countries
|
||||
or self.industries
|
||||
or self.companies
|
||||
or self.min_experience is not None
|
||||
or self.max_experience is not None
|
||||
)
|
||||
|
||||
def as_log_context(self) -> dict[str, Any]:
|
||||
"""Compact representation for the audit log (no free text explosion)."""
|
||||
return {
|
||||
"query": self.q[:120],
|
||||
"filters": {
|
||||
"skills": list(self.skills),
|
||||
"roles": list(self.roles),
|
||||
"seniority": list(self.seniority),
|
||||
"countries": list(self.countries),
|
||||
"industries": list(self.industries),
|
||||
"companies": list(self.companies),
|
||||
"min_experience": self.min_experience,
|
||||
"max_experience": self.max_experience,
|
||||
},
|
||||
"sort": self.sort,
|
||||
"page": self.page,
|
||||
"page_size": self.page_size,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SearchHit:
|
||||
"""One result card - identical shape whichever backend produced it."""
|
||||
|
||||
id: int
|
||||
score: float
|
||||
full_name: str
|
||||
headline: str
|
||||
job_title: str
|
||||
job_title_role: str
|
||||
seniority_levels: list[str]
|
||||
company: str
|
||||
industry: str
|
||||
location_name: str
|
||||
location_country: str
|
||||
years_experience: int | None
|
||||
linkedin_url: str
|
||||
skills: list[str]
|
||||
highlight: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FacetBucket:
|
||||
key: str
|
||||
count: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SearchResult:
|
||||
total: int
|
||||
hits: list[SearchHit]
|
||||
facets: dict[str, list[FacetBucket]]
|
||||
took_ms: int
|
||||
backend: str
|
||||
@@ -0,0 +1,11 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.search.views import ReindexView, SearchView, SuggestView
|
||||
|
||||
app_name = "search"
|
||||
|
||||
urlpatterns = [
|
||||
path("", SearchView.as_view(), name="search"),
|
||||
path("suggest/", SuggestView.as_view(), name="suggest"),
|
||||
path("reindex/", ReindexView.as_view(), name="reindex"),
|
||||
]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from drf_spectacular.utils import OpenApiParameter, extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from apps.core.permissions import IsAdminRole
|
||||
from apps.core.throttling import SearchRateThrottle
|
||||
from apps.search.backends import get_search_backend
|
||||
from apps.search.serializers import (
|
||||
SearchRequestSerializer,
|
||||
SearchResponseSerializer,
|
||||
SuggestResponseSerializer,
|
||||
)
|
||||
|
||||
audit = logging.getLogger("audit.search")
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
parameters=[SearchRequestSerializer],
|
||||
responses={200: SearchResponseSerializer},
|
||||
summary="Keyword search with filters and facets",
|
||||
description=(
|
||||
"Ranked keyword search across name, job title, skills, employer, education, "
|
||||
"location and summary, combined with structured filters. Filter values are "
|
||||
"OR-ed inside one filter and AND-ed across filters."
|
||||
),
|
||||
)
|
||||
class SearchView(APIView):
|
||||
"""
|
||||
``GET /api/v1/search/``
|
||||
|
||||
Thin by design: validate the query string, hand a normalised request to the
|
||||
configured backend, shape the response, emit an audit event. All ranking
|
||||
lives in the backend, all validation lives in the serializer.
|
||||
"""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
throttle_classes = [SearchRateThrottle]
|
||||
serializer_class = SearchRequestSerializer
|
||||
|
||||
def get(self, request: Request) -> Response:
|
||||
serializer = SearchRequestSerializer(data=request.query_params)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
search_request = serializer.to_search_request()
|
||||
|
||||
backend = get_search_backend()
|
||||
result = backend.search(search_request)
|
||||
|
||||
audit.info(
|
||||
"search_executed",
|
||||
extra={
|
||||
"event": "search_executed",
|
||||
"user_id": request.user.id,
|
||||
"backend": result.backend,
|
||||
"result_count": result.total,
|
||||
"took_ms": result.took_ms,
|
||||
**search_request.as_log_context(),
|
||||
},
|
||||
)
|
||||
return Response(SearchResponseSerializer.from_result(result, search_request))
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
parameters=[
|
||||
OpenApiParameter("q", str, description="Partial name or skill.", required=True),
|
||||
OpenApiParameter("limit", int, description="Maximum suggestions (1-20)."),
|
||||
],
|
||||
responses={200: SuggestResponseSerializer},
|
||||
summary="Type-ahead suggestions",
|
||||
)
|
||||
class SuggestView(APIView):
|
||||
"""Edge n-gram completions used by the search box."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
throttle_classes = [SearchRateThrottle]
|
||||
|
||||
def get(self, request: Request) -> Response:
|
||||
prefix = request.query_params.get("q", "").strip()[:64]
|
||||
try:
|
||||
limit = min(max(int(request.query_params.get("limit", 8)), 1), 20)
|
||||
except ValueError:
|
||||
limit = 8
|
||||
suggestions = get_search_backend().suggest(prefix, limit=limit) if prefix else []
|
||||
return Response({"suggestions": suggestions})
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
request=None,
|
||||
responses={202: dict},
|
||||
summary="Rebuild the search index (admin only)",
|
||||
)
|
||||
class ReindexView(APIView):
|
||||
"""
|
||||
Triggers a zero-downtime rebuild of the Elasticsearch index.
|
||||
|
||||
Restricted to the admin role: it is an expensive, cluster-wide operation.
|
||||
"""
|
||||
|
||||
permission_classes = [IsAuthenticated, IsAdminRole]
|
||||
|
||||
def post(self, request: Request) -> Response:
|
||||
from apps.search.indexing import rebuild_index
|
||||
|
||||
result = rebuild_index()
|
||||
audit.info(
|
||||
"index_rebuilt",
|
||||
extra={"event": "index_rebuilt", "user_id": request.user.id, **result},
|
||||
)
|
||||
return Response(result, status=status.HTTP_202_ACCEPTED)
|
||||
Reference in New Issue
Block a user