feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Domain model for the LinkedIn dataset.
|
||||
|
||||
The raw export is one very wide CSV row per person with several columns holding
|
||||
serialised lists of dictionaries. Storing that blob as-is would make filtering
|
||||
and aggregation painful, so it is normalised into five tables:
|
||||
|
||||
Company <--- Experience ---> Profile <--- Education ---> School
|
||||
|
|
||||
+--- ProfileSkill ---> Skill
|
||||
|
||||
PostgreSQL is the source of truth. Elasticsearch holds a denormalised
|
||||
projection of the same data for search (see ``apps.search``); it can always be
|
||||
rebuilt from these tables, which is why the index is disposable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.contrib.postgres.indexes import GinIndex
|
||||
from django.contrib.postgres.search import SearchVectorField
|
||||
from django.db import models
|
||||
from django.utils.text import slugify
|
||||
|
||||
|
||||
class TimeStampedModel(models.Model):
|
||||
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class Company(TimeStampedModel):
|
||||
"""An employer referenced by the current job or by a past experience."""
|
||||
|
||||
external_id = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
size = models.CharField(max_length=32, blank=True, help_text="Head-count bucket, e.g. 51-200")
|
||||
founded = models.PositiveIntegerField(null=True, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
verbose_name_plural = "companies"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class School(TimeStampedModel):
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
type = models.CharField(max_length=64, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class Skill(TimeStampedModel):
|
||||
"""
|
||||
A normalised skill.
|
||||
|
||||
``slug`` is the deduplication key: the raw data contains the same skill with
|
||||
different casing and spacing ("Team Building" / "team building").
|
||||
"""
|
||||
|
||||
name = models.CharField(max_length=128)
|
||||
slug = models.SlugField(max_length=140, unique=True)
|
||||
profile_count = models.PositiveIntegerField(
|
||||
default=0, db_index=True, help_text="Denormalised counter used to rank filter suggestions."
|
||||
)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-profile_count", "name")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
@staticmethod
|
||||
def make_slug(name: str) -> str:
|
||||
return slugify(name)[:140] or name.strip().lower()[:140]
|
||||
|
||||
|
||||
class Profile(TimeStampedModel):
|
||||
"""A single person in the dataset."""
|
||||
|
||||
class Gender(models.TextChoices):
|
||||
MALE = "male", "Male"
|
||||
FEMALE = "female", "Female"
|
||||
UNKNOWN = "", "Unknown"
|
||||
|
||||
# -- Identity ----------------------------------------------------------
|
||||
external_id = models.CharField(
|
||||
max_length=64,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
)
|
||||
full_name = models.CharField(max_length=255, db_index=True)
|
||||
first_name = models.CharField(max_length=128, blank=True)
|
||||
last_name = models.CharField(max_length=128, blank=True)
|
||||
gender = models.CharField(max_length=16, blank=True, choices=Gender.choices)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
linkedin_username = models.CharField(max_length=128, blank=True)
|
||||
github_url = models.CharField(max_length=255, blank=True)
|
||||
twitter_url = models.CharField(max_length=255, blank=True)
|
||||
|
||||
# -- Current position --------------------------------------------------
|
||||
job_title = models.CharField(max_length=255, blank=True, db_index=True)
|
||||
job_title_role = models.CharField(
|
||||
max_length=64, blank=True, db_index=True, help_text="Coarse job family, e.g. engineering."
|
||||
)
|
||||
job_title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
seniority_levels = ArrayField(
|
||||
models.CharField(max_length=32),
|
||||
default=list,
|
||||
blank=True,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
)
|
||||
current_company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="current_employees"
|
||||
)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
job_summary = models.TextField(blank=True)
|
||||
job_start_date = models.CharField(max_length=10, blank=True)
|
||||
|
||||
# -- Location ----------------------------------------------------------
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
location_locality = models.CharField(max_length=128, blank=True)
|
||||
location_region = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_continent = models.CharField(max_length=64, blank=True)
|
||||
|
||||
# -- Narrative & metrics ----------------------------------------------
|
||||
summary = models.TextField(blank=True)
|
||||
years_experience = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
inferred_salary = models.CharField(max_length=32, blank=True)
|
||||
connections = models.PositiveIntegerField(null=True, blank=True)
|
||||
interests = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
languages = ArrayField(models.CharField(max_length=64), default=list, blank=True)
|
||||
|
||||
# -- Contact details (PII: masked unless the caller is privileged) ------
|
||||
emails = models.JSONField(default=list, blank=True)
|
||||
phone_numbers = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
|
||||
skills = models.ManyToManyField(Skill, through="ProfileSkill", related_name="profiles")
|
||||
|
||||
# -- Denormalised text, maintained by the importer ---------------------
|
||||
# PostgreSQL's ``SearchVector`` cannot traverse a many-to-many relation, so
|
||||
# the searchable text of the related rows is flattened here once at import
|
||||
# time. This keeps the tsvector update to a single UPDATE statement and
|
||||
# keeps query time O(index lookup) instead of O(joins).
|
||||
skills_text = models.TextField(blank=True, editable=False)
|
||||
companies_text = models.TextField(blank=True, editable=False)
|
||||
education_text = models.TextField(blank=True, editable=False)
|
||||
|
||||
# Materialised tsvector powering the PostgreSQL search backend.
|
||||
search_vector = SearchVectorField(null=True, editable=False)
|
||||
|
||||
class Meta:
|
||||
ordering = ("full_name", "id")
|
||||
indexes = [
|
||||
GinIndex(fields=["search_vector"], name="profile_search_vector_gin"),
|
||||
GinIndex(fields=["seniority_levels"], name="profile_seniority_gin"),
|
||||
models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["external_id"],
|
||||
condition=models.Q(external_id__gt=""),
|
||||
name="uniq_profile_external_id",
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.full_name
|
||||
|
||||
@property
|
||||
def headline(self) -> str:
|
||||
"""Short "Title at Company" line used in result cards."""
|
||||
company = self.current_company.name if self.current_company_id else ""
|
||||
if self.job_title and company:
|
||||
return f"{self.job_title} at {company}"
|
||||
return self.job_title or company
|
||||
|
||||
|
||||
class ProfileSkill(models.Model):
|
||||
"""Through table - keeps the original ordering of the skill list."""
|
||||
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
skill = models.ForeignKey(Skill, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
position = models.PositiveSmallIntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
ordering = ("position",)
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["profile", "skill"], name="uniq_profile_skill")
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.profile_id}:{self.skill_id}"
|
||||
|
||||
|
||||
class Experience(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="experiences")
|
||||
company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="experiences"
|
||||
)
|
||||
company_name = models.CharField(max_length=255, blank=True)
|
||||
title = models.CharField(max_length=255, blank=True)
|
||||
title_role = models.CharField(max_length=64, blank=True)
|
||||
title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
levels = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
start_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True)
|
||||
is_primary = models.BooleanField(default=False, db_index=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-is_primary", "-start_year", "id")
|
||||
indexes = [models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.title} @ {self.company_name}"
|
||||
|
||||
|
||||
class Education(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="educations")
|
||||
school = models.ForeignKey(
|
||||
School, null=True, blank=True, on_delete=models.SET_NULL, related_name="educations"
|
||||
)
|
||||
school_name = models.CharField(max_length=255, blank=True)
|
||||
degrees = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
majors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
minors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
gpa = models.DecimalField(max_digits=4, decimal_places=2, null=True, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-end_year", "id")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.school_name}"
|
||||
Reference in New Issue
Block a user