feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)

Full-stack implementation of the Cyberyan full-stack trial task.

Backend (Django 5 / DRF)
- Normalised PostgreSQL model (Profile, Company, School, Skill, Experience,
  Education) as the source of truth, with GIN + trigram indexes.
- Self-healing CSV reader: the supplied export carries an injected watermark
  column, missing columns, rows split by broken quoting and a duplicated
  header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean
  records out of 336 physical rows; only 10 rows parse correctly without it.
- Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL
  full-text search as a fallback selected by a single environment variable.
- Keyword search with per-field boosts, fuzziness and phrase boosting, six
  faceted filters (OR within a filter, AND across filters), an experience
  range, four sort orders, highlighting and facet aggregations.
- JWT auth with rotation + blacklist, three roles, PII masking, scoped rate
  limits, a single error envelope, OpenAPI schema and health probes.

Frontend (React 18 / TypeScript / Vite)
- Debounced search with type-ahead, faceted filter panel, result cards,
  profile drawer, URL-synced search state, automatic token refresh.
- Highlight snippets rendered without dangerouslySetInnerHTML.

Infrastructure
- Hardened multi-stage images (non-root, read-only rootfs, healthchecks).
- docker-compose for the full stack plus dev and production overlays.
- ELK: structured JSON logs shipped asynchronously to Logstash, routed per
  event type, with a composable index template and a Kibana setup script.

Quality
- 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc,
  GitHub Actions running lint, tests and both image builds.
- Persian documentation: README plus architecture, search, API, data,
  security and operations guides.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
2026-09-03 08:13:10 +00:00
commit a464e550c9
133 changed files with 17019 additions and 0 deletions
View File
+72
View File
@@ -0,0 +1,72 @@
from django.contrib import admin
from apps.profiles.models import (
Company,
Education,
Experience,
Profile,
ProfileSkill,
School,
Skill,
)
class ExperienceInline(admin.TabularInline):
model = Experience
extra = 0
fields = ("title", "company_name", "start_date", "end_date", "is_primary")
readonly_fields = fields
can_delete = False
show_change_link = True
class EducationInline(admin.TabularInline):
model = Education
extra = 0
fields = ("school_name", "degrees", "majors", "end_date")
readonly_fields = fields
can_delete = False
class ProfileSkillInline(admin.TabularInline):
model = ProfileSkill
extra = 0
autocomplete_fields = ("skill",)
@admin.register(Profile)
class ProfileAdmin(admin.ModelAdmin):
list_display = (
"full_name", "job_title", "current_company", "location_country", "years_experience",
) # fmt: skip
list_filter = ("location_country", "job_title_role", "industry")
search_fields = ("full_name", "job_title", "skills_text", "companies_text")
list_select_related = ("current_company",)
inlines = (ProfileSkillInline, ExperienceInline, EducationInline)
readonly_fields = (
"created_at",
"updated_at",
"skills_text",
"companies_text",
"education_text",
)
@admin.register(Skill)
class SkillAdmin(admin.ModelAdmin):
list_display = ("name", "slug", "profile_count")
search_fields = ("name", "slug")
ordering = ("-profile_count",)
@admin.register(Company)
class CompanyAdmin(admin.ModelAdmin):
list_display = ("name", "industry", "size", "country")
search_fields = ("name",)
list_filter = ("size", "industry")
@admin.register(School)
class SchoolAdmin(admin.ModelAdmin):
list_display = ("name", "type", "country")
search_fields = ("name",)
+7
View File
@@ -0,0 +1,7 @@
from django.apps import AppConfig
class ProfilesConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "apps.profiles"
verbose_name = "Profiles"
+394
View File
@@ -0,0 +1,394 @@
"""
Resilient reader for the supplied LinkedIn CSV export.
Why this module exists
----------------------
The provided file is a real-world dirty export, not a clean CSV. Feeding it to
``csv.DictReader`` silently produces garbage. Four separate defects are present:
1. **Injected watermark** - many rows start with an extra, unheadered column
containing ``H:\\New folder\\leak\\...@BreachedData1 ...``. Everything after
it is shifted one position to the right.
2. **Missing columns** - most records omit one or more optional fields
(``job_title_role``, ``job_title_sub_role``, ...) instead of writing an empty
value, so the remaining values slide left relative to the header.
3. **Broken quoting** - a handful of ``summary`` values contain an unescaped
double quote, which splits a single logical record across several physical
rows.
4. **Concatenated exports** - the header line appears a second time in the
middle of the file and ~40 people are duplicated.
Repair strategy
---------------
Every column has a *signature*: ``linkedin_url`` contains ``linkedin.com/in/``,
``skills`` is a Python list literal, ``location_geo`` looks like ``lat,lon`` and
so on. We score each (column, value) pair against those signatures and then run
a Needleman-Wunsch style alignment between the row's values and the header
columns, allowing two edit operations:
* *skip column* - the record simply does not carry that field;
* *drop value* - a stray fragment that belongs to no column.
The alignment that maximises the total score is the reconstructed record. The
result is validated a second time and any value that still fails its signature
is discarded rather than persisted as junk - it is better to store nothing than
to store a phone number in the e-mail column.
``ast.literal_eval`` is used (never ``eval``) so the nested Python-repr columns
can be parsed without executing anything from the file.
"""
from __future__ import annotations
import ast
import csv
import re
import sys
from collections.abc import Callable, Iterator
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
# The nested columns are large; the stdlib default (128 KiB) is not enough.
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))
WATERMARK_MARKERS = ("@BreachedData", "H:\\New folder")
CONTINENTS = frozenset(
{"north america", "south america", "europe", "asia", "africa", "oceania", "antarctica"}
)
GENDERS = frozenset({"male", "female"})
Validator = Callable[[str], bool | None]
# ---------------------------------------------------------------------------
# Column signatures
# ---------------------------------------------------------------------------
def _list_of_str(value: str) -> bool | None:
parsed = _safe_literal(value)
if parsed is None:
return None
return isinstance(parsed, list) and all(isinstance(item, str) for item in parsed)
def _list_of_dicts(required_key: str) -> Validator:
def check(value: str) -> bool | None:
parsed = _safe_literal(value)
if parsed is None:
return None
if not isinstance(parsed, list):
return None
if not parsed:
return True
return all(isinstance(item, dict) for item in parsed) and required_key in parsed[0]
return check
def _dict_with(required_key: str) -> Validator:
def check(value: str) -> bool | None:
parsed = _safe_literal(value)
if parsed is None:
return None
return isinstance(parsed, dict) and required_key in parsed
return check
def _matches(pattern: str) -> Validator:
compiled = re.compile(pattern)
return lambda value: bool(compiled.fullmatch(value))
def _contains(needle: str) -> Validator:
return lambda value: needle in value
def _one_of(options: frozenset[str]) -> Validator:
return lambda value: value.lower() in options
def _safe_literal(value: str) -> Any | None:
"""``ast.literal_eval`` that returns ``None`` instead of raising."""
try:
return ast.literal_eval(value)
except (ValueError, SyntaxError, MemoryError, RecursionError):
return None
#: Columns without an entry here accept any text (name, title, industry, ...).
FIELD_VALIDATORS: dict[str, Validator] = {
"gender": _one_of(GENDERS),
"linkedin_url": _contains("linkedin.com/in/"),
"linkedin_id": _matches(r"\d{3,20}"),
"facebook_url": _contains("facebook.com/"),
"facebook_id": _matches(r"\d{5,25}"),
"job_title_role": _matches(r"[a-z_]{3,40}"),
"job_title_sub_role": _matches(r"[a-z_]{3,40}"),
"job_title_levels": _list_of_str,
"job_company_website": _matches(r"[\w.-]+\.[a-z]{2,10}"),
"job_company_size": _matches(
r"(1-10|11-50|51-200|201-500|501-1000|1001-5000|5001-10000|10001\+)"
),
"job_company_founded": _matches(r"(1[6-9]|20)\d\d(\.0)?"),
"job_company_linkedin_url": _contains("linkedin.com/company/"),
"job_company_linkedin_id": _matches(r"\d{3,20}"),
"job_company_facebook_url": _contains("facebook.com/"),
"job_company_twitter_url": _contains("twitter.com/"),
"job_company_location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
"job_company_location_continent": _one_of(CONTINENTS),
"job_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
"job_start_date": _matches(r"\d{4}(-\d{2})?"),
"location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
"location_continent": _one_of(CONTINENTS),
"location_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
"linkedin_connections": _matches(r"\d+(\.\d+)?"),
"inferred_salary": _matches(r"[\d,]+-[\d,]+|>?\d[\d,]*\+?"),
"inferred_years_experience": _matches(r"\d{1,2}(\.\d+)?"),
"phone_numbers": _list_of_str,
"emails": _list_of_dicts("address"),
"interests": _list_of_str,
"skills": _list_of_str,
"location_names": _list_of_str,
"regions": _list_of_str,
"countries": _list_of_str,
"street_addresses": _list_of_dicts("street_address"),
"experience": _list_of_dicts("company"),
"education": _list_of_dicts("school"),
"profiles": _list_of_dicts("network"),
"certifications": _list_of_dicts("name"),
"languages": _list_of_dicts("name"),
"version_status": _dict_with("status"),
"work_email": _contains("@"),
"middle_initial": _matches(r"[a-z]"),
"birth_year": _matches(r"(19|20)\d\d"),
"birth_date": _matches(r"\d{4}-\d{2}(-\d{2})?"),
"twitter_url": _contains("twitter.com/"),
"github_url": _contains("github.com/"),
"mobile_phone": _matches(r"\+?\d[\d -]{6,20}"),
}
# Alignment weights. Tuned so that a confirmed signature match outweighs any
# combination of edits, while losing a non-empty value is expensive.
SCORE_MATCH = 3.0
SCORE_MISMATCH = -4.0
SCORE_FREE_TEXT = 0.3
SCORE_EMPTY = 0.15
PENALTY_MISSING_COLUMN = -0.5
PENALTY_DROP_EMPTY = -0.3
PENALTY_DROP_VALUE = -3.5
@dataclass
class ImportReport:
"""Counters surfaced by the management command so imports are auditable."""
physical_rows: int = 0
duplicate_headers: int = 0
repaired_rows: int = 0
orphan_fragments: int = 0
duplicate_records: int = 0
discarded_values: int = 0
records: int = 0
issues: list[str] = field(default_factory=list)
def as_dict(self) -> dict[str, Any]:
return {
"physical_rows": self.physical_rows,
"duplicate_headers": self.duplicate_headers,
"repaired_rows": self.repaired_rows,
"orphan_fragments": self.orphan_fragments,
"duplicate_records": self.duplicate_records,
"discarded_values": self.discarded_values,
"records": self.records,
}
class DatasetReader:
"""Yields clean, de-duplicated records from the raw CSV export."""
def __init__(self, path: str | Path) -> None:
self.path = Path(path)
def read(self) -> tuple[list[dict[str, str]], ImportReport]:
report = ImportReport()
with self.path.open(encoding="utf-8", newline="") as handle:
rows = list(csv.reader(handle))
if not rows:
return [], report
header = rows[0]
report.physical_rows = len(rows) - 1
logical_rows = self._reassemble(rows[1:], header, report)
seen: set[str] = set()
records: list[dict[str, str]] = []
for values in logical_rows:
record = self._align(values, header)
self._sanitize(record, report)
key = self._identity(record)
if key in seen:
report.duplicate_records += 1
continue
seen.add(key)
records.append(record)
report.records = len(records)
return records, report
# -- Step 1: turn physical rows into logical records --------------------
def _reassemble(
self, rows: list[list[str]], header: list[str], report: ImportReport
) -> list[list[str]]:
expected = len(header)
logical: list[list[str]] = []
for raw_row in rows:
row = self._strip_watermark(list(raw_row))
if not row:
continue
if self._is_repeated_header(row, header):
report.duplicate_headers += 1
continue
if self._starts_record(row) or not logical:
if len(row) != expected:
report.repaired_rows += 1
logical.append(row)
elif len(logical[-1]) < expected - 7:
# The previous record is short: this fragment is its tail,
# split by a stray quote inside a free-text column.
logical[-1].extend(row)
report.repaired_rows += 1
else:
# Cannot be attributed to any record without guessing.
report.orphan_fragments += 1
return logical
@staticmethod
def _strip_watermark(row: list[str]) -> list[str]:
while row and any(marker in row[0] for marker in WATERMARK_MARKERS):
row = row[1:]
return row
@staticmethod
def _is_repeated_header(row: list[str], header: list[str]) -> bool:
return len(row) > 3 and row[:3] == header[:3]
@staticmethod
def _starts_record(row: list[str]) -> bool:
"""
A record always begins with ``full_name`` followed by ``first_name``,
and the full name starts with the first name. That is a strong,
cheap anchor for telling a new record from a continuation fragment.
"""
if len(row) < 9:
return False
full_name, first_name = row[0].strip(), row[1].strip()
return bool(
full_name
and first_name
and len(full_name) < 80
and "@" not in full_name
and full_name.startswith(first_name)
)
# -- Step 2: align values to columns ------------------------------------
@staticmethod
def _score(column: str, value: str) -> float:
if value == "":
return SCORE_EMPTY
validator = FIELD_VALIDATORS.get(column)
if validator is None:
return SCORE_FREE_TEXT
try:
verdict = validator(value)
except Exception: # noqa: BLE001 - a broken value is just a mismatch
verdict = False
if verdict is None:
return SCORE_MISMATCH
return SCORE_MATCH if verdict else SCORE_MISMATCH
def _align(self, values: list[str], header: list[str]) -> dict[str, str]:
"""Needleman-Wunsch alignment of row values against header columns."""
n, m = len(values), len(header)
neg_inf = float("-inf")
best = [[neg_inf] * (m + 1) for _ in range(n + 1)]
move = [[0] * (m + 1) for _ in range(n + 1)]
best[0][0] = 0.0
for j in range(1, m + 1):
best[0][j] = best[0][j - 1] + PENALTY_MISSING_COLUMN
move[0][j] = 2
for i in range(1, n + 1):
best[i][0] = best[i - 1][0] + self._drop_penalty(values[i - 1])
move[i][0] = 3
for i in range(1, n + 1):
value = values[i - 1]
drop = self._drop_penalty(value)
for j in range(1, m + 1):
score = best[i - 1][j - 1] + self._score(header[j - 1], value)
choice = 1
skip_column = best[i][j - 1] + PENALTY_MISSING_COLUMN
if skip_column > score:
score, choice = skip_column, 2
drop_value = best[i - 1][j] + drop
if drop_value > score:
score, choice = drop_value, 3
best[i][j], move[i][j] = score, choice
record: dict[str, str] = {}
i, j = n, m
while i > 0 or j > 0:
choice = move[i][j]
if choice == 1:
record[header[j - 1]] = values[i - 1].strip()
i, j = i - 1, j - 1
elif choice == 2:
record.setdefault(header[j - 1], "")
j -= 1
else:
i -= 1
return record
@staticmethod
def _drop_penalty(value: str) -> float:
return PENALTY_DROP_EMPTY if value == "" else PENALTY_DROP_VALUE
# -- Step 3: final validation ------------------------------------------
@staticmethod
def _sanitize(record: dict[str, str], report: ImportReport) -> None:
for column, value in list(record.items()):
if not value:
continue
validator = FIELD_VALIDATORS.get(column)
if validator is None:
continue
try:
verdict = validator(value)
except Exception: # noqa: BLE001
verdict = False
if not verdict:
record[column] = ""
report.discarded_values += 1
@staticmethod
def _identity(record: dict[str, str]) -> str:
return (
record.get("linkedin_id") or record.get("linkedin_url") or record.get("full_name") or ""
).lower()
def parse_literal(value: str | None, default: Any) -> Any:
"""Public helper for the importer: parse a nested column defensively."""
if not value:
return default
parsed = _safe_literal(value)
return default if parsed is None else parsed
def read_dataset(path: str | Path) -> Iterator[dict[str, str]]:
"""Convenience wrapper for callers that do not need the report."""
records, _ = DatasetReader(path).read()
yield from records
+500
View File
@@ -0,0 +1,500 @@
"""
Maps the repaired CSV records onto the relational model.
Design notes
------------
* **Idempotent.** The natural key is the LinkedIn id (falling back to the
profile URL), so re-running the import updates instead of duplicating.
* **Bulk friendly.** Companies, schools and skills are resolved through
in-memory caches, and the child rows of a profile are replaced wholesale with
``bulk_create`` - 284 profiles import in a couple of seconds.
* **Self healing input.** The top level columns of this export are frequently
missing, but the same information usually exists inside the nested
``experience`` blob. ``_derive_current_position`` falls back to the primary
experience entry, which raises coverage of ``job_title_role`` from 108 to 183
profiles and of the current employer from 219 to 271.
"""
from __future__ import annotations
import logging
import re
from dataclasses import dataclass, field
from typing import Any
from django.contrib.postgres.search import SearchVector
from django.db import transaction
from apps.profiles.dataset import DatasetReader, ImportReport, parse_literal
from apps.profiles.models import (
Company,
Education,
Experience,
Profile,
ProfileSkill,
School,
Skill,
)
logger = logging.getLogger(__name__)
_YEAR_RE = re.compile(r"(\d{4})")
_MAX_SKILLS = 60
@dataclass
class ImportStats:
created: int = 0
updated: int = 0
skills: int = 0
companies: int = 0
schools: int = 0
experiences: int = 0
educations: int = 0
report: ImportReport = field(default_factory=ImportReport)
def as_dict(self) -> dict[str, Any]:
return {
"profiles_created": self.created,
"profiles_updated": self.updated,
"skills": self.skills,
"companies": self.companies,
"schools": self.schools,
"experiences": self.experiences,
"educations": self.educations,
**self.report.as_dict(),
}
class ProfileImporter:
"""Loads the dataset into PostgreSQL."""
def __init__(self, path: str, limit: int | None = None) -> None:
self.path = path
self.limit = limit
self._companies: dict[str, Company] = {}
self._schools: dict[str, School] = {}
self._skills: dict[str, Skill] = {}
# -- Entry point --------------------------------------------------------
def run(self) -> ImportStats:
records, report = DatasetReader(self.path).read()
if self.limit:
records = records[: self.limit]
stats = ImportStats(report=report)
self._warm_caches()
for record in records:
try:
with transaction.atomic():
self._import_record(record, stats)
except Exception: # noqa: BLE001 - one bad row must not abort the run
logger.exception(
"profile_import_failed",
extra={"event": "profile_import_failed", "name": record.get("full_name")},
)
report.issues.append(record.get("full_name", "<unknown>"))
self._refresh_skill_counters()
self._refresh_search_vectors()
stats.skills = len(self._skills)
stats.companies = len(self._companies)
stats.schools = len(self._schools)
return stats
# -- Caches -------------------------------------------------------------
def _warm_caches(self) -> None:
self._companies = {c.name: c for c in Company.objects.all()}
self._schools = {s.name: s for s in School.objects.all()}
self._skills = {s.slug: s for s in Skill.objects.all()}
# -- Per record ---------------------------------------------------------
def _import_record(self, record: dict[str, str], stats: ImportStats) -> None:
experiences = parse_literal(record.get("experience"), [])
educations = parse_literal(record.get("education"), [])
primary = self._primary_experience(experiences)
title, role, sub_role, levels = self._derive_current_position(record, primary)
company = self._resolve_current_company(record, primary)
skills = self._clean_str_list(parse_literal(record.get("skills"), []))[:_MAX_SKILLS]
past_companies = [
str(item.get("company", {}).get("name") or "")
for item in experiences
if isinstance(item, dict) and isinstance(item.get("company"), dict)
]
schools = [
str(item.get("school", {}).get("name") or "")
for item in educations
if isinstance(item, dict) and isinstance(item.get("school"), dict)
]
majors = [
major
for item in educations
if isinstance(item, dict)
for major in self._clean_str_list(item.get("majors") or [])
]
defaults: dict[str, Any] = {
"full_name": record.get("full_name", "")[:255],
"first_name": record.get("first_name", "")[:128],
"last_name": record.get("last_name", "")[:128],
"gender": record.get("gender", "")[:16],
"linkedin_url": record.get("linkedin_url", "")[:255],
"linkedin_username": record.get("linkedin_username", "")[:128],
"github_url": record.get("github_url", "")[:255],
"twitter_url": record.get("twitter_url", "")[:255],
"job_title": title[:255],
"job_title_role": role[:64],
"job_title_sub_role": sub_role[:64],
"seniority_levels": levels,
"current_company": company,
"industry": (record.get("industry") or "")[:128],
"job_summary": record.get("job_summary", ""),
"job_start_date": record.get("job_start_date", "")[:10],
"location_name": record.get("location_name", "")[:255],
"location_locality": record.get("location_locality", "")[:128],
"location_region": record.get("location_region", "")[:128],
"location_country": record.get("location_country", "")[:128],
"location_continent": record.get("location_continent", "")[:64],
"summary": record.get("summary", ""),
"years_experience": self._to_int(record.get("inferred_years_experience"), 0, 70),
"inferred_salary": record.get("inferred_salary", "")[:32],
"connections": self._to_int(record.get("linkedin_connections"), 0, 100_000),
"interests": self._clean_str_list(parse_literal(record.get("interests"), []))[:40],
"languages": self._language_names(parse_literal(record.get("languages"), [])),
"emails": self._emails(parse_literal(record.get("emails"), [])),
"phone_numbers": self._clean_str_list(parse_literal(record.get("phone_numbers"), []))[
:10
],
"skills_text": " ".join(skills),
"companies_text": " ".join(
filter(None, {*past_companies, company.name if company else ""})
),
"education_text": " ".join(filter(None, [*schools, *majors])),
}
external_id = (record.get("linkedin_id") or "")[:64]
lookup = (
{"external_id": external_id}
if external_id
else {"linkedin_url": defaults["linkedin_url"]}
)
profile, created = Profile.objects.update_or_create(**lookup, defaults=defaults)
if created:
stats.created += 1
else:
stats.updated += 1
self._sync_skills(profile, skills)
stats.experiences += self._sync_experiences(profile, experiences)
stats.educations += self._sync_educations(profile, educations)
# -- Current position ---------------------------------------------------
@staticmethod
def _primary_experience(experiences: list[Any]) -> dict[str, Any]:
entries = [item for item in experiences if isinstance(item, dict)]
for item in entries:
if item.get("is_primary"):
return item
return entries[0] if entries else {}
def _derive_current_position(
self, record: dict[str, str], primary: dict[str, Any]
) -> tuple[str, str, str, list[str]]:
title_blob = primary.get("title") if isinstance(primary.get("title"), dict) else {}
title = record.get("job_title") or str(title_blob.get("name") or "")
role = record.get("job_title_role") or str(title_blob.get("role") or "")
sub_role = record.get("job_title_sub_role") or str(title_blob.get("sub_role") or "")
levels = self._clean_str_list(
parse_literal(record.get("job_title_levels"), None) or title_blob.get("levels") or []
)
return title, role, sub_role, levels[:5]
def _resolve_current_company(
self, record: dict[str, str], primary: dict[str, Any]
) -> Company | None:
name = (record.get("job_company_name") or "").strip()
payload: dict[str, Any] = {
"external_id": record.get("job_company_id", ""),
"industry": record.get("job_company_industry", ""),
"size": record.get("job_company_size", ""),
"founded": self._to_int(record.get("job_company_founded"), 1600, 2100),
"website": record.get("job_company_website", ""),
"linkedin_url": record.get("job_company_linkedin_url", ""),
"location_name": record.get("job_company_location_name", ""),
"country": record.get("job_company_location_country", ""),
}
if not name and isinstance(primary.get("company"), dict):
blob = primary["company"]
name = str(blob.get("name") or "").strip()
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
payload = {
"external_id": str(blob.get("id") or ""),
"industry": str(blob.get("industry") or ""),
"size": str(blob.get("size") or ""),
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
"website": str(blob.get("website") or ""),
"linkedin_url": str(blob.get("linkedin_url") or ""),
"location_name": str(location.get("name") or ""),
"country": str(location.get("country") or ""),
}
return self._get_or_create_company(name, payload)
def _get_or_create_company(self, name: str, payload: dict[str, Any]) -> Company | None:
name = name.strip()[:255]
if not name:
return None
cached = self._companies.get(name)
if cached:
return cached
company = Company.objects.create(
name=name,
external_id=str(payload.get("external_id") or "")[:128],
industry=str(payload.get("industry") or "")[:128],
size=str(payload.get("size") or "")[:32],
founded=payload.get("founded"),
website=str(payload.get("website") or "")[:255],
linkedin_url=str(payload.get("linkedin_url") or "")[:255],
location_name=str(payload.get("location_name") or "")[:255],
country=str(payload.get("country") or "")[:128],
)
self._companies[name] = company
return company
def _get_or_create_school(self, blob: dict[str, Any]) -> School | None:
name = str(blob.get("name") or "").strip()[:255]
if not name:
return None
cached = self._schools.get(name)
if cached:
return cached
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
school = School.objects.create(
name=name,
type=str(blob.get("type") or "")[:64],
website=str(blob.get("website") or "")[:255],
linkedin_url=str(blob.get("linkedin_url") or "")[:255],
location_name=str(location.get("name") or "")[:255],
country=str(location.get("country") or "")[:128],
)
self._schools[name] = school
return school
# -- Children -----------------------------------------------------------
def _sync_skills(self, profile: Profile, skills: list[str]) -> None:
ProfileSkill.objects.filter(profile=profile).delete()
links: list[ProfileSkill] = []
used: set[int] = set()
for position, name in enumerate(skills):
slug = Skill.make_slug(name)
if not slug:
continue
skill = self._skills.get(slug)
if skill is None:
skill, _ = Skill.objects.get_or_create(slug=slug, defaults={"name": name[:128]})
self._skills[slug] = skill
if skill.pk in used:
continue
used.add(skill.pk)
links.append(ProfileSkill(profile=profile, skill=skill, position=position))
ProfileSkill.objects.bulk_create(links, ignore_conflicts=True)
def _sync_experiences(self, profile: Profile, experiences: list[Any]) -> int:
Experience.objects.filter(profile=profile).delete()
rows: list[Experience] = []
for item in experiences:
if not isinstance(item, dict):
continue
blob = item.get("company") if isinstance(item.get("company"), dict) else {}
title = item.get("title") if isinstance(item.get("title"), dict) else {}
company = self._get_or_create_company(
str(blob.get("name") or ""),
{
"external_id": blob.get("id"),
"industry": blob.get("industry"),
"size": blob.get("size"),
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
"website": blob.get("website"),
"linkedin_url": blob.get("linkedin_url"),
"location_name": (
(blob.get("location") or {}).get("name")
if isinstance(blob.get("location"), dict)
else ""
),
"country": (
(blob.get("location") or {}).get("country")
if isinstance(blob.get("location"), dict)
else ""
),
},
)
rows.append(
Experience(
profile=profile,
company=company,
company_name=str(blob.get("name") or "")[:255],
title=str(title.get("name") or "")[:255],
title_role=str(title.get("role") or "")[:64],
title_sub_role=str(title.get("sub_role") or "")[:64],
levels=self._clean_str_list(title.get("levels") or [])[:5],
start_date=str(item.get("start_date") or "")[:10],
end_date=str(item.get("end_date") or "")[:10],
start_year=self._year(item.get("start_date")),
end_year=self._year(item.get("end_date")),
is_primary=bool(item.get("is_primary")),
location_name=self._first_location(item.get("location_names")),
summary=str(item.get("summary") or ""),
)
)
Experience.objects.bulk_create(rows)
return len(rows)
def _sync_educations(self, profile: Profile, educations: list[Any]) -> int:
Education.objects.filter(profile=profile).delete()
rows: list[Education] = []
for item in educations:
if not isinstance(item, dict):
continue
blob = item.get("school") if isinstance(item.get("school"), dict) else {}
rows.append(
Education(
profile=profile,
school=self._get_or_create_school(blob),
school_name=str(blob.get("name") or "")[:255],
degrees=self._clean_str_list(item.get("degrees") or [])[:5],
majors=self._clean_str_list(item.get("majors") or [])[:5],
minors=self._clean_str_list(item.get("minors") or [])[:5],
gpa=self._to_decimal(item.get("gpa")),
start_date=str(item.get("start_date") or "")[:10],
end_date=str(item.get("end_date") or "")[:10],
end_year=self._year(item.get("end_date")),
)
)
Education.objects.bulk_create(rows)
return len(rows)
# -- Post processing ----------------------------------------------------
@staticmethod
def _refresh_skill_counters() -> None:
"""Single UPDATE ... FROM keeps the facet ordering cheap at query time."""
from django.db import connection
with connection.cursor() as cursor:
cursor.execute(
"""
UPDATE profiles_skill AS s
SET profile_count = COALESCE(counts.total, 0)
FROM (
SELECT skill_id, COUNT(*) AS total
FROM profiles_profileskill
GROUP BY skill_id
) AS counts
WHERE counts.skill_id = s.id
"""
)
cursor.execute(
"""
UPDATE profiles_skill SET profile_count = 0
WHERE id NOT IN (SELECT skill_id FROM profiles_profileskill)
"""
)
@staticmethod
def _refresh_search_vectors() -> None:
"""
Rebuild the tsvector for every profile in one statement.
Weights encode the ranking intent: a hit on the person's name beats a
hit on their title or skills, which in turn beats a hit in the free-text
summary.
"""
Profile.objects.update(
search_vector=(
SearchVector("full_name", weight="A", config="english")
+ SearchVector("job_title", weight="B", config="english")
+ SearchVector("skills_text", weight="B", config="english")
+ SearchVector("companies_text", weight="B", config="english")
+ SearchVector("industry", weight="C", config="english")
+ SearchVector("education_text", weight="C", config="english")
+ SearchVector("location_name", weight="C", config="english")
+ SearchVector("summary", weight="D", config="english")
)
)
# -- Small helpers ------------------------------------------------------
@staticmethod
def _clean_str_list(values: Any) -> list[str]:
if not isinstance(values, list):
return []
seen: set[str] = set()
out: list[str] = []
for value in values:
if not isinstance(value, str):
continue
cleaned = value.strip().lower()[:128]
if cleaned and cleaned not in seen:
seen.add(cleaned)
out.append(cleaned)
return out
@staticmethod
def _language_names(values: Any) -> list[str]:
if not isinstance(values, list):
return []
return [
str(item.get("name")).strip().lower()[:64]
for item in values
if isinstance(item, dict) and item.get("name")
][:10]
@staticmethod
def _emails(values: Any) -> list[dict[str, str]]:
if not isinstance(values, list):
return []
return [
{
"address": str(item.get("address"))[:254],
"type": str(item.get("type") or "other")[:32],
}
for item in values
if isinstance(item, dict) and item.get("address")
][:10]
@staticmethod
def _first_location(values: Any) -> str:
if isinstance(values, list) and values and isinstance(values[0], str):
return values[0][:255]
return ""
@staticmethod
def _to_int(value: str | None, minimum: int, maximum: int) -> int | None:
if not value:
return None
try:
number = int(float(str(value).replace(",", "")))
except (TypeError, ValueError):
return None
return number if minimum <= number <= maximum else None
@staticmethod
def _to_decimal(value: Any) -> Any:
try:
number = float(value)
except (TypeError, ValueError):
return None
return round(number, 2) if 0 <= number <= 10 else None
@staticmethod
def _year(value: Any) -> int | None:
if not value:
return None
match = _YEAR_RE.search(str(value))
if not match:
return None
year = int(match.group(1))
return year if 1900 <= year <= 2100 else None
@@ -0,0 +1,87 @@
"""``python manage.py import_profiles`` - load the CSV export into PostgreSQL."""
from __future__ import annotations
import json
import time
from typing import Any
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from apps.profiles.importers import ProfileImporter
from apps.profiles.models import (
Company,
Education,
Experience,
Profile,
ProfileSkill,
School,
Skill,
)
class Command(BaseCommand):
help = "Import (or refresh) LinkedIn profiles from the raw CSV export."
def add_arguments(self, parser) -> None: # noqa: ANN001
parser.add_argument(
"--path",
default=settings.DATASET_PATH,
help="Path to the CSV export (defaults to DATASET_PATH).",
)
parser.add_argument(
"--limit", type=int, default=None, help="Import at most N profiles (smoke tests)."
)
parser.add_argument(
"--truncate",
action="store_true",
help="Delete existing profile data before importing.",
)
parser.add_argument(
"--reindex",
action="store_true",
help="Rebuild the Elasticsearch index once the import finishes.",
)
def handle(self, *args: Any, **options: Any) -> None:
path = options["path"]
started = time.perf_counter()
if options["truncate"]:
self.stdout.write("Truncating existing profile data ...")
with transaction.atomic():
ProfileSkill.objects.all().delete()
Experience.objects.all().delete()
Education.objects.all().delete()
Profile.objects.all().delete()
Skill.objects.all().delete()
Company.objects.all().delete()
School.objects.all().delete()
self.stdout.write(f"Reading dataset from {path} ...")
try:
stats = ProfileImporter(path, limit=options["limit"]).run()
except FileNotFoundError as exc:
raise CommandError(f"Dataset not found: {path}") from exc
elapsed = round(time.perf_counter() - started, 2)
summary = {**stats.as_dict(), "elapsed_seconds": elapsed}
self.stdout.write(json.dumps(summary, indent=2))
if stats.report.issues:
self.stdout.write(
self.style.WARNING(
f"{len(stats.report.issues)} record(s) failed: "
f"{', '.join(stats.report.issues[:10])}"
)
)
if options["reindex"]:
self.stdout.write("Rebuilding the search index ...")
from django.core.management import call_command
call_command("search_index", "--rebuild")
self.stdout.write(self.style.SUCCESS(f"Import finished in {elapsed}s."))
@@ -0,0 +1,416 @@
# Generated by Django 5.1.15 on 2026-09-03 03:36
import django.contrib.postgres.fields
import django.contrib.postgres.indexes
import django.contrib.postgres.search
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="Company",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("external_id", models.CharField(blank=True, db_index=True, max_length=128)),
("name", models.CharField(max_length=255, unique=True)),
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
(
"size",
models.CharField(
blank=True, help_text="Head-count bucket, e.g. 51-200", max_length=32
),
),
("founded", models.PositiveIntegerField(blank=True, null=True)),
("website", models.CharField(blank=True, max_length=255)),
("linkedin_url", models.CharField(blank=True, max_length=255)),
("location_name", models.CharField(blank=True, max_length=255)),
("country", models.CharField(blank=True, db_index=True, max_length=128)),
],
options={
"verbose_name_plural": "companies",
"ordering": ("name",),
},
),
migrations.CreateModel(
name="School",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("name", models.CharField(max_length=255, unique=True)),
("type", models.CharField(blank=True, max_length=64)),
("website", models.CharField(blank=True, max_length=255)),
("linkedin_url", models.CharField(blank=True, max_length=255)),
("location_name", models.CharField(blank=True, max_length=255)),
("country", models.CharField(blank=True, max_length=128)),
],
options={
"ordering": ("name",),
},
),
migrations.CreateModel(
name="Skill",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("name", models.CharField(max_length=128)),
("slug", models.SlugField(max_length=140, unique=True)),
(
"profile_count",
models.PositiveIntegerField(
db_index=True,
default=0,
help_text="Denormalised counter used to rank filter suggestions.",
),
),
],
options={
"ordering": ("-profile_count", "name"),
},
),
migrations.CreateModel(
name="Profile",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"external_id",
models.CharField(
blank=True,
db_index=True,
help_text="LinkedIn numeric id from the source export.",
max_length=64,
),
),
("full_name", models.CharField(db_index=True, max_length=255)),
("first_name", models.CharField(blank=True, max_length=128)),
("last_name", models.CharField(blank=True, max_length=128)),
(
"gender",
models.CharField(
blank=True,
choices=[("male", "Male"), ("female", "Female"), ("", "Unknown")],
max_length=16,
),
),
("linkedin_url", models.CharField(blank=True, max_length=255)),
("linkedin_username", models.CharField(blank=True, max_length=128)),
("github_url", models.CharField(blank=True, max_length=255)),
("twitter_url", models.CharField(blank=True, max_length=255)),
("job_title", models.CharField(blank=True, db_index=True, max_length=255)),
(
"job_title_role",
models.CharField(
blank=True,
db_index=True,
help_text="Coarse job family, e.g. engineering.",
max_length=64,
),
),
("job_title_sub_role", models.CharField(blank=True, max_length=64)),
(
"seniority_levels",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=32),
blank=True,
default=list,
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
size=None,
),
),
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
("job_summary", models.TextField(blank=True)),
("job_start_date", models.CharField(blank=True, max_length=10)),
("location_name", models.CharField(blank=True, max_length=255)),
("location_locality", models.CharField(blank=True, max_length=128)),
("location_region", models.CharField(blank=True, db_index=True, max_length=128)),
("location_country", models.CharField(blank=True, db_index=True, max_length=128)),
("location_continent", models.CharField(blank=True, max_length=64)),
("summary", models.TextField(blank=True)),
(
"years_experience",
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
),
("inferred_salary", models.CharField(blank=True, max_length=32)),
("connections", models.PositiveIntegerField(blank=True, null=True)),
(
"interests",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=128),
blank=True,
default=list,
size=None,
),
),
(
"languages",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=64),
blank=True,
default=list,
size=None,
),
),
("emails", models.JSONField(blank=True, default=list)),
(
"phone_numbers",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=32),
blank=True,
default=list,
size=None,
),
),
("skills_text", models.TextField(blank=True, editable=False)),
("companies_text", models.TextField(blank=True, editable=False)),
("education_text", models.TextField(blank=True, editable=False)),
(
"search_vector",
django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
),
(
"current_company",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="current_employees",
to="profiles.company",
),
),
],
options={
"ordering": ("full_name", "id"),
},
),
migrations.CreateModel(
name="Education",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("school_name", models.CharField(blank=True, max_length=255)),
(
"degrees",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=128),
blank=True,
default=list,
size=None,
),
),
(
"majors",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=128),
blank=True,
default=list,
size=None,
),
),
(
"minors",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=128),
blank=True,
default=list,
size=None,
),
),
("gpa", models.DecimalField(blank=True, decimal_places=2, max_digits=4, null=True)),
("start_date", models.CharField(blank=True, max_length=10)),
("end_date", models.CharField(blank=True, max_length=10)),
(
"end_year",
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
),
(
"profile",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="educations",
to="profiles.profile",
),
),
(
"school",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="educations",
to="profiles.school",
),
),
],
options={
"ordering": ("-end_year", "id"),
},
),
migrations.CreateModel(
name="ProfileSkill",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("position", models.PositiveSmallIntegerField(default=0)),
(
"profile",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="profile_skills",
to="profiles.profile",
),
),
(
"skill",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="profile_skills",
to="profiles.skill",
),
),
],
options={
"ordering": ("position",),
},
),
migrations.AddField(
model_name="profile",
name="skills",
field=models.ManyToManyField(
related_name="profiles", through="profiles.ProfileSkill", to="profiles.skill"
),
),
migrations.CreateModel(
name="Experience",
fields=[
(
"id",
models.BigAutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
("updated_at", models.DateTimeField(auto_now=True)),
("company_name", models.CharField(blank=True, max_length=255)),
("title", models.CharField(blank=True, max_length=255)),
("title_role", models.CharField(blank=True, max_length=64)),
("title_sub_role", models.CharField(blank=True, max_length=64)),
(
"levels",
django.contrib.postgres.fields.ArrayField(
base_field=models.CharField(max_length=32),
blank=True,
default=list,
size=None,
),
),
("start_date", models.CharField(blank=True, max_length=10)),
("end_date", models.CharField(blank=True, max_length=10)),
(
"start_year",
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
),
("end_year", models.PositiveSmallIntegerField(blank=True, null=True)),
("is_primary", models.BooleanField(db_index=True, default=False)),
("location_name", models.CharField(blank=True, max_length=255)),
("summary", models.TextField(blank=True)),
(
"company",
models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.SET_NULL,
related_name="experiences",
to="profiles.company",
),
),
(
"profile",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="experiences",
to="profiles.profile",
),
),
],
options={
"ordering": ("-is_primary", "-start_year", "id"),
"indexes": [
models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")
],
},
),
migrations.AddConstraint(
model_name="profileskill",
constraint=models.UniqueConstraint(
fields=("profile", "skill"), name="uniq_profile_skill"
),
),
migrations.AddIndex(
model_name="profile",
index=django.contrib.postgres.indexes.GinIndex(
fields=["search_vector"], name="profile_search_vector_gin"
),
),
migrations.AddIndex(
model_name="profile",
index=django.contrib.postgres.indexes.GinIndex(
fields=["seniority_levels"], name="profile_seniority_gin"
),
),
migrations.AddIndex(
model_name="profile",
index=models.Index(
fields=["location_country", "job_title_role"], name="profile_country_role"
),
),
migrations.AddConstraint(
model_name="profile",
constraint=models.UniqueConstraint(
condition=models.Q(("external_id__gt", "")),
fields=("external_id",),
name="uniq_profile_external_id",
),
),
]
@@ -0,0 +1,33 @@
"""
Enable the PostgreSQL extensions the search fallback relies on.
``pg_trgm`` powers ``TrigramSimilarity`` (typo tolerant name matching) and the
GIN trigram index below makes ``ILIKE '%term%'`` on skill names fast enough for
the type-ahead endpoint.
"""
from django.contrib.postgres.operations import TrigramExtension, UnaccentExtension
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [("profiles", "0001_initial")]
operations = [
TrigramExtension(),
UnaccentExtension(),
migrations.RunSQL(
sql=(
"CREATE INDEX IF NOT EXISTS profile_full_name_trgm "
"ON profiles_profile USING gin (full_name gin_trgm_ops);"
),
reverse_sql="DROP INDEX IF EXISTS profile_full_name_trgm;",
),
migrations.RunSQL(
sql=(
"CREATE INDEX IF NOT EXISTS skill_name_trgm "
"ON profiles_skill USING gin (name gin_trgm_ops);"
),
reverse_sql="DROP INDEX IF EXISTS skill_name_trgm;",
),
]
+261
View File
@@ -0,0 +1,261 @@
"""
Domain model for the LinkedIn dataset.
The raw export is one very wide CSV row per person with several columns holding
serialised lists of dictionaries. Storing that blob as-is would make filtering
and aggregation painful, so it is normalised into five tables:
Company <--- Experience ---> Profile <--- Education ---> School
|
+--- ProfileSkill ---> Skill
PostgreSQL is the source of truth. Elasticsearch holds a denormalised
projection of the same data for search (see ``apps.search``); it can always be
rebuilt from these tables, which is why the index is disposable.
"""
from __future__ import annotations
from django.contrib.postgres.fields import ArrayField
from django.contrib.postgres.indexes import GinIndex
from django.contrib.postgres.search import SearchVectorField
from django.db import models
from django.utils.text import slugify
class TimeStampedModel(models.Model):
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
updated_at = models.DateTimeField(auto_now=True)
class Meta:
abstract = True
class Company(TimeStampedModel):
"""An employer referenced by the current job or by a past experience."""
external_id = models.CharField(max_length=128, blank=True, db_index=True)
name = models.CharField(max_length=255, unique=True)
industry = models.CharField(max_length=128, blank=True, db_index=True)
size = models.CharField(max_length=32, blank=True, help_text="Head-count bucket, e.g. 51-200")
founded = models.PositiveIntegerField(null=True, blank=True)
website = models.CharField(max_length=255, blank=True)
linkedin_url = models.CharField(max_length=255, blank=True)
location_name = models.CharField(max_length=255, blank=True)
country = models.CharField(max_length=128, blank=True, db_index=True)
class Meta:
ordering = ("name",)
verbose_name_plural = "companies"
def __str__(self) -> str:
return self.name
class School(TimeStampedModel):
name = models.CharField(max_length=255, unique=True)
type = models.CharField(max_length=64, blank=True)
website = models.CharField(max_length=255, blank=True)
linkedin_url = models.CharField(max_length=255, blank=True)
location_name = models.CharField(max_length=255, blank=True)
country = models.CharField(max_length=128, blank=True)
class Meta:
ordering = ("name",)
def __str__(self) -> str:
return self.name
class Skill(TimeStampedModel):
"""
A normalised skill.
``slug`` is the deduplication key: the raw data contains the same skill with
different casing and spacing ("Team Building" / "team building").
"""
name = models.CharField(max_length=128)
slug = models.SlugField(max_length=140, unique=True)
profile_count = models.PositiveIntegerField(
default=0, db_index=True, help_text="Denormalised counter used to rank filter suggestions."
)
class Meta:
ordering = ("-profile_count", "name")
def __str__(self) -> str:
return self.name
@staticmethod
def make_slug(name: str) -> str:
return slugify(name)[:140] or name.strip().lower()[:140]
class Profile(TimeStampedModel):
"""A single person in the dataset."""
class Gender(models.TextChoices):
MALE = "male", "Male"
FEMALE = "female", "Female"
UNKNOWN = "", "Unknown"
# -- Identity ----------------------------------------------------------
external_id = models.CharField(
max_length=64,
blank=True,
db_index=True,
help_text="LinkedIn numeric id from the source export.",
)
full_name = models.CharField(max_length=255, db_index=True)
first_name = models.CharField(max_length=128, blank=True)
last_name = models.CharField(max_length=128, blank=True)
gender = models.CharField(max_length=16, blank=True, choices=Gender.choices)
linkedin_url = models.CharField(max_length=255, blank=True)
linkedin_username = models.CharField(max_length=128, blank=True)
github_url = models.CharField(max_length=255, blank=True)
twitter_url = models.CharField(max_length=255, blank=True)
# -- Current position --------------------------------------------------
job_title = models.CharField(max_length=255, blank=True, db_index=True)
job_title_role = models.CharField(
max_length=64, blank=True, db_index=True, help_text="Coarse job family, e.g. engineering."
)
job_title_sub_role = models.CharField(max_length=64, blank=True)
seniority_levels = ArrayField(
models.CharField(max_length=32),
default=list,
blank=True,
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
)
current_company = models.ForeignKey(
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="current_employees"
)
industry = models.CharField(max_length=128, blank=True, db_index=True)
job_summary = models.TextField(blank=True)
job_start_date = models.CharField(max_length=10, blank=True)
# -- Location ----------------------------------------------------------
location_name = models.CharField(max_length=255, blank=True)
location_locality = models.CharField(max_length=128, blank=True)
location_region = models.CharField(max_length=128, blank=True, db_index=True)
location_country = models.CharField(max_length=128, blank=True, db_index=True)
location_continent = models.CharField(max_length=64, blank=True)
# -- Narrative & metrics ----------------------------------------------
summary = models.TextField(blank=True)
years_experience = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
inferred_salary = models.CharField(max_length=32, blank=True)
connections = models.PositiveIntegerField(null=True, blank=True)
interests = ArrayField(models.CharField(max_length=128), default=list, blank=True)
languages = ArrayField(models.CharField(max_length=64), default=list, blank=True)
# -- Contact details (PII: masked unless the caller is privileged) ------
emails = models.JSONField(default=list, blank=True)
phone_numbers = ArrayField(models.CharField(max_length=32), default=list, blank=True)
skills = models.ManyToManyField(Skill, through="ProfileSkill", related_name="profiles")
# -- Denormalised text, maintained by the importer ---------------------
# PostgreSQL's ``SearchVector`` cannot traverse a many-to-many relation, so
# the searchable text of the related rows is flattened here once at import
# time. This keeps the tsvector update to a single UPDATE statement and
# keeps query time O(index lookup) instead of O(joins).
skills_text = models.TextField(blank=True, editable=False)
companies_text = models.TextField(blank=True, editable=False)
education_text = models.TextField(blank=True, editable=False)
# Materialised tsvector powering the PostgreSQL search backend.
search_vector = SearchVectorField(null=True, editable=False)
class Meta:
ordering = ("full_name", "id")
indexes = [
GinIndex(fields=["search_vector"], name="profile_search_vector_gin"),
GinIndex(fields=["seniority_levels"], name="profile_seniority_gin"),
models.Index(
fields=["location_country", "job_title_role"], name="profile_country_role"
),
]
constraints = [
models.UniqueConstraint(
fields=["external_id"],
condition=models.Q(external_id__gt=""),
name="uniq_profile_external_id",
)
]
def __str__(self) -> str:
return self.full_name
@property
def headline(self) -> str:
"""Short "Title at Company" line used in result cards."""
company = self.current_company.name if self.current_company_id else ""
if self.job_title and company:
return f"{self.job_title} at {company}"
return self.job_title or company
class ProfileSkill(models.Model):
"""Through table - keeps the original ordering of the skill list."""
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="profile_skills")
skill = models.ForeignKey(Skill, on_delete=models.CASCADE, related_name="profile_skills")
position = models.PositiveSmallIntegerField(default=0)
class Meta:
ordering = ("position",)
constraints = [
models.UniqueConstraint(fields=["profile", "skill"], name="uniq_profile_skill")
]
def __str__(self) -> str:
return f"{self.profile_id}:{self.skill_id}"
class Experience(TimeStampedModel):
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="experiences")
company = models.ForeignKey(
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="experiences"
)
company_name = models.CharField(max_length=255, blank=True)
title = models.CharField(max_length=255, blank=True)
title_role = models.CharField(max_length=64, blank=True)
title_sub_role = models.CharField(max_length=64, blank=True)
levels = ArrayField(models.CharField(max_length=32), default=list, blank=True)
start_date = models.CharField(max_length=10, blank=True)
end_date = models.CharField(max_length=10, blank=True)
start_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
end_year = models.PositiveSmallIntegerField(null=True, blank=True)
is_primary = models.BooleanField(default=False, db_index=True)
location_name = models.CharField(max_length=255, blank=True)
summary = models.TextField(blank=True)
class Meta:
ordering = ("-is_primary", "-start_year", "id")
indexes = [models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")]
def __str__(self) -> str:
return f"{self.title} @ {self.company_name}"
class Education(TimeStampedModel):
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="educations")
school = models.ForeignKey(
School, null=True, blank=True, on_delete=models.SET_NULL, related_name="educations"
)
school_name = models.CharField(max_length=255, blank=True)
degrees = ArrayField(models.CharField(max_length=128), default=list, blank=True)
majors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
minors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
gpa = models.DecimalField(max_digits=4, decimal_places=2, null=True, blank=True)
start_date = models.CharField(max_length=10, blank=True)
end_date = models.CharField(max_length=10, blank=True)
end_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
class Meta:
ordering = ("-end_year", "id")
def __str__(self) -> str:
return f"{self.school_name}"
+46
View File
@@ -0,0 +1,46 @@
"""
Contact-detail masking.
The dataset contains real personal data (e-mail addresses, mobile numbers,
street addresses). Exposing it to every authenticated user would be both a
privacy and a compliance problem, so contact details are masked unless the
caller holds a role that is explicitly allowed to see them
(:pyattr:`apps.accounts.models.User.can_view_pii`).
Masking happens in the serialisation layer - the raw values never leave the
database for an unprivileged caller, and the search index stores them in a
non-searchable field so a keyword query can never be used to confirm an e-mail
address by brute force.
"""
from __future__ import annotations
import re
_EMAIL_RE = re.compile(r"^([^@]+)@(.+)$")
def mask_email(address: str) -> str:
"""``joseph.holland@garver.com`` -> ``jo************@garver.com``."""
match = _EMAIL_RE.match(address or "")
if not match:
return "***"
local, domain = match.groups()
keep = min(2, max(len(local) - 1, 0))
return f"{local[:keep]}{'*' * max(len(local) - keep, 1)}@{domain}"
def mask_phone(number: str) -> str:
"""``+19402058928`` -> ``+1940*****28``."""
digits = (number or "").strip()
if len(digits) < 6:
return "***"
return f"{digits[:5]}{'*' * (len(digits) - 7)}{digits[-2:]}"
def mask_emails(emails: list[dict[str, str]]) -> list[dict[str, str]]:
return [{**item, "address": mask_email(item.get("address", ""))} for item in emails or []]
def mask_phones(numbers: list[str]) -> list[str]:
return [mask_phone(number) for number in numbers or []]
+119
View File
@@ -0,0 +1,119 @@
"""Read-only serializers for profile payloads."""
from __future__ import annotations
from typing import Any
from rest_framework import serializers
from apps.profiles.models import Company, Education, Experience, Profile, School, Skill
from apps.profiles.privacy import mask_emails, mask_phones
class SkillSerializer(serializers.ModelSerializer):
class Meta:
model = Skill
fields = ("id", "name", "slug", "profile_count")
class CompanySerializer(serializers.ModelSerializer):
class Meta:
model = Company
fields = ("id", "name", "industry", "size", "founded", "website", "location_name")
class SchoolSerializer(serializers.ModelSerializer):
class Meta:
model = School
fields = ("id", "name", "type", "location_name")
class ExperienceSerializer(serializers.ModelSerializer):
company = CompanySerializer(read_only=True)
class Meta:
model = Experience
fields = (
"id", "title", "title_role", "title_sub_role", "levels", "company", "company_name",
"start_date", "end_date", "start_year", "end_year", "is_primary", "location_name",
"summary",
) # fmt: skip
class EducationSerializer(serializers.ModelSerializer):
school = SchoolSerializer(read_only=True)
class Meta:
model = Education
fields = (
"id", "school", "school_name", "degrees", "majors", "minors", "gpa",
"start_date", "end_date", "end_year",
) # fmt: skip
class PIIMixin:
"""Masks contact details unless the caller is allowed to see them."""
def _can_view_pii(self) -> bool:
request = self.context.get("request") # type: ignore[attr-defined]
user = getattr(request, "user", None)
return bool(user and user.is_authenticated and user.can_view_pii)
def get_emails(self, obj: Profile) -> list[dict[str, str]]:
return obj.emails if self._can_view_pii() else mask_emails(obj.emails)
def get_phone_numbers(self, obj: Profile) -> list[str]:
return obj.phone_numbers if self._can_view_pii() else mask_phones(obj.phone_numbers)
class ProfileListSerializer(PIIMixin, serializers.ModelSerializer):
"""Compact representation used for result cards."""
headline = serializers.CharField(read_only=True)
company = serializers.CharField(source="current_company.name", default="", read_only=True)
skills = serializers.SerializerMethodField()
emails = serializers.SerializerMethodField()
class Meta:
model = Profile
fields = (
"id", "full_name", "headline", "job_title", "job_title_role", "seniority_levels",
"company", "industry", "location_name", "location_country", "years_experience",
"linkedin_url", "skills", "emails",
) # fmt: skip
def get_skills(self, obj: Profile) -> list[str]:
# ``prefetch_related`` in the view keeps this at one extra query total.
return [link.skill.name for link in obj.profile_skills.all()[:10]]
class ProfileDetailSerializer(PIIMixin, serializers.ModelSerializer):
headline = serializers.CharField(read_only=True)
current_company = CompanySerializer(read_only=True)
skills = serializers.SerializerMethodField()
experiences = ExperienceSerializer(many=True, read_only=True)
educations = EducationSerializer(many=True, read_only=True)
emails = serializers.SerializerMethodField()
phone_numbers = serializers.SerializerMethodField()
pii_masked = serializers.SerializerMethodField()
class Meta:
model = Profile
fields = (
"id", "external_id", "full_name", "first_name", "last_name", "gender", "headline",
"job_title", "job_title_role", "job_title_sub_role", "seniority_levels",
"current_company", "industry", "job_summary", "job_start_date",
"location_name", "location_locality", "location_region", "location_country",
"location_continent", "summary", "years_experience", "inferred_salary",
"connections", "interests", "languages", "linkedin_url", "linkedin_username",
"github_url", "twitter_url", "skills", "experiences", "educations",
"emails", "phone_numbers", "pii_masked",
) # fmt: skip
def get_skills(self, obj: Profile) -> list[dict[str, Any]]:
return [
{"name": link.skill.name, "slug": link.skill.slug} for link in obj.profile_skills.all()
]
def get_pii_masked(self, obj: Profile) -> bool: # noqa: ARG002
return not self._can_view_pii()
+11
View File
@@ -0,0 +1,11 @@
from django.urls import path
from apps.profiles.views import ProfileDetailView, ProfileListView, SkillListView
app_name = "profiles"
urlpatterns = [
path("", ProfileListView.as_view(), name="list"),
path("skills/", SkillListView.as_view(), name="skills"),
path("<int:pk>/", ProfileDetailView.as_view(), name="detail"),
]
+72
View File
@@ -0,0 +1,72 @@
"""Profile read endpoints (the search endpoints live in ``apps.search``)."""
from __future__ import annotations
from django.db.models import QuerySet
from drf_spectacular.utils import extend_schema
from rest_framework import generics
from rest_framework.permissions import IsAuthenticated
from apps.profiles.models import Profile, Skill
from apps.profiles.serializers import (
ProfileDetailSerializer,
ProfileListSerializer,
SkillSerializer,
)
@extend_schema(tags=["profiles"])
class ProfileListView(generics.ListAPIView):
"""
Plain, unranked listing.
Useful for browsing and for verifying the dataset; ranked keyword search is
served by ``/api/v1/search/``.
"""
serializer_class = ProfileListSerializer
permission_classes = [IsAuthenticated]
def get_queryset(self) -> QuerySet[Profile]:
return (
Profile.objects.select_related("current_company")
.prefetch_related("profile_skills__skill")
.order_by("full_name", "id")
)
@extend_schema(tags=["profiles"])
class ProfileDetailView(generics.RetrieveAPIView):
"""Full profile including work history, education and (masked) contacts."""
serializer_class = ProfileDetailSerializer
permission_classes = [IsAuthenticated]
def get_queryset(self) -> QuerySet[Profile]:
return Profile.objects.select_related("current_company").prefetch_related(
"profile_skills__skill",
"experiences__company",
"educations__school",
)
@extend_schema(tags=["profiles"])
class SkillListView(generics.ListAPIView):
"""
Skill dictionary ordered by popularity.
The frontend uses it to power the skill filter's type-ahead without hitting
Elasticsearch for every keystroke.
"""
serializer_class = SkillSerializer
permission_classes = [IsAuthenticated]
pagination_class = None
def get_queryset(self) -> QuerySet[Skill]:
queryset = Skill.objects.filter(profile_count__gt=0)
term = self.request.query_params.get("q", "").strip().lower()
if term:
queryset = queryset.filter(name__icontains=term[:64])
limit = min(int(self.request.query_params.get("limit", 50) or 50), 200)
return queryset.order_by("-profile_count", "name")[:limit]