feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,72 @@
|
||||
from django.contrib import admin
|
||||
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
|
||||
class ExperienceInline(admin.TabularInline):
|
||||
model = Experience
|
||||
extra = 0
|
||||
fields = ("title", "company_name", "start_date", "end_date", "is_primary")
|
||||
readonly_fields = fields
|
||||
can_delete = False
|
||||
show_change_link = True
|
||||
|
||||
|
||||
class EducationInline(admin.TabularInline):
|
||||
model = Education
|
||||
extra = 0
|
||||
fields = ("school_name", "degrees", "majors", "end_date")
|
||||
readonly_fields = fields
|
||||
can_delete = False
|
||||
|
||||
|
||||
class ProfileSkillInline(admin.TabularInline):
|
||||
model = ProfileSkill
|
||||
extra = 0
|
||||
autocomplete_fields = ("skill",)
|
||||
|
||||
|
||||
@admin.register(Profile)
|
||||
class ProfileAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
"full_name", "job_title", "current_company", "location_country", "years_experience",
|
||||
) # fmt: skip
|
||||
list_filter = ("location_country", "job_title_role", "industry")
|
||||
search_fields = ("full_name", "job_title", "skills_text", "companies_text")
|
||||
list_select_related = ("current_company",)
|
||||
inlines = (ProfileSkillInline, ExperienceInline, EducationInline)
|
||||
readonly_fields = (
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"skills_text",
|
||||
"companies_text",
|
||||
"education_text",
|
||||
)
|
||||
|
||||
|
||||
@admin.register(Skill)
|
||||
class SkillAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "slug", "profile_count")
|
||||
search_fields = ("name", "slug")
|
||||
ordering = ("-profile_count",)
|
||||
|
||||
|
||||
@admin.register(Company)
|
||||
class CompanyAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "industry", "size", "country")
|
||||
search_fields = ("name",)
|
||||
list_filter = ("size", "industry")
|
||||
|
||||
|
||||
@admin.register(School)
|
||||
class SchoolAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "type", "country")
|
||||
search_fields = ("name",)
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ProfilesConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.profiles"
|
||||
verbose_name = "Profiles"
|
||||
@@ -0,0 +1,394 @@
|
||||
"""
|
||||
Resilient reader for the supplied LinkedIn CSV export.
|
||||
|
||||
Why this module exists
|
||||
----------------------
|
||||
The provided file is a real-world dirty export, not a clean CSV. Feeding it to
|
||||
``csv.DictReader`` silently produces garbage. Four separate defects are present:
|
||||
|
||||
1. **Injected watermark** - many rows start with an extra, unheadered column
|
||||
containing ``H:\\New folder\\leak\\...@BreachedData1 ...``. Everything after
|
||||
it is shifted one position to the right.
|
||||
2. **Missing columns** - most records omit one or more optional fields
|
||||
(``job_title_role``, ``job_title_sub_role``, ...) instead of writing an empty
|
||||
value, so the remaining values slide left relative to the header.
|
||||
3. **Broken quoting** - a handful of ``summary`` values contain an unescaped
|
||||
double quote, which splits a single logical record across several physical
|
||||
rows.
|
||||
4. **Concatenated exports** - the header line appears a second time in the
|
||||
middle of the file and ~40 people are duplicated.
|
||||
|
||||
Repair strategy
|
||||
---------------
|
||||
Every column has a *signature*: ``linkedin_url`` contains ``linkedin.com/in/``,
|
||||
``skills`` is a Python list literal, ``location_geo`` looks like ``lat,lon`` and
|
||||
so on. We score each (column, value) pair against those signatures and then run
|
||||
a Needleman-Wunsch style alignment between the row's values and the header
|
||||
columns, allowing two edit operations:
|
||||
|
||||
* *skip column* - the record simply does not carry that field;
|
||||
* *drop value* - a stray fragment that belongs to no column.
|
||||
|
||||
The alignment that maximises the total score is the reconstructed record. The
|
||||
result is validated a second time and any value that still fails its signature
|
||||
is discarded rather than persisted as junk - it is better to store nothing than
|
||||
to store a phone number in the e-mail column.
|
||||
|
||||
``ast.literal_eval`` is used (never ``eval``) so the nested Python-repr columns
|
||||
can be parsed without executing anything from the file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import csv
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# The nested columns are large; the stdlib default (128 KiB) is not enough.
|
||||
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))
|
||||
|
||||
WATERMARK_MARKERS = ("@BreachedData", "H:\\New folder")
|
||||
CONTINENTS = frozenset(
|
||||
{"north america", "south america", "europe", "asia", "africa", "oceania", "antarctica"}
|
||||
)
|
||||
GENDERS = frozenset({"male", "female"})
|
||||
|
||||
Validator = Callable[[str], bool | None]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Column signatures
|
||||
# ---------------------------------------------------------------------------
|
||||
def _list_of_str(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
return isinstance(parsed, list) and all(isinstance(item, str) for item in parsed)
|
||||
|
||||
|
||||
def _list_of_dicts(required_key: str) -> Validator:
|
||||
def check(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
if not isinstance(parsed, list):
|
||||
return None
|
||||
if not parsed:
|
||||
return True
|
||||
return all(isinstance(item, dict) for item in parsed) and required_key in parsed[0]
|
||||
|
||||
return check
|
||||
|
||||
|
||||
def _dict_with(required_key: str) -> Validator:
|
||||
def check(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
return isinstance(parsed, dict) and required_key in parsed
|
||||
|
||||
return check
|
||||
|
||||
|
||||
def _matches(pattern: str) -> Validator:
|
||||
compiled = re.compile(pattern)
|
||||
return lambda value: bool(compiled.fullmatch(value))
|
||||
|
||||
|
||||
def _contains(needle: str) -> Validator:
|
||||
return lambda value: needle in value
|
||||
|
||||
|
||||
def _one_of(options: frozenset[str]) -> Validator:
|
||||
return lambda value: value.lower() in options
|
||||
|
||||
|
||||
def _safe_literal(value: str) -> Any | None:
|
||||
"""``ast.literal_eval`` that returns ``None`` instead of raising."""
|
||||
try:
|
||||
return ast.literal_eval(value)
|
||||
except (ValueError, SyntaxError, MemoryError, RecursionError):
|
||||
return None
|
||||
|
||||
|
||||
#: Columns without an entry here accept any text (name, title, industry, ...).
|
||||
FIELD_VALIDATORS: dict[str, Validator] = {
|
||||
"gender": _one_of(GENDERS),
|
||||
"linkedin_url": _contains("linkedin.com/in/"),
|
||||
"linkedin_id": _matches(r"\d{3,20}"),
|
||||
"facebook_url": _contains("facebook.com/"),
|
||||
"facebook_id": _matches(r"\d{5,25}"),
|
||||
"job_title_role": _matches(r"[a-z_]{3,40}"),
|
||||
"job_title_sub_role": _matches(r"[a-z_]{3,40}"),
|
||||
"job_title_levels": _list_of_str,
|
||||
"job_company_website": _matches(r"[\w.-]+\.[a-z]{2,10}"),
|
||||
"job_company_size": _matches(
|
||||
r"(1-10|11-50|51-200|201-500|501-1000|1001-5000|5001-10000|10001\+)"
|
||||
),
|
||||
"job_company_founded": _matches(r"(1[6-9]|20)\d\d(\.0)?"),
|
||||
"job_company_linkedin_url": _contains("linkedin.com/company/"),
|
||||
"job_company_linkedin_id": _matches(r"\d{3,20}"),
|
||||
"job_company_facebook_url": _contains("facebook.com/"),
|
||||
"job_company_twitter_url": _contains("twitter.com/"),
|
||||
"job_company_location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"job_company_location_continent": _one_of(CONTINENTS),
|
||||
"job_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
|
||||
"job_start_date": _matches(r"\d{4}(-\d{2})?"),
|
||||
"location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"location_continent": _one_of(CONTINENTS),
|
||||
"location_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
|
||||
"linkedin_connections": _matches(r"\d+(\.\d+)?"),
|
||||
"inferred_salary": _matches(r"[\d,]+-[\d,]+|>?\d[\d,]*\+?"),
|
||||
"inferred_years_experience": _matches(r"\d{1,2}(\.\d+)?"),
|
||||
"phone_numbers": _list_of_str,
|
||||
"emails": _list_of_dicts("address"),
|
||||
"interests": _list_of_str,
|
||||
"skills": _list_of_str,
|
||||
"location_names": _list_of_str,
|
||||
"regions": _list_of_str,
|
||||
"countries": _list_of_str,
|
||||
"street_addresses": _list_of_dicts("street_address"),
|
||||
"experience": _list_of_dicts("company"),
|
||||
"education": _list_of_dicts("school"),
|
||||
"profiles": _list_of_dicts("network"),
|
||||
"certifications": _list_of_dicts("name"),
|
||||
"languages": _list_of_dicts("name"),
|
||||
"version_status": _dict_with("status"),
|
||||
"work_email": _contains("@"),
|
||||
"middle_initial": _matches(r"[a-z]"),
|
||||
"birth_year": _matches(r"(19|20)\d\d"),
|
||||
"birth_date": _matches(r"\d{4}-\d{2}(-\d{2})?"),
|
||||
"twitter_url": _contains("twitter.com/"),
|
||||
"github_url": _contains("github.com/"),
|
||||
"mobile_phone": _matches(r"\+?\d[\d -]{6,20}"),
|
||||
}
|
||||
|
||||
# Alignment weights. Tuned so that a confirmed signature match outweighs any
|
||||
# combination of edits, while losing a non-empty value is expensive.
|
||||
SCORE_MATCH = 3.0
|
||||
SCORE_MISMATCH = -4.0
|
||||
SCORE_FREE_TEXT = 0.3
|
||||
SCORE_EMPTY = 0.15
|
||||
PENALTY_MISSING_COLUMN = -0.5
|
||||
PENALTY_DROP_EMPTY = -0.3
|
||||
PENALTY_DROP_VALUE = -3.5
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportReport:
|
||||
"""Counters surfaced by the management command so imports are auditable."""
|
||||
|
||||
physical_rows: int = 0
|
||||
duplicate_headers: int = 0
|
||||
repaired_rows: int = 0
|
||||
orphan_fragments: int = 0
|
||||
duplicate_records: int = 0
|
||||
discarded_values: int = 0
|
||||
records: int = 0
|
||||
issues: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"physical_rows": self.physical_rows,
|
||||
"duplicate_headers": self.duplicate_headers,
|
||||
"repaired_rows": self.repaired_rows,
|
||||
"orphan_fragments": self.orphan_fragments,
|
||||
"duplicate_records": self.duplicate_records,
|
||||
"discarded_values": self.discarded_values,
|
||||
"records": self.records,
|
||||
}
|
||||
|
||||
|
||||
class DatasetReader:
|
||||
"""Yields clean, de-duplicated records from the raw CSV export."""
|
||||
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path)
|
||||
|
||||
def read(self) -> tuple[list[dict[str, str]], ImportReport]:
|
||||
report = ImportReport()
|
||||
with self.path.open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.reader(handle))
|
||||
|
||||
if not rows:
|
||||
return [], report
|
||||
|
||||
header = rows[0]
|
||||
report.physical_rows = len(rows) - 1
|
||||
|
||||
logical_rows = self._reassemble(rows[1:], header, report)
|
||||
|
||||
seen: set[str] = set()
|
||||
records: list[dict[str, str]] = []
|
||||
for values in logical_rows:
|
||||
record = self._align(values, header)
|
||||
self._sanitize(record, report)
|
||||
key = self._identity(record)
|
||||
if key in seen:
|
||||
report.duplicate_records += 1
|
||||
continue
|
||||
seen.add(key)
|
||||
records.append(record)
|
||||
|
||||
report.records = len(records)
|
||||
return records, report
|
||||
|
||||
# -- Step 1: turn physical rows into logical records --------------------
|
||||
def _reassemble(
|
||||
self, rows: list[list[str]], header: list[str], report: ImportReport
|
||||
) -> list[list[str]]:
|
||||
expected = len(header)
|
||||
logical: list[list[str]] = []
|
||||
for raw_row in rows:
|
||||
row = self._strip_watermark(list(raw_row))
|
||||
if not row:
|
||||
continue
|
||||
if self._is_repeated_header(row, header):
|
||||
report.duplicate_headers += 1
|
||||
continue
|
||||
if self._starts_record(row) or not logical:
|
||||
if len(row) != expected:
|
||||
report.repaired_rows += 1
|
||||
logical.append(row)
|
||||
elif len(logical[-1]) < expected - 7:
|
||||
# The previous record is short: this fragment is its tail,
|
||||
# split by a stray quote inside a free-text column.
|
||||
logical[-1].extend(row)
|
||||
report.repaired_rows += 1
|
||||
else:
|
||||
# Cannot be attributed to any record without guessing.
|
||||
report.orphan_fragments += 1
|
||||
return logical
|
||||
|
||||
@staticmethod
|
||||
def _strip_watermark(row: list[str]) -> list[str]:
|
||||
while row and any(marker in row[0] for marker in WATERMARK_MARKERS):
|
||||
row = row[1:]
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def _is_repeated_header(row: list[str], header: list[str]) -> bool:
|
||||
return len(row) > 3 and row[:3] == header[:3]
|
||||
|
||||
@staticmethod
|
||||
def _starts_record(row: list[str]) -> bool:
|
||||
"""
|
||||
A record always begins with ``full_name`` followed by ``first_name``,
|
||||
and the full name starts with the first name. That is a strong,
|
||||
cheap anchor for telling a new record from a continuation fragment.
|
||||
"""
|
||||
if len(row) < 9:
|
||||
return False
|
||||
full_name, first_name = row[0].strip(), row[1].strip()
|
||||
return bool(
|
||||
full_name
|
||||
and first_name
|
||||
and len(full_name) < 80
|
||||
and "@" not in full_name
|
||||
and full_name.startswith(first_name)
|
||||
)
|
||||
|
||||
# -- Step 2: align values to columns ------------------------------------
|
||||
@staticmethod
|
||||
def _score(column: str, value: str) -> float:
|
||||
if value == "":
|
||||
return SCORE_EMPTY
|
||||
validator = FIELD_VALIDATORS.get(column)
|
||||
if validator is None:
|
||||
return SCORE_FREE_TEXT
|
||||
try:
|
||||
verdict = validator(value)
|
||||
except Exception: # noqa: BLE001 - a broken value is just a mismatch
|
||||
verdict = False
|
||||
if verdict is None:
|
||||
return SCORE_MISMATCH
|
||||
return SCORE_MATCH if verdict else SCORE_MISMATCH
|
||||
|
||||
def _align(self, values: list[str], header: list[str]) -> dict[str, str]:
|
||||
"""Needleman-Wunsch alignment of row values against header columns."""
|
||||
n, m = len(values), len(header)
|
||||
neg_inf = float("-inf")
|
||||
best = [[neg_inf] * (m + 1) for _ in range(n + 1)]
|
||||
move = [[0] * (m + 1) for _ in range(n + 1)]
|
||||
best[0][0] = 0.0
|
||||
|
||||
for j in range(1, m + 1):
|
||||
best[0][j] = best[0][j - 1] + PENALTY_MISSING_COLUMN
|
||||
move[0][j] = 2
|
||||
for i in range(1, n + 1):
|
||||
best[i][0] = best[i - 1][0] + self._drop_penalty(values[i - 1])
|
||||
move[i][0] = 3
|
||||
|
||||
for i in range(1, n + 1):
|
||||
value = values[i - 1]
|
||||
drop = self._drop_penalty(value)
|
||||
for j in range(1, m + 1):
|
||||
score = best[i - 1][j - 1] + self._score(header[j - 1], value)
|
||||
choice = 1
|
||||
skip_column = best[i][j - 1] + PENALTY_MISSING_COLUMN
|
||||
if skip_column > score:
|
||||
score, choice = skip_column, 2
|
||||
drop_value = best[i - 1][j] + drop
|
||||
if drop_value > score:
|
||||
score, choice = drop_value, 3
|
||||
best[i][j], move[i][j] = score, choice
|
||||
|
||||
record: dict[str, str] = {}
|
||||
i, j = n, m
|
||||
while i > 0 or j > 0:
|
||||
choice = move[i][j]
|
||||
if choice == 1:
|
||||
record[header[j - 1]] = values[i - 1].strip()
|
||||
i, j = i - 1, j - 1
|
||||
elif choice == 2:
|
||||
record.setdefault(header[j - 1], "")
|
||||
j -= 1
|
||||
else:
|
||||
i -= 1
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def _drop_penalty(value: str) -> float:
|
||||
return PENALTY_DROP_EMPTY if value == "" else PENALTY_DROP_VALUE
|
||||
|
||||
# -- Step 3: final validation ------------------------------------------
|
||||
@staticmethod
|
||||
def _sanitize(record: dict[str, str], report: ImportReport) -> None:
|
||||
for column, value in list(record.items()):
|
||||
if not value:
|
||||
continue
|
||||
validator = FIELD_VALIDATORS.get(column)
|
||||
if validator is None:
|
||||
continue
|
||||
try:
|
||||
verdict = validator(value)
|
||||
except Exception: # noqa: BLE001
|
||||
verdict = False
|
||||
if not verdict:
|
||||
record[column] = ""
|
||||
report.discarded_values += 1
|
||||
|
||||
@staticmethod
|
||||
def _identity(record: dict[str, str]) -> str:
|
||||
return (
|
||||
record.get("linkedin_id") or record.get("linkedin_url") or record.get("full_name") or ""
|
||||
).lower()
|
||||
|
||||
|
||||
def parse_literal(value: str | None, default: Any) -> Any:
|
||||
"""Public helper for the importer: parse a nested column defensively."""
|
||||
if not value:
|
||||
return default
|
||||
parsed = _safe_literal(value)
|
||||
return default if parsed is None else parsed
|
||||
|
||||
|
||||
def read_dataset(path: str | Path) -> Iterator[dict[str, str]]:
|
||||
"""Convenience wrapper for callers that do not need the report."""
|
||||
records, _ = DatasetReader(path).read()
|
||||
yield from records
|
||||
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Maps the repaired CSV records onto the relational model.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* **Idempotent.** The natural key is the LinkedIn id (falling back to the
|
||||
profile URL), so re-running the import updates instead of duplicating.
|
||||
* **Bulk friendly.** Companies, schools and skills are resolved through
|
||||
in-memory caches, and the child rows of a profile are replaced wholesale with
|
||||
``bulk_create`` - 284 profiles import in a couple of seconds.
|
||||
* **Self healing input.** The top level columns of this export are frequently
|
||||
missing, but the same information usually exists inside the nested
|
||||
``experience`` blob. ``_derive_current_position`` falls back to the primary
|
||||
experience entry, which raises coverage of ``job_title_role`` from 108 to 183
|
||||
profiles and of the current employer from 219 to 271.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.postgres.search import SearchVector
|
||||
from django.db import transaction
|
||||
|
||||
from apps.profiles.dataset import DatasetReader, ImportReport, parse_literal
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_YEAR_RE = re.compile(r"(\d{4})")
|
||||
_MAX_SKILLS = 60
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportStats:
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skills: int = 0
|
||||
companies: int = 0
|
||||
schools: int = 0
|
||||
experiences: int = 0
|
||||
educations: int = 0
|
||||
report: ImportReport = field(default_factory=ImportReport)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"profiles_created": self.created,
|
||||
"profiles_updated": self.updated,
|
||||
"skills": self.skills,
|
||||
"companies": self.companies,
|
||||
"schools": self.schools,
|
||||
"experiences": self.experiences,
|
||||
"educations": self.educations,
|
||||
**self.report.as_dict(),
|
||||
}
|
||||
|
||||
|
||||
class ProfileImporter:
|
||||
"""Loads the dataset into PostgreSQL."""
|
||||
|
||||
def __init__(self, path: str, limit: int | None = None) -> None:
|
||||
self.path = path
|
||||
self.limit = limit
|
||||
self._companies: dict[str, Company] = {}
|
||||
self._schools: dict[str, School] = {}
|
||||
self._skills: dict[str, Skill] = {}
|
||||
|
||||
# -- Entry point --------------------------------------------------------
|
||||
def run(self) -> ImportStats:
|
||||
records, report = DatasetReader(self.path).read()
|
||||
if self.limit:
|
||||
records = records[: self.limit]
|
||||
|
||||
stats = ImportStats(report=report)
|
||||
self._warm_caches()
|
||||
|
||||
for record in records:
|
||||
try:
|
||||
with transaction.atomic():
|
||||
self._import_record(record, stats)
|
||||
except Exception: # noqa: BLE001 - one bad row must not abort the run
|
||||
logger.exception(
|
||||
"profile_import_failed",
|
||||
extra={"event": "profile_import_failed", "name": record.get("full_name")},
|
||||
)
|
||||
report.issues.append(record.get("full_name", "<unknown>"))
|
||||
|
||||
self._refresh_skill_counters()
|
||||
self._refresh_search_vectors()
|
||||
|
||||
stats.skills = len(self._skills)
|
||||
stats.companies = len(self._companies)
|
||||
stats.schools = len(self._schools)
|
||||
return stats
|
||||
|
||||
# -- Caches -------------------------------------------------------------
|
||||
def _warm_caches(self) -> None:
|
||||
self._companies = {c.name: c for c in Company.objects.all()}
|
||||
self._schools = {s.name: s for s in School.objects.all()}
|
||||
self._skills = {s.slug: s for s in Skill.objects.all()}
|
||||
|
||||
# -- Per record ---------------------------------------------------------
|
||||
def _import_record(self, record: dict[str, str], stats: ImportStats) -> None:
|
||||
experiences = parse_literal(record.get("experience"), [])
|
||||
educations = parse_literal(record.get("education"), [])
|
||||
primary = self._primary_experience(experiences)
|
||||
|
||||
title, role, sub_role, levels = self._derive_current_position(record, primary)
|
||||
company = self._resolve_current_company(record, primary)
|
||||
|
||||
skills = self._clean_str_list(parse_literal(record.get("skills"), []))[:_MAX_SKILLS]
|
||||
past_companies = [
|
||||
str(item.get("company", {}).get("name") or "")
|
||||
for item in experiences
|
||||
if isinstance(item, dict) and isinstance(item.get("company"), dict)
|
||||
]
|
||||
schools = [
|
||||
str(item.get("school", {}).get("name") or "")
|
||||
for item in educations
|
||||
if isinstance(item, dict) and isinstance(item.get("school"), dict)
|
||||
]
|
||||
majors = [
|
||||
major
|
||||
for item in educations
|
||||
if isinstance(item, dict)
|
||||
for major in self._clean_str_list(item.get("majors") or [])
|
||||
]
|
||||
|
||||
defaults: dict[str, Any] = {
|
||||
"full_name": record.get("full_name", "")[:255],
|
||||
"first_name": record.get("first_name", "")[:128],
|
||||
"last_name": record.get("last_name", "")[:128],
|
||||
"gender": record.get("gender", "")[:16],
|
||||
"linkedin_url": record.get("linkedin_url", "")[:255],
|
||||
"linkedin_username": record.get("linkedin_username", "")[:128],
|
||||
"github_url": record.get("github_url", "")[:255],
|
||||
"twitter_url": record.get("twitter_url", "")[:255],
|
||||
"job_title": title[:255],
|
||||
"job_title_role": role[:64],
|
||||
"job_title_sub_role": sub_role[:64],
|
||||
"seniority_levels": levels,
|
||||
"current_company": company,
|
||||
"industry": (record.get("industry") or "")[:128],
|
||||
"job_summary": record.get("job_summary", ""),
|
||||
"job_start_date": record.get("job_start_date", "")[:10],
|
||||
"location_name": record.get("location_name", "")[:255],
|
||||
"location_locality": record.get("location_locality", "")[:128],
|
||||
"location_region": record.get("location_region", "")[:128],
|
||||
"location_country": record.get("location_country", "")[:128],
|
||||
"location_continent": record.get("location_continent", "")[:64],
|
||||
"summary": record.get("summary", ""),
|
||||
"years_experience": self._to_int(record.get("inferred_years_experience"), 0, 70),
|
||||
"inferred_salary": record.get("inferred_salary", "")[:32],
|
||||
"connections": self._to_int(record.get("linkedin_connections"), 0, 100_000),
|
||||
"interests": self._clean_str_list(parse_literal(record.get("interests"), []))[:40],
|
||||
"languages": self._language_names(parse_literal(record.get("languages"), [])),
|
||||
"emails": self._emails(parse_literal(record.get("emails"), [])),
|
||||
"phone_numbers": self._clean_str_list(parse_literal(record.get("phone_numbers"), []))[
|
||||
:10
|
||||
],
|
||||
"skills_text": " ".join(skills),
|
||||
"companies_text": " ".join(
|
||||
filter(None, {*past_companies, company.name if company else ""})
|
||||
),
|
||||
"education_text": " ".join(filter(None, [*schools, *majors])),
|
||||
}
|
||||
|
||||
external_id = (record.get("linkedin_id") or "")[:64]
|
||||
lookup = (
|
||||
{"external_id": external_id}
|
||||
if external_id
|
||||
else {"linkedin_url": defaults["linkedin_url"]}
|
||||
)
|
||||
profile, created = Profile.objects.update_or_create(**lookup, defaults=defaults)
|
||||
if created:
|
||||
stats.created += 1
|
||||
else:
|
||||
stats.updated += 1
|
||||
|
||||
self._sync_skills(profile, skills)
|
||||
stats.experiences += self._sync_experiences(profile, experiences)
|
||||
stats.educations += self._sync_educations(profile, educations)
|
||||
|
||||
# -- Current position ---------------------------------------------------
|
||||
@staticmethod
|
||||
def _primary_experience(experiences: list[Any]) -> dict[str, Any]:
|
||||
entries = [item for item in experiences if isinstance(item, dict)]
|
||||
for item in entries:
|
||||
if item.get("is_primary"):
|
||||
return item
|
||||
return entries[0] if entries else {}
|
||||
|
||||
def _derive_current_position(
|
||||
self, record: dict[str, str], primary: dict[str, Any]
|
||||
) -> tuple[str, str, str, list[str]]:
|
||||
title_blob = primary.get("title") if isinstance(primary.get("title"), dict) else {}
|
||||
title = record.get("job_title") or str(title_blob.get("name") or "")
|
||||
role = record.get("job_title_role") or str(title_blob.get("role") or "")
|
||||
sub_role = record.get("job_title_sub_role") or str(title_blob.get("sub_role") or "")
|
||||
levels = self._clean_str_list(
|
||||
parse_literal(record.get("job_title_levels"), None) or title_blob.get("levels") or []
|
||||
)
|
||||
return title, role, sub_role, levels[:5]
|
||||
|
||||
def _resolve_current_company(
|
||||
self, record: dict[str, str], primary: dict[str, Any]
|
||||
) -> Company | None:
|
||||
name = (record.get("job_company_name") or "").strip()
|
||||
payload: dict[str, Any] = {
|
||||
"external_id": record.get("job_company_id", ""),
|
||||
"industry": record.get("job_company_industry", ""),
|
||||
"size": record.get("job_company_size", ""),
|
||||
"founded": self._to_int(record.get("job_company_founded"), 1600, 2100),
|
||||
"website": record.get("job_company_website", ""),
|
||||
"linkedin_url": record.get("job_company_linkedin_url", ""),
|
||||
"location_name": record.get("job_company_location_name", ""),
|
||||
"country": record.get("job_company_location_country", ""),
|
||||
}
|
||||
|
||||
if not name and isinstance(primary.get("company"), dict):
|
||||
blob = primary["company"]
|
||||
name = str(blob.get("name") or "").strip()
|
||||
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
|
||||
payload = {
|
||||
"external_id": str(blob.get("id") or ""),
|
||||
"industry": str(blob.get("industry") or ""),
|
||||
"size": str(blob.get("size") or ""),
|
||||
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
|
||||
"website": str(blob.get("website") or ""),
|
||||
"linkedin_url": str(blob.get("linkedin_url") or ""),
|
||||
"location_name": str(location.get("name") or ""),
|
||||
"country": str(location.get("country") or ""),
|
||||
}
|
||||
|
||||
return self._get_or_create_company(name, payload)
|
||||
|
||||
def _get_or_create_company(self, name: str, payload: dict[str, Any]) -> Company | None:
|
||||
name = name.strip()[:255]
|
||||
if not name:
|
||||
return None
|
||||
cached = self._companies.get(name)
|
||||
if cached:
|
||||
return cached
|
||||
company = Company.objects.create(
|
||||
name=name,
|
||||
external_id=str(payload.get("external_id") or "")[:128],
|
||||
industry=str(payload.get("industry") or "")[:128],
|
||||
size=str(payload.get("size") or "")[:32],
|
||||
founded=payload.get("founded"),
|
||||
website=str(payload.get("website") or "")[:255],
|
||||
linkedin_url=str(payload.get("linkedin_url") or "")[:255],
|
||||
location_name=str(payload.get("location_name") or "")[:255],
|
||||
country=str(payload.get("country") or "")[:128],
|
||||
)
|
||||
self._companies[name] = company
|
||||
return company
|
||||
|
||||
def _get_or_create_school(self, blob: dict[str, Any]) -> School | None:
|
||||
name = str(blob.get("name") or "").strip()[:255]
|
||||
if not name:
|
||||
return None
|
||||
cached = self._schools.get(name)
|
||||
if cached:
|
||||
return cached
|
||||
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
|
||||
school = School.objects.create(
|
||||
name=name,
|
||||
type=str(blob.get("type") or "")[:64],
|
||||
website=str(blob.get("website") or "")[:255],
|
||||
linkedin_url=str(blob.get("linkedin_url") or "")[:255],
|
||||
location_name=str(location.get("name") or "")[:255],
|
||||
country=str(location.get("country") or "")[:128],
|
||||
)
|
||||
self._schools[name] = school
|
||||
return school
|
||||
|
||||
# -- Children -----------------------------------------------------------
|
||||
def _sync_skills(self, profile: Profile, skills: list[str]) -> None:
|
||||
ProfileSkill.objects.filter(profile=profile).delete()
|
||||
links: list[ProfileSkill] = []
|
||||
used: set[int] = set()
|
||||
for position, name in enumerate(skills):
|
||||
slug = Skill.make_slug(name)
|
||||
if not slug:
|
||||
continue
|
||||
skill = self._skills.get(slug)
|
||||
if skill is None:
|
||||
skill, _ = Skill.objects.get_or_create(slug=slug, defaults={"name": name[:128]})
|
||||
self._skills[slug] = skill
|
||||
if skill.pk in used:
|
||||
continue
|
||||
used.add(skill.pk)
|
||||
links.append(ProfileSkill(profile=profile, skill=skill, position=position))
|
||||
ProfileSkill.objects.bulk_create(links, ignore_conflicts=True)
|
||||
|
||||
def _sync_experiences(self, profile: Profile, experiences: list[Any]) -> int:
|
||||
Experience.objects.filter(profile=profile).delete()
|
||||
rows: list[Experience] = []
|
||||
for item in experiences:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
blob = item.get("company") if isinstance(item.get("company"), dict) else {}
|
||||
title = item.get("title") if isinstance(item.get("title"), dict) else {}
|
||||
company = self._get_or_create_company(
|
||||
str(blob.get("name") or ""),
|
||||
{
|
||||
"external_id": blob.get("id"),
|
||||
"industry": blob.get("industry"),
|
||||
"size": blob.get("size"),
|
||||
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
|
||||
"website": blob.get("website"),
|
||||
"linkedin_url": blob.get("linkedin_url"),
|
||||
"location_name": (
|
||||
(blob.get("location") or {}).get("name")
|
||||
if isinstance(blob.get("location"), dict)
|
||||
else ""
|
||||
),
|
||||
"country": (
|
||||
(blob.get("location") or {}).get("country")
|
||||
if isinstance(blob.get("location"), dict)
|
||||
else ""
|
||||
),
|
||||
},
|
||||
)
|
||||
rows.append(
|
||||
Experience(
|
||||
profile=profile,
|
||||
company=company,
|
||||
company_name=str(blob.get("name") or "")[:255],
|
||||
title=str(title.get("name") or "")[:255],
|
||||
title_role=str(title.get("role") or "")[:64],
|
||||
title_sub_role=str(title.get("sub_role") or "")[:64],
|
||||
levels=self._clean_str_list(title.get("levels") or [])[:5],
|
||||
start_date=str(item.get("start_date") or "")[:10],
|
||||
end_date=str(item.get("end_date") or "")[:10],
|
||||
start_year=self._year(item.get("start_date")),
|
||||
end_year=self._year(item.get("end_date")),
|
||||
is_primary=bool(item.get("is_primary")),
|
||||
location_name=self._first_location(item.get("location_names")),
|
||||
summary=str(item.get("summary") or ""),
|
||||
)
|
||||
)
|
||||
Experience.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
def _sync_educations(self, profile: Profile, educations: list[Any]) -> int:
|
||||
Education.objects.filter(profile=profile).delete()
|
||||
rows: list[Education] = []
|
||||
for item in educations:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
blob = item.get("school") if isinstance(item.get("school"), dict) else {}
|
||||
rows.append(
|
||||
Education(
|
||||
profile=profile,
|
||||
school=self._get_or_create_school(blob),
|
||||
school_name=str(blob.get("name") or "")[:255],
|
||||
degrees=self._clean_str_list(item.get("degrees") or [])[:5],
|
||||
majors=self._clean_str_list(item.get("majors") or [])[:5],
|
||||
minors=self._clean_str_list(item.get("minors") or [])[:5],
|
||||
gpa=self._to_decimal(item.get("gpa")),
|
||||
start_date=str(item.get("start_date") or "")[:10],
|
||||
end_date=str(item.get("end_date") or "")[:10],
|
||||
end_year=self._year(item.get("end_date")),
|
||||
)
|
||||
)
|
||||
Education.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
# -- Post processing ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _refresh_skill_counters() -> None:
|
||||
"""Single UPDATE ... FROM keeps the facet ordering cheap at query time."""
|
||||
from django.db import connection
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE profiles_skill AS s
|
||||
SET profile_count = COALESCE(counts.total, 0)
|
||||
FROM (
|
||||
SELECT skill_id, COUNT(*) AS total
|
||||
FROM profiles_profileskill
|
||||
GROUP BY skill_id
|
||||
) AS counts
|
||||
WHERE counts.skill_id = s.id
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE profiles_skill SET profile_count = 0
|
||||
WHERE id NOT IN (SELECT skill_id FROM profiles_profileskill)
|
||||
"""
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _refresh_search_vectors() -> None:
|
||||
"""
|
||||
Rebuild the tsvector for every profile in one statement.
|
||||
|
||||
Weights encode the ranking intent: a hit on the person's name beats a
|
||||
hit on their title or skills, which in turn beats a hit in the free-text
|
||||
summary.
|
||||
"""
|
||||
Profile.objects.update(
|
||||
search_vector=(
|
||||
SearchVector("full_name", weight="A", config="english")
|
||||
+ SearchVector("job_title", weight="B", config="english")
|
||||
+ SearchVector("skills_text", weight="B", config="english")
|
||||
+ SearchVector("companies_text", weight="B", config="english")
|
||||
+ SearchVector("industry", weight="C", config="english")
|
||||
+ SearchVector("education_text", weight="C", config="english")
|
||||
+ SearchVector("location_name", weight="C", config="english")
|
||||
+ SearchVector("summary", weight="D", config="english")
|
||||
)
|
||||
)
|
||||
|
||||
# -- Small helpers ------------------------------------------------------
|
||||
@staticmethod
|
||||
def _clean_str_list(values: Any) -> list[str]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for value in values:
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
cleaned = value.strip().lower()[:128]
|
||||
if cleaned and cleaned not in seen:
|
||||
seen.add(cleaned)
|
||||
out.append(cleaned)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _language_names(values: Any) -> list[str]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
return [
|
||||
str(item.get("name")).strip().lower()[:64]
|
||||
for item in values
|
||||
if isinstance(item, dict) and item.get("name")
|
||||
][:10]
|
||||
|
||||
@staticmethod
|
||||
def _emails(values: Any) -> list[dict[str, str]]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"address": str(item.get("address"))[:254],
|
||||
"type": str(item.get("type") or "other")[:32],
|
||||
}
|
||||
for item in values
|
||||
if isinstance(item, dict) and item.get("address")
|
||||
][:10]
|
||||
|
||||
@staticmethod
|
||||
def _first_location(values: Any) -> str:
|
||||
if isinstance(values, list) and values and isinstance(values[0], str):
|
||||
return values[0][:255]
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _to_int(value: str | None, minimum: int, maximum: int) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
number = int(float(str(value).replace(",", "")))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if minimum <= number <= maximum else None
|
||||
|
||||
@staticmethod
|
||||
def _to_decimal(value: Any) -> Any:
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return round(number, 2) if 0 <= number <= 10 else None
|
||||
|
||||
@staticmethod
|
||||
def _year(value: Any) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
match = _YEAR_RE.search(str(value))
|
||||
if not match:
|
||||
return None
|
||||
year = int(match.group(1))
|
||||
return year if 1900 <= year <= 2100 else None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""``python manage.py import_profiles`` - load the CSV export into PostgreSQL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import transaction
|
||||
|
||||
from apps.profiles.importers import ProfileImporter
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Import (or refresh) LinkedIn profiles from the raw CSV export."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
parser.add_argument(
|
||||
"--path",
|
||||
default=settings.DATASET_PATH,
|
||||
help="Path to the CSV export (defaults to DATASET_PATH).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", type=int, default=None, help="Import at most N profiles (smoke tests)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--truncate",
|
||||
action="store_true",
|
||||
help="Delete existing profile data before importing.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reindex",
|
||||
action="store_true",
|
||||
help="Rebuild the Elasticsearch index once the import finishes.",
|
||||
)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
path = options["path"]
|
||||
started = time.perf_counter()
|
||||
|
||||
if options["truncate"]:
|
||||
self.stdout.write("Truncating existing profile data ...")
|
||||
with transaction.atomic():
|
||||
ProfileSkill.objects.all().delete()
|
||||
Experience.objects.all().delete()
|
||||
Education.objects.all().delete()
|
||||
Profile.objects.all().delete()
|
||||
Skill.objects.all().delete()
|
||||
Company.objects.all().delete()
|
||||
School.objects.all().delete()
|
||||
|
||||
self.stdout.write(f"Reading dataset from {path} ...")
|
||||
try:
|
||||
stats = ProfileImporter(path, limit=options["limit"]).run()
|
||||
except FileNotFoundError as exc:
|
||||
raise CommandError(f"Dataset not found: {path}") from exc
|
||||
|
||||
elapsed = round(time.perf_counter() - started, 2)
|
||||
summary = {**stats.as_dict(), "elapsed_seconds": elapsed}
|
||||
self.stdout.write(json.dumps(summary, indent=2))
|
||||
|
||||
if stats.report.issues:
|
||||
self.stdout.write(
|
||||
self.style.WARNING(
|
||||
f"{len(stats.report.issues)} record(s) failed: "
|
||||
f"{', '.join(stats.report.issues[:10])}"
|
||||
)
|
||||
)
|
||||
|
||||
if options["reindex"]:
|
||||
self.stdout.write("Rebuilding the search index ...")
|
||||
from django.core.management import call_command
|
||||
|
||||
call_command("search_index", "--rebuild")
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(f"Import finished in {elapsed}s."))
|
||||
@@ -0,0 +1,416 @@
|
||||
# Generated by Django 5.1.15 on 2026-09-03 03:36
|
||||
|
||||
import django.contrib.postgres.fields
|
||||
import django.contrib.postgres.indexes
|
||||
import django.contrib.postgres.search
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = []
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Company",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("external_id", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("name", models.CharField(max_length=255, unique=True)),
|
||||
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
(
|
||||
"size",
|
||||
models.CharField(
|
||||
blank=True, help_text="Head-count bucket, e.g. 51-200", max_length=32
|
||||
),
|
||||
),
|
||||
("founded", models.PositiveIntegerField(blank=True, null=True)),
|
||||
("website", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("country", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
],
|
||||
options={
|
||||
"verbose_name_plural": "companies",
|
||||
"ordering": ("name",),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="School",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("name", models.CharField(max_length=255, unique=True)),
|
||||
("type", models.CharField(blank=True, max_length=64)),
|
||||
("website", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("country", models.CharField(blank=True, max_length=128)),
|
||||
],
|
||||
options={
|
||||
"ordering": ("name",),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Skill",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("name", models.CharField(max_length=128)),
|
||||
("slug", models.SlugField(max_length=140, unique=True)),
|
||||
(
|
||||
"profile_count",
|
||||
models.PositiveIntegerField(
|
||||
db_index=True,
|
||||
default=0,
|
||||
help_text="Denormalised counter used to rank filter suggestions.",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-profile_count", "name"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Profile",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"external_id",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
max_length=64,
|
||||
),
|
||||
),
|
||||
("full_name", models.CharField(db_index=True, max_length=255)),
|
||||
("first_name", models.CharField(blank=True, max_length=128)),
|
||||
("last_name", models.CharField(blank=True, max_length=128)),
|
||||
(
|
||||
"gender",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
choices=[("male", "Male"), ("female", "Female"), ("", "Unknown")],
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_username", models.CharField(blank=True, max_length=128)),
|
||||
("github_url", models.CharField(blank=True, max_length=255)),
|
||||
("twitter_url", models.CharField(blank=True, max_length=255)),
|
||||
("job_title", models.CharField(blank=True, db_index=True, max_length=255)),
|
||||
(
|
||||
"job_title_role",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="Coarse job family, e.g. engineering.",
|
||||
max_length=64,
|
||||
),
|
||||
),
|
||||
("job_title_sub_role", models.CharField(blank=True, max_length=64)),
|
||||
(
|
||||
"seniority_levels",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("job_summary", models.TextField(blank=True)),
|
||||
("job_start_date", models.CharField(blank=True, max_length=10)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("location_locality", models.CharField(blank=True, max_length=128)),
|
||||
("location_region", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("location_country", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("location_continent", models.CharField(blank=True, max_length=64)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
(
|
||||
"years_experience",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
("inferred_salary", models.CharField(blank=True, max_length=32)),
|
||||
("connections", models.PositiveIntegerField(blank=True, null=True)),
|
||||
(
|
||||
"interests",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"languages",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=64),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("emails", models.JSONField(blank=True, default=list)),
|
||||
(
|
||||
"phone_numbers",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("skills_text", models.TextField(blank=True, editable=False)),
|
||||
("companies_text", models.TextField(blank=True, editable=False)),
|
||||
("education_text", models.TextField(blank=True, editable=False)),
|
||||
(
|
||||
"search_vector",
|
||||
django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
|
||||
),
|
||||
(
|
||||
"current_company",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="current_employees",
|
||||
to="profiles.company",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("full_name", "id"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Education",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("school_name", models.CharField(blank=True, max_length=255)),
|
||||
(
|
||||
"degrees",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"majors",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"minors",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("gpa", models.DecimalField(blank=True, decimal_places=2, max_digits=4, null=True)),
|
||||
("start_date", models.CharField(blank=True, max_length=10)),
|
||||
("end_date", models.CharField(blank=True, max_length=10)),
|
||||
(
|
||||
"end_year",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="educations",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
(
|
||||
"school",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="educations",
|
||||
to="profiles.school",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-end_year", "id"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ProfileSkill",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("position", models.PositiveSmallIntegerField(default=0)),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="profile_skills",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
(
|
||||
"skill",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="profile_skills",
|
||||
to="profiles.skill",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("position",),
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="profile",
|
||||
name="skills",
|
||||
field=models.ManyToManyField(
|
||||
related_name="profiles", through="profiles.ProfileSkill", to="profiles.skill"
|
||||
),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Experience",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("company_name", models.CharField(blank=True, max_length=255)),
|
||||
("title", models.CharField(blank=True, max_length=255)),
|
||||
("title_role", models.CharField(blank=True, max_length=64)),
|
||||
("title_sub_role", models.CharField(blank=True, max_length=64)),
|
||||
(
|
||||
"levels",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("start_date", models.CharField(blank=True, max_length=10)),
|
||||
("end_date", models.CharField(blank=True, max_length=10)),
|
||||
(
|
||||
"start_year",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
("end_year", models.PositiveSmallIntegerField(blank=True, null=True)),
|
||||
("is_primary", models.BooleanField(db_index=True, default=False)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
(
|
||||
"company",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="experiences",
|
||||
to="profiles.company",
|
||||
),
|
||||
),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="experiences",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-is_primary", "-start_year", "id"),
|
||||
"indexes": [
|
||||
models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")
|
||||
],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="profileskill",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("profile", "skill"), name="uniq_profile_skill"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=["search_vector"], name="profile_search_vector_gin"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=["seniority_levels"], name="profile_seniority_gin"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="profile",
|
||||
constraint=models.UniqueConstraint(
|
||||
condition=models.Q(("external_id__gt", "")),
|
||||
fields=("external_id",),
|
||||
name="uniq_profile_external_id",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
Enable the PostgreSQL extensions the search fallback relies on.
|
||||
|
||||
``pg_trgm`` powers ``TrigramSimilarity`` (typo tolerant name matching) and the
|
||||
GIN trigram index below makes ``ILIKE '%term%'`` on skill names fast enough for
|
||||
the type-ahead endpoint.
|
||||
"""
|
||||
|
||||
from django.contrib.postgres.operations import TrigramExtension, UnaccentExtension
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("profiles", "0001_initial")]
|
||||
|
||||
operations = [
|
||||
TrigramExtension(),
|
||||
UnaccentExtension(),
|
||||
migrations.RunSQL(
|
||||
sql=(
|
||||
"CREATE INDEX IF NOT EXISTS profile_full_name_trgm "
|
||||
"ON profiles_profile USING gin (full_name gin_trgm_ops);"
|
||||
),
|
||||
reverse_sql="DROP INDEX IF EXISTS profile_full_name_trgm;",
|
||||
),
|
||||
migrations.RunSQL(
|
||||
sql=(
|
||||
"CREATE INDEX IF NOT EXISTS skill_name_trgm "
|
||||
"ON profiles_skill USING gin (name gin_trgm_ops);"
|
||||
),
|
||||
reverse_sql="DROP INDEX IF EXISTS skill_name_trgm;",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Domain model for the LinkedIn dataset.
|
||||
|
||||
The raw export is one very wide CSV row per person with several columns holding
|
||||
serialised lists of dictionaries. Storing that blob as-is would make filtering
|
||||
and aggregation painful, so it is normalised into five tables:
|
||||
|
||||
Company <--- Experience ---> Profile <--- Education ---> School
|
||||
|
|
||||
+--- ProfileSkill ---> Skill
|
||||
|
||||
PostgreSQL is the source of truth. Elasticsearch holds a denormalised
|
||||
projection of the same data for search (see ``apps.search``); it can always be
|
||||
rebuilt from these tables, which is why the index is disposable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.contrib.postgres.indexes import GinIndex
|
||||
from django.contrib.postgres.search import SearchVectorField
|
||||
from django.db import models
|
||||
from django.utils.text import slugify
|
||||
|
||||
|
||||
class TimeStampedModel(models.Model):
|
||||
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class Company(TimeStampedModel):
|
||||
"""An employer referenced by the current job or by a past experience."""
|
||||
|
||||
external_id = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
size = models.CharField(max_length=32, blank=True, help_text="Head-count bucket, e.g. 51-200")
|
||||
founded = models.PositiveIntegerField(null=True, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
verbose_name_plural = "companies"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class School(TimeStampedModel):
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
type = models.CharField(max_length=64, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class Skill(TimeStampedModel):
|
||||
"""
|
||||
A normalised skill.
|
||||
|
||||
``slug`` is the deduplication key: the raw data contains the same skill with
|
||||
different casing and spacing ("Team Building" / "team building").
|
||||
"""
|
||||
|
||||
name = models.CharField(max_length=128)
|
||||
slug = models.SlugField(max_length=140, unique=True)
|
||||
profile_count = models.PositiveIntegerField(
|
||||
default=0, db_index=True, help_text="Denormalised counter used to rank filter suggestions."
|
||||
)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-profile_count", "name")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
@staticmethod
|
||||
def make_slug(name: str) -> str:
|
||||
return slugify(name)[:140] or name.strip().lower()[:140]
|
||||
|
||||
|
||||
class Profile(TimeStampedModel):
|
||||
"""A single person in the dataset."""
|
||||
|
||||
class Gender(models.TextChoices):
|
||||
MALE = "male", "Male"
|
||||
FEMALE = "female", "Female"
|
||||
UNKNOWN = "", "Unknown"
|
||||
|
||||
# -- Identity ----------------------------------------------------------
|
||||
external_id = models.CharField(
|
||||
max_length=64,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
)
|
||||
full_name = models.CharField(max_length=255, db_index=True)
|
||||
first_name = models.CharField(max_length=128, blank=True)
|
||||
last_name = models.CharField(max_length=128, blank=True)
|
||||
gender = models.CharField(max_length=16, blank=True, choices=Gender.choices)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
linkedin_username = models.CharField(max_length=128, blank=True)
|
||||
github_url = models.CharField(max_length=255, blank=True)
|
||||
twitter_url = models.CharField(max_length=255, blank=True)
|
||||
|
||||
# -- Current position --------------------------------------------------
|
||||
job_title = models.CharField(max_length=255, blank=True, db_index=True)
|
||||
job_title_role = models.CharField(
|
||||
max_length=64, blank=True, db_index=True, help_text="Coarse job family, e.g. engineering."
|
||||
)
|
||||
job_title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
seniority_levels = ArrayField(
|
||||
models.CharField(max_length=32),
|
||||
default=list,
|
||||
blank=True,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
)
|
||||
current_company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="current_employees"
|
||||
)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
job_summary = models.TextField(blank=True)
|
||||
job_start_date = models.CharField(max_length=10, blank=True)
|
||||
|
||||
# -- Location ----------------------------------------------------------
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
location_locality = models.CharField(max_length=128, blank=True)
|
||||
location_region = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_continent = models.CharField(max_length=64, blank=True)
|
||||
|
||||
# -- Narrative & metrics ----------------------------------------------
|
||||
summary = models.TextField(blank=True)
|
||||
years_experience = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
inferred_salary = models.CharField(max_length=32, blank=True)
|
||||
connections = models.PositiveIntegerField(null=True, blank=True)
|
||||
interests = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
languages = ArrayField(models.CharField(max_length=64), default=list, blank=True)
|
||||
|
||||
# -- Contact details (PII: masked unless the caller is privileged) ------
|
||||
emails = models.JSONField(default=list, blank=True)
|
||||
phone_numbers = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
|
||||
skills = models.ManyToManyField(Skill, through="ProfileSkill", related_name="profiles")
|
||||
|
||||
# -- Denormalised text, maintained by the importer ---------------------
|
||||
# PostgreSQL's ``SearchVector`` cannot traverse a many-to-many relation, so
|
||||
# the searchable text of the related rows is flattened here once at import
|
||||
# time. This keeps the tsvector update to a single UPDATE statement and
|
||||
# keeps query time O(index lookup) instead of O(joins).
|
||||
skills_text = models.TextField(blank=True, editable=False)
|
||||
companies_text = models.TextField(blank=True, editable=False)
|
||||
education_text = models.TextField(blank=True, editable=False)
|
||||
|
||||
# Materialised tsvector powering the PostgreSQL search backend.
|
||||
search_vector = SearchVectorField(null=True, editable=False)
|
||||
|
||||
class Meta:
|
||||
ordering = ("full_name", "id")
|
||||
indexes = [
|
||||
GinIndex(fields=["search_vector"], name="profile_search_vector_gin"),
|
||||
GinIndex(fields=["seniority_levels"], name="profile_seniority_gin"),
|
||||
models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["external_id"],
|
||||
condition=models.Q(external_id__gt=""),
|
||||
name="uniq_profile_external_id",
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.full_name
|
||||
|
||||
@property
|
||||
def headline(self) -> str:
|
||||
"""Short "Title at Company" line used in result cards."""
|
||||
company = self.current_company.name if self.current_company_id else ""
|
||||
if self.job_title and company:
|
||||
return f"{self.job_title} at {company}"
|
||||
return self.job_title or company
|
||||
|
||||
|
||||
class ProfileSkill(models.Model):
|
||||
"""Through table - keeps the original ordering of the skill list."""
|
||||
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
skill = models.ForeignKey(Skill, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
position = models.PositiveSmallIntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
ordering = ("position",)
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["profile", "skill"], name="uniq_profile_skill")
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.profile_id}:{self.skill_id}"
|
||||
|
||||
|
||||
class Experience(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="experiences")
|
||||
company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="experiences"
|
||||
)
|
||||
company_name = models.CharField(max_length=255, blank=True)
|
||||
title = models.CharField(max_length=255, blank=True)
|
||||
title_role = models.CharField(max_length=64, blank=True)
|
||||
title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
levels = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
start_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True)
|
||||
is_primary = models.BooleanField(default=False, db_index=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-is_primary", "-start_year", "id")
|
||||
indexes = [models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.title} @ {self.company_name}"
|
||||
|
||||
|
||||
class Education(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="educations")
|
||||
school = models.ForeignKey(
|
||||
School, null=True, blank=True, on_delete=models.SET_NULL, related_name="educations"
|
||||
)
|
||||
school_name = models.CharField(max_length=255, blank=True)
|
||||
degrees = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
majors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
minors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
gpa = models.DecimalField(max_digits=4, decimal_places=2, null=True, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-end_year", "id")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.school_name}"
|
||||
@@ -0,0 +1,46 @@
|
||||
"""
|
||||
Contact-detail masking.
|
||||
|
||||
The dataset contains real personal data (e-mail addresses, mobile numbers,
|
||||
street addresses). Exposing it to every authenticated user would be both a
|
||||
privacy and a compliance problem, so contact details are masked unless the
|
||||
caller holds a role that is explicitly allowed to see them
|
||||
(:pyattr:`apps.accounts.models.User.can_view_pii`).
|
||||
|
||||
Masking happens in the serialisation layer - the raw values never leave the
|
||||
database for an unprivileged caller, and the search index stores them in a
|
||||
non-searchable field so a keyword query can never be used to confirm an e-mail
|
||||
address by brute force.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_EMAIL_RE = re.compile(r"^([^@]+)@(.+)$")
|
||||
|
||||
|
||||
def mask_email(address: str) -> str:
|
||||
"""``joseph.holland@garver.com`` -> ``jo************@garver.com``."""
|
||||
match = _EMAIL_RE.match(address or "")
|
||||
if not match:
|
||||
return "***"
|
||||
local, domain = match.groups()
|
||||
keep = min(2, max(len(local) - 1, 0))
|
||||
return f"{local[:keep]}{'*' * max(len(local) - keep, 1)}@{domain}"
|
||||
|
||||
|
||||
def mask_phone(number: str) -> str:
|
||||
"""``+19402058928`` -> ``+1940*****28``."""
|
||||
digits = (number or "").strip()
|
||||
if len(digits) < 6:
|
||||
return "***"
|
||||
return f"{digits[:5]}{'*' * (len(digits) - 7)}{digits[-2:]}"
|
||||
|
||||
|
||||
def mask_emails(emails: list[dict[str, str]]) -> list[dict[str, str]]:
|
||||
return [{**item, "address": mask_email(item.get("address", ""))} for item in emails or []]
|
||||
|
||||
|
||||
def mask_phones(numbers: list[str]) -> list[str]:
|
||||
return [mask_phone(number) for number in numbers or []]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Read-only serializers for profile payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.profiles.models import Company, Education, Experience, Profile, School, Skill
|
||||
from apps.profiles.privacy import mask_emails, mask_phones
|
||||
|
||||
|
||||
class SkillSerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = Skill
|
||||
fields = ("id", "name", "slug", "profile_count")
|
||||
|
||||
|
||||
class CompanySerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = Company
|
||||
fields = ("id", "name", "industry", "size", "founded", "website", "location_name")
|
||||
|
||||
|
||||
class SchoolSerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = School
|
||||
fields = ("id", "name", "type", "location_name")
|
||||
|
||||
|
||||
class ExperienceSerializer(serializers.ModelSerializer):
|
||||
company = CompanySerializer(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = Experience
|
||||
fields = (
|
||||
"id", "title", "title_role", "title_sub_role", "levels", "company", "company_name",
|
||||
"start_date", "end_date", "start_year", "end_year", "is_primary", "location_name",
|
||||
"summary",
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class EducationSerializer(serializers.ModelSerializer):
|
||||
school = SchoolSerializer(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = Education
|
||||
fields = (
|
||||
"id", "school", "school_name", "degrees", "majors", "minors", "gpa",
|
||||
"start_date", "end_date", "end_year",
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class PIIMixin:
|
||||
"""Masks contact details unless the caller is allowed to see them."""
|
||||
|
||||
def _can_view_pii(self) -> bool:
|
||||
request = self.context.get("request") # type: ignore[attr-defined]
|
||||
user = getattr(request, "user", None)
|
||||
return bool(user and user.is_authenticated and user.can_view_pii)
|
||||
|
||||
def get_emails(self, obj: Profile) -> list[dict[str, str]]:
|
||||
return obj.emails if self._can_view_pii() else mask_emails(obj.emails)
|
||||
|
||||
def get_phone_numbers(self, obj: Profile) -> list[str]:
|
||||
return obj.phone_numbers if self._can_view_pii() else mask_phones(obj.phone_numbers)
|
||||
|
||||
|
||||
class ProfileListSerializer(PIIMixin, serializers.ModelSerializer):
|
||||
"""Compact representation used for result cards."""
|
||||
|
||||
headline = serializers.CharField(read_only=True)
|
||||
company = serializers.CharField(source="current_company.name", default="", read_only=True)
|
||||
skills = serializers.SerializerMethodField()
|
||||
emails = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Profile
|
||||
fields = (
|
||||
"id", "full_name", "headline", "job_title", "job_title_role", "seniority_levels",
|
||||
"company", "industry", "location_name", "location_country", "years_experience",
|
||||
"linkedin_url", "skills", "emails",
|
||||
) # fmt: skip
|
||||
|
||||
def get_skills(self, obj: Profile) -> list[str]:
|
||||
# ``prefetch_related`` in the view keeps this at one extra query total.
|
||||
return [link.skill.name for link in obj.profile_skills.all()[:10]]
|
||||
|
||||
|
||||
class ProfileDetailSerializer(PIIMixin, serializers.ModelSerializer):
|
||||
headline = serializers.CharField(read_only=True)
|
||||
current_company = CompanySerializer(read_only=True)
|
||||
skills = serializers.SerializerMethodField()
|
||||
experiences = ExperienceSerializer(many=True, read_only=True)
|
||||
educations = EducationSerializer(many=True, read_only=True)
|
||||
emails = serializers.SerializerMethodField()
|
||||
phone_numbers = serializers.SerializerMethodField()
|
||||
pii_masked = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Profile
|
||||
fields = (
|
||||
"id", "external_id", "full_name", "first_name", "last_name", "gender", "headline",
|
||||
"job_title", "job_title_role", "job_title_sub_role", "seniority_levels",
|
||||
"current_company", "industry", "job_summary", "job_start_date",
|
||||
"location_name", "location_locality", "location_region", "location_country",
|
||||
"location_continent", "summary", "years_experience", "inferred_salary",
|
||||
"connections", "interests", "languages", "linkedin_url", "linkedin_username",
|
||||
"github_url", "twitter_url", "skills", "experiences", "educations",
|
||||
"emails", "phone_numbers", "pii_masked",
|
||||
) # fmt: skip
|
||||
|
||||
def get_skills(self, obj: Profile) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": link.skill.name, "slug": link.skill.slug} for link in obj.profile_skills.all()
|
||||
]
|
||||
|
||||
def get_pii_masked(self, obj: Profile) -> bool: # noqa: ARG002
|
||||
return not self._can_view_pii()
|
||||
@@ -0,0 +1,11 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.profiles.views import ProfileDetailView, ProfileListView, SkillListView
|
||||
|
||||
app_name = "profiles"
|
||||
|
||||
urlpatterns = [
|
||||
path("", ProfileListView.as_view(), name="list"),
|
||||
path("skills/", SkillListView.as_view(), name="skills"),
|
||||
path("<int:pk>/", ProfileDetailView.as_view(), name="detail"),
|
||||
]
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Profile read endpoints (the search endpoints live in ``apps.search``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.db.models import QuerySet
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import generics
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
|
||||
from apps.profiles.models import Profile, Skill
|
||||
from apps.profiles.serializers import (
|
||||
ProfileDetailSerializer,
|
||||
ProfileListSerializer,
|
||||
SkillSerializer,
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class ProfileListView(generics.ListAPIView):
|
||||
"""
|
||||
Plain, unranked listing.
|
||||
|
||||
Useful for browsing and for verifying the dataset; ranked keyword search is
|
||||
served by ``/api/v1/search/``.
|
||||
"""
|
||||
|
||||
serializer_class = ProfileListSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_queryset(self) -> QuerySet[Profile]:
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill")
|
||||
.order_by("full_name", "id")
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class ProfileDetailView(generics.RetrieveAPIView):
|
||||
"""Full profile including work history, education and (masked) contacts."""
|
||||
|
||||
serializer_class = ProfileDetailSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_queryset(self) -> QuerySet[Profile]:
|
||||
return Profile.objects.select_related("current_company").prefetch_related(
|
||||
"profile_skills__skill",
|
||||
"experiences__company",
|
||||
"educations__school",
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class SkillListView(generics.ListAPIView):
|
||||
"""
|
||||
Skill dictionary ordered by popularity.
|
||||
|
||||
The frontend uses it to power the skill filter's type-ahead without hitting
|
||||
Elasticsearch for every keystroke.
|
||||
"""
|
||||
|
||||
serializer_class = SkillSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
pagination_class = None
|
||||
|
||||
def get_queryset(self) -> QuerySet[Skill]:
|
||||
queryset = Skill.objects.filter(profile_count__gt=0)
|
||||
term = self.request.query_params.get("q", "").strip().lower()
|
||||
if term:
|
||||
queryset = queryset.filter(name__icontains=term[:64])
|
||||
limit = min(int(self.request.query_params.get("limit", 50) or 50), 200)
|
||||
return queryset.order_by("-profile_count", "name")[:limit]
|
||||
Reference in New Issue
Block a user