262 lines
11 KiB
Python
262 lines
11 KiB
Python
|
"""
|
||
|
|
Domain model for the LinkedIn dataset.
|
||
|
|
|
||
|
|
The raw export is one very wide CSV row per person with several columns holding
|
||
|
|
serialised lists of dictionaries. Storing that blob as-is would make filtering
|
||
|
|
and aggregation painful, so it is normalised into five tables:
|
||
|
|
|
||
|
|
Company <--- Experience ---> Profile <--- Education ---> School
|
||
|
|
|
|
||
|
|
+--- ProfileSkill ---> Skill
|
||
|
|
|
||
|
|
PostgreSQL is the source of truth. Elasticsearch holds a denormalised
|
||
|
|
projection of the same data for search (see ``apps.search``); it can always be
|
||
|
|
rebuilt from these tables, which is why the index is disposable.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from django.contrib.postgres.fields import ArrayField
|
||
|
|
from django.contrib.postgres.indexes import GinIndex
|
||
|
|
from django.contrib.postgres.search import SearchVectorField
|
||
|
|
from django.db import models
|
||
|
|
from django.utils.text import slugify
|
||
|
|
|
||
|
|
|
||
|
|
class TimeStampedModel(models.Model):
|
||
|
|
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
|
||
|
|
updated_at = models.DateTimeField(auto_now=True)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
abstract = True
|
||
|
|
|
||
|
|
|
||
|
|
class Company(TimeStampedModel):
|
||
|
|
"""An employer referenced by the current job or by a past experience."""
|
||
|
|
|
||
|
|
external_id = models.CharField(max_length=128, blank=True, db_index=True)
|
||
|
|
name = models.CharField(max_length=255, unique=True)
|
||
|
|
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||
|
|
size = models.CharField(max_length=32, blank=True, help_text="Head-count bucket, e.g. 51-200")
|
||
|
|
founded = models.PositiveIntegerField(null=True, blank=True)
|
||
|
|
website = models.CharField(max_length=255, blank=True)
|
||
|
|
linkedin_url = models.CharField(max_length=255, blank=True)
|
||
|
|
location_name = models.CharField(max_length=255, blank=True)
|
||
|
|
country = models.CharField(max_length=128, blank=True, db_index=True)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("name",)
|
||
|
|
verbose_name_plural = "companies"
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return self.name
|
||
|
|
|
||
|
|
|
||
|
|
class School(TimeStampedModel):
|
||
|
|
name = models.CharField(max_length=255, unique=True)
|
||
|
|
type = models.CharField(max_length=64, blank=True)
|
||
|
|
website = models.CharField(max_length=255, blank=True)
|
||
|
|
linkedin_url = models.CharField(max_length=255, blank=True)
|
||
|
|
location_name = models.CharField(max_length=255, blank=True)
|
||
|
|
country = models.CharField(max_length=128, blank=True)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("name",)
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return self.name
|
||
|
|
|
||
|
|
|
||
|
|
class Skill(TimeStampedModel):
|
||
|
|
"""
|
||
|
|
A normalised skill.
|
||
|
|
|
||
|
|
``slug`` is the deduplication key: the raw data contains the same skill with
|
||
|
|
different casing and spacing ("Team Building" / "team building").
|
||
|
|
"""
|
||
|
|
|
||
|
|
name = models.CharField(max_length=128)
|
||
|
|
slug = models.SlugField(max_length=140, unique=True)
|
||
|
|
profile_count = models.PositiveIntegerField(
|
||
|
|
default=0, db_index=True, help_text="Denormalised counter used to rank filter suggestions."
|
||
|
|
)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("-profile_count", "name")
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return self.name
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def make_slug(name: str) -> str:
|
||
|
|
return slugify(name)[:140] or name.strip().lower()[:140]
|
||
|
|
|
||
|
|
|
||
|
|
class Profile(TimeStampedModel):
|
||
|
|
"""A single person in the dataset."""
|
||
|
|
|
||
|
|
class Gender(models.TextChoices):
|
||
|
|
MALE = "male", "Male"
|
||
|
|
FEMALE = "female", "Female"
|
||
|
|
UNKNOWN = "", "Unknown"
|
||
|
|
|
||
|
|
# -- Identity ----------------------------------------------------------
|
||
|
|
external_id = models.CharField(
|
||
|
|
max_length=64,
|
||
|
|
blank=True,
|
||
|
|
db_index=True,
|
||
|
|
help_text="LinkedIn numeric id from the source export.",
|
||
|
|
)
|
||
|
|
full_name = models.CharField(max_length=255, db_index=True)
|
||
|
|
first_name = models.CharField(max_length=128, blank=True)
|
||
|
|
last_name = models.CharField(max_length=128, blank=True)
|
||
|
|
gender = models.CharField(max_length=16, blank=True, choices=Gender.choices)
|
||
|
|
linkedin_url = models.CharField(max_length=255, blank=True)
|
||
|
|
linkedin_username = models.CharField(max_length=128, blank=True)
|
||
|
|
github_url = models.CharField(max_length=255, blank=True)
|
||
|
|
twitter_url = models.CharField(max_length=255, blank=True)
|
||
|
|
|
||
|
|
# -- Current position --------------------------------------------------
|
||
|
|
job_title = models.CharField(max_length=255, blank=True, db_index=True)
|
||
|
|
job_title_role = models.CharField(
|
||
|
|
max_length=64, blank=True, db_index=True, help_text="Coarse job family, e.g. engineering."
|
||
|
|
)
|
||
|
|
job_title_sub_role = models.CharField(max_length=64, blank=True)
|
||
|
|
seniority_levels = ArrayField(
|
||
|
|
models.CharField(max_length=32),
|
||
|
|
default=list,
|
||
|
|
blank=True,
|
||
|
|
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||
|
|
)
|
||
|
|
current_company = models.ForeignKey(
|
||
|
|
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="current_employees"
|
||
|
|
)
|
||
|
|
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||
|
|
job_summary = models.TextField(blank=True)
|
||
|
|
job_start_date = models.CharField(max_length=10, blank=True)
|
||
|
|
|
||
|
|
# -- Location ----------------------------------------------------------
|
||
|
|
location_name = models.CharField(max_length=255, blank=True)
|
||
|
|
location_locality = models.CharField(max_length=128, blank=True)
|
||
|
|
location_region = models.CharField(max_length=128, blank=True, db_index=True)
|
||
|
|
location_country = models.CharField(max_length=128, blank=True, db_index=True)
|
||
|
|
location_continent = models.CharField(max_length=64, blank=True)
|
||
|
|
|
||
|
|
# -- Narrative & metrics ----------------------------------------------
|
||
|
|
summary = models.TextField(blank=True)
|
||
|
|
years_experience = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||
|
|
inferred_salary = models.CharField(max_length=32, blank=True)
|
||
|
|
connections = models.PositiveIntegerField(null=True, blank=True)
|
||
|
|
interests = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||
|
|
languages = ArrayField(models.CharField(max_length=64), default=list, blank=True)
|
||
|
|
|
||
|
|
# -- Contact details (PII: masked unless the caller is privileged) ------
|
||
|
|
emails = models.JSONField(default=list, blank=True)
|
||
|
|
phone_numbers = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||
|
|
|
||
|
|
skills = models.ManyToManyField(Skill, through="ProfileSkill", related_name="profiles")
|
||
|
|
|
||
|
|
# -- Denormalised text, maintained by the importer ---------------------
|
||
|
|
# PostgreSQL's ``SearchVector`` cannot traverse a many-to-many relation, so
|
||
|
|
# the searchable text of the related rows is flattened here once at import
|
||
|
|
# time. This keeps the tsvector update to a single UPDATE statement and
|
||
|
|
# keeps query time O(index lookup) instead of O(joins).
|
||
|
|
skills_text = models.TextField(blank=True, editable=False)
|
||
|
|
companies_text = models.TextField(blank=True, editable=False)
|
||
|
|
education_text = models.TextField(blank=True, editable=False)
|
||
|
|
|
||
|
|
# Materialised tsvector powering the PostgreSQL search backend.
|
||
|
|
search_vector = SearchVectorField(null=True, editable=False)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("full_name", "id")
|
||
|
|
indexes = [
|
||
|
|
GinIndex(fields=["search_vector"], name="profile_search_vector_gin"),
|
||
|
|
GinIndex(fields=["seniority_levels"], name="profile_seniority_gin"),
|
||
|
|
models.Index(
|
||
|
|
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||
|
|
),
|
||
|
|
]
|
||
|
|
constraints = [
|
||
|
|
models.UniqueConstraint(
|
||
|
|
fields=["external_id"],
|
||
|
|
condition=models.Q(external_id__gt=""),
|
||
|
|
name="uniq_profile_external_id",
|
||
|
|
)
|
||
|
|
]
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return self.full_name
|
||
|
|
|
||
|
|
@property
|
||
|
|
def headline(self) -> str:
|
||
|
|
"""Short "Title at Company" line used in result cards."""
|
||
|
|
company = self.current_company.name if self.current_company_id else ""
|
||
|
|
if self.job_title and company:
|
||
|
|
return f"{self.job_title} at {company}"
|
||
|
|
return self.job_title or company
|
||
|
|
|
||
|
|
|
||
|
|
class ProfileSkill(models.Model):
|
||
|
|
"""Through table - keeps the original ordering of the skill list."""
|
||
|
|
|
||
|
|
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="profile_skills")
|
||
|
|
skill = models.ForeignKey(Skill, on_delete=models.CASCADE, related_name="profile_skills")
|
||
|
|
position = models.PositiveSmallIntegerField(default=0)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("position",)
|
||
|
|
constraints = [
|
||
|
|
models.UniqueConstraint(fields=["profile", "skill"], name="uniq_profile_skill")
|
||
|
|
]
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return f"{self.profile_id}:{self.skill_id}"
|
||
|
|
|
||
|
|
|
||
|
|
class Experience(TimeStampedModel):
|
||
|
|
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="experiences")
|
||
|
|
company = models.ForeignKey(
|
||
|
|
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="experiences"
|
||
|
|
)
|
||
|
|
company_name = models.CharField(max_length=255, blank=True)
|
||
|
|
title = models.CharField(max_length=255, blank=True)
|
||
|
|
title_role = models.CharField(max_length=64, blank=True)
|
||
|
|
title_sub_role = models.CharField(max_length=64, blank=True)
|
||
|
|
levels = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||
|
|
start_date = models.CharField(max_length=10, blank=True)
|
||
|
|
end_date = models.CharField(max_length=10, blank=True)
|
||
|
|
start_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||
|
|
end_year = models.PositiveSmallIntegerField(null=True, blank=True)
|
||
|
|
is_primary = models.BooleanField(default=False, db_index=True)
|
||
|
|
location_name = models.CharField(max_length=255, blank=True)
|
||
|
|
summary = models.TextField(blank=True)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("-is_primary", "-start_year", "id")
|
||
|
|
indexes = [models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")]
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return f"{self.title} @ {self.company_name}"
|
||
|
|
|
||
|
|
|
||
|
|
class Education(TimeStampedModel):
|
||
|
|
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="educations")
|
||
|
|
school = models.ForeignKey(
|
||
|
|
School, null=True, blank=True, on_delete=models.SET_NULL, related_name="educations"
|
||
|
|
)
|
||
|
|
school_name = models.CharField(max_length=255, blank=True)
|
||
|
|
degrees = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||
|
|
majors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||
|
|
minors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||
|
|
gpa = models.DecimalField(max_digits=4, decimal_places=2, null=True, blank=True)
|
||
|
|
start_date = models.CharField(max_length=10, blank=True)
|
||
|
|
end_date = models.CharField(max_length=10, blank=True)
|
||
|
|
end_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||
|
|
|
||
|
|
class Meta:
|
||
|
|
ordering = ("-end_year", "id")
|
||
|
|
|
||
|
|
def __str__(self) -> str:
|
||
|
|
return f"{self.school_name}"
|