feat: LinkedIn profile search engine (Django + React + Elasticsearch + ELK)
Full-stack implementation of the Cyberyan full-stack trial task. Backend (Django 5 / DRF) - Normalised PostgreSQL model (Profile, Company, School, Skill, Experience, Education) as the source of truth, with GIN + trigram indexes. - Self-healing CSV reader: the supplied export carries an injected watermark column, missing columns, rows split by broken quoting and a duplicated header. A signature-scored Needleman-Wunsch alignment reconstructs 284 clean records out of 336 physical rows; only 10 rows parse correctly without it. - Pluggable search layer (Strategy): Elasticsearch by default, PostgreSQL full-text search as a fallback selected by a single environment variable. - Keyword search with per-field boosts, fuzziness and phrase boosting, six faceted filters (OR within a filter, AND across filters), an experience range, four sort orders, highlighting and facet aggregations. - JWT auth with rotation + blacklist, three roles, PII masking, scoped rate limits, a single error envelope, OpenAPI schema and health probes. Frontend (React 18 / TypeScript / Vite) - Debounced search with type-ahead, faceted filter panel, result cards, profile drawer, URL-synced search state, automatic token refresh. - Highlight snippets rendered without dangerouslySetInnerHTML. Infrastructure - Hardened multi-stage images (non-root, read-only rootfs, healthchecks). - docker-compose for the full stack plus dev and production overlays. - ELK: structured JSON logs shipped asynchronously to Logstash, routed per event type, with a composable index template and a Kibana setup script. Quality - 61 backend tests (85% coverage), ruff/black/bandit/mypy, ESLint + tsc, GitHub Actions running lint, tests and both image builds. - Persian documentation: README plus architecture, search, API, data, security and operations guides. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01GnFo6fNaHCTAkJbfcdkqtG
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
# Keep build contexts small and secrets out of images.
|
||||
.git/
|
||||
.github/
|
||||
docs/
|
||||
**/node_modules/
|
||||
**/dist/
|
||||
**/__pycache__/
|
||||
**/*.py[cod]
|
||||
**/.venv/
|
||||
**/venv/
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
**/staticfiles/
|
||||
**/htmlcov/
|
||||
**/.coverage
|
||||
**/.pytest_cache/
|
||||
**/.mypy_cache/
|
||||
**/.ruff_cache/
|
||||
backend/tests/
|
||||
*.log
|
||||
*.zip
|
||||
@@ -0,0 +1,86 @@
|
||||
# =============================================================================
|
||||
# Copy to .env and adjust. Never commit the real .env.
|
||||
# cp .env.example .env
|
||||
#
|
||||
# Generate a strong secret:
|
||||
# python -c "import secrets; print(secrets.token_urlsafe(64))"
|
||||
# =============================================================================
|
||||
|
||||
# --- Django ------------------------------------------------------------------
|
||||
DJANGO_SETTINGS_MODULE=config.settings.prod
|
||||
DJANGO_SECRET_KEY=change-me-to-a-64-character-random-string-before-any-real-deployment
|
||||
DJANGO_DEBUG=false
|
||||
DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1,backend
|
||||
DJANGO_ADMIN_URL=admin/
|
||||
ENVIRONMENT=development
|
||||
SERVICE_NAME=linkedin-search-api
|
||||
SECURE_SSL_REDIRECT=false
|
||||
|
||||
# The SPA is served from the same origin through nginx, so this list only
|
||||
# matters when you run `npm run dev` against the containerised API.
|
||||
CORS_ALLOWED_ORIGINS=http://localhost:8080,http://localhost:5173
|
||||
CSRF_TRUSTED_ORIGINS=http://localhost:8080,http://localhost:5173
|
||||
|
||||
# --- PostgreSQL --------------------------------------------------------------
|
||||
POSTGRES_DB=linkedin
|
||||
POSTGRES_USER=linkedin
|
||||
POSTGRES_PASSWORD=change-me-postgres
|
||||
POSTGRES_HOST=postgres
|
||||
POSTGRES_PORT=5432
|
||||
|
||||
# --- Redis (cache + throttling counters) -------------------------------------
|
||||
REDIS_URL=redis://redis:6379/1
|
||||
|
||||
# --- JWT ---------------------------------------------------------------------
|
||||
JWT_ACCESS_MINUTES=15
|
||||
JWT_REFRESH_DAYS=7
|
||||
|
||||
# --- Search ------------------------------------------------------------------
|
||||
# "elasticsearch" (default) or "postgres" to run without a cluster.
|
||||
SEARCH_BACKEND=elasticsearch
|
||||
ELASTICSEARCH_HOST=elasticsearch
|
||||
ELASTICSEARCH_PORT=9200
|
||||
ELASTICSEARCH_HOSTS=http://elasticsearch:9200
|
||||
ELASTICSEARCH_INDEX=linkedin-profiles
|
||||
ELASTICSEARCH_TIMEOUT=10
|
||||
# Only needed when the cluster has authentication enabled (docker-compose.prod.yml).
|
||||
ELASTIC_PASSWORD=change-me-elastic
|
||||
ELASTICSEARCH_USERNAME=
|
||||
ELASTICSEARCH_PASSWORD=
|
||||
|
||||
# --- Logging / ELK -----------------------------------------------------------
|
||||
LOG_LEVEL=INFO
|
||||
LOGSTASH_ENABLED=true
|
||||
LOGSTASH_HOST=logstash
|
||||
LOGSTASH_PORT=5959
|
||||
|
||||
# --- Rate limiting -----------------------------------------------------------
|
||||
THROTTLE_ANON=30/min
|
||||
THROTTLE_USER=300/min
|
||||
THROTTLE_SEARCH=120/min
|
||||
THROTTLE_LOGIN=10/min
|
||||
|
||||
# --- Bootstrap ---------------------------------------------------------------
|
||||
# The API container imports the dataset and builds the index on first start.
|
||||
SEED_DATASET=true
|
||||
SEED_USERS=true
|
||||
RUN_MIGRATIONS=true
|
||||
DATASET_PATH=/data/linkedin_profiles.csv
|
||||
|
||||
# Demo accounts created by `manage.py seed_users` (used by the README walkthrough).
|
||||
DEMO_ADMIN_EMAIL=admin@example.com
|
||||
DEMO_ADMIN_PASSWORD=Admin-Passw0rd!
|
||||
DEMO_ANALYST_EMAIL=analyst@example.com
|
||||
DEMO_ANALYST_PASSWORD=Analyst-Passw0rd!
|
||||
DEMO_VIEWER_EMAIL=viewer@example.com
|
||||
DEMO_VIEWER_PASSWORD=Viewer-Passw0rd!
|
||||
|
||||
# --- Resource sizing ---------------------------------------------------------
|
||||
ES_JAVA_OPTS=-Xms512m -Xmx512m
|
||||
LS_JAVA_OPTS=-Xms256m -Xmx256m
|
||||
GUNICORN_WORKERS=3
|
||||
|
||||
# --- Published ports ---------------------------------------------------------
|
||||
FRONTEND_PORT=8080
|
||||
BACKEND_PORT=8000
|
||||
KIBANA_PORT=5601
|
||||
@@ -0,0 +1,148 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, master]
|
||||
pull_request:
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.12"
|
||||
NODE_VERSION: "22"
|
||||
|
||||
jobs:
|
||||
backend:
|
||||
name: Backend (lint, types, tests)
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: linkedin_test
|
||||
ports: ["5432:5432"]
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
|
||||
env:
|
||||
DJANGO_SECRET_KEY: ci-secret-key-only-used-by-the-test-suite-0123456789
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/linkedin_test
|
||||
DJANGO_SETTINGS_MODULE: config.settings.test
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: backend
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
cache: pip
|
||||
cache-dependency-path: backend/requirements/*.txt
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install -r requirements/dev.txt
|
||||
|
||||
- name: Ruff
|
||||
run: ruff check .
|
||||
|
||||
- name: Black
|
||||
run: black --check .
|
||||
|
||||
- name: Mypy
|
||||
run: mypy apps config
|
||||
continue-on-error: true # advisory until the codebase is fully annotated
|
||||
|
||||
- name: Bandit (security lint)
|
||||
run: bandit -q -c pyproject.toml -r apps config
|
||||
|
||||
- name: Django system checks
|
||||
run: python manage.py check --deploy --fail-level ERROR
|
||||
env:
|
||||
DJANGO_SETTINGS_MODULE: config.settings.prod
|
||||
DJANGO_SECRET_KEY: ci-secret-key-long-enough-for-the-production-guard-0123456789abcdef
|
||||
DJANGO_DEBUG: "false"
|
||||
|
||||
- name: Missing migrations?
|
||||
run: python manage.py makemigrations --check --dry-run
|
||||
|
||||
- name: Tests
|
||||
run: pytest --cov=apps --cov-report=xml --cov-report=term-missing
|
||||
|
||||
- name: Upload coverage
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: backend-coverage
|
||||
path: backend/coverage.xml
|
||||
if-no-files-found: ignore
|
||||
|
||||
frontend:
|
||||
name: Frontend (lint, types, build)
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
cache: npm
|
||||
cache-dependency-path: frontend/package-lock.json
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci --no-audit --no-fund
|
||||
|
||||
- name: ESLint
|
||||
run: npm run lint
|
||||
|
||||
- name: TypeScript
|
||||
run: npm run typecheck
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
images:
|
||||
name: Docker images build
|
||||
runs-on: ubuntu-latest
|
||||
needs: [backend, frontend]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Validate the compose files
|
||||
run: |
|
||||
cp .env.example .env
|
||||
docker compose config --quiet
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml config --quiet
|
||||
|
||||
- name: Build the API image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: backend/Dockerfile
|
||||
push: false
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Build the SPA image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: frontend/Dockerfile
|
||||
push: false
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
# Environment / secrets
|
||||
.env
|
||||
.env.local
|
||||
*.pem
|
||||
*.key
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.venv/
|
||||
venv/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.pytest_cache/
|
||||
htmlcov/
|
||||
.coverage
|
||||
coverage.xml
|
||||
backend/staticfiles/
|
||||
|
||||
# Node
|
||||
node_modules/
|
||||
frontend/dist/
|
||||
npm-debug.log*
|
||||
|
||||
# Editors / OS
|
||||
.vscode/
|
||||
.idea/
|
||||
.DS_Store
|
||||
|
||||
# Build artefacts
|
||||
*.zip
|
||||
*.tar.gz
|
||||
@@ -0,0 +1,39 @@
|
||||
default_language_version:
|
||||
python: python3.12
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v4.6.0
|
||||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
- id: check-yaml
|
||||
args: [--allow-multiple-documents]
|
||||
- id: check-json
|
||||
- id: check-merge-conflict
|
||||
- id: check-added-large-files
|
||||
args: [--maxkb=6000]
|
||||
- id: detect-private-key
|
||||
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.6.9
|
||||
hooks:
|
||||
# Ruff lints and sorts imports; formatting is left to black so there is
|
||||
# exactly one formatter in the toolchain.
|
||||
- id: ruff
|
||||
args: [--fix]
|
||||
files: ^backend/
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.10.0
|
||||
hooks:
|
||||
- id: black
|
||||
files: ^backend/
|
||||
|
||||
- repo: https://github.com/PyCQA/bandit
|
||||
rev: 1.7.10
|
||||
hooks:
|
||||
- id: bandit
|
||||
args: [-q, -c, backend/pyproject.toml]
|
||||
files: ^backend/apps/
|
||||
additional_dependencies: ["bandit[toml]"]
|
||||
@@ -0,0 +1,106 @@
|
||||
# =============================================================================
|
||||
# Task runner. `make help` lists everything.
|
||||
# =============================================================================
|
||||
.DEFAULT_GOAL := help
|
||||
COMPOSE := docker compose
|
||||
COMPOSE_DEV := docker compose -f docker-compose.yml -f docker-compose.dev.yml
|
||||
COMPOSE_PROD := docker compose -f docker-compose.yml -f docker-compose.prod.yml
|
||||
|
||||
.PHONY: help
|
||||
help: ## Show this help
|
||||
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \
|
||||
| awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}'
|
||||
|
||||
# --- Lifecycle ---------------------------------------------------------------
|
||||
.PHONY: env
|
||||
env: ## Create .env from the template (never overwrites an existing one)
|
||||
@test -f .env || (cp .env.example .env && echo "Created .env - review the secrets before deploying.")
|
||||
|
||||
.PHONY: up
|
||||
up: env ## Build and start the whole stack
|
||||
$(COMPOSE) up --build -d
|
||||
@echo "SPA http://localhost:$${FRONTEND_PORT:-8080}"
|
||||
@echo "API docs http://localhost:$${FRONTEND_PORT:-8080}/api/docs/"
|
||||
@echo "Kibana http://localhost:$${KIBANA_PORT:-5601}"
|
||||
|
||||
.PHONY: dev
|
||||
dev: env ## Start the stack with hot reload (Vite on :5173, runserver on :8000)
|
||||
$(COMPOSE_DEV) up --build
|
||||
|
||||
.PHONY: prod
|
||||
prod: env ## Start with the production overlay
|
||||
$(COMPOSE_PROD) up --build -d
|
||||
|
||||
.PHONY: down
|
||||
down: ## Stop the stack (keeps volumes)
|
||||
$(COMPOSE) down
|
||||
|
||||
.PHONY: clean
|
||||
clean: ## Stop the stack and delete all volumes
|
||||
$(COMPOSE) down -v --remove-orphans
|
||||
|
||||
.PHONY: logs
|
||||
logs: ## Follow the logs of every service
|
||||
$(COMPOSE) logs -f --tail=100
|
||||
|
||||
.PHONY: ps
|
||||
ps: ## Show container status
|
||||
$(COMPOSE) ps
|
||||
|
||||
# --- Data --------------------------------------------------------------------
|
||||
.PHONY: seed
|
||||
seed: ## Import the dataset and rebuild the search index
|
||||
$(COMPOSE) exec backend python manage.py import_profiles --truncate --reindex
|
||||
|
||||
.PHONY: reindex
|
||||
reindex: ## Rebuild the Elasticsearch index from PostgreSQL
|
||||
$(COMPOSE) exec backend python manage.py search_index --rebuild
|
||||
|
||||
.PHONY: index-status
|
||||
index-status: ## Show index/alias status and document count
|
||||
$(COMPOSE) exec backend python manage.py search_index --status
|
||||
|
||||
.PHONY: users
|
||||
users: ## Create the demo admin / analyst / viewer accounts
|
||||
$(COMPOSE) exec backend python manage.py seed_users
|
||||
|
||||
.PHONY: superuser
|
||||
superuser: ## Create a Django superuser interactively
|
||||
$(COMPOSE) exec backend python manage.py createsuperuser
|
||||
|
||||
.PHONY: shell
|
||||
shell: ## Open a Django shell in the API container
|
||||
$(COMPOSE) exec backend python manage.py shell
|
||||
|
||||
.PHONY: kibana-setup
|
||||
kibana-setup: ## Register the app-logs data view in Kibana
|
||||
./scripts/kibana_setup.sh
|
||||
|
||||
# --- Quality -----------------------------------------------------------------
|
||||
.PHONY: test
|
||||
test: ## Run the backend test suite inside the stack
|
||||
$(COMPOSE) exec -e DJANGO_SETTINGS_MODULE=config.settings.test backend pytest
|
||||
|
||||
.PHONY: test-local
|
||||
test-local: ## Run the backend test suite against a local PostgreSQL
|
||||
cd backend && pytest
|
||||
|
||||
.PHONY: lint
|
||||
lint: ## Lint and type-check both halves of the project
|
||||
cd backend && ruff check . && black --check . && mypy apps config
|
||||
cd frontend && npm run lint && npm run typecheck
|
||||
|
||||
.PHONY: format
|
||||
format: ## Auto-format the codebase
|
||||
cd backend && ruff check --fix . && black .
|
||||
cd frontend && npm run format
|
||||
|
||||
.PHONY: audit
|
||||
audit: ## Dependency and static security scan
|
||||
cd backend && pip-audit -r requirements/base.txt && bandit -q -r apps config
|
||||
cd frontend && npm audit --omit=dev
|
||||
|
||||
.PHONY: schema
|
||||
schema: ## Write the OpenAPI schema to docs/openapi.yaml
|
||||
$(COMPOSE) exec backend python manage.py spectacular --api-version v1 > docs/openapi.yaml
|
||||
@echo "Wrote docs/openapi.yaml"
|
||||
@@ -0,0 +1,573 @@
|
||||
# موتور جستجوی دیتاست لینکدین
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
یک وباپلیکیشن کامل برای **جستجو و فیلتر روی دیتاست پروفایلهای لینکدین**؛ بکاند Django/DRF، فرانتاند React/TypeScript، موتور جستجو Elasticsearch (با fallback روی PostgreSQL Full-Text Search) و مانیتورینگ با استک ELK — همه بهصورت داکرایزشده و قابل اجرا با یک دستور.
|
||||
|
||||
> این پروژه پاسخ تسک آزمایشی «متخصص فولاستک» مؤسسه Cyberyan است.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## فهرست
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
1. [اجرای سریع](#اجرای-سریع)
|
||||
2. [معماری کلی](#معماری-کلی)
|
||||
3. [منطق سرچ و فیلترها](#منطق-سرچ-و-فیلترها)
|
||||
4. [ساختار ذخیرهسازی داده](#ساختار-ذخیرهسازی-داده)
|
||||
5. [کیفیت دیتاست ورودی](#کیفیت-دیتاست-ورودی)
|
||||
6. [API](#api)
|
||||
7. [امنیت](#امنیت)
|
||||
8. [مانیتورینگ و لاگ (ELK)](#مانیتورینگ-و-لاگ-elk)
|
||||
9. [تست و کیفیت کد](#تست-و-کیفیت-کد)
|
||||
10. [ساختار پروژه](#ساختار-پروژه)
|
||||
11. [مستندات تکمیلی](#مستندات-تکمیلی)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## اجرای سریع
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**پیشنیاز:** فقط Docker و Docker Compose (نسخه ۲ به بالا). هیچ چیز دیگری لازم نیست.
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
git clone <repo> && cd cyberyan-linkedin-search
|
||||
|
||||
cp .env.example .env # مقادیر پیشفرض برای اجرای محلی آماده است
|
||||
docker compose up --build # یا: make up
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
اولین اجرا حدود ۳ تا ۵ دقیقه طول میکشد (بیلد ایمیجها + بالا آمدن Elasticsearch). کانتینر بکاند بهصورت خودکار:
|
||||
|
||||
1. منتظر آماده شدن Postgres و Elasticsearch میماند،
|
||||
2. مایگریشنها را اجرا میکند،
|
||||
3. دیتاست CSV را ایمپورت میکند (۲۸۴ پروفایل یکتا)،
|
||||
4. ایندکس Elasticsearch را میسازد،
|
||||
5. سه کاربر نمونه میسازد.
|
||||
|
||||
سپس:
|
||||
|
||||
| سرویس | آدرس |
|
||||
|---|---|
|
||||
| وباپلیکیشن (SPA) | <http://localhost:8080> |
|
||||
| مستندات تعاملی API (Swagger) | <http://localhost:8080/api/docs/> |
|
||||
| مستندات ReDoc | <http://localhost:8080/api/redoc/> |
|
||||
| پنل ادمین جنگو | <http://localhost:8080/admin/> |
|
||||
| Kibana | <http://localhost:5601> |
|
||||
|
||||
**کاربران نمونه** (از `.env` خوانده میشوند):
|
||||
|
||||
| ایمیل | رمز | نقش | دسترسی |
|
||||
|---|---|---|---|
|
||||
| `admin@example.com` | `Admin-Passw0rd!` | admin | همهچیز + بازسازی ایندکس + دیدن اطلاعات تماس |
|
||||
| `analyst@example.com` | `Analyst-Passw0rd!` | analyst | سرچ + دیدن اطلاعات تماس |
|
||||
| `viewer@example.com` | `Viewer-Passw0rd!` | viewer | سرچ، اطلاعات تماس **ماسکشده** |
|
||||
|
||||
> در محیط واقعی حتماً `DJANGO_SECRET_KEY`، `POSTGRES_PASSWORD` و رمزهای نمونه را عوض کنید. فایل `.env.example` دستور تولید کلید امن را هم دارد.
|
||||
|
||||
### دستورهای پرکاربرد
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
make help # فهرست همه دستورها
|
||||
make up # بالا آوردن کل استک
|
||||
make dev # اجرا با hot-reload (Vite روی :5173، runserver روی :8000)
|
||||
make down # خاموش کردن
|
||||
make clean # خاموش کردن + پاک کردن volumeها
|
||||
make logs # دیدن لاگ همه سرویسها
|
||||
make seed # ایمپورت مجدد دیتاست + بازسازی ایندکس
|
||||
make reindex # فقط بازسازی ایندکس Elasticsearch
|
||||
make test # اجرای تستهای بکاند
|
||||
make lint # lint + type-check هر دو بخش
|
||||
make kibana-setup # ساخت data view لاگها در Kibana
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### اجرا بدون Elasticsearch
|
||||
|
||||
اگر میخواهید سبکتر اجرا کنید (مثلاً برای مرور سریع کد)، کافی است `SEARCH_BACKEND=postgres` را در `.env` بگذارید. در این حالت سرچ روی Full-Text Search خود PostgreSQL اجرا میشود و کل API و فرانتاند بدون هیچ تغییری کار میکنند. دلیل این طراحی در [بخش سرچ](#منطق-سرچ-و-فیلترها) توضیح داده شده است.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## معماری کلی
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────────┐
|
||||
مرورگر ────────▶│ nginx (frontend) :8080 │
|
||||
│ • سرو کردن باندل React │
|
||||
│ • پراکسی /api → backend (بدون CORS) │
|
||||
└───────────────┬──────────────────────────────┘
|
||||
│
|
||||
┌───────────────▼──────────────────────────────┐
|
||||
│ Django + DRF + gunicorn :8000 │
|
||||
│ ┌────────────────────────────────────────┐ │
|
||||
│ │ apps.accounts JWT / نقشها │ │
|
||||
│ │ apps.profiles مدل دامنه + ایمپورتر │ │
|
||||
│ │ apps.search لایه سرچ (Strategy) │ │
|
||||
│ │ apps.core لاگ، خطا، سلامت، throttle│ │
|
||||
│ └────────────────────────────────────────┘ │
|
||||
└───┬───────────┬──────────────┬───────────────┘
|
||||
│ │ │
|
||||
┌─────────▼──┐ ┌─────▼──────┐ ┌────▼──────────┐
|
||||
│ PostgreSQL │ │ Redis │ │ Elasticsearch │
|
||||
│ منبع حقیقت │ │ cache + │ │ ایندکس سرچ │
|
||||
│ + FTS │ │ throttle │ └────┬──────────┘
|
||||
└────────────┘ └────────────┘ │
|
||||
│ │
|
||||
│ لاگ JSON (async TCP) │
|
||||
└────────▶ Logstash ───────┘
|
||||
│
|
||||
Kibana :5601
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**اصول طراحی:**
|
||||
|
||||
- **PostgreSQL منبع حقیقت است، Elasticsearch یک projection دورریختنی.** ایندکس همیشه از دیتابیس قابل بازسازی است (`make reindex`)، پس از دست رفتنش فاجعه نیست.
|
||||
- **لایه سرچ پشت یک interface است** (`SearchBackend`). دو پیادهسازی دارد: Elasticsearch و PostgreSQL. ویو، سریالایزر و فرانتاند اصلاً نمیدانند کدام موتور جواب داده است.
|
||||
- **ویوها نازکاند.** اعتبارسنجی در سریالایزر، منطق رتبهبندی در بکاند سرچ، ماسککردن داده حساس در لایه سریالایز.
|
||||
- **هیچ اطلاعات تماسی وارد Elasticsearch نمیشود.** ایمیل و شماره تلفن فقط در Postgres میمانند.
|
||||
|
||||
جزئیات کامل: [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## منطق سرچ و فیلترها
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### یک درخواست، سه بخش
|
||||
|
||||
هر درخواست سرچ از سه چیز تشکیل شده است:
|
||||
|
||||
| بخش | نقش | تأثیر روی امتیاز |
|
||||
|---|---|---|
|
||||
| `q` (کلیدواژه) | چه کسی «مرتبط» است | ✅ امتیاز میدهد |
|
||||
| فیلترها | چه کسی اصلاً «واجد شرایط» است | ❌ فقط حذف میکند |
|
||||
| `sort` / صفحهبندی | ترتیب و برش نتایج | — |
|
||||
|
||||
این تفکیک عمدی است: در Elasticsearch فیلترها در **filter context** قرار میگیرند، پس کش میشوند و روی scoring اثر نمیگذارند.
|
||||
|
||||
### کلیدواژه چطور امتیاز میگیرد
|
||||
|
||||
</div>
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"bool": {
|
||||
"must": [{ // باید جایی مچ شود
|
||||
"multi_match": {
|
||||
"query": "product manager",
|
||||
"type": "best_fields",
|
||||
"fields": ["full_name^6", "job_title^4", "skills^3.5", "company^2.5",
|
||||
"headline^2", "past_companies^1.5", "industry^1.5",
|
||||
"majors^1.5", "education", "location_name", "summary", "interests"],
|
||||
"minimum_should_match": "2<70%", // یککلمهای: اجباری. چندکلمهای: ۷۰٪ کافی است
|
||||
"fuzziness": "AUTO" // «pyhton» هم «python» را پیدا میکند
|
||||
}
|
||||
}],
|
||||
"should": [{ // فقط امتیاز اضافه میکند
|
||||
"multi_match": { "type": "phrase", "fields": ["full_name^8", "job_title^6",
|
||||
"skills^5", "company^4", "headline^3"], "boost": 4 }
|
||||
}],
|
||||
"filter": [ /* فیلترهای ساختاریافته - بدون امتیاز */ ]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
سه ایده پشت این کوئری:
|
||||
|
||||
1. **بوست هر فیلد یک تصمیم محصولی است.** وقتی کاربر «سارا» را جستجو میکند، منظورش یک آدم است نه کسی که در خلاصهاش کلمه «سارا» آمده — پس `full_name^6` در برابر `summary` بدون بوست.
|
||||
2. **`fuzziness: AUTO`** غلط تایپی را میبخشد، ولی `prefix_length: 1` جلوی انفجار عبارتهای بیربط را میگیرد.
|
||||
3. **بوست عبارت دقیق (phrase)** باعث میشود «product manager» بهعنوان یک عبارت، بالاتر از پروفایلی بیاید که این دو کلمه را جدا و بیربط دارد.
|
||||
|
||||
### فیلترها
|
||||
|
||||
| پارامتر | مثال | معنی |
|
||||
|---|---|---|
|
||||
| `skills` | `skills=python,django` | مهارتها |
|
||||
| `roles` | `roles=engineering` | خانواده شغلی |
|
||||
| `seniority` | `seniority=senior,manager` | سطح ارشدیت |
|
||||
| `countries` | `countries=united states` | کشور |
|
||||
| `industries` | `industries=banking` | صنعت کارفرمای فعلی |
|
||||
| `companies` | `companies=garver` | نام کارفرمای فعلی |
|
||||
| `min_experience` / `max_experience` | `min_experience=5` | بازه سالهای تجربه |
|
||||
|
||||
**قاعده ترکیب:** مقادیر **داخل** یک فیلتر با OR و فیلترهای **مختلف** با AND ترکیب میشوند.
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
skills=python,django & countries=germany
|
||||
⇒ (python OR django) AND (germany)
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
این دقیقاً همان چیزی است که کاربر از یک پنل فیلتر انتظار دارد: تیک زدن مهارت دوم نتایج را **بیشتر** میکند، ولی تیک زدن یک کشور نتایج را **محدودتر**.
|
||||
|
||||
### Facetها (شمارنده کنار هر فیلتر)
|
||||
|
||||
هر پاسخ سرچ، بههمراه نتایج، شمارش aggregation هر فیلتر را هم برمیگرداند. یعنی کاربر **قبل از کلیک** میداند «germany» چند نتیجه دارد و بنبست نمیخورد. مقادیر انتخابشده حتی اگر شمارش صفر داشته باشند در پنل میمانند تا کاربر بتواند تیکشان را بردارد.
|
||||
|
||||
### fallback روی PostgreSQL
|
||||
|
||||
بکاند دوم از `tsvector` وزندار استفاده میکند که ایمپورتر آن را میسازد:
|
||||
|
||||
| وزن | فیلدها |
|
||||
|---|---|
|
||||
| A | نام کامل |
|
||||
| B | عنوان شغلی، مهارتها، شرکتها |
|
||||
| C | صنعت، تحصیلات، موقعیت مکانی |
|
||||
| D | خلاصه |
|
||||
|
||||
بهعلاوه یک pass با `pg_trgm` روی نام، تا غلط تایپی (`jhon` → `john`) هم کار کند.
|
||||
|
||||
**چرا اصلاً این بکاند دوم وجود دارد؟** چون یک API سرچ که بدون Elasticsearch اصلاً بالا نمیآید شکننده است: تستها به کلاستر نیاز پیدا میکنند، مرورکننده کد برای اجرای پروژه به کلاستر نیاز دارد، و یک قطعی ES کل محصول را میخواباند. با یک متغیر محیطی، سیستم روی حالت کمتوانتر ولی سالم میرود.
|
||||
|
||||
توضیح کاملتر همراه با مثال کوئریها: [`docs/SEARCH.md`](docs/SEARCH.md)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ساختار ذخیرهسازی داده
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
دیتاست خام، هر آدم را در یک ردیف بسیار پهن CSV نگه میدارد که چند ستونش خودشان لیست دیکشنری سریالایزشدهاند. ذخیره کردن همان بلوب، فیلتر و aggregation را غیرممکن میکند؛ پس داده به پنج جدول نرمال شده است:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
Company ◀──── Experience ────▶ Profile ◀──── Education ────▶ School
|
||||
│
|
||||
└──── ProfileSkill ────▶ Skill
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| جدول | نکته طراحی |
|
||||
|---|---|
|
||||
| `Profile` | ایندکس GIN روی `search_vector` و `seniority_levels`؛ ایندکس ترکیبی روی (کشور، خانواده شغلی) |
|
||||
| `Skill` | کلید یکتا `slug` است؛ «Team Building» و «team building» یک رکورد میشوند. `profile_count` denormalize شده تا مرتبسازی فیلترها ارزان باشد |
|
||||
| `Company` / `School` | یکتا بر اساس نام، با کش درونحافظهای در ایمپورتر (بدون N+1) |
|
||||
| `ProfileSkill` | جدول واسط با `position` تا ترتیب اصلی مهارتها حفظ شود |
|
||||
|
||||
سه ستون متنی denormalizeشده (`skills_text`، `companies_text`، `education_text`) روی `Profile` نگهداری میشوند، چون `SearchVector` جنگو نمیتواند از رابطه many-to-many عبور کند. نتیجه: بهروزرسانی کل `tsvector` با **یک** کوئری `UPDATE` انجام میشود.
|
||||
|
||||
ایمپورتر **idempotent** است: کلید طبیعی، شناسه لینکدین (و در نبودش URL پروفایل) است، پس اجرای دوبارهاش بهجای تکرار داده، آن را بهروز میکند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## کیفیت دیتاست ورودی
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
فایل CSV دادهشده یک export واقعی و خراب است، نه یک CSV تمیز. اگر مستقیم به `csv.DictReader` بدهید، بیسروصدا دادهی غلط تولید میکند. چهار نقص مستقل در فایل هست:
|
||||
|
||||
| نقص | نشانه |
|
||||
|---|---|
|
||||
| **واترمارک تزریقشده** | خیلی از ردیفها با یک ستون اضافه شروع میشوند: `H:\New folder\leak\...@BreachedData1` — همهچیز یک خانه به راست میلغزد |
|
||||
| **ستونهای غایب** | اکثر رکوردها بهجای مقدار خالی، فیلد اختیاری (مثل `job_title_role`) را اصلاً ننوشتهاند — بقیه مقادیر به چپ میلغزند |
|
||||
| **کوتیشن شکسته** | چند مقدار `summary` کوتیشن escapeنشده دارند و یک رکورد را بین چند خط فیزیکی میشکنند |
|
||||
| **دو export به هم چسبیده** | خط هدر یک بار دیگر وسط فایل تکرار شده و حدود ۴۰ نفر تکراریاند |
|
||||
|
||||
اثر عملی: با خواندن ساده، فقط **۱۰ ردیف از ۳۲۳** کاملاً سالماند.
|
||||
|
||||
### راهکار: خواندن خودترمیمگر
|
||||
|
||||
هر ستون یک **امضا** دارد (`linkedin_url` باید شامل `linkedin.com/in/` باشد، `skills` باید یک literal لیست پایتون باشد، `location_geo` باید شبیه `lat,lon` باشد و…). ماژول [`apps/profiles/dataset.py`](backend/apps/profiles/dataset.py) هر جفت (ستون، مقدار) را با این امضاها امتیاز میدهد و بعد یک **الگوریتم همترازی بهسبک Needleman–Wunsch** اجرا میکند که دو عمل ویرایشی مجاز دارد:
|
||||
|
||||
- *رد کردن ستون* — رکورد اصلاً این فیلد را ندارد؛
|
||||
- *دور انداختن مقدار* — یک تکه سرگردان که به هیچ ستونی تعلق ندارد.
|
||||
|
||||
همترازی با بیشترین امتیاز، رکورد بازسازیشده است. سپس هر مقدار دوباره اعتبارسنجی میشود و هر چیزی که هنوز امضایش را رد نکند **دور انداخته میشود** — ذخیره نکردن بهتر از ذخیره کردن شماره تلفن در ستون ایمیل است.
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
$ docker compose exec backend python manage.py import_profiles --truncate
|
||||
{
|
||||
"profiles_created": 284, "skills": 2376, "companies": 1371, "schools": 610,
|
||||
"experiences": 2128, "educations": 870,
|
||||
"physical_rows": 336, "duplicate_headers": 1, "repaired_rows": 16,
|
||||
"orphan_fragments": 12, "duplicate_records": 39, "discarded_values": 175,
|
||||
"records": 284, "elapsed_seconds": 15.4
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**غنیسازی از داده تودرتو:** وقتی فیلدهای سطحبالا غایباند، ایمپورتر همان اطلاعات را از داخل بلوب `experience` بیرون میکشد. نتیجه:
|
||||
|
||||
| فیلد | فقط ستون سطحبالا | بعد از غنیسازی |
|
||||
|---|---|---|
|
||||
| `job_title_role` | ۱۰۸ | **۱۸۴** |
|
||||
| کارفرمای فعلی | ۲۱۹ | **۲۶۵** |
|
||||
|
||||
هفت تست اختصاصی (`tests/test_dataset_repair.py`) هر کدام یکی از این نقصها را بازتولید میکنند تا رگرسیون در منطق ترمیم، بیصدا رد نشود.
|
||||
|
||||
جزئیات کامل: [`docs/DATA.md`](docs/DATA.md)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## API
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
پایه: `/api/v1/` — مستندات تعاملی و همیشه بهروز: <http://localhost:8080/api/docs/>
|
||||
|
||||
| متد | مسیر | توضیح | دسترسی |
|
||||
|---|---|---|---|
|
||||
| `POST` | `/auth/register/` | ثبتنام (همیشه نقش `viewer`) | عمومی |
|
||||
| `POST` | `/auth/login/` | گرفتن access/refresh token | عمومی |
|
||||
| `POST` | `/auth/refresh/` | چرخش توکن | عمومی |
|
||||
| `POST` | `/auth/logout/` | بلاکلیست کردن refresh token | لاگین |
|
||||
| `GET` | `/auth/me/` | هویت و سطح دسترسی فعلی | لاگین |
|
||||
| `GET` | **`/search/`** | **سرچ کلیدواژهای + فیلتر + facet** | لاگین |
|
||||
| `GET` | `/search/suggest/` | پیشنهاد تایپآهسته (autocomplete) | لاگین |
|
||||
| `POST` | `/search/reindex/` | بازسازی ایندکس | فقط admin |
|
||||
| `GET` | `/profiles/` | لیست ساده پروفایلها | لاگین |
|
||||
| `GET` | `/profiles/{id}/` | پروفایل کامل + سوابق | لاگین |
|
||||
| `GET` | `/profiles/skills/` | دیکشنری مهارتها برای type-ahead | لاگین |
|
||||
| `GET` | `/health/live/` | liveness probe | عمومی |
|
||||
| `GET` | `/health/ready/` | readiness (چک DB و موتور سرچ) | عمومی |
|
||||
|
||||
### نمونه
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
# ۱) لاگین
|
||||
TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/auth/login/ \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"email":"analyst@example.com","password":"Analyst-Passw0rd!"}' | jq -r .access)
|
||||
|
||||
# ۲) سرچ: کلیدواژه + دو فیلتر + مرتبسازی
|
||||
curl -s -G http://localhost:8080/api/v1/search/ \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
--data-urlencode 'q=leadership' \
|
||||
--data-urlencode 'skills=management,training' \
|
||||
--data-urlencode 'countries=united states' \
|
||||
--data-urlencode 'min_experience=10' \
|
||||
--data-urlencode 'sort=-experience' | jq
|
||||
```
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"count": 41, "page": 1, "page_size": 20, "total_pages": 3,
|
||||
"took_ms": 12, "backend": "elasticsearch",
|
||||
"results": [{
|
||||
"id": 118, "score": 14.82,
|
||||
"full_name": "joseph holland",
|
||||
"headline": "recruiting manager at garver",
|
||||
"job_title_role": "human_resources",
|
||||
"seniority_levels": ["manager"],
|
||||
"location_country": "united states",
|
||||
"years_experience": 12,
|
||||
"skills": ["recruiting", "leadership", "human resources"],
|
||||
"highlight": { "skills": ["<mark>leadership</mark>"] }
|
||||
}],
|
||||
"facets": {
|
||||
"skills": [{ "key": "leadership", "count": 41 }, { "key": "management", "count": 33 }],
|
||||
"country": [{ "key": "united states", "count": 41 }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### قالب یکسان خطاها
|
||||
|
||||
هر خطا — اعتبارسنجی، دسترسی، rate limit یا خطای پیشبینینشده — دقیقاً یک شکل دارد:
|
||||
|
||||
</div>
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": "The request payload failed validation.",
|
||||
"details": { "min_experience": ["min_experience cannot be greater than max_experience."] },
|
||||
"request_id": "9f2c1a7e8b4d4c0e"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`request_id` همان مقداری است که در هدر `X-Request-ID` پاسخ برمیگردد؛ کپی کردنش در Kibana، کل ردپای آن درخواست را نشان میدهد.
|
||||
|
||||
مرجع کامل: [`docs/API.md`](docs/API.md) — اسکیمای OpenAPI: [`docs/openapi.yaml`](docs/openapi.yaml)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## امنیت
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| لایه | اقدام |
|
||||
|---|---|
|
||||
| احراز هویت | JWT با عمر کوتاه (۱۵ دقیقه) + refresh چرخشی + بلاکلیست بعد از چرخش |
|
||||
| رمز عبور | هش Argon2، حداقل ۱۰ کاراکتر، اعتبارسنجهای استاندارد جنگو |
|
||||
| مجوزدهی | پیشفرض **deny**؛ سه نقش (admin / analyst / viewer)؛ بازسازی ایندکس فقط admin |
|
||||
| حریم خصوصی | ایمیل و تلفن برای نقش `viewer` **ماسک** میشود و **اصلاً وارد Elasticsearch نمیشود** |
|
||||
| Rate limiting | سقف جدا برای anon / user / search / login (روی IP برای مسیرهای اعتبارسنجی) |
|
||||
| ورودی | همه پارامترها از سریالایزر رد میشوند؛ سقف طول کوئری، تعداد فیلترها و اندازه صفحه |
|
||||
| هدرها | HSTS، `nosniff`، `X-Frame-Options: DENY`، Referrer-Policy، CSP سختگیرانه در nginx |
|
||||
| XSS | snippetهای highlight **بدون** `dangerouslySetInnerHTML` رندر میشوند |
|
||||
| اسرار | همه از متغیر محیطی؛ `settings/prod.py` اگر کلید ضعیف یا `DEBUG=true` ببیند بالا نمیآید |
|
||||
| کانتینر | کاربر غیرroot، `no-new-privileges`، فایلسیستم read-only برای nginx، بیلد چندمرحلهای |
|
||||
| شبکه | Postgres/Redis/Elasticsearch روی شبکهی `internal` داکر؛ فقط SPA و Kibana منتشر میشوند |
|
||||
| CI | `bandit`، `pip-audit`، `npm audit` و `manage.py check --deploy` |
|
||||
|
||||
توضیح تصمیمها و trade-offها (از جمله اینکه چرا احراز هویت Elasticsearch در حالت dev خاموش است): [`docs/SECURITY.md`](docs/SECURITY.md)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## مانیتورینگ و لاگ (ELK)
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
هر لاگ یک شیء JSON است، نه یک خط متن. هر رکورد `request_id` درخواستی که تولیدش کرده را حمل میکند، پس یک کلیک کاربر تا انتها قابل ردیابی است.
|
||||
|
||||
</div>
|
||||
|
||||
```json
|
||||
{"@timestamp":"2026-09-03T04:00:08Z","level":"INFO","logger":"audit.search",
|
||||
"event":"search_executed","request_id":"bb3c21c5","user_id":2,
|
||||
"backend":"postgres","result_count":80,"took_ms":31,
|
||||
"query":"leadership","filters":{"skills":["management"],"countries":["united states"]},
|
||||
"sort":"relevance","page":1,"page_size":3}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
مسیر: **Django → (TCP غیرمسدودکننده) → Logstash → Elasticsearch → Kibana**
|
||||
|
||||
- ارسال با `python-logstash-async` انجام میشود: بافر درونحافظهای و ترد جدا، پس قطعی Logstash هیچوقت یک ترد درخواست را بلاک نمیکند (این رفتار در همین ریپو تست شده: با خاموش بودن Logstash فقط یک warning میآید و API سالم میماند).
|
||||
- pipeline لاگها را بر اساس نوع رویداد به ایندکسهای جدا مسیردهی میکند: `app-logs-search-*`، `app-logs-audit-*`، `app-logs-error-*`، `app-logs-app-*` — پس «هر چیز قدیمیتر از ۳۰ روز را پاک کن» یک دستور حذف ایندکس است.
|
||||
- سرچ کندتر از ۵۰۰ms برچسب `slow_search` و سرچ بدون نتیجه برچسب `zero_results` میگیرد.
|
||||
- یک index template قالببندی فیلدها را قفل میکند تا Kibana مجبور به حدس زدن نوع داده نشود.
|
||||
|
||||
`make kibana-setup` را اجرا کنید تا data view ساخته شود، بعد در Discover:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
event: "search_executed" and result_count: 0 # کوئریهایی که چیزی پیدا نکردند
|
||||
event: "search_executed" and took_ms > 200 # سرچهای کند
|
||||
level: "ERROR" # خطاها با stack trace
|
||||
request_id: "bb3c21c5..." # ردیابی یک درخواست خاص
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
جزئیات و runbook: [`docs/OPERATIONS.md`](docs/OPERATIONS.md)
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## تست و کیفیت کد
|
||||
|
||||
```bash
|
||||
make test # ۶۱ تست بکاند
|
||||
make lint # ruff + black + mypy + eslint + tsc
|
||||
make audit # bandit + pip-audit + npm audit
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| فایل تست | چه چیزی را تضمین میکند |
|
||||
|---|---|
|
||||
| `test_dataset_repair.py` | هر چهار نقص واقعی فایل CSV درست ترمیم میشوند |
|
||||
| `test_importer.py` | نرمالسازی، غنیسازی از داده تودرتو، idempotent بودن |
|
||||
| `test_query_builder.py` | کوئری Elasticsearch (تابع خالص — بدون نیاز به کلاستر) |
|
||||
| `test_search_api.py` | قرارداد کامل API: کلیدواژه، OR/AND فیلترها، facet، مرتبسازی، صفحهبندی، خطاها |
|
||||
| `test_auth_api.py` | ثبتنام، لاگین، سطح دسترسی، ماسک شدن PII بر اساس نقش |
|
||||
| `test_indexing.py` | داکیومنت ایندکس با mapping همخوان است و **هرگز** PII حمل نمیکند |
|
||||
| `test_privacy.py` | توابع ماسککردن هیچوقت مقدار اصلی را لو نمیدهند |
|
||||
|
||||
تستها روی بکاند PostgreSQL اجرا میشوند و به Elasticsearch نیاز ندارند؛ pipeline گیتهاب (`.github/workflows/ci.yml`) همینها را بهعلاوه بیلد هر دو ایمیج داکر اجرا میکند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ساختار پروژه
|
||||
|
||||
```
|
||||
.
|
||||
├── backend/
|
||||
│ ├── config/settings/{base,dev,prod,test}.py تنظیمات تفکیکشده بر اساس محیط
|
||||
│ ├── apps/
|
||||
│ │ ├── core/ لاگ ساختیافته، middleware، قالب خطا، throttle، health
|
||||
│ │ ├── accounts/ کاربر سفارشی، نقشها، JWT
|
||||
│ │ ├── profiles/ مدل دامنه، خواننده خودترمیم CSV، ایمپورتر، ماسک PII
|
||||
│ │ └── search/ types، ایندکس ES، دو بکاند سرچ، ویو و سریالایزر
|
||||
│ ├── tests/ ۶۱ تست
|
||||
│ ├── Dockerfile بیلد چندمرحلهای، کاربر غیرroot
|
||||
│ └── entrypoint.sh انتظار برای وابستگیها، migrate، seed
|
||||
├── frontend/
|
||||
│ ├── src/
|
||||
│ │ ├── api/ کلاینت axios با refresh خودکار توکن
|
||||
│ │ ├── components/ SearchBar، FilterPanel، FacetGroup، ResultCard، Drawer
|
||||
│ │ ├── hooks/ debounce، همگامسازی وضعیت سرچ با URL
|
||||
│ │ └── pages/ Login، Search
|
||||
│ ├── Dockerfile بیلد Node → سرو با nginx
|
||||
│ └── nginx.conf پراکسی API + هدرهای امنیتی + CSP
|
||||
├── elk/
|
||||
│ └── logstash/ pipeline، تنظیمات، index template
|
||||
├── data/ دیتاست خام CSV
|
||||
├── docs/ معماری، API، سرچ، داده، امنیت، عملیات، openapi.yaml
|
||||
├── docker-compose.yml استک کامل
|
||||
├── docker-compose.dev.yml overlay توسعه (hot reload)
|
||||
├── docker-compose.prod.yml overlay پروداکشن (auth روی ES، محدودیت منابع)
|
||||
└── Makefile
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## مستندات تکمیلی
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| سند | محتوا |
|
||||
|---|---|
|
||||
| [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) | تصمیمهای معماری، مسیر یک درخواست، الگوی Strategy در لایه سرچ |
|
||||
| [`docs/SEARCH.md`](docs/SEARCH.md) | منطق سرچ و فیلتر، آنالایزرها، رتبهبندی، مقایسه دو بکاند |
|
||||
| [`docs/API.md`](docs/API.md) | مرجع کامل endpointها با نمونه درخواست/پاسخ |
|
||||
| [`docs/DATA.md`](docs/DATA.md) | تحلیل کیفیت دیتاست و الگوریتم ترمیم |
|
||||
| [`docs/SECURITY.md`](docs/SECURITY.md) | مدل تهدید، لایههای دفاعی، trade-offها |
|
||||
| [`docs/OPERATIONS.md`](docs/OPERATIONS.md) | اجرا، ELK، runbook خطاهای رایج |
|
||||
|
||||
</div>
|
||||
@@ -0,0 +1,15 @@
|
||||
**/__pycache__/
|
||||
**/*.py[cod]
|
||||
.venv/
|
||||
venv/
|
||||
.env
|
||||
.env.*
|
||||
staticfiles/
|
||||
htmlcov/
|
||||
.coverage
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
tests/
|
||||
*.sqlite3
|
||||
*.log
|
||||
@@ -0,0 +1,71 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# =============================================================================
|
||||
# Django API image.
|
||||
#
|
||||
# Two stages: wheels are built with a compiler present, the runtime image gets
|
||||
# only the compiled artefacts. The result is small, has no build toolchain to
|
||||
# exploit, and runs as an unprivileged user on a read-only-friendly layout.
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stage 1 - build wheels
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM python:3.12-slim-bookworm AS builder
|
||||
|
||||
ENV PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y build-essential libpq-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# The build context is the repository root so the image can also carry the
|
||||
# dataset; paths below are therefore prefixed with the directory name.
|
||||
WORKDIR /build
|
||||
COPY backend/requirements/ requirements/
|
||||
RUN pip wheel --wheel-dir /wheels -r requirements/base.txt
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stage 2 - runtime
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM python:3.12-slim-bookworm AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
DJANGO_SETTINGS_MODULE=config.settings.prod \
|
||||
PATH="/home/app/.local/bin:${PATH}"
|
||||
|
||||
# libpq5 is the only runtime library psycopg needs; curl backs the healthcheck.
|
||||
RUN apt-get update \
|
||||
&& apt-get install --no-install-recommends -y libpq5 curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
COPY --from=builder /wheels /wheels
|
||||
COPY backend/requirements/ /tmp/requirements/
|
||||
RUN pip install --no-index --find-links=/wheels -r /tmp/requirements/base.txt \
|
||||
&& rm -rf /wheels /tmp/requirements
|
||||
|
||||
WORKDIR /app
|
||||
COPY --chown=app:app backend/ /app/
|
||||
COPY --chown=app:app data/ /data/
|
||||
|
||||
# Static assets are only needed for the admin and the API docs, but baking them
|
||||
# in keeps the container stateless.
|
||||
RUN DJANGO_SECRET_KEY=build-only-not-a-real-secret-000000000000000000000000000000000000 \
|
||||
DATABASE_URL=postgres://build:build@localhost:5432/build \
|
||||
DJANGO_SETTINGS_MODULE=config.settings.dev \
|
||||
python manage.py collectstatic --noinput --clear \
|
||||
&& chmod +x /app/entrypoint.sh
|
||||
|
||||
USER app
|
||||
EXPOSE 8000
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=40s --retries=3 \
|
||||
CMD curl -fsS http://localhost:8000/api/v1/health/live/ || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
CMD ["gunicorn", "config.wsgi:application", "--config", "/app/gunicorn.conf.py"]
|
||||
@@ -0,0 +1,30 @@
|
||||
from django.contrib import admin
|
||||
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from apps.accounts.models import User
|
||||
|
||||
|
||||
@admin.register(User)
|
||||
class UserAdmin(BaseUserAdmin):
|
||||
ordering = ("-date_joined",)
|
||||
list_display = ("email", "full_name", "role", "is_active", "is_staff", "date_joined")
|
||||
list_filter = ("role", "is_active", "is_staff", "is_superuser")
|
||||
search_fields = ("email", "full_name")
|
||||
readonly_fields = ("date_joined", "last_login")
|
||||
|
||||
fieldsets = (
|
||||
(None, {"fields": ("email", "password")}),
|
||||
(_("Personal info"), {"fields": ("full_name",)}),
|
||||
(_("Permissions"), {"fields": ("role", "is_active", "is_staff", "is_superuser", "groups")}),
|
||||
(_("Important dates"), {"fields": ("last_login", "date_joined")}),
|
||||
)
|
||||
add_fieldsets = (
|
||||
(
|
||||
None,
|
||||
{
|
||||
"classes": ("wide",),
|
||||
"fields": ("email", "full_name", "role", "password1", "password2"),
|
||||
},
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class AccountsConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.accounts"
|
||||
verbose_name = "Accounts"
|
||||
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
``python manage.py seed_users`` - create the demo accounts used by the README.
|
||||
|
||||
Credentials come from the environment so no password is ever committed; the
|
||||
command is idempotent and refuses to run with production settings unless
|
||||
``--force`` is given.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
|
||||
from apps.accounts.models import Role, User
|
||||
|
||||
DEMO_USERS = (
|
||||
("DEMO_ADMIN_EMAIL", "DEMO_ADMIN_PASSWORD", Role.ADMIN, True),
|
||||
("DEMO_ANALYST_EMAIL", "DEMO_ANALYST_PASSWORD", Role.ANALYST, False),
|
||||
("DEMO_VIEWER_EMAIL", "DEMO_VIEWER_PASSWORD", Role.VIEWER, False),
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create the demo admin / analyst / viewer accounts."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
parser.add_argument("--force", action="store_true", help="Run even outside development.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
for email_var, password_var, role, is_staff in DEMO_USERS:
|
||||
email = os.environ.get(email_var)
|
||||
password = os.environ.get(password_var)
|
||||
if not email or not password:
|
||||
self.stdout.write(
|
||||
self.style.WARNING(f"Skipping {role}: {email_var}/{password_var} not set.")
|
||||
)
|
||||
continue
|
||||
|
||||
user = User.objects.filter(email=email.lower()).first()
|
||||
if user:
|
||||
user.role = role
|
||||
user.is_staff = is_staff
|
||||
user.is_superuser = is_staff
|
||||
user.set_password(password)
|
||||
user.save(update_fields=["role", "is_staff", "is_superuser", "password"])
|
||||
self.stdout.write(f"Updated {email} ({role}).")
|
||||
continue
|
||||
|
||||
User.objects.create_user(
|
||||
email=email,
|
||||
password=password,
|
||||
role=role,
|
||||
full_name=f"Demo {role.label}",
|
||||
is_staff=is_staff,
|
||||
is_superuser=is_staff,
|
||||
)
|
||||
self.stdout.write(self.style.SUCCESS(f"Created {email} ({role})."))
|
||||
@@ -0,0 +1,103 @@
|
||||
# Generated by Django 5.1.15 on 2026-09-03 03:36
|
||||
|
||||
import apps.accounts.models
|
||||
import django.utils.timezone
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
("auth", "0012_alter_user_first_name_max_length"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="User",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("password", models.CharField(max_length=128, verbose_name="password")),
|
||||
(
|
||||
"last_login",
|
||||
models.DateTimeField(blank=True, null=True, verbose_name="last login"),
|
||||
),
|
||||
(
|
||||
"is_superuser",
|
||||
models.BooleanField(
|
||||
default=False,
|
||||
help_text="Designates that this user has all permissions without explicitly assigning them.",
|
||||
verbose_name="superuser status",
|
||||
),
|
||||
),
|
||||
(
|
||||
"email",
|
||||
models.EmailField(
|
||||
db_index=True, max_length=254, unique=True, verbose_name="email address"
|
||||
),
|
||||
),
|
||||
(
|
||||
"full_name",
|
||||
models.CharField(blank=True, max_length=150, verbose_name="full name"),
|
||||
),
|
||||
(
|
||||
"role",
|
||||
models.CharField(
|
||||
choices=[
|
||||
("admin", "Administrator"),
|
||||
("analyst", "Analyst"),
|
||||
("viewer", "Viewer"),
|
||||
],
|
||||
default="viewer",
|
||||
help_text="Determines which fields and actions the user has access to.",
|
||||
max_length=16,
|
||||
verbose_name="role",
|
||||
),
|
||||
),
|
||||
("is_active", models.BooleanField(default=True, verbose_name="active")),
|
||||
("is_staff", models.BooleanField(default=False, verbose_name="staff status")),
|
||||
(
|
||||
"date_joined",
|
||||
models.DateTimeField(
|
||||
default=django.utils.timezone.now, verbose_name="date joined"
|
||||
),
|
||||
),
|
||||
(
|
||||
"groups",
|
||||
models.ManyToManyField(
|
||||
blank=True,
|
||||
help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
|
||||
related_name="user_set",
|
||||
related_query_name="user",
|
||||
to="auth.group",
|
||||
verbose_name="groups",
|
||||
),
|
||||
),
|
||||
(
|
||||
"user_permissions",
|
||||
models.ManyToManyField(
|
||||
blank=True,
|
||||
help_text="Specific permissions for this user.",
|
||||
related_name="user_set",
|
||||
related_query_name="user",
|
||||
to="auth.permission",
|
||||
verbose_name="user permissions",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "user",
|
||||
"verbose_name_plural": "users",
|
||||
"ordering": ("-date_joined",),
|
||||
},
|
||||
managers=[
|
||||
("objects", apps.accounts.models.UserManager()),
|
||||
],
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
"""
|
||||
Custom user model.
|
||||
|
||||
Two things are worth pointing out:
|
||||
|
||||
* The username field is the e-mail address - the dataset users are internal
|
||||
analysts, and an e-mail is the identifier they already have.
|
||||
* Authorisation is expressed as a single ``role`` field instead of a pile of
|
||||
Django groups. With three well defined roles this is easier to reason about,
|
||||
easier to test, and it keeps the JWT payload small.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib.auth.models import AbstractBaseUser, BaseUserManager, PermissionsMixin
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
||||
class Role(models.TextChoices):
|
||||
ADMIN = "admin", _("Administrator")
|
||||
ANALYST = "analyst", _("Analyst")
|
||||
VIEWER = "viewer", _("Viewer")
|
||||
|
||||
|
||||
class UserManager(BaseUserManager):
|
||||
"""Manager that normalises e-mails and refuses to create users without one."""
|
||||
|
||||
use_in_migrations = True
|
||||
|
||||
def create_user(self, email: str, password: str | None = None, **extra) -> User:
|
||||
if not email:
|
||||
raise ValueError("Users must have an email address.")
|
||||
email = self.normalize_email(email).lower()
|
||||
user = self.model(email=email, **extra)
|
||||
# ``set_password`` hashes with Argon2 (see PASSWORD_HASHERS).
|
||||
user.set_password(password)
|
||||
user.full_clean(exclude=["password"])
|
||||
user.save(using=self._db)
|
||||
return user
|
||||
|
||||
def create_superuser(self, email: str, password: str | None = None, **extra) -> User:
|
||||
extra.setdefault("is_staff", True)
|
||||
extra.setdefault("is_superuser", True)
|
||||
extra.setdefault("role", Role.ADMIN)
|
||||
if extra.get("is_staff") is not True or extra.get("is_superuser") is not True:
|
||||
raise ValueError("Superuser must have is_staff=True and is_superuser=True.")
|
||||
return self.create_user(email, password, **extra)
|
||||
|
||||
|
||||
class User(AbstractBaseUser, PermissionsMixin):
|
||||
email = models.EmailField(_("email address"), unique=True, db_index=True)
|
||||
full_name = models.CharField(_("full name"), max_length=150, blank=True)
|
||||
role = models.CharField(
|
||||
_("role"),
|
||||
max_length=16,
|
||||
choices=Role.choices,
|
||||
default=Role.VIEWER,
|
||||
help_text=_("Determines which fields and actions the user has access to."),
|
||||
)
|
||||
is_active = models.BooleanField(_("active"), default=True)
|
||||
is_staff = models.BooleanField(_("staff status"), default=False)
|
||||
date_joined = models.DateTimeField(_("date joined"), default=timezone.now)
|
||||
|
||||
objects = UserManager()
|
||||
|
||||
USERNAME_FIELD = "email"
|
||||
REQUIRED_FIELDS: list[str] = []
|
||||
|
||||
class Meta:
|
||||
verbose_name = _("user")
|
||||
verbose_name_plural = _("users")
|
||||
ordering = ("-date_joined",)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.email
|
||||
|
||||
# -- Authorisation helpers ---------------------------------------------
|
||||
@property
|
||||
def is_admin_role(self) -> bool:
|
||||
return self.role == Role.ADMIN or self.is_superuser
|
||||
|
||||
@property
|
||||
def can_view_pii(self) -> bool:
|
||||
"""
|
||||
Whether raw contact details may be returned to this user.
|
||||
|
||||
Viewers only ever see masked e-mails and phone numbers; unmasking is a
|
||||
deliberate privilege granted to analysts and administrators.
|
||||
"""
|
||||
return self.role in (Role.ADMIN, Role.ANALYST) or self.is_superuser
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Serializers for registration, login and profile of the API's own users."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.auth import password_validation
|
||||
from django.db import IntegrityError
|
||||
from rest_framework import serializers
|
||||
from rest_framework_simplejwt.serializers import (
|
||||
TokenObtainPairSerializer as BaseTokenObtainPairSerializer,
|
||||
)
|
||||
|
||||
from apps.accounts.models import Role, User
|
||||
|
||||
audit = logging.getLogger("audit.auth")
|
||||
|
||||
|
||||
class UserSerializer(serializers.ModelSerializer):
|
||||
can_view_pii = serializers.BooleanField(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = User
|
||||
fields = ("id", "email", "full_name", "role", "can_view_pii", "date_joined")
|
||||
read_only_fields = fields
|
||||
|
||||
|
||||
class RegisterSerializer(serializers.ModelSerializer):
|
||||
password = serializers.CharField(write_only=True, style={"input_type": "password"})
|
||||
password_confirm = serializers.CharField(write_only=True, style={"input_type": "password"})
|
||||
|
||||
class Meta:
|
||||
model = User
|
||||
fields = ("email", "full_name", "password", "password_confirm")
|
||||
|
||||
def validate_email(self, value: str) -> str:
|
||||
return value.strip().lower()
|
||||
|
||||
def validate_password(self, value: str) -> str:
|
||||
# Runs Django's configured validators: length, common passwords, etc.
|
||||
password_validation.validate_password(value)
|
||||
return value
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
if attrs["password"] != attrs["password_confirm"]:
|
||||
raise serializers.ValidationError({"password_confirm": "Passwords do not match."})
|
||||
return attrs
|
||||
|
||||
def create(self, validated_data: dict[str, Any]) -> User:
|
||||
validated_data.pop("password_confirm")
|
||||
password = validated_data.pop("password")
|
||||
try:
|
||||
# Self-service sign-up always yields the least privileged role;
|
||||
# elevation is an explicit admin action.
|
||||
return User.objects.create_user(password=password, role=Role.VIEWER, **validated_data)
|
||||
except IntegrityError as exc: # pragma: no cover - race with unique index
|
||||
raise serializers.ValidationError(
|
||||
{"email": "A user with this email already exists."}
|
||||
) from exc
|
||||
|
||||
|
||||
class TokenObtainPairSerializer(BaseTokenObtainPairSerializer):
|
||||
"""Adds identity claims to the access token and records an audit event."""
|
||||
|
||||
@classmethod
|
||||
def get_token(cls, user: User): # noqa: ANN206
|
||||
token = super().get_token(user)
|
||||
token["role"] = user.role
|
||||
token["email"] = user.email
|
||||
return token
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
data = super().validate(attrs)
|
||||
audit.info(
|
||||
"login_succeeded",
|
||||
extra={
|
||||
"event": "login_succeeded",
|
||||
"user_id": self.user.id,
|
||||
"user_role": self.user.role,
|
||||
},
|
||||
)
|
||||
data["user"] = UserSerializer(self.user).data
|
||||
return data
|
||||
|
||||
|
||||
class LogoutSerializer(serializers.Serializer):
|
||||
refresh = serializers.CharField()
|
||||
@@ -0,0 +1,13 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.accounts.views import LoginView, LogoutView, MeView, RefreshView, RegisterView
|
||||
|
||||
app_name = "accounts"
|
||||
|
||||
urlpatterns = [
|
||||
path("register/", RegisterView.as_view(), name="register"),
|
||||
path("login/", LoginView.as_view(), name="login"),
|
||||
path("refresh/", RefreshView.as_view(), name="refresh"),
|
||||
path("logout/", LogoutView.as_view(), name="logout"),
|
||||
path("me/", MeView.as_view(), name="me"),
|
||||
]
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Authentication endpoints: register, login, refresh, logout, me."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import generics, status
|
||||
from rest_framework.permissions import AllowAny, IsAuthenticated
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
from rest_framework_simplejwt.exceptions import TokenError
|
||||
from rest_framework_simplejwt.tokens import RefreshToken
|
||||
from rest_framework_simplejwt.views import TokenObtainPairView as BaseTokenObtainPairView
|
||||
from rest_framework_simplejwt.views import TokenRefreshView as BaseTokenRefreshView
|
||||
|
||||
from apps.accounts.models import User
|
||||
from apps.accounts.serializers import LogoutSerializer, RegisterSerializer, UserSerializer
|
||||
from apps.core.throttling import LoginRateThrottle
|
||||
|
||||
audit = logging.getLogger("audit.auth")
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class RegisterView(generics.CreateAPIView):
|
||||
"""Self-service sign-up. New accounts always get the ``viewer`` role."""
|
||||
|
||||
serializer_class = RegisterSerializer
|
||||
permission_classes = [AllowAny]
|
||||
throttle_classes = [LoginRateThrottle]
|
||||
queryset = User.objects.all()
|
||||
|
||||
def create(self, request: Request, *args, **kwargs) -> Response:
|
||||
serializer = self.get_serializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
user = serializer.save()
|
||||
audit.info("user_registered", extra={"event": "user_registered", "user_id": user.id})
|
||||
return Response(UserSerializer(user).data, status=status.HTTP_201_CREATED)
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class LoginView(BaseTokenObtainPairView):
|
||||
"""Exchange credentials for an access/refresh token pair."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
throttle_classes = [LoginRateThrottle]
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class RefreshView(BaseTokenRefreshView):
|
||||
"""Rotate a refresh token; the presented token is blacklisted afterwards."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
throttle_classes = [LoginRateThrottle]
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"], request=LogoutSerializer, responses={205: None})
|
||||
class LogoutView(APIView):
|
||||
"""Blacklist a refresh token so it can no longer be rotated."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
serializer_class = LogoutSerializer
|
||||
|
||||
def post(self, request: Request) -> Response:
|
||||
serializer = LogoutSerializer(data=request.data)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
# An already-invalid token is not an error worth surfacing: the
|
||||
# caller's intent (be logged out) is satisfied either way.
|
||||
with contextlib.suppress(TokenError):
|
||||
RefreshToken(serializer.validated_data["refresh"]).blacklist()
|
||||
audit.info("logout", extra={"event": "logout", "user_id": request.user.id})
|
||||
return Response(status=status.HTTP_205_RESET_CONTENT)
|
||||
|
||||
|
||||
@extend_schema(tags=["auth"])
|
||||
class MeView(generics.RetrieveAPIView):
|
||||
"""Identity and effective privileges of the calling user."""
|
||||
|
||||
serializer_class = UserSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_object(self) -> User:
|
||||
return self.request.user
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class CoreConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.core"
|
||||
verbose_name = "Core"
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
A single, predictable error envelope for the whole API.
|
||||
|
||||
Every failure - validation, permission, throttling, or an unexpected crash -
|
||||
comes back in the same shape:
|
||||
|
||||
{"error": {"code": "validation_error", "message": "...", "details": {...},
|
||||
"request_id": "..."}}
|
||||
|
||||
Unhandled exceptions are logged with their stack trace but never leak internals
|
||||
to the client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from django.core.exceptions import PermissionDenied
|
||||
from django.http import Http404
|
||||
from rest_framework import exceptions, status
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import exception_handler as drf_exception_handler
|
||||
|
||||
from apps.core.logging import get_request_id
|
||||
|
||||
logger = logging.getLogger("apps.errors")
|
||||
|
||||
_CODE_BY_STATUS = {
|
||||
status.HTTP_400_BAD_REQUEST: "validation_error",
|
||||
status.HTTP_401_UNAUTHORIZED: "authentication_failed",
|
||||
status.HTTP_403_FORBIDDEN: "permission_denied",
|
||||
status.HTTP_404_NOT_FOUND: "not_found",
|
||||
status.HTTP_405_METHOD_NOT_ALLOWED: "method_not_allowed",
|
||||
status.HTTP_429_TOO_MANY_REQUESTS: "throttled",
|
||||
status.HTTP_503_SERVICE_UNAVAILABLE: "service_unavailable",
|
||||
}
|
||||
|
||||
|
||||
class ServiceUnavailable(exceptions.APIException):
|
||||
status_code = status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
default_detail = "A downstream dependency is unavailable. Please retry shortly."
|
||||
default_code = "service_unavailable"
|
||||
|
||||
|
||||
class SearchBackendError(ServiceUnavailable):
|
||||
default_detail = "The search backend is currently unavailable."
|
||||
default_code = "search_unavailable"
|
||||
|
||||
|
||||
def api_exception_handler(exc: Exception, context: dict[str, Any]) -> Response | None:
|
||||
response = drf_exception_handler(exc, context)
|
||||
|
||||
if response is None:
|
||||
# Anything DRF does not know about is a bug on our side.
|
||||
if isinstance(exc, Http404):
|
||||
response = Response(status=status.HTTP_404_NOT_FOUND)
|
||||
elif isinstance(exc, PermissionDenied):
|
||||
response = Response(status=status.HTTP_403_FORBIDDEN)
|
||||
else:
|
||||
logger.exception(
|
||||
"unhandled_exception",
|
||||
extra={"event": "unhandled_exception", "view": str(context.get("view"))},
|
||||
)
|
||||
return Response(
|
||||
_envelope("internal_error", "An unexpected error occurred.", None),
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
detail = getattr(exc, "detail", None)
|
||||
if isinstance(exc, exceptions.ValidationError):
|
||||
# DRF labels every field error "invalid"; one stable code per failure
|
||||
# mode is far easier for a client to branch on.
|
||||
code = "validation_error"
|
||||
else:
|
||||
code = getattr(exc, "default_code", None) or _CODE_BY_STATUS.get(
|
||||
response.status_code, "error"
|
||||
)
|
||||
|
||||
if isinstance(detail, dict):
|
||||
message = "The request payload failed validation."
|
||||
details: Any = detail
|
||||
elif isinstance(detail, list):
|
||||
message = str(detail[0]) if detail else "Request failed."
|
||||
details = detail
|
||||
elif detail is not None:
|
||||
message = str(detail)
|
||||
details = None
|
||||
else:
|
||||
message = response.status_text
|
||||
details = None
|
||||
|
||||
response.data = _envelope(code, message, details)
|
||||
return response
|
||||
|
||||
|
||||
def _envelope(code: str, message: str, details: Any) -> dict[str, Any]:
|
||||
error: dict[str, Any] = {"code": code, "message": message, "request_id": get_request_id()}
|
||||
if details:
|
||||
error["details"] = details
|
||||
return {"error": error}
|
||||
@@ -0,0 +1,90 @@
|
||||
"""
|
||||
Structured logging primitives.
|
||||
|
||||
Every log record is emitted as a single JSON object so Logstash can ingest it
|
||||
without grok patterns, and every record carries the request id of the HTTP
|
||||
request that produced it, which makes a single user action traceable end to end
|
||||
in Kibana.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import traceback
|
||||
from contextvars import ContextVar
|
||||
from typing import Any
|
||||
|
||||
# Propagates the current request id to any code running in the same context,
|
||||
# including background threads started per request.
|
||||
_request_id: ContextVar[str] = ContextVar("request_id", default="-")
|
||||
|
||||
# Attributes that ``logging`` puts on every record; anything else the caller
|
||||
# passed through ``extra=`` is treated as structured context and serialised.
|
||||
_STANDARD_ATTRS = frozenset(
|
||||
{
|
||||
"args", "asctime", "created", "exc_info", "exc_text", "filename",
|
||||
"funcName", "levelname", "levelno", "lineno", "module", "msecs",
|
||||
"message", "msg", "name", "pathname", "process", "processName",
|
||||
"relativeCreated", "stack_info", "thread", "threadName", "taskName",
|
||||
"request_id",
|
||||
}
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
def set_request_id(value: str) -> None:
|
||||
_request_id.set(value)
|
||||
|
||||
|
||||
def get_request_id() -> str:
|
||||
return _request_id.get()
|
||||
|
||||
|
||||
class RequestIDFilter(logging.Filter):
|
||||
"""Attaches the ambient request id to every record."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
record.request_id = getattr(record, "request_id", None) or get_request_id()
|
||||
return True
|
||||
|
||||
|
||||
class JSONFormatter(logging.Formatter):
|
||||
"""Renders a ``LogRecord`` as one line of JSON."""
|
||||
|
||||
def __init__(self, service: str = "app", environment: str = "development") -> None:
|
||||
super().__init__()
|
||||
self.service = service
|
||||
self.environment = environment
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
"@timestamp": dt.datetime.fromtimestamp(record.created, tz=dt.UTC).isoformat(),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
"service": self.service,
|
||||
"environment": self.environment,
|
||||
"request_id": getattr(record, "request_id", "-"),
|
||||
}
|
||||
|
||||
if record.exc_info:
|
||||
exc_type, exc_value, exc_tb = record.exc_info
|
||||
payload["error"] = {
|
||||
"type": getattr(exc_type, "__name__", str(exc_type)),
|
||||
"message": str(exc_value),
|
||||
"stack": "".join(traceback.format_exception(exc_type, exc_value, exc_tb))[:8000],
|
||||
}
|
||||
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in _STANDARD_ATTRS and not key.startswith("_"):
|
||||
payload[key] = _coerce(value)
|
||||
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _coerce(value: Any) -> Any:
|
||||
"""Keep the payload JSON serialisable without blowing up on odd types."""
|
||||
if isinstance(value, str | int | float | bool | type(None) | list | dict):
|
||||
return value
|
||||
return str(value)
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Request scoped middleware: correlation ids and structured access logs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
|
||||
from django.http import HttpRequest, HttpResponse
|
||||
|
||||
from apps.core.logging import set_request_id
|
||||
|
||||
logger = logging.getLogger("apps.access")
|
||||
|
||||
REQUEST_ID_HEADER = "HTTP_X_REQUEST_ID"
|
||||
RESPONSE_HEADER = "X-Request-ID"
|
||||
_MAX_REQUEST_ID_LEN = 64
|
||||
|
||||
|
||||
class RequestIDMiddleware:
|
||||
"""
|
||||
Reads ``X-Request-ID`` from the incoming request or generates one.
|
||||
|
||||
The value is echoed back on the response and injected into every log record
|
||||
produced while handling the request, so a browser network tab entry can be
|
||||
pasted straight into Kibana.
|
||||
"""
|
||||
|
||||
def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request: HttpRequest) -> HttpResponse:
|
||||
incoming = request.META.get(REQUEST_ID_HEADER, "")
|
||||
# Never trust client input verbatim: keep it short and alphanumeric.
|
||||
request_id = (
|
||||
incoming[:_MAX_REQUEST_ID_LEN]
|
||||
if incoming and incoming.replace("-", "").replace("_", "").isalnum()
|
||||
else uuid.uuid4().hex
|
||||
)
|
||||
request.request_id = request_id # type: ignore[attr-defined]
|
||||
set_request_id(request_id)
|
||||
|
||||
response = self.get_response(request)
|
||||
response[RESPONSE_HEADER] = request_id
|
||||
return response
|
||||
|
||||
|
||||
class AccessLogMiddleware:
|
||||
"""Emits one structured log line per request with timing and identity."""
|
||||
|
||||
def __init__(self, get_response: Callable[[HttpRequest], HttpResponse]) -> None:
|
||||
self.get_response = get_response
|
||||
|
||||
def __call__(self, request: HttpRequest) -> HttpResponse:
|
||||
started = time.perf_counter()
|
||||
response = self.get_response(request)
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
|
||||
user = getattr(request, "user", None)
|
||||
logger.info(
|
||||
"http_request",
|
||||
extra={
|
||||
"event": "http_request",
|
||||
"http_method": request.method,
|
||||
"http_path": request.path,
|
||||
"http_status": response.status_code,
|
||||
"duration_ms": duration_ms,
|
||||
"client_ip": _client_ip(request),
|
||||
"user_agent": request.META.get("HTTP_USER_AGENT", "")[:200],
|
||||
"user_id": getattr(user, "id", None) if user and user.is_authenticated else None,
|
||||
},
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def _client_ip(request: HttpRequest) -> str:
|
||||
"""
|
||||
Resolve the client IP.
|
||||
|
||||
``X-Forwarded-For`` is only consulted because the API always sits behind a
|
||||
reverse proxy we control; the left-most entry is the original client.
|
||||
"""
|
||||
forwarded = request.META.get("HTTP_X_FORWARDED_FOR")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()[:45]
|
||||
return request.META.get("REMOTE_ADDR", "")[:45]
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Shared pagination classes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import Any
|
||||
|
||||
from rest_framework import pagination
|
||||
from rest_framework.response import Response
|
||||
|
||||
|
||||
class PageNumberPagination(pagination.PageNumberPagination):
|
||||
"""Page/page_size pagination with a hard upper bound on page size."""
|
||||
|
||||
page_size = 20
|
||||
page_size_query_param = "page_size"
|
||||
max_page_size = 100
|
||||
|
||||
def get_paginated_response(self, data: Any) -> Response:
|
||||
return Response(
|
||||
OrderedDict(
|
||||
[
|
||||
("count", self.page.paginator.count),
|
||||
("page", self.page.number),
|
||||
("page_size", self.get_page_size(self.request)),
|
||||
("total_pages", self.page.paginator.num_pages),
|
||||
("next", self.get_next_link()),
|
||||
("previous", self.get_previous_link()),
|
||||
("results", data),
|
||||
]
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
"""Reusable permission classes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from rest_framework.permissions import SAFE_METHODS, BasePermission
|
||||
|
||||
|
||||
class IsAdminRole(BasePermission):
|
||||
"""Only users with the ADMIN role (or Django superusers)."""
|
||||
|
||||
message = "This action requires the admin role."
|
||||
|
||||
def has_permission(self, request, view) -> bool: # noqa: ANN001, ARG002
|
||||
user = request.user
|
||||
return bool(user and user.is_authenticated and user.is_admin_role)
|
||||
|
||||
|
||||
class ReadOnly(BasePermission):
|
||||
def has_permission(self, request, view) -> bool: # noqa: ANN001, ARG002
|
||||
return request.method in SAFE_METHODS
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Scoped throttles used by the search and auth endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from rest_framework.throttling import ScopedRateThrottle, SimpleRateThrottle
|
||||
|
||||
|
||||
class SearchRateThrottle(ScopedRateThrottle):
|
||||
"""Rate limit for the (relatively expensive) search endpoints."""
|
||||
|
||||
scope = "search"
|
||||
|
||||
|
||||
class LoginRateThrottle(SimpleRateThrottle):
|
||||
"""
|
||||
Brute-force protection for credential endpoints.
|
||||
|
||||
Keyed on the client IP rather than the user, because the attacker controls
|
||||
the username field.
|
||||
"""
|
||||
|
||||
scope = "login"
|
||||
|
||||
def get_cache_key(self, request, view) -> str | None: # noqa: ANN001, ARG002
|
||||
return self.cache_format % {"scope": self.scope, "ident": self.get_ident(request)}
|
||||
@@ -0,0 +1,10 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.core.views import LivenessView, ReadinessView
|
||||
|
||||
app_name = "core"
|
||||
|
||||
urlpatterns = [
|
||||
path("live/", LivenessView.as_view(), name="liveness"),
|
||||
path("ready/", ReadinessView.as_view(), name="readiness"),
|
||||
]
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Operational endpoints used by Docker/Kubernetes probes and by humans."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.permissions import AllowAny
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from apps.search.backends import get_search_backend
|
||||
|
||||
|
||||
class LivenessView(APIView):
|
||||
"""Cheap check: the process is up and can answer HTTP."""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes: list = []
|
||||
throttle_classes: list = []
|
||||
|
||||
@extend_schema(tags=["health"], responses={200: dict})
|
||||
def get(self, request: Request) -> Response: # noqa: ARG002
|
||||
return Response({"status": "ok", "service": settings.SERVICE_NAME})
|
||||
|
||||
|
||||
class ReadinessView(APIView):
|
||||
"""
|
||||
Deep check: verifies every dependency the API needs to serve traffic.
|
||||
|
||||
Returns 503 when any dependency is down so an orchestrator stops routing
|
||||
traffic to this instance.
|
||||
"""
|
||||
|
||||
permission_classes = [AllowAny]
|
||||
authentication_classes: list = []
|
||||
throttle_classes: list = []
|
||||
|
||||
@extend_schema(tags=["health"], responses={200: dict, 503: dict})
|
||||
def get(self, request: Request) -> Response: # noqa: ARG002
|
||||
checks: dict[str, Any] = {
|
||||
"database": _check(_database),
|
||||
"search": _check(_search),
|
||||
}
|
||||
healthy = all(c["healthy"] for c in checks.values())
|
||||
return Response(
|
||||
{"status": "ok" if healthy else "degraded", "checks": checks},
|
||||
status=status.HTTP_200_OK if healthy else status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
)
|
||||
|
||||
|
||||
def _check(fn) -> dict[str, Any]: # noqa: ANN001
|
||||
try:
|
||||
detail = fn()
|
||||
except Exception as exc: # noqa: BLE001 - health checks must never raise
|
||||
return {"healthy": False, "detail": str(exc)[:200]}
|
||||
return {"healthy": True, "detail": detail}
|
||||
|
||||
|
||||
def _database() -> str:
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT 1")
|
||||
cursor.fetchone()
|
||||
return "reachable"
|
||||
|
||||
|
||||
def _search() -> str:
|
||||
backend = get_search_backend()
|
||||
backend.ping()
|
||||
return f"{backend.name} reachable"
|
||||
@@ -0,0 +1,72 @@
|
||||
from django.contrib import admin
|
||||
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
|
||||
class ExperienceInline(admin.TabularInline):
|
||||
model = Experience
|
||||
extra = 0
|
||||
fields = ("title", "company_name", "start_date", "end_date", "is_primary")
|
||||
readonly_fields = fields
|
||||
can_delete = False
|
||||
show_change_link = True
|
||||
|
||||
|
||||
class EducationInline(admin.TabularInline):
|
||||
model = Education
|
||||
extra = 0
|
||||
fields = ("school_name", "degrees", "majors", "end_date")
|
||||
readonly_fields = fields
|
||||
can_delete = False
|
||||
|
||||
|
||||
class ProfileSkillInline(admin.TabularInline):
|
||||
model = ProfileSkill
|
||||
extra = 0
|
||||
autocomplete_fields = ("skill",)
|
||||
|
||||
|
||||
@admin.register(Profile)
|
||||
class ProfileAdmin(admin.ModelAdmin):
|
||||
list_display = (
|
||||
"full_name", "job_title", "current_company", "location_country", "years_experience",
|
||||
) # fmt: skip
|
||||
list_filter = ("location_country", "job_title_role", "industry")
|
||||
search_fields = ("full_name", "job_title", "skills_text", "companies_text")
|
||||
list_select_related = ("current_company",)
|
||||
inlines = (ProfileSkillInline, ExperienceInline, EducationInline)
|
||||
readonly_fields = (
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"skills_text",
|
||||
"companies_text",
|
||||
"education_text",
|
||||
)
|
||||
|
||||
|
||||
@admin.register(Skill)
|
||||
class SkillAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "slug", "profile_count")
|
||||
search_fields = ("name", "slug")
|
||||
ordering = ("-profile_count",)
|
||||
|
||||
|
||||
@admin.register(Company)
|
||||
class CompanyAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "industry", "size", "country")
|
||||
search_fields = ("name",)
|
||||
list_filter = ("size", "industry")
|
||||
|
||||
|
||||
@admin.register(School)
|
||||
class SchoolAdmin(admin.ModelAdmin):
|
||||
list_display = ("name", "type", "country")
|
||||
search_fields = ("name",)
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class ProfilesConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.profiles"
|
||||
verbose_name = "Profiles"
|
||||
@@ -0,0 +1,394 @@
|
||||
"""
|
||||
Resilient reader for the supplied LinkedIn CSV export.
|
||||
|
||||
Why this module exists
|
||||
----------------------
|
||||
The provided file is a real-world dirty export, not a clean CSV. Feeding it to
|
||||
``csv.DictReader`` silently produces garbage. Four separate defects are present:
|
||||
|
||||
1. **Injected watermark** - many rows start with an extra, unheadered column
|
||||
containing ``H:\\New folder\\leak\\...@BreachedData1 ...``. Everything after
|
||||
it is shifted one position to the right.
|
||||
2. **Missing columns** - most records omit one or more optional fields
|
||||
(``job_title_role``, ``job_title_sub_role``, ...) instead of writing an empty
|
||||
value, so the remaining values slide left relative to the header.
|
||||
3. **Broken quoting** - a handful of ``summary`` values contain an unescaped
|
||||
double quote, which splits a single logical record across several physical
|
||||
rows.
|
||||
4. **Concatenated exports** - the header line appears a second time in the
|
||||
middle of the file and ~40 people are duplicated.
|
||||
|
||||
Repair strategy
|
||||
---------------
|
||||
Every column has a *signature*: ``linkedin_url`` contains ``linkedin.com/in/``,
|
||||
``skills`` is a Python list literal, ``location_geo`` looks like ``lat,lon`` and
|
||||
so on. We score each (column, value) pair against those signatures and then run
|
||||
a Needleman-Wunsch style alignment between the row's values and the header
|
||||
columns, allowing two edit operations:
|
||||
|
||||
* *skip column* - the record simply does not carry that field;
|
||||
* *drop value* - a stray fragment that belongs to no column.
|
||||
|
||||
The alignment that maximises the total score is the reconstructed record. The
|
||||
result is validated a second time and any value that still fails its signature
|
||||
is discarded rather than persisted as junk - it is better to store nothing than
|
||||
to store a phone number in the e-mail column.
|
||||
|
||||
``ast.literal_eval`` is used (never ``eval``) so the nested Python-repr columns
|
||||
can be parsed without executing anything from the file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import csv
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# The nested columns are large; the stdlib default (128 KiB) is not enough.
|
||||
csv.field_size_limit(min(sys.maxsize, 2**31 - 1))
|
||||
|
||||
WATERMARK_MARKERS = ("@BreachedData", "H:\\New folder")
|
||||
CONTINENTS = frozenset(
|
||||
{"north america", "south america", "europe", "asia", "africa", "oceania", "antarctica"}
|
||||
)
|
||||
GENDERS = frozenset({"male", "female"})
|
||||
|
||||
Validator = Callable[[str], bool | None]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Column signatures
|
||||
# ---------------------------------------------------------------------------
|
||||
def _list_of_str(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
return isinstance(parsed, list) and all(isinstance(item, str) for item in parsed)
|
||||
|
||||
|
||||
def _list_of_dicts(required_key: str) -> Validator:
|
||||
def check(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
if not isinstance(parsed, list):
|
||||
return None
|
||||
if not parsed:
|
||||
return True
|
||||
return all(isinstance(item, dict) for item in parsed) and required_key in parsed[0]
|
||||
|
||||
return check
|
||||
|
||||
|
||||
def _dict_with(required_key: str) -> Validator:
|
||||
def check(value: str) -> bool | None:
|
||||
parsed = _safe_literal(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
return isinstance(parsed, dict) and required_key in parsed
|
||||
|
||||
return check
|
||||
|
||||
|
||||
def _matches(pattern: str) -> Validator:
|
||||
compiled = re.compile(pattern)
|
||||
return lambda value: bool(compiled.fullmatch(value))
|
||||
|
||||
|
||||
def _contains(needle: str) -> Validator:
|
||||
return lambda value: needle in value
|
||||
|
||||
|
||||
def _one_of(options: frozenset[str]) -> Validator:
|
||||
return lambda value: value.lower() in options
|
||||
|
||||
|
||||
def _safe_literal(value: str) -> Any | None:
|
||||
"""``ast.literal_eval`` that returns ``None`` instead of raising."""
|
||||
try:
|
||||
return ast.literal_eval(value)
|
||||
except (ValueError, SyntaxError, MemoryError, RecursionError):
|
||||
return None
|
||||
|
||||
|
||||
#: Columns without an entry here accept any text (name, title, industry, ...).
|
||||
FIELD_VALIDATORS: dict[str, Validator] = {
|
||||
"gender": _one_of(GENDERS),
|
||||
"linkedin_url": _contains("linkedin.com/in/"),
|
||||
"linkedin_id": _matches(r"\d{3,20}"),
|
||||
"facebook_url": _contains("facebook.com/"),
|
||||
"facebook_id": _matches(r"\d{5,25}"),
|
||||
"job_title_role": _matches(r"[a-z_]{3,40}"),
|
||||
"job_title_sub_role": _matches(r"[a-z_]{3,40}"),
|
||||
"job_title_levels": _list_of_str,
|
||||
"job_company_website": _matches(r"[\w.-]+\.[a-z]{2,10}"),
|
||||
"job_company_size": _matches(
|
||||
r"(1-10|11-50|51-200|201-500|501-1000|1001-5000|5001-10000|10001\+)"
|
||||
),
|
||||
"job_company_founded": _matches(r"(1[6-9]|20)\d\d(\.0)?"),
|
||||
"job_company_linkedin_url": _contains("linkedin.com/company/"),
|
||||
"job_company_linkedin_id": _matches(r"\d{3,20}"),
|
||||
"job_company_facebook_url": _contains("facebook.com/"),
|
||||
"job_company_twitter_url": _contains("twitter.com/"),
|
||||
"job_company_location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"job_company_location_continent": _one_of(CONTINENTS),
|
||||
"job_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
|
||||
"job_start_date": _matches(r"\d{4}(-\d{2})?"),
|
||||
"location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"location_continent": _one_of(CONTINENTS),
|
||||
"location_last_updated": _matches(r"\d{4}-\d{2}-\d{2}"),
|
||||
"linkedin_connections": _matches(r"\d+(\.\d+)?"),
|
||||
"inferred_salary": _matches(r"[\d,]+-[\d,]+|>?\d[\d,]*\+?"),
|
||||
"inferred_years_experience": _matches(r"\d{1,2}(\.\d+)?"),
|
||||
"phone_numbers": _list_of_str,
|
||||
"emails": _list_of_dicts("address"),
|
||||
"interests": _list_of_str,
|
||||
"skills": _list_of_str,
|
||||
"location_names": _list_of_str,
|
||||
"regions": _list_of_str,
|
||||
"countries": _list_of_str,
|
||||
"street_addresses": _list_of_dicts("street_address"),
|
||||
"experience": _list_of_dicts("company"),
|
||||
"education": _list_of_dicts("school"),
|
||||
"profiles": _list_of_dicts("network"),
|
||||
"certifications": _list_of_dicts("name"),
|
||||
"languages": _list_of_dicts("name"),
|
||||
"version_status": _dict_with("status"),
|
||||
"work_email": _contains("@"),
|
||||
"middle_initial": _matches(r"[a-z]"),
|
||||
"birth_year": _matches(r"(19|20)\d\d"),
|
||||
"birth_date": _matches(r"\d{4}-\d{2}(-\d{2})?"),
|
||||
"twitter_url": _contains("twitter.com/"),
|
||||
"github_url": _contains("github.com/"),
|
||||
"mobile_phone": _matches(r"\+?\d[\d -]{6,20}"),
|
||||
}
|
||||
|
||||
# Alignment weights. Tuned so that a confirmed signature match outweighs any
|
||||
# combination of edits, while losing a non-empty value is expensive.
|
||||
SCORE_MATCH = 3.0
|
||||
SCORE_MISMATCH = -4.0
|
||||
SCORE_FREE_TEXT = 0.3
|
||||
SCORE_EMPTY = 0.15
|
||||
PENALTY_MISSING_COLUMN = -0.5
|
||||
PENALTY_DROP_EMPTY = -0.3
|
||||
PENALTY_DROP_VALUE = -3.5
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportReport:
|
||||
"""Counters surfaced by the management command so imports are auditable."""
|
||||
|
||||
physical_rows: int = 0
|
||||
duplicate_headers: int = 0
|
||||
repaired_rows: int = 0
|
||||
orphan_fragments: int = 0
|
||||
duplicate_records: int = 0
|
||||
discarded_values: int = 0
|
||||
records: int = 0
|
||||
issues: list[str] = field(default_factory=list)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"physical_rows": self.physical_rows,
|
||||
"duplicate_headers": self.duplicate_headers,
|
||||
"repaired_rows": self.repaired_rows,
|
||||
"orphan_fragments": self.orphan_fragments,
|
||||
"duplicate_records": self.duplicate_records,
|
||||
"discarded_values": self.discarded_values,
|
||||
"records": self.records,
|
||||
}
|
||||
|
||||
|
||||
class DatasetReader:
|
||||
"""Yields clean, de-duplicated records from the raw CSV export."""
|
||||
|
||||
def __init__(self, path: str | Path) -> None:
|
||||
self.path = Path(path)
|
||||
|
||||
def read(self) -> tuple[list[dict[str, str]], ImportReport]:
|
||||
report = ImportReport()
|
||||
with self.path.open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.reader(handle))
|
||||
|
||||
if not rows:
|
||||
return [], report
|
||||
|
||||
header = rows[0]
|
||||
report.physical_rows = len(rows) - 1
|
||||
|
||||
logical_rows = self._reassemble(rows[1:], header, report)
|
||||
|
||||
seen: set[str] = set()
|
||||
records: list[dict[str, str]] = []
|
||||
for values in logical_rows:
|
||||
record = self._align(values, header)
|
||||
self._sanitize(record, report)
|
||||
key = self._identity(record)
|
||||
if key in seen:
|
||||
report.duplicate_records += 1
|
||||
continue
|
||||
seen.add(key)
|
||||
records.append(record)
|
||||
|
||||
report.records = len(records)
|
||||
return records, report
|
||||
|
||||
# -- Step 1: turn physical rows into logical records --------------------
|
||||
def _reassemble(
|
||||
self, rows: list[list[str]], header: list[str], report: ImportReport
|
||||
) -> list[list[str]]:
|
||||
expected = len(header)
|
||||
logical: list[list[str]] = []
|
||||
for raw_row in rows:
|
||||
row = self._strip_watermark(list(raw_row))
|
||||
if not row:
|
||||
continue
|
||||
if self._is_repeated_header(row, header):
|
||||
report.duplicate_headers += 1
|
||||
continue
|
||||
if self._starts_record(row) or not logical:
|
||||
if len(row) != expected:
|
||||
report.repaired_rows += 1
|
||||
logical.append(row)
|
||||
elif len(logical[-1]) < expected - 7:
|
||||
# The previous record is short: this fragment is its tail,
|
||||
# split by a stray quote inside a free-text column.
|
||||
logical[-1].extend(row)
|
||||
report.repaired_rows += 1
|
||||
else:
|
||||
# Cannot be attributed to any record without guessing.
|
||||
report.orphan_fragments += 1
|
||||
return logical
|
||||
|
||||
@staticmethod
|
||||
def _strip_watermark(row: list[str]) -> list[str]:
|
||||
while row and any(marker in row[0] for marker in WATERMARK_MARKERS):
|
||||
row = row[1:]
|
||||
return row
|
||||
|
||||
@staticmethod
|
||||
def _is_repeated_header(row: list[str], header: list[str]) -> bool:
|
||||
return len(row) > 3 and row[:3] == header[:3]
|
||||
|
||||
@staticmethod
|
||||
def _starts_record(row: list[str]) -> bool:
|
||||
"""
|
||||
A record always begins with ``full_name`` followed by ``first_name``,
|
||||
and the full name starts with the first name. That is a strong,
|
||||
cheap anchor for telling a new record from a continuation fragment.
|
||||
"""
|
||||
if len(row) < 9:
|
||||
return False
|
||||
full_name, first_name = row[0].strip(), row[1].strip()
|
||||
return bool(
|
||||
full_name
|
||||
and first_name
|
||||
and len(full_name) < 80
|
||||
and "@" not in full_name
|
||||
and full_name.startswith(first_name)
|
||||
)
|
||||
|
||||
# -- Step 2: align values to columns ------------------------------------
|
||||
@staticmethod
|
||||
def _score(column: str, value: str) -> float:
|
||||
if value == "":
|
||||
return SCORE_EMPTY
|
||||
validator = FIELD_VALIDATORS.get(column)
|
||||
if validator is None:
|
||||
return SCORE_FREE_TEXT
|
||||
try:
|
||||
verdict = validator(value)
|
||||
except Exception: # noqa: BLE001 - a broken value is just a mismatch
|
||||
verdict = False
|
||||
if verdict is None:
|
||||
return SCORE_MISMATCH
|
||||
return SCORE_MATCH if verdict else SCORE_MISMATCH
|
||||
|
||||
def _align(self, values: list[str], header: list[str]) -> dict[str, str]:
|
||||
"""Needleman-Wunsch alignment of row values against header columns."""
|
||||
n, m = len(values), len(header)
|
||||
neg_inf = float("-inf")
|
||||
best = [[neg_inf] * (m + 1) for _ in range(n + 1)]
|
||||
move = [[0] * (m + 1) for _ in range(n + 1)]
|
||||
best[0][0] = 0.0
|
||||
|
||||
for j in range(1, m + 1):
|
||||
best[0][j] = best[0][j - 1] + PENALTY_MISSING_COLUMN
|
||||
move[0][j] = 2
|
||||
for i in range(1, n + 1):
|
||||
best[i][0] = best[i - 1][0] + self._drop_penalty(values[i - 1])
|
||||
move[i][0] = 3
|
||||
|
||||
for i in range(1, n + 1):
|
||||
value = values[i - 1]
|
||||
drop = self._drop_penalty(value)
|
||||
for j in range(1, m + 1):
|
||||
score = best[i - 1][j - 1] + self._score(header[j - 1], value)
|
||||
choice = 1
|
||||
skip_column = best[i][j - 1] + PENALTY_MISSING_COLUMN
|
||||
if skip_column > score:
|
||||
score, choice = skip_column, 2
|
||||
drop_value = best[i - 1][j] + drop
|
||||
if drop_value > score:
|
||||
score, choice = drop_value, 3
|
||||
best[i][j], move[i][j] = score, choice
|
||||
|
||||
record: dict[str, str] = {}
|
||||
i, j = n, m
|
||||
while i > 0 or j > 0:
|
||||
choice = move[i][j]
|
||||
if choice == 1:
|
||||
record[header[j - 1]] = values[i - 1].strip()
|
||||
i, j = i - 1, j - 1
|
||||
elif choice == 2:
|
||||
record.setdefault(header[j - 1], "")
|
||||
j -= 1
|
||||
else:
|
||||
i -= 1
|
||||
return record
|
||||
|
||||
@staticmethod
|
||||
def _drop_penalty(value: str) -> float:
|
||||
return PENALTY_DROP_EMPTY if value == "" else PENALTY_DROP_VALUE
|
||||
|
||||
# -- Step 3: final validation ------------------------------------------
|
||||
@staticmethod
|
||||
def _sanitize(record: dict[str, str], report: ImportReport) -> None:
|
||||
for column, value in list(record.items()):
|
||||
if not value:
|
||||
continue
|
||||
validator = FIELD_VALIDATORS.get(column)
|
||||
if validator is None:
|
||||
continue
|
||||
try:
|
||||
verdict = validator(value)
|
||||
except Exception: # noqa: BLE001
|
||||
verdict = False
|
||||
if not verdict:
|
||||
record[column] = ""
|
||||
report.discarded_values += 1
|
||||
|
||||
@staticmethod
|
||||
def _identity(record: dict[str, str]) -> str:
|
||||
return (
|
||||
record.get("linkedin_id") or record.get("linkedin_url") or record.get("full_name") or ""
|
||||
).lower()
|
||||
|
||||
|
||||
def parse_literal(value: str | None, default: Any) -> Any:
|
||||
"""Public helper for the importer: parse a nested column defensively."""
|
||||
if not value:
|
||||
return default
|
||||
parsed = _safe_literal(value)
|
||||
return default if parsed is None else parsed
|
||||
|
||||
|
||||
def read_dataset(path: str | Path) -> Iterator[dict[str, str]]:
|
||||
"""Convenience wrapper for callers that do not need the report."""
|
||||
records, _ = DatasetReader(path).read()
|
||||
yield from records
|
||||
@@ -0,0 +1,500 @@
|
||||
"""
|
||||
Maps the repaired CSV records onto the relational model.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* **Idempotent.** The natural key is the LinkedIn id (falling back to the
|
||||
profile URL), so re-running the import updates instead of duplicating.
|
||||
* **Bulk friendly.** Companies, schools and skills are resolved through
|
||||
in-memory caches, and the child rows of a profile are replaced wholesale with
|
||||
``bulk_create`` - 284 profiles import in a couple of seconds.
|
||||
* **Self healing input.** The top level columns of this export are frequently
|
||||
missing, but the same information usually exists inside the nested
|
||||
``experience`` blob. ``_derive_current_position`` falls back to the primary
|
||||
experience entry, which raises coverage of ``job_title_role`` from 108 to 183
|
||||
profiles and of the current employer from 219 to 271.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from django.contrib.postgres.search import SearchVector
|
||||
from django.db import transaction
|
||||
|
||||
from apps.profiles.dataset import DatasetReader, ImportReport, parse_literal
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_YEAR_RE = re.compile(r"(\d{4})")
|
||||
_MAX_SKILLS = 60
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportStats:
|
||||
created: int = 0
|
||||
updated: int = 0
|
||||
skills: int = 0
|
||||
companies: int = 0
|
||||
schools: int = 0
|
||||
experiences: int = 0
|
||||
educations: int = 0
|
||||
report: ImportReport = field(default_factory=ImportReport)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"profiles_created": self.created,
|
||||
"profiles_updated": self.updated,
|
||||
"skills": self.skills,
|
||||
"companies": self.companies,
|
||||
"schools": self.schools,
|
||||
"experiences": self.experiences,
|
||||
"educations": self.educations,
|
||||
**self.report.as_dict(),
|
||||
}
|
||||
|
||||
|
||||
class ProfileImporter:
|
||||
"""Loads the dataset into PostgreSQL."""
|
||||
|
||||
def __init__(self, path: str, limit: int | None = None) -> None:
|
||||
self.path = path
|
||||
self.limit = limit
|
||||
self._companies: dict[str, Company] = {}
|
||||
self._schools: dict[str, School] = {}
|
||||
self._skills: dict[str, Skill] = {}
|
||||
|
||||
# -- Entry point --------------------------------------------------------
|
||||
def run(self) -> ImportStats:
|
||||
records, report = DatasetReader(self.path).read()
|
||||
if self.limit:
|
||||
records = records[: self.limit]
|
||||
|
||||
stats = ImportStats(report=report)
|
||||
self._warm_caches()
|
||||
|
||||
for record in records:
|
||||
try:
|
||||
with transaction.atomic():
|
||||
self._import_record(record, stats)
|
||||
except Exception: # noqa: BLE001 - one bad row must not abort the run
|
||||
logger.exception(
|
||||
"profile_import_failed",
|
||||
extra={"event": "profile_import_failed", "name": record.get("full_name")},
|
||||
)
|
||||
report.issues.append(record.get("full_name", "<unknown>"))
|
||||
|
||||
self._refresh_skill_counters()
|
||||
self._refresh_search_vectors()
|
||||
|
||||
stats.skills = len(self._skills)
|
||||
stats.companies = len(self._companies)
|
||||
stats.schools = len(self._schools)
|
||||
return stats
|
||||
|
||||
# -- Caches -------------------------------------------------------------
|
||||
def _warm_caches(self) -> None:
|
||||
self._companies = {c.name: c for c in Company.objects.all()}
|
||||
self._schools = {s.name: s for s in School.objects.all()}
|
||||
self._skills = {s.slug: s for s in Skill.objects.all()}
|
||||
|
||||
# -- Per record ---------------------------------------------------------
|
||||
def _import_record(self, record: dict[str, str], stats: ImportStats) -> None:
|
||||
experiences = parse_literal(record.get("experience"), [])
|
||||
educations = parse_literal(record.get("education"), [])
|
||||
primary = self._primary_experience(experiences)
|
||||
|
||||
title, role, sub_role, levels = self._derive_current_position(record, primary)
|
||||
company = self._resolve_current_company(record, primary)
|
||||
|
||||
skills = self._clean_str_list(parse_literal(record.get("skills"), []))[:_MAX_SKILLS]
|
||||
past_companies = [
|
||||
str(item.get("company", {}).get("name") or "")
|
||||
for item in experiences
|
||||
if isinstance(item, dict) and isinstance(item.get("company"), dict)
|
||||
]
|
||||
schools = [
|
||||
str(item.get("school", {}).get("name") or "")
|
||||
for item in educations
|
||||
if isinstance(item, dict) and isinstance(item.get("school"), dict)
|
||||
]
|
||||
majors = [
|
||||
major
|
||||
for item in educations
|
||||
if isinstance(item, dict)
|
||||
for major in self._clean_str_list(item.get("majors") or [])
|
||||
]
|
||||
|
||||
defaults: dict[str, Any] = {
|
||||
"full_name": record.get("full_name", "")[:255],
|
||||
"first_name": record.get("first_name", "")[:128],
|
||||
"last_name": record.get("last_name", "")[:128],
|
||||
"gender": record.get("gender", "")[:16],
|
||||
"linkedin_url": record.get("linkedin_url", "")[:255],
|
||||
"linkedin_username": record.get("linkedin_username", "")[:128],
|
||||
"github_url": record.get("github_url", "")[:255],
|
||||
"twitter_url": record.get("twitter_url", "")[:255],
|
||||
"job_title": title[:255],
|
||||
"job_title_role": role[:64],
|
||||
"job_title_sub_role": sub_role[:64],
|
||||
"seniority_levels": levels,
|
||||
"current_company": company,
|
||||
"industry": (record.get("industry") or "")[:128],
|
||||
"job_summary": record.get("job_summary", ""),
|
||||
"job_start_date": record.get("job_start_date", "")[:10],
|
||||
"location_name": record.get("location_name", "")[:255],
|
||||
"location_locality": record.get("location_locality", "")[:128],
|
||||
"location_region": record.get("location_region", "")[:128],
|
||||
"location_country": record.get("location_country", "")[:128],
|
||||
"location_continent": record.get("location_continent", "")[:64],
|
||||
"summary": record.get("summary", ""),
|
||||
"years_experience": self._to_int(record.get("inferred_years_experience"), 0, 70),
|
||||
"inferred_salary": record.get("inferred_salary", "")[:32],
|
||||
"connections": self._to_int(record.get("linkedin_connections"), 0, 100_000),
|
||||
"interests": self._clean_str_list(parse_literal(record.get("interests"), []))[:40],
|
||||
"languages": self._language_names(parse_literal(record.get("languages"), [])),
|
||||
"emails": self._emails(parse_literal(record.get("emails"), [])),
|
||||
"phone_numbers": self._clean_str_list(parse_literal(record.get("phone_numbers"), []))[
|
||||
:10
|
||||
],
|
||||
"skills_text": " ".join(skills),
|
||||
"companies_text": " ".join(
|
||||
filter(None, {*past_companies, company.name if company else ""})
|
||||
),
|
||||
"education_text": " ".join(filter(None, [*schools, *majors])),
|
||||
}
|
||||
|
||||
external_id = (record.get("linkedin_id") or "")[:64]
|
||||
lookup = (
|
||||
{"external_id": external_id}
|
||||
if external_id
|
||||
else {"linkedin_url": defaults["linkedin_url"]}
|
||||
)
|
||||
profile, created = Profile.objects.update_or_create(**lookup, defaults=defaults)
|
||||
if created:
|
||||
stats.created += 1
|
||||
else:
|
||||
stats.updated += 1
|
||||
|
||||
self._sync_skills(profile, skills)
|
||||
stats.experiences += self._sync_experiences(profile, experiences)
|
||||
stats.educations += self._sync_educations(profile, educations)
|
||||
|
||||
# -- Current position ---------------------------------------------------
|
||||
@staticmethod
|
||||
def _primary_experience(experiences: list[Any]) -> dict[str, Any]:
|
||||
entries = [item for item in experiences if isinstance(item, dict)]
|
||||
for item in entries:
|
||||
if item.get("is_primary"):
|
||||
return item
|
||||
return entries[0] if entries else {}
|
||||
|
||||
def _derive_current_position(
|
||||
self, record: dict[str, str], primary: dict[str, Any]
|
||||
) -> tuple[str, str, str, list[str]]:
|
||||
title_blob = primary.get("title") if isinstance(primary.get("title"), dict) else {}
|
||||
title = record.get("job_title") or str(title_blob.get("name") or "")
|
||||
role = record.get("job_title_role") or str(title_blob.get("role") or "")
|
||||
sub_role = record.get("job_title_sub_role") or str(title_blob.get("sub_role") or "")
|
||||
levels = self._clean_str_list(
|
||||
parse_literal(record.get("job_title_levels"), None) or title_blob.get("levels") or []
|
||||
)
|
||||
return title, role, sub_role, levels[:5]
|
||||
|
||||
def _resolve_current_company(
|
||||
self, record: dict[str, str], primary: dict[str, Any]
|
||||
) -> Company | None:
|
||||
name = (record.get("job_company_name") or "").strip()
|
||||
payload: dict[str, Any] = {
|
||||
"external_id": record.get("job_company_id", ""),
|
||||
"industry": record.get("job_company_industry", ""),
|
||||
"size": record.get("job_company_size", ""),
|
||||
"founded": self._to_int(record.get("job_company_founded"), 1600, 2100),
|
||||
"website": record.get("job_company_website", ""),
|
||||
"linkedin_url": record.get("job_company_linkedin_url", ""),
|
||||
"location_name": record.get("job_company_location_name", ""),
|
||||
"country": record.get("job_company_location_country", ""),
|
||||
}
|
||||
|
||||
if not name and isinstance(primary.get("company"), dict):
|
||||
blob = primary["company"]
|
||||
name = str(blob.get("name") or "").strip()
|
||||
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
|
||||
payload = {
|
||||
"external_id": str(blob.get("id") or ""),
|
||||
"industry": str(blob.get("industry") or ""),
|
||||
"size": str(blob.get("size") or ""),
|
||||
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
|
||||
"website": str(blob.get("website") or ""),
|
||||
"linkedin_url": str(blob.get("linkedin_url") or ""),
|
||||
"location_name": str(location.get("name") or ""),
|
||||
"country": str(location.get("country") or ""),
|
||||
}
|
||||
|
||||
return self._get_or_create_company(name, payload)
|
||||
|
||||
def _get_or_create_company(self, name: str, payload: dict[str, Any]) -> Company | None:
|
||||
name = name.strip()[:255]
|
||||
if not name:
|
||||
return None
|
||||
cached = self._companies.get(name)
|
||||
if cached:
|
||||
return cached
|
||||
company = Company.objects.create(
|
||||
name=name,
|
||||
external_id=str(payload.get("external_id") or "")[:128],
|
||||
industry=str(payload.get("industry") or "")[:128],
|
||||
size=str(payload.get("size") or "")[:32],
|
||||
founded=payload.get("founded"),
|
||||
website=str(payload.get("website") or "")[:255],
|
||||
linkedin_url=str(payload.get("linkedin_url") or "")[:255],
|
||||
location_name=str(payload.get("location_name") or "")[:255],
|
||||
country=str(payload.get("country") or "")[:128],
|
||||
)
|
||||
self._companies[name] = company
|
||||
return company
|
||||
|
||||
def _get_or_create_school(self, blob: dict[str, Any]) -> School | None:
|
||||
name = str(blob.get("name") or "").strip()[:255]
|
||||
if not name:
|
||||
return None
|
||||
cached = self._schools.get(name)
|
||||
if cached:
|
||||
return cached
|
||||
location = blob.get("location") if isinstance(blob.get("location"), dict) else {}
|
||||
school = School.objects.create(
|
||||
name=name,
|
||||
type=str(blob.get("type") or "")[:64],
|
||||
website=str(blob.get("website") or "")[:255],
|
||||
linkedin_url=str(blob.get("linkedin_url") or "")[:255],
|
||||
location_name=str(location.get("name") or "")[:255],
|
||||
country=str(location.get("country") or "")[:128],
|
||||
)
|
||||
self._schools[name] = school
|
||||
return school
|
||||
|
||||
# -- Children -----------------------------------------------------------
|
||||
def _sync_skills(self, profile: Profile, skills: list[str]) -> None:
|
||||
ProfileSkill.objects.filter(profile=profile).delete()
|
||||
links: list[ProfileSkill] = []
|
||||
used: set[int] = set()
|
||||
for position, name in enumerate(skills):
|
||||
slug = Skill.make_slug(name)
|
||||
if not slug:
|
||||
continue
|
||||
skill = self._skills.get(slug)
|
||||
if skill is None:
|
||||
skill, _ = Skill.objects.get_or_create(slug=slug, defaults={"name": name[:128]})
|
||||
self._skills[slug] = skill
|
||||
if skill.pk in used:
|
||||
continue
|
||||
used.add(skill.pk)
|
||||
links.append(ProfileSkill(profile=profile, skill=skill, position=position))
|
||||
ProfileSkill.objects.bulk_create(links, ignore_conflicts=True)
|
||||
|
||||
def _sync_experiences(self, profile: Profile, experiences: list[Any]) -> int:
|
||||
Experience.objects.filter(profile=profile).delete()
|
||||
rows: list[Experience] = []
|
||||
for item in experiences:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
blob = item.get("company") if isinstance(item.get("company"), dict) else {}
|
||||
title = item.get("title") if isinstance(item.get("title"), dict) else {}
|
||||
company = self._get_or_create_company(
|
||||
str(blob.get("name") or ""),
|
||||
{
|
||||
"external_id": blob.get("id"),
|
||||
"industry": blob.get("industry"),
|
||||
"size": blob.get("size"),
|
||||
"founded": self._to_int(str(blob.get("founded") or ""), 1600, 2100),
|
||||
"website": blob.get("website"),
|
||||
"linkedin_url": blob.get("linkedin_url"),
|
||||
"location_name": (
|
||||
(blob.get("location") or {}).get("name")
|
||||
if isinstance(blob.get("location"), dict)
|
||||
else ""
|
||||
),
|
||||
"country": (
|
||||
(blob.get("location") or {}).get("country")
|
||||
if isinstance(blob.get("location"), dict)
|
||||
else ""
|
||||
),
|
||||
},
|
||||
)
|
||||
rows.append(
|
||||
Experience(
|
||||
profile=profile,
|
||||
company=company,
|
||||
company_name=str(blob.get("name") or "")[:255],
|
||||
title=str(title.get("name") or "")[:255],
|
||||
title_role=str(title.get("role") or "")[:64],
|
||||
title_sub_role=str(title.get("sub_role") or "")[:64],
|
||||
levels=self._clean_str_list(title.get("levels") or [])[:5],
|
||||
start_date=str(item.get("start_date") or "")[:10],
|
||||
end_date=str(item.get("end_date") or "")[:10],
|
||||
start_year=self._year(item.get("start_date")),
|
||||
end_year=self._year(item.get("end_date")),
|
||||
is_primary=bool(item.get("is_primary")),
|
||||
location_name=self._first_location(item.get("location_names")),
|
||||
summary=str(item.get("summary") or ""),
|
||||
)
|
||||
)
|
||||
Experience.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
def _sync_educations(self, profile: Profile, educations: list[Any]) -> int:
|
||||
Education.objects.filter(profile=profile).delete()
|
||||
rows: list[Education] = []
|
||||
for item in educations:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
blob = item.get("school") if isinstance(item.get("school"), dict) else {}
|
||||
rows.append(
|
||||
Education(
|
||||
profile=profile,
|
||||
school=self._get_or_create_school(blob),
|
||||
school_name=str(blob.get("name") or "")[:255],
|
||||
degrees=self._clean_str_list(item.get("degrees") or [])[:5],
|
||||
majors=self._clean_str_list(item.get("majors") or [])[:5],
|
||||
minors=self._clean_str_list(item.get("minors") or [])[:5],
|
||||
gpa=self._to_decimal(item.get("gpa")),
|
||||
start_date=str(item.get("start_date") or "")[:10],
|
||||
end_date=str(item.get("end_date") or "")[:10],
|
||||
end_year=self._year(item.get("end_date")),
|
||||
)
|
||||
)
|
||||
Education.objects.bulk_create(rows)
|
||||
return len(rows)
|
||||
|
||||
# -- Post processing ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _refresh_skill_counters() -> None:
|
||||
"""Single UPDATE ... FROM keeps the facet ordering cheap at query time."""
|
||||
from django.db import connection
|
||||
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE profiles_skill AS s
|
||||
SET profile_count = COALESCE(counts.total, 0)
|
||||
FROM (
|
||||
SELECT skill_id, COUNT(*) AS total
|
||||
FROM profiles_profileskill
|
||||
GROUP BY skill_id
|
||||
) AS counts
|
||||
WHERE counts.skill_id = s.id
|
||||
"""
|
||||
)
|
||||
cursor.execute(
|
||||
"""
|
||||
UPDATE profiles_skill SET profile_count = 0
|
||||
WHERE id NOT IN (SELECT skill_id FROM profiles_profileskill)
|
||||
"""
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _refresh_search_vectors() -> None:
|
||||
"""
|
||||
Rebuild the tsvector for every profile in one statement.
|
||||
|
||||
Weights encode the ranking intent: a hit on the person's name beats a
|
||||
hit on their title or skills, which in turn beats a hit in the free-text
|
||||
summary.
|
||||
"""
|
||||
Profile.objects.update(
|
||||
search_vector=(
|
||||
SearchVector("full_name", weight="A", config="english")
|
||||
+ SearchVector("job_title", weight="B", config="english")
|
||||
+ SearchVector("skills_text", weight="B", config="english")
|
||||
+ SearchVector("companies_text", weight="B", config="english")
|
||||
+ SearchVector("industry", weight="C", config="english")
|
||||
+ SearchVector("education_text", weight="C", config="english")
|
||||
+ SearchVector("location_name", weight="C", config="english")
|
||||
+ SearchVector("summary", weight="D", config="english")
|
||||
)
|
||||
)
|
||||
|
||||
# -- Small helpers ------------------------------------------------------
|
||||
@staticmethod
|
||||
def _clean_str_list(values: Any) -> list[str]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for value in values:
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
cleaned = value.strip().lower()[:128]
|
||||
if cleaned and cleaned not in seen:
|
||||
seen.add(cleaned)
|
||||
out.append(cleaned)
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _language_names(values: Any) -> list[str]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
return [
|
||||
str(item.get("name")).strip().lower()[:64]
|
||||
for item in values
|
||||
if isinstance(item, dict) and item.get("name")
|
||||
][:10]
|
||||
|
||||
@staticmethod
|
||||
def _emails(values: Any) -> list[dict[str, str]]:
|
||||
if not isinstance(values, list):
|
||||
return []
|
||||
return [
|
||||
{
|
||||
"address": str(item.get("address"))[:254],
|
||||
"type": str(item.get("type") or "other")[:32],
|
||||
}
|
||||
for item in values
|
||||
if isinstance(item, dict) and item.get("address")
|
||||
][:10]
|
||||
|
||||
@staticmethod
|
||||
def _first_location(values: Any) -> str:
|
||||
if isinstance(values, list) and values and isinstance(values[0], str):
|
||||
return values[0][:255]
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _to_int(value: str | None, minimum: int, maximum: int) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
number = int(float(str(value).replace(",", "")))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if minimum <= number <= maximum else None
|
||||
|
||||
@staticmethod
|
||||
def _to_decimal(value: Any) -> Any:
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return round(number, 2) if 0 <= number <= 10 else None
|
||||
|
||||
@staticmethod
|
||||
def _year(value: Any) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
match = _YEAR_RE.search(str(value))
|
||||
if not match:
|
||||
return None
|
||||
year = int(match.group(1))
|
||||
return year if 1900 <= year <= 2100 else None
|
||||
@@ -0,0 +1,87 @@
|
||||
"""``python manage.py import_profiles`` - load the CSV export into PostgreSQL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
from django.db import transaction
|
||||
|
||||
from apps.profiles.importers import ProfileImporter
|
||||
from apps.profiles.models import (
|
||||
Company,
|
||||
Education,
|
||||
Experience,
|
||||
Profile,
|
||||
ProfileSkill,
|
||||
School,
|
||||
Skill,
|
||||
)
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Import (or refresh) LinkedIn profiles from the raw CSV export."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
parser.add_argument(
|
||||
"--path",
|
||||
default=settings.DATASET_PATH,
|
||||
help="Path to the CSV export (defaults to DATASET_PATH).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--limit", type=int, default=None, help="Import at most N profiles (smoke tests)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--truncate",
|
||||
action="store_true",
|
||||
help="Delete existing profile data before importing.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reindex",
|
||||
action="store_true",
|
||||
help="Rebuild the Elasticsearch index once the import finishes.",
|
||||
)
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
path = options["path"]
|
||||
started = time.perf_counter()
|
||||
|
||||
if options["truncate"]:
|
||||
self.stdout.write("Truncating existing profile data ...")
|
||||
with transaction.atomic():
|
||||
ProfileSkill.objects.all().delete()
|
||||
Experience.objects.all().delete()
|
||||
Education.objects.all().delete()
|
||||
Profile.objects.all().delete()
|
||||
Skill.objects.all().delete()
|
||||
Company.objects.all().delete()
|
||||
School.objects.all().delete()
|
||||
|
||||
self.stdout.write(f"Reading dataset from {path} ...")
|
||||
try:
|
||||
stats = ProfileImporter(path, limit=options["limit"]).run()
|
||||
except FileNotFoundError as exc:
|
||||
raise CommandError(f"Dataset not found: {path}") from exc
|
||||
|
||||
elapsed = round(time.perf_counter() - started, 2)
|
||||
summary = {**stats.as_dict(), "elapsed_seconds": elapsed}
|
||||
self.stdout.write(json.dumps(summary, indent=2))
|
||||
|
||||
if stats.report.issues:
|
||||
self.stdout.write(
|
||||
self.style.WARNING(
|
||||
f"{len(stats.report.issues)} record(s) failed: "
|
||||
f"{', '.join(stats.report.issues[:10])}"
|
||||
)
|
||||
)
|
||||
|
||||
if options["reindex"]:
|
||||
self.stdout.write("Rebuilding the search index ...")
|
||||
from django.core.management import call_command
|
||||
|
||||
call_command("search_index", "--rebuild")
|
||||
|
||||
self.stdout.write(self.style.SUCCESS(f"Import finished in {elapsed}s."))
|
||||
@@ -0,0 +1,416 @@
|
||||
# Generated by Django 5.1.15 on 2026-09-03 03:36
|
||||
|
||||
import django.contrib.postgres.fields
|
||||
import django.contrib.postgres.indexes
|
||||
import django.contrib.postgres.search
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = []
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="Company",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("external_id", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("name", models.CharField(max_length=255, unique=True)),
|
||||
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
(
|
||||
"size",
|
||||
models.CharField(
|
||||
blank=True, help_text="Head-count bucket, e.g. 51-200", max_length=32
|
||||
),
|
||||
),
|
||||
("founded", models.PositiveIntegerField(blank=True, null=True)),
|
||||
("website", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("country", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
],
|
||||
options={
|
||||
"verbose_name_plural": "companies",
|
||||
"ordering": ("name",),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="School",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("name", models.CharField(max_length=255, unique=True)),
|
||||
("type", models.CharField(blank=True, max_length=64)),
|
||||
("website", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("country", models.CharField(blank=True, max_length=128)),
|
||||
],
|
||||
options={
|
||||
"ordering": ("name",),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Skill",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("name", models.CharField(max_length=128)),
|
||||
("slug", models.SlugField(max_length=140, unique=True)),
|
||||
(
|
||||
"profile_count",
|
||||
models.PositiveIntegerField(
|
||||
db_index=True,
|
||||
default=0,
|
||||
help_text="Denormalised counter used to rank filter suggestions.",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-profile_count", "name"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Profile",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
(
|
||||
"external_id",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
max_length=64,
|
||||
),
|
||||
),
|
||||
("full_name", models.CharField(db_index=True, max_length=255)),
|
||||
("first_name", models.CharField(blank=True, max_length=128)),
|
||||
("last_name", models.CharField(blank=True, max_length=128)),
|
||||
(
|
||||
"gender",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
choices=[("male", "Male"), ("female", "Female"), ("", "Unknown")],
|
||||
max_length=16,
|
||||
),
|
||||
),
|
||||
("linkedin_url", models.CharField(blank=True, max_length=255)),
|
||||
("linkedin_username", models.CharField(blank=True, max_length=128)),
|
||||
("github_url", models.CharField(blank=True, max_length=255)),
|
||||
("twitter_url", models.CharField(blank=True, max_length=255)),
|
||||
("job_title", models.CharField(blank=True, db_index=True, max_length=255)),
|
||||
(
|
||||
"job_title_role",
|
||||
models.CharField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="Coarse job family, e.g. engineering.",
|
||||
max_length=64,
|
||||
),
|
||||
),
|
||||
("job_title_sub_role", models.CharField(blank=True, max_length=64)),
|
||||
(
|
||||
"seniority_levels",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("industry", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("job_summary", models.TextField(blank=True)),
|
||||
("job_start_date", models.CharField(blank=True, max_length=10)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("location_locality", models.CharField(blank=True, max_length=128)),
|
||||
("location_region", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("location_country", models.CharField(blank=True, db_index=True, max_length=128)),
|
||||
("location_continent", models.CharField(blank=True, max_length=64)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
(
|
||||
"years_experience",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
("inferred_salary", models.CharField(blank=True, max_length=32)),
|
||||
("connections", models.PositiveIntegerField(blank=True, null=True)),
|
||||
(
|
||||
"interests",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"languages",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=64),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("emails", models.JSONField(blank=True, default=list)),
|
||||
(
|
||||
"phone_numbers",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("skills_text", models.TextField(blank=True, editable=False)),
|
||||
("companies_text", models.TextField(blank=True, editable=False)),
|
||||
("education_text", models.TextField(blank=True, editable=False)),
|
||||
(
|
||||
"search_vector",
|
||||
django.contrib.postgres.search.SearchVectorField(editable=False, null=True),
|
||||
),
|
||||
(
|
||||
"current_company",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="current_employees",
|
||||
to="profiles.company",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("full_name", "id"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Education",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("school_name", models.CharField(blank=True, max_length=255)),
|
||||
(
|
||||
"degrees",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"majors",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
(
|
||||
"minors",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=128),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("gpa", models.DecimalField(blank=True, decimal_places=2, max_digits=4, null=True)),
|
||||
("start_date", models.CharField(blank=True, max_length=10)),
|
||||
("end_date", models.CharField(blank=True, max_length=10)),
|
||||
(
|
||||
"end_year",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="educations",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
(
|
||||
"school",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="educations",
|
||||
to="profiles.school",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-end_year", "id"),
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="ProfileSkill",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("position", models.PositiveSmallIntegerField(default=0)),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="profile_skills",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
(
|
||||
"skill",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="profile_skills",
|
||||
to="profiles.skill",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("position",),
|
||||
},
|
||||
),
|
||||
migrations.AddField(
|
||||
model_name="profile",
|
||||
name="skills",
|
||||
field=models.ManyToManyField(
|
||||
related_name="profiles", through="profiles.ProfileSkill", to="profiles.skill"
|
||||
),
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name="Experience",
|
||||
fields=[
|
||||
(
|
||||
"id",
|
||||
models.BigAutoField(
|
||||
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
|
||||
),
|
||||
),
|
||||
("created_at", models.DateTimeField(auto_now_add=True, db_index=True)),
|
||||
("updated_at", models.DateTimeField(auto_now=True)),
|
||||
("company_name", models.CharField(blank=True, max_length=255)),
|
||||
("title", models.CharField(blank=True, max_length=255)),
|
||||
("title_role", models.CharField(blank=True, max_length=64)),
|
||||
("title_sub_role", models.CharField(blank=True, max_length=64)),
|
||||
(
|
||||
"levels",
|
||||
django.contrib.postgres.fields.ArrayField(
|
||||
base_field=models.CharField(max_length=32),
|
||||
blank=True,
|
||||
default=list,
|
||||
size=None,
|
||||
),
|
||||
),
|
||||
("start_date", models.CharField(blank=True, max_length=10)),
|
||||
("end_date", models.CharField(blank=True, max_length=10)),
|
||||
(
|
||||
"start_year",
|
||||
models.PositiveSmallIntegerField(blank=True, db_index=True, null=True),
|
||||
),
|
||||
("end_year", models.PositiveSmallIntegerField(blank=True, null=True)),
|
||||
("is_primary", models.BooleanField(db_index=True, default=False)),
|
||||
("location_name", models.CharField(blank=True, max_length=255)),
|
||||
("summary", models.TextField(blank=True)),
|
||||
(
|
||||
"company",
|
||||
models.ForeignKey(
|
||||
blank=True,
|
||||
null=True,
|
||||
on_delete=django.db.models.deletion.SET_NULL,
|
||||
related_name="experiences",
|
||||
to="profiles.company",
|
||||
),
|
||||
),
|
||||
(
|
||||
"profile",
|
||||
models.ForeignKey(
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
related_name="experiences",
|
||||
to="profiles.profile",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"ordering": ("-is_primary", "-start_year", "id"),
|
||||
"indexes": [
|
||||
models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")
|
||||
],
|
||||
},
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="profileskill",
|
||||
constraint=models.UniqueConstraint(
|
||||
fields=("profile", "skill"), name="uniq_profile_skill"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=["search_vector"], name="profile_search_vector_gin"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=django.contrib.postgres.indexes.GinIndex(
|
||||
fields=["seniority_levels"], name="profile_seniority_gin"
|
||||
),
|
||||
),
|
||||
migrations.AddIndex(
|
||||
model_name="profile",
|
||||
index=models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
),
|
||||
migrations.AddConstraint(
|
||||
model_name="profile",
|
||||
constraint=models.UniqueConstraint(
|
||||
condition=models.Q(("external_id__gt", "")),
|
||||
fields=("external_id",),
|
||||
name="uniq_profile_external_id",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
Enable the PostgreSQL extensions the search fallback relies on.
|
||||
|
||||
``pg_trgm`` powers ``TrigramSimilarity`` (typo tolerant name matching) and the
|
||||
GIN trigram index below makes ``ILIKE '%term%'`` on skill names fast enough for
|
||||
the type-ahead endpoint.
|
||||
"""
|
||||
|
||||
from django.contrib.postgres.operations import TrigramExtension, UnaccentExtension
|
||||
from django.db import migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("profiles", "0001_initial")]
|
||||
|
||||
operations = [
|
||||
TrigramExtension(),
|
||||
UnaccentExtension(),
|
||||
migrations.RunSQL(
|
||||
sql=(
|
||||
"CREATE INDEX IF NOT EXISTS profile_full_name_trgm "
|
||||
"ON profiles_profile USING gin (full_name gin_trgm_ops);"
|
||||
),
|
||||
reverse_sql="DROP INDEX IF EXISTS profile_full_name_trgm;",
|
||||
),
|
||||
migrations.RunSQL(
|
||||
sql=(
|
||||
"CREATE INDEX IF NOT EXISTS skill_name_trgm "
|
||||
"ON profiles_skill USING gin (name gin_trgm_ops);"
|
||||
),
|
||||
reverse_sql="DROP INDEX IF EXISTS skill_name_trgm;",
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,261 @@
|
||||
"""
|
||||
Domain model for the LinkedIn dataset.
|
||||
|
||||
The raw export is one very wide CSV row per person with several columns holding
|
||||
serialised lists of dictionaries. Storing that blob as-is would make filtering
|
||||
and aggregation painful, so it is normalised into five tables:
|
||||
|
||||
Company <--- Experience ---> Profile <--- Education ---> School
|
||||
|
|
||||
+--- ProfileSkill ---> Skill
|
||||
|
||||
PostgreSQL is the source of truth. Elasticsearch holds a denormalised
|
||||
projection of the same data for search (see ``apps.search``); it can always be
|
||||
rebuilt from these tables, which is why the index is disposable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.contrib.postgres.fields import ArrayField
|
||||
from django.contrib.postgres.indexes import GinIndex
|
||||
from django.contrib.postgres.search import SearchVectorField
|
||||
from django.db import models
|
||||
from django.utils.text import slugify
|
||||
|
||||
|
||||
class TimeStampedModel(models.Model):
|
||||
created_at = models.DateTimeField(auto_now_add=True, db_index=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
abstract = True
|
||||
|
||||
|
||||
class Company(TimeStampedModel):
|
||||
"""An employer referenced by the current job or by a past experience."""
|
||||
|
||||
external_id = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
size = models.CharField(max_length=32, blank=True, help_text="Head-count bucket, e.g. 51-200")
|
||||
founded = models.PositiveIntegerField(null=True, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
verbose_name_plural = "companies"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class School(TimeStampedModel):
|
||||
name = models.CharField(max_length=255, unique=True)
|
||||
type = models.CharField(max_length=64, blank=True)
|
||||
website = models.CharField(max_length=255, blank=True)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
country = models.CharField(max_length=128, blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("name",)
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class Skill(TimeStampedModel):
|
||||
"""
|
||||
A normalised skill.
|
||||
|
||||
``slug`` is the deduplication key: the raw data contains the same skill with
|
||||
different casing and spacing ("Team Building" / "team building").
|
||||
"""
|
||||
|
||||
name = models.CharField(max_length=128)
|
||||
slug = models.SlugField(max_length=140, unique=True)
|
||||
profile_count = models.PositiveIntegerField(
|
||||
default=0, db_index=True, help_text="Denormalised counter used to rank filter suggestions."
|
||||
)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-profile_count", "name")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.name
|
||||
|
||||
@staticmethod
|
||||
def make_slug(name: str) -> str:
|
||||
return slugify(name)[:140] or name.strip().lower()[:140]
|
||||
|
||||
|
||||
class Profile(TimeStampedModel):
|
||||
"""A single person in the dataset."""
|
||||
|
||||
class Gender(models.TextChoices):
|
||||
MALE = "male", "Male"
|
||||
FEMALE = "female", "Female"
|
||||
UNKNOWN = "", "Unknown"
|
||||
|
||||
# -- Identity ----------------------------------------------------------
|
||||
external_id = models.CharField(
|
||||
max_length=64,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text="LinkedIn numeric id from the source export.",
|
||||
)
|
||||
full_name = models.CharField(max_length=255, db_index=True)
|
||||
first_name = models.CharField(max_length=128, blank=True)
|
||||
last_name = models.CharField(max_length=128, blank=True)
|
||||
gender = models.CharField(max_length=16, blank=True, choices=Gender.choices)
|
||||
linkedin_url = models.CharField(max_length=255, blank=True)
|
||||
linkedin_username = models.CharField(max_length=128, blank=True)
|
||||
github_url = models.CharField(max_length=255, blank=True)
|
||||
twitter_url = models.CharField(max_length=255, blank=True)
|
||||
|
||||
# -- Current position --------------------------------------------------
|
||||
job_title = models.CharField(max_length=255, blank=True, db_index=True)
|
||||
job_title_role = models.CharField(
|
||||
max_length=64, blank=True, db_index=True, help_text="Coarse job family, e.g. engineering."
|
||||
)
|
||||
job_title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
seniority_levels = ArrayField(
|
||||
models.CharField(max_length=32),
|
||||
default=list,
|
||||
blank=True,
|
||||
help_text="e.g. ['manager'], ['senior'], ['cxo'].",
|
||||
)
|
||||
current_company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="current_employees"
|
||||
)
|
||||
industry = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
job_summary = models.TextField(blank=True)
|
||||
job_start_date = models.CharField(max_length=10, blank=True)
|
||||
|
||||
# -- Location ----------------------------------------------------------
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
location_locality = models.CharField(max_length=128, blank=True)
|
||||
location_region = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_country = models.CharField(max_length=128, blank=True, db_index=True)
|
||||
location_continent = models.CharField(max_length=64, blank=True)
|
||||
|
||||
# -- Narrative & metrics ----------------------------------------------
|
||||
summary = models.TextField(blank=True)
|
||||
years_experience = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
inferred_salary = models.CharField(max_length=32, blank=True)
|
||||
connections = models.PositiveIntegerField(null=True, blank=True)
|
||||
interests = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
languages = ArrayField(models.CharField(max_length=64), default=list, blank=True)
|
||||
|
||||
# -- Contact details (PII: masked unless the caller is privileged) ------
|
||||
emails = models.JSONField(default=list, blank=True)
|
||||
phone_numbers = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
|
||||
skills = models.ManyToManyField(Skill, through="ProfileSkill", related_name="profiles")
|
||||
|
||||
# -- Denormalised text, maintained by the importer ---------------------
|
||||
# PostgreSQL's ``SearchVector`` cannot traverse a many-to-many relation, so
|
||||
# the searchable text of the related rows is flattened here once at import
|
||||
# time. This keeps the tsvector update to a single UPDATE statement and
|
||||
# keeps query time O(index lookup) instead of O(joins).
|
||||
skills_text = models.TextField(blank=True, editable=False)
|
||||
companies_text = models.TextField(blank=True, editable=False)
|
||||
education_text = models.TextField(blank=True, editable=False)
|
||||
|
||||
# Materialised tsvector powering the PostgreSQL search backend.
|
||||
search_vector = SearchVectorField(null=True, editable=False)
|
||||
|
||||
class Meta:
|
||||
ordering = ("full_name", "id")
|
||||
indexes = [
|
||||
GinIndex(fields=["search_vector"], name="profile_search_vector_gin"),
|
||||
GinIndex(fields=["seniority_levels"], name="profile_seniority_gin"),
|
||||
models.Index(
|
||||
fields=["location_country", "job_title_role"], name="profile_country_role"
|
||||
),
|
||||
]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["external_id"],
|
||||
condition=models.Q(external_id__gt=""),
|
||||
name="uniq_profile_external_id",
|
||||
)
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.full_name
|
||||
|
||||
@property
|
||||
def headline(self) -> str:
|
||||
"""Short "Title at Company" line used in result cards."""
|
||||
company = self.current_company.name if self.current_company_id else ""
|
||||
if self.job_title and company:
|
||||
return f"{self.job_title} at {company}"
|
||||
return self.job_title or company
|
||||
|
||||
|
||||
class ProfileSkill(models.Model):
|
||||
"""Through table - keeps the original ordering of the skill list."""
|
||||
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
skill = models.ForeignKey(Skill, on_delete=models.CASCADE, related_name="profile_skills")
|
||||
position = models.PositiveSmallIntegerField(default=0)
|
||||
|
||||
class Meta:
|
||||
ordering = ("position",)
|
||||
constraints = [
|
||||
models.UniqueConstraint(fields=["profile", "skill"], name="uniq_profile_skill")
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.profile_id}:{self.skill_id}"
|
||||
|
||||
|
||||
class Experience(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="experiences")
|
||||
company = models.ForeignKey(
|
||||
Company, null=True, blank=True, on_delete=models.SET_NULL, related_name="experiences"
|
||||
)
|
||||
company_name = models.CharField(max_length=255, blank=True)
|
||||
title = models.CharField(max_length=255, blank=True)
|
||||
title_role = models.CharField(max_length=64, blank=True)
|
||||
title_sub_role = models.CharField(max_length=64, blank=True)
|
||||
levels = ArrayField(models.CharField(max_length=32), default=list, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
start_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True)
|
||||
is_primary = models.BooleanField(default=False, db_index=True)
|
||||
location_name = models.CharField(max_length=255, blank=True)
|
||||
summary = models.TextField(blank=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-is_primary", "-start_year", "id")
|
||||
indexes = [models.Index(fields=["profile", "is_primary"], name="exp_profile_primary")]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.title} @ {self.company_name}"
|
||||
|
||||
|
||||
class Education(TimeStampedModel):
|
||||
profile = models.ForeignKey(Profile, on_delete=models.CASCADE, related_name="educations")
|
||||
school = models.ForeignKey(
|
||||
School, null=True, blank=True, on_delete=models.SET_NULL, related_name="educations"
|
||||
)
|
||||
school_name = models.CharField(max_length=255, blank=True)
|
||||
degrees = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
majors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
minors = ArrayField(models.CharField(max_length=128), default=list, blank=True)
|
||||
gpa = models.DecimalField(max_digits=4, decimal_places=2, null=True, blank=True)
|
||||
start_date = models.CharField(max_length=10, blank=True)
|
||||
end_date = models.CharField(max_length=10, blank=True)
|
||||
end_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True)
|
||||
|
||||
class Meta:
|
||||
ordering = ("-end_year", "id")
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.school_name}"
|
||||
@@ -0,0 +1,46 @@
|
||||
"""
|
||||
Contact-detail masking.
|
||||
|
||||
The dataset contains real personal data (e-mail addresses, mobile numbers,
|
||||
street addresses). Exposing it to every authenticated user would be both a
|
||||
privacy and a compliance problem, so contact details are masked unless the
|
||||
caller holds a role that is explicitly allowed to see them
|
||||
(:pyattr:`apps.accounts.models.User.can_view_pii`).
|
||||
|
||||
Masking happens in the serialisation layer - the raw values never leave the
|
||||
database for an unprivileged caller, and the search index stores them in a
|
||||
non-searchable field so a keyword query can never be used to confirm an e-mail
|
||||
address by brute force.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_EMAIL_RE = re.compile(r"^([^@]+)@(.+)$")
|
||||
|
||||
|
||||
def mask_email(address: str) -> str:
|
||||
"""``joseph.holland@garver.com`` -> ``jo************@garver.com``."""
|
||||
match = _EMAIL_RE.match(address or "")
|
||||
if not match:
|
||||
return "***"
|
||||
local, domain = match.groups()
|
||||
keep = min(2, max(len(local) - 1, 0))
|
||||
return f"{local[:keep]}{'*' * max(len(local) - keep, 1)}@{domain}"
|
||||
|
||||
|
||||
def mask_phone(number: str) -> str:
|
||||
"""``+19402058928`` -> ``+1940*****28``."""
|
||||
digits = (number or "").strip()
|
||||
if len(digits) < 6:
|
||||
return "***"
|
||||
return f"{digits[:5]}{'*' * (len(digits) - 7)}{digits[-2:]}"
|
||||
|
||||
|
||||
def mask_emails(emails: list[dict[str, str]]) -> list[dict[str, str]]:
|
||||
return [{**item, "address": mask_email(item.get("address", ""))} for item in emails or []]
|
||||
|
||||
|
||||
def mask_phones(numbers: list[str]) -> list[str]:
|
||||
return [mask_phone(number) for number in numbers or []]
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Read-only serializers for profile payloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.profiles.models import Company, Education, Experience, Profile, School, Skill
|
||||
from apps.profiles.privacy import mask_emails, mask_phones
|
||||
|
||||
|
||||
class SkillSerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = Skill
|
||||
fields = ("id", "name", "slug", "profile_count")
|
||||
|
||||
|
||||
class CompanySerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = Company
|
||||
fields = ("id", "name", "industry", "size", "founded", "website", "location_name")
|
||||
|
||||
|
||||
class SchoolSerializer(serializers.ModelSerializer):
|
||||
class Meta:
|
||||
model = School
|
||||
fields = ("id", "name", "type", "location_name")
|
||||
|
||||
|
||||
class ExperienceSerializer(serializers.ModelSerializer):
|
||||
company = CompanySerializer(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = Experience
|
||||
fields = (
|
||||
"id", "title", "title_role", "title_sub_role", "levels", "company", "company_name",
|
||||
"start_date", "end_date", "start_year", "end_year", "is_primary", "location_name",
|
||||
"summary",
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class EducationSerializer(serializers.ModelSerializer):
|
||||
school = SchoolSerializer(read_only=True)
|
||||
|
||||
class Meta:
|
||||
model = Education
|
||||
fields = (
|
||||
"id", "school", "school_name", "degrees", "majors", "minors", "gpa",
|
||||
"start_date", "end_date", "end_year",
|
||||
) # fmt: skip
|
||||
|
||||
|
||||
class PIIMixin:
|
||||
"""Masks contact details unless the caller is allowed to see them."""
|
||||
|
||||
def _can_view_pii(self) -> bool:
|
||||
request = self.context.get("request") # type: ignore[attr-defined]
|
||||
user = getattr(request, "user", None)
|
||||
return bool(user and user.is_authenticated and user.can_view_pii)
|
||||
|
||||
def get_emails(self, obj: Profile) -> list[dict[str, str]]:
|
||||
return obj.emails if self._can_view_pii() else mask_emails(obj.emails)
|
||||
|
||||
def get_phone_numbers(self, obj: Profile) -> list[str]:
|
||||
return obj.phone_numbers if self._can_view_pii() else mask_phones(obj.phone_numbers)
|
||||
|
||||
|
||||
class ProfileListSerializer(PIIMixin, serializers.ModelSerializer):
|
||||
"""Compact representation used for result cards."""
|
||||
|
||||
headline = serializers.CharField(read_only=True)
|
||||
company = serializers.CharField(source="current_company.name", default="", read_only=True)
|
||||
skills = serializers.SerializerMethodField()
|
||||
emails = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Profile
|
||||
fields = (
|
||||
"id", "full_name", "headline", "job_title", "job_title_role", "seniority_levels",
|
||||
"company", "industry", "location_name", "location_country", "years_experience",
|
||||
"linkedin_url", "skills", "emails",
|
||||
) # fmt: skip
|
||||
|
||||
def get_skills(self, obj: Profile) -> list[str]:
|
||||
# ``prefetch_related`` in the view keeps this at one extra query total.
|
||||
return [link.skill.name for link in obj.profile_skills.all()[:10]]
|
||||
|
||||
|
||||
class ProfileDetailSerializer(PIIMixin, serializers.ModelSerializer):
|
||||
headline = serializers.CharField(read_only=True)
|
||||
current_company = CompanySerializer(read_only=True)
|
||||
skills = serializers.SerializerMethodField()
|
||||
experiences = ExperienceSerializer(many=True, read_only=True)
|
||||
educations = EducationSerializer(many=True, read_only=True)
|
||||
emails = serializers.SerializerMethodField()
|
||||
phone_numbers = serializers.SerializerMethodField()
|
||||
pii_masked = serializers.SerializerMethodField()
|
||||
|
||||
class Meta:
|
||||
model = Profile
|
||||
fields = (
|
||||
"id", "external_id", "full_name", "first_name", "last_name", "gender", "headline",
|
||||
"job_title", "job_title_role", "job_title_sub_role", "seniority_levels",
|
||||
"current_company", "industry", "job_summary", "job_start_date",
|
||||
"location_name", "location_locality", "location_region", "location_country",
|
||||
"location_continent", "summary", "years_experience", "inferred_salary",
|
||||
"connections", "interests", "languages", "linkedin_url", "linkedin_username",
|
||||
"github_url", "twitter_url", "skills", "experiences", "educations",
|
||||
"emails", "phone_numbers", "pii_masked",
|
||||
) # fmt: skip
|
||||
|
||||
def get_skills(self, obj: Profile) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"name": link.skill.name, "slug": link.skill.slug} for link in obj.profile_skills.all()
|
||||
]
|
||||
|
||||
def get_pii_masked(self, obj: Profile) -> bool: # noqa: ARG002
|
||||
return not self._can_view_pii()
|
||||
@@ -0,0 +1,11 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.profiles.views import ProfileDetailView, ProfileListView, SkillListView
|
||||
|
||||
app_name = "profiles"
|
||||
|
||||
urlpatterns = [
|
||||
path("", ProfileListView.as_view(), name="list"),
|
||||
path("skills/", SkillListView.as_view(), name="skills"),
|
||||
path("<int:pk>/", ProfileDetailView.as_view(), name="detail"),
|
||||
]
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Profile read endpoints (the search endpoints live in ``apps.search``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from django.db.models import QuerySet
|
||||
from drf_spectacular.utils import extend_schema
|
||||
from rest_framework import generics
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
|
||||
from apps.profiles.models import Profile, Skill
|
||||
from apps.profiles.serializers import (
|
||||
ProfileDetailSerializer,
|
||||
ProfileListSerializer,
|
||||
SkillSerializer,
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class ProfileListView(generics.ListAPIView):
|
||||
"""
|
||||
Plain, unranked listing.
|
||||
|
||||
Useful for browsing and for verifying the dataset; ranked keyword search is
|
||||
served by ``/api/v1/search/``.
|
||||
"""
|
||||
|
||||
serializer_class = ProfileListSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_queryset(self) -> QuerySet[Profile]:
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill")
|
||||
.order_by("full_name", "id")
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class ProfileDetailView(generics.RetrieveAPIView):
|
||||
"""Full profile including work history, education and (masked) contacts."""
|
||||
|
||||
serializer_class = ProfileDetailSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def get_queryset(self) -> QuerySet[Profile]:
|
||||
return Profile.objects.select_related("current_company").prefetch_related(
|
||||
"profile_skills__skill",
|
||||
"experiences__company",
|
||||
"educations__school",
|
||||
)
|
||||
|
||||
|
||||
@extend_schema(tags=["profiles"])
|
||||
class SkillListView(generics.ListAPIView):
|
||||
"""
|
||||
Skill dictionary ordered by popularity.
|
||||
|
||||
The frontend uses it to power the skill filter's type-ahead without hitting
|
||||
Elasticsearch for every keystroke.
|
||||
"""
|
||||
|
||||
serializer_class = SkillSerializer
|
||||
permission_classes = [IsAuthenticated]
|
||||
pagination_class = None
|
||||
|
||||
def get_queryset(self) -> QuerySet[Skill]:
|
||||
queryset = Skill.objects.filter(profile_count__gt=0)
|
||||
term = self.request.query_params.get("q", "").strip().lower()
|
||||
if term:
|
||||
queryset = queryset.filter(name__icontains=term[:64])
|
||||
limit = min(int(self.request.query_params.get("limit", 50) or 50), 200)
|
||||
return queryset.order_by("-profile_count", "name")[:limit]
|
||||
@@ -0,0 +1,7 @@
|
||||
from django.apps import AppConfig
|
||||
|
||||
|
||||
class SearchConfig(AppConfig):
|
||||
default_auto_field = "django.db.models.BigAutoField"
|
||||
name = "apps.search"
|
||||
verbose_name = "Search"
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Backend factory - resolves ``SEARCH["BACKEND"]`` to a concrete strategy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import cache
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
|
||||
from apps.search.backends.base import SearchBackend
|
||||
|
||||
__all__ = ["SearchBackend", "get_search_backend"]
|
||||
|
||||
|
||||
@cache
|
||||
def get_search_backend(name: str | None = None) -> SearchBackend:
|
||||
"""
|
||||
Return the configured backend.
|
||||
|
||||
Cached because both implementations hold a connection pool that should be
|
||||
created once per process, not once per request.
|
||||
"""
|
||||
backend = (name or settings.SEARCH["BACKEND"]).lower()
|
||||
|
||||
if backend == "elasticsearch":
|
||||
from apps.search.backends.elastic import ElasticsearchBackend
|
||||
|
||||
return ElasticsearchBackend()
|
||||
if backend == "postgres":
|
||||
from apps.search.backends.postgres import PostgresBackend
|
||||
|
||||
return PostgresBackend()
|
||||
|
||||
raise ImproperlyConfigured(
|
||||
f"Unknown SEARCH['BACKEND'] {backend!r}; expected 'elasticsearch' or 'postgres'."
|
||||
)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""The contract every search backend implements."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from apps.search.types import SearchRequest, SearchResult
|
||||
|
||||
|
||||
class SearchBackend(ABC):
|
||||
"""
|
||||
Strategy interface for keyword search.
|
||||
|
||||
Two implementations ship with the project:
|
||||
|
||||
* :class:`apps.search.backends.elastic.ElasticsearchBackend` - the default,
|
||||
used in every environment that runs the ELK stack.
|
||||
* :class:`apps.search.backends.postgres.PostgresBackend` - a full-text
|
||||
fallback that keeps the API (and the test suite) working without any
|
||||
external search service.
|
||||
|
||||
Keeping the surface this small is deliberate: the views, the serializers and
|
||||
the frontend are completely unaware of which engine answered the query.
|
||||
"""
|
||||
|
||||
#: Human readable identifier, surfaced in the API response and health check.
|
||||
name: str = "base"
|
||||
|
||||
@abstractmethod
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
"""Execute a search and return ranked hits plus facet counts."""
|
||||
|
||||
@abstractmethod
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
"""Return type-ahead completions for a partial name or skill."""
|
||||
|
||||
@abstractmethod
|
||||
def ping(self) -> bool:
|
||||
"""Raise or return ``False`` when the backend is not usable."""
|
||||
@@ -0,0 +1,279 @@
|
||||
"""
|
||||
Elasticsearch search backend.
|
||||
|
||||
The query builder is a pure function (:func:`build_query`) that turns a
|
||||
:class:`~apps.search.types.SearchRequest` into an Elasticsearch request body.
|
||||
Keeping it free of I/O means the ranking logic is unit-testable without a
|
||||
running cluster - see ``tests/test_query_builder.py``.
|
||||
|
||||
Ranking model
|
||||
-------------
|
||||
``must`` the keyword must match somewhere, with per-field boosts that encode
|
||||
"a name hit is worth more than a summary hit", plus ``fuzziness:
|
||||
AUTO`` so ``pyhton`` still finds ``python``.
|
||||
``should`` an exact phrase match on the strong fields adds a large bonus, which
|
||||
pushes "product manager" above documents that merely contain both
|
||||
words far apart.
|
||||
``filter`` structured constraints (skills, role, seniority, country, industry,
|
||||
employer, years of experience). They live in the filter context, so
|
||||
they are cached by Elasticsearch and contribute nothing to the score.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import ApiError, Elasticsearch, TransportError
|
||||
|
||||
from apps.core.exceptions import SearchBackendError
|
||||
from apps.search.backends.base import SearchBackend
|
||||
from apps.search.indexing import alias_name, get_client
|
||||
from apps.search.types import (
|
||||
FACET_FIELDS,
|
||||
SORT_EXPERIENCE_ASC,
|
||||
SORT_EXPERIENCE_DESC,
|
||||
SORT_NAME,
|
||||
FacetBucket,
|
||||
SearchHit,
|
||||
SearchRequest,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Field boosts for the free-text part of the query.
|
||||
SEARCH_FIELDS = [
|
||||
"full_name^6",
|
||||
"job_title^4",
|
||||
"skills^3.5",
|
||||
"company^2.5",
|
||||
"past_companies^1.5",
|
||||
"industry^1.5",
|
||||
"majors^1.5",
|
||||
"education",
|
||||
"location_name",
|
||||
"headline^2",
|
||||
"summary",
|
||||
"interests",
|
||||
]
|
||||
|
||||
#: Narrower field set for the phrase bonus - phrases in a summary are noise.
|
||||
PHRASE_FIELDS = ["full_name^8", "job_title^6", "skills^5", "company^4", "headline^3"]
|
||||
|
||||
HIGHLIGHT_FIELDS = {
|
||||
"full_name": {"number_of_fragments": 0},
|
||||
"job_title": {"number_of_fragments": 0},
|
||||
"skills": {"number_of_fragments": 3},
|
||||
"summary": {"fragment_size": 160, "number_of_fragments": 1},
|
||||
}
|
||||
|
||||
#: Maps an API filter to the indexed keyword field it constrains.
|
||||
FILTER_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"roles": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"countries": "location_country",
|
||||
"industries": "industry.keyword",
|
||||
"companies": "company.keyword",
|
||||
}
|
||||
|
||||
|
||||
def build_query(request: SearchRequest) -> dict[str, Any]:
|
||||
"""Translate a search request into an Elasticsearch request body."""
|
||||
must: list[dict[str, Any]] = []
|
||||
should: list[dict[str, Any]] = []
|
||||
|
||||
if request.q:
|
||||
must.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": request.q,
|
||||
"type": "best_fields",
|
||||
"fields": SEARCH_FIELDS,
|
||||
"operator": "or",
|
||||
# One-word queries must match; longer queries tolerate a
|
||||
# missing term so "senior python django" still returns
|
||||
# people who only list two of the three.
|
||||
"minimum_should_match": "2<70%",
|
||||
"fuzziness": "AUTO",
|
||||
"prefix_length": 1,
|
||||
"max_expansions": 30,
|
||||
}
|
||||
}
|
||||
)
|
||||
should.append(
|
||||
{
|
||||
"multi_match": {
|
||||
"query": request.q,
|
||||
"type": "phrase",
|
||||
"fields": PHRASE_FIELDS,
|
||||
"slop": 1,
|
||||
"boost": 4,
|
||||
}
|
||||
}
|
||||
)
|
||||
else:
|
||||
must.append({"match_all": {}})
|
||||
|
||||
body: dict[str, Any] = {
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": must,
|
||||
"should": should,
|
||||
"filter": _build_filters(request),
|
||||
}
|
||||
},
|
||||
"from": request.offset,
|
||||
"size": request.page_size,
|
||||
"track_total_hits": True,
|
||||
"sort": _build_sort(request),
|
||||
}
|
||||
|
||||
if request.q:
|
||||
body["highlight"] = {
|
||||
"pre_tags": ["<mark>"],
|
||||
"post_tags": ["</mark>"],
|
||||
"fields": HIGHLIGHT_FIELDS,
|
||||
}
|
||||
|
||||
if request.include_facets:
|
||||
body["aggs"] = _build_aggregations()
|
||||
|
||||
return body
|
||||
|
||||
|
||||
def _build_filters(request: SearchRequest) -> list[dict[str, Any]]:
|
||||
filters: list[dict[str, Any]] = []
|
||||
|
||||
for attribute, field_name in FILTER_FIELDS.items():
|
||||
values = getattr(request, attribute)
|
||||
if values:
|
||||
# ``terms`` is OR within one filter; separate filters are ANDed,
|
||||
# which matches how users read the UI: "python OR django" skills,
|
||||
# AND located in Germany.
|
||||
filters.append({"terms": {field_name: list(values)}})
|
||||
|
||||
if request.min_experience is not None or request.max_experience is not None:
|
||||
bounds: dict[str, int] = {}
|
||||
if request.min_experience is not None:
|
||||
bounds["gte"] = request.min_experience
|
||||
if request.max_experience is not None:
|
||||
bounds["lte"] = request.max_experience
|
||||
filters.append({"range": {"years_experience": bounds}})
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
def _build_sort(request: SearchRequest) -> list[Any]:
|
||||
if request.sort == SORT_NAME:
|
||||
return [{"full_name.keyword": "asc"}, {"id": "asc"}]
|
||||
if request.sort == SORT_EXPERIENCE_DESC:
|
||||
return [{"years_experience": {"order": "desc", "missing": "_last"}}, "_score"]
|
||||
if request.sort == SORT_EXPERIENCE_ASC:
|
||||
return [{"years_experience": {"order": "asc", "missing": "_last"}}, "_score"]
|
||||
return ["_score", {"id": "asc"}] # deterministic tie-break
|
||||
|
||||
|
||||
def _build_aggregations() -> dict[str, Any]:
|
||||
size = settings.SEARCH["FACET_SIZE"]
|
||||
return {
|
||||
name: {"terms": {"field": field_name, "size": size, "order": {"_count": "desc"}}}
|
||||
for name, field_name in FACET_FIELDS.items()
|
||||
}
|
||||
|
||||
|
||||
class ElasticsearchBackend(SearchBackend):
|
||||
name = "elasticsearch"
|
||||
|
||||
def __init__(self, client: Elasticsearch | None = None) -> None:
|
||||
self._client = client or get_client()
|
||||
self._index = alias_name()
|
||||
|
||||
# -- SearchBackend ------------------------------------------------------
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
body = build_query(request)
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = self._client.search(index=self._index, **body)
|
||||
except (ApiError, TransportError) as exc:
|
||||
logger.exception("search_backend_error", extra={"event": "search_backend_error"})
|
||||
raise SearchBackendError() from exc
|
||||
|
||||
took_ms = int((time.perf_counter() - started) * 1000)
|
||||
hits = [self._to_hit(raw) for raw in response["hits"]["hits"]]
|
||||
return SearchResult(
|
||||
total=response["hits"]["total"]["value"],
|
||||
hits=hits,
|
||||
facets=self._to_facets(response.get("aggregations", {})),
|
||||
took_ms=took_ms,
|
||||
backend=self.name,
|
||||
)
|
||||
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
if not prefix:
|
||||
return []
|
||||
try:
|
||||
response = self._client.search(
|
||||
index=self._index,
|
||||
size=limit,
|
||||
query={
|
||||
"bool": {
|
||||
"should": [
|
||||
{"match": {"full_name.autocomplete": {"query": prefix, "boost": 2}}},
|
||||
{"match": {"skills.autocomplete": prefix}},
|
||||
],
|
||||
"minimum_should_match": 1,
|
||||
}
|
||||
},
|
||||
_source=["full_name", "job_title"],
|
||||
)
|
||||
except (ApiError, TransportError) as exc:
|
||||
raise SearchBackendError() from exc
|
||||
|
||||
seen: list[str] = []
|
||||
for raw in response["hits"]["hits"]:
|
||||
value = raw["_source"].get("full_name", "")
|
||||
if value and value not in seen:
|
||||
seen.append(value)
|
||||
return seen[:limit]
|
||||
|
||||
def ping(self) -> bool:
|
||||
if not self._client.ping():
|
||||
raise SearchBackendError("Elasticsearch did not answer a ping.")
|
||||
return True
|
||||
|
||||
# -- Mapping helpers ----------------------------------------------------
|
||||
@staticmethod
|
||||
def _to_hit(raw: dict[str, Any]) -> SearchHit:
|
||||
source = raw["_source"]
|
||||
return SearchHit(
|
||||
id=source["id"],
|
||||
score=raw.get("_score") or 0.0,
|
||||
full_name=source.get("full_name", ""),
|
||||
headline=source.get("headline", ""),
|
||||
job_title=source.get("job_title", ""),
|
||||
job_title_role=source.get("job_title_role", ""),
|
||||
seniority_levels=source.get("seniority_levels", []),
|
||||
company=source.get("company", ""),
|
||||
industry=source.get("industry", ""),
|
||||
location_name=source.get("location_name", ""),
|
||||
location_country=source.get("location_country", ""),
|
||||
years_experience=source.get("years_experience"),
|
||||
linkedin_url=source.get("linkedin_url", ""),
|
||||
skills=source.get("skills", [])[:12],
|
||||
highlight=raw.get("highlight", {}),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _to_facets(aggregations: dict[str, Any]) -> dict[str, list[FacetBucket]]:
|
||||
return {
|
||||
name: [
|
||||
FacetBucket(key=str(bucket["key"]), count=bucket["doc_count"])
|
||||
for bucket in payload.get("buckets", [])
|
||||
if bucket["key"] != ""
|
||||
]
|
||||
for name, payload in aggregations.items()
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
"""
|
||||
PostgreSQL full-text search backend.
|
||||
|
||||
Why it exists
|
||||
-------------
|
||||
Elasticsearch is the primary engine, but a search API that hard-fails without
|
||||
it is fragile: the test suite would need a cluster, a reviewer would need one to
|
||||
run the project, and an Elasticsearch outage would take the whole product down.
|
||||
Setting ``SEARCH_BACKEND=postgres`` switches to this implementation, which is
|
||||
built on the ``tsvector`` maintained by the importer plus a trigram similarity
|
||||
pass for typo tolerance.
|
||||
|
||||
It is intentionally *not* a re-implementation of Elasticsearch: ranking is
|
||||
coarser and there is no stemming beyond what the ``english`` text search
|
||||
configuration provides. For a 300 profile dataset the difference is barely
|
||||
visible, which is exactly why it is a good fallback.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from collections import Counter
|
||||
|
||||
from django.contrib.postgres.search import SearchQuery, SearchRank, TrigramSimilarity
|
||||
from django.db.models import Case, F, FloatField, Q, QuerySet, Value, When
|
||||
from django.db.models.functions import Greatest
|
||||
|
||||
from apps.profiles.models import Profile, Skill
|
||||
from apps.search.backends.base import SearchBackend
|
||||
from apps.search.types import (
|
||||
SORT_EXPERIENCE_ASC,
|
||||
SORT_EXPERIENCE_DESC,
|
||||
SORT_NAME,
|
||||
FacetBucket,
|
||||
SearchHit,
|
||||
SearchRequest,
|
||||
SearchResult,
|
||||
)
|
||||
|
||||
_TOKEN_RE = re.compile(r"[\w']+", re.UNICODE)
|
||||
_FACET_LIMIT = 25
|
||||
_MIN_TRIGRAM_SIMILARITY = 0.3
|
||||
|
||||
|
||||
class PostgresBackend(SearchBackend):
|
||||
name = "postgres"
|
||||
|
||||
def search(self, request: SearchRequest) -> SearchResult:
|
||||
started = time.perf_counter()
|
||||
queryset = self._apply_filters(Profile.objects.all(), request)
|
||||
queryset = self._apply_keyword(queryset, request.q)
|
||||
total = queryset.count()
|
||||
|
||||
page = (
|
||||
self._apply_sort(queryset, request)
|
||||
.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill")[
|
||||
request.offset : request.offset + request.page_size
|
||||
]
|
||||
)
|
||||
hits = [self._to_hit(profile, request.q) for profile in page]
|
||||
facets = self._facets(queryset) if request.include_facets else {}
|
||||
|
||||
return SearchResult(
|
||||
total=total,
|
||||
hits=hits,
|
||||
facets=facets,
|
||||
took_ms=int((time.perf_counter() - started) * 1000),
|
||||
backend=self.name,
|
||||
)
|
||||
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]:
|
||||
if not prefix:
|
||||
return []
|
||||
return list(
|
||||
Profile.objects.filter(
|
||||
Q(full_name__istartswith=prefix) | Q(full_name__icontains=f" {prefix}")
|
||||
)
|
||||
.order_by("full_name")
|
||||
.values_list("full_name", flat=True)[:limit]
|
||||
)
|
||||
|
||||
def ping(self) -> bool:
|
||||
Profile.objects.exists()
|
||||
return True
|
||||
|
||||
# -- Query construction -------------------------------------------------
|
||||
@staticmethod
|
||||
def _apply_filters(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
|
||||
if request.skills:
|
||||
# Every selected skill widens the result set (OR), mirroring the
|
||||
# ``terms`` filter used by the Elasticsearch backend.
|
||||
queryset = queryset.filter(
|
||||
skills__slug__in=[Skill.make_slug(s) for s in request.skills]
|
||||
)
|
||||
if request.roles:
|
||||
queryset = queryset.filter(job_title_role__in=request.roles)
|
||||
if request.seniority:
|
||||
queryset = queryset.filter(seniority_levels__overlap=list(request.seniority))
|
||||
if request.countries:
|
||||
queryset = queryset.filter(location_country__in=request.countries)
|
||||
if request.industries:
|
||||
queryset = queryset.filter(industry__in=request.industries)
|
||||
if request.companies:
|
||||
queryset = queryset.filter(current_company__name__in=request.companies)
|
||||
if request.min_experience is not None:
|
||||
queryset = queryset.filter(years_experience__gte=request.min_experience)
|
||||
if request.max_experience is not None:
|
||||
queryset = queryset.filter(years_experience__lte=request.max_experience)
|
||||
return queryset.distinct()
|
||||
|
||||
@staticmethod
|
||||
def _apply_keyword(queryset: QuerySet[Profile], term: str) -> QuerySet[Profile]:
|
||||
if not term:
|
||||
return queryset.annotate(rank=Value(0.0, output_field=FloatField()))
|
||||
|
||||
# ``websearch`` accepts the syntax users already know from search
|
||||
# engines: quoted phrases, ``or``, and ``-excluded``.
|
||||
query = SearchQuery(term, search_type="websearch", config="english")
|
||||
return (
|
||||
queryset.annotate(
|
||||
text_rank=SearchRank(F("search_vector"), query),
|
||||
name_similarity=TrigramSimilarity("full_name", term),
|
||||
)
|
||||
.annotate(
|
||||
rank=Greatest(
|
||||
F("text_rank"),
|
||||
# A close name match is worth as much as a strong text hit,
|
||||
# which is what makes "jospeh" still find "joseph".
|
||||
Case(
|
||||
When(
|
||||
name_similarity__gte=_MIN_TRIGRAM_SIMILARITY, then=F("name_similarity")
|
||||
),
|
||||
default=Value(0.0),
|
||||
output_field=FloatField(),
|
||||
),
|
||||
output_field=FloatField(),
|
||||
)
|
||||
)
|
||||
.filter(Q(search_vector=query) | Q(name_similarity__gte=_MIN_TRIGRAM_SIMILARITY))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _apply_sort(queryset: QuerySet[Profile], request: SearchRequest) -> QuerySet[Profile]:
|
||||
if request.sort == SORT_NAME:
|
||||
return queryset.order_by("full_name", "id")
|
||||
if request.sort == SORT_EXPERIENCE_DESC:
|
||||
return queryset.order_by(F("years_experience").desc(nulls_last=True), "id")
|
||||
if request.sort == SORT_EXPERIENCE_ASC:
|
||||
return queryset.order_by(F("years_experience").asc(nulls_last=True), "id")
|
||||
return queryset.order_by("-rank", "full_name", "id")
|
||||
|
||||
# -- Facets -------------------------------------------------------------
|
||||
def _facets(self, queryset: QuerySet[Profile]) -> dict[str, list[FacetBucket]]:
|
||||
ids = list(queryset.values_list("id", flat=True))
|
||||
if not ids:
|
||||
return {
|
||||
name: []
|
||||
for name in (
|
||||
"skills",
|
||||
"job_title_role",
|
||||
"seniority",
|
||||
"country",
|
||||
"industry",
|
||||
"company",
|
||||
)
|
||||
}
|
||||
|
||||
skills = Skill.objects.filter(profile_skills__profile_id__in=ids).values_list(
|
||||
"name", flat=True
|
||||
)
|
||||
rows = Profile.objects.filter(id__in=ids).values(
|
||||
"job_title_role", "location_country", "industry", "current_company__name",
|
||||
"seniority_levels",
|
||||
) # fmt: skip
|
||||
|
||||
counters: dict[str, Counter] = {
|
||||
"skills": Counter(skills),
|
||||
"job_title_role": Counter(),
|
||||
"seniority": Counter(),
|
||||
"country": Counter(),
|
||||
"industry": Counter(),
|
||||
"company": Counter(),
|
||||
}
|
||||
for row in rows:
|
||||
counters["job_title_role"][row["job_title_role"]] += 1
|
||||
counters["country"][row["location_country"]] += 1
|
||||
counters["industry"][row["industry"]] += 1
|
||||
counters["company"][row["current_company__name"] or ""] += 1
|
||||
for level in row["seniority_levels"] or []:
|
||||
counters["seniority"][level] += 1
|
||||
|
||||
return {
|
||||
name: [
|
||||
FacetBucket(key=key, count=count)
|
||||
for key, count in counter.most_common(_FACET_LIMIT)
|
||||
if key
|
||||
]
|
||||
for name, counter in counters.items()
|
||||
}
|
||||
|
||||
# -- Result mapping -----------------------------------------------------
|
||||
@classmethod
|
||||
def _to_hit(cls, profile: Profile, term: str) -> SearchHit:
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()[:12]]
|
||||
return SearchHit(
|
||||
id=profile.id,
|
||||
score=round(float(getattr(profile, "rank", 0.0) or 0.0), 4),
|
||||
full_name=profile.full_name,
|
||||
headline=profile.headline,
|
||||
job_title=profile.job_title,
|
||||
job_title_role=profile.job_title_role,
|
||||
seniority_levels=profile.seniority_levels,
|
||||
company=profile.current_company.name if profile.current_company_id else "",
|
||||
industry=profile.industry,
|
||||
location_name=profile.location_name,
|
||||
location_country=profile.location_country,
|
||||
years_experience=profile.years_experience,
|
||||
linkedin_url=profile.linkedin_url,
|
||||
skills=skills,
|
||||
highlight=cls._highlight(term, profile, skills),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _highlight(term: str, profile: Profile, skills: list[str]) -> dict[str, list[str]]:
|
||||
"""
|
||||
Approximate Elasticsearch's highlighter.
|
||||
|
||||
PostgreSQL offers ``ts_headline`` but it needs an extra round trip per
|
||||
row; for the fields shown on a result card a simple token wrap gives the
|
||||
same visual result at no query cost.
|
||||
"""
|
||||
tokens = [token for token in _TOKEN_RE.findall(term.lower()) if len(token) > 1]
|
||||
if not tokens:
|
||||
return {}
|
||||
pattern = re.compile("|".join(re.escape(token) for token in tokens), re.IGNORECASE)
|
||||
|
||||
highlight: dict[str, list[str]] = {}
|
||||
for field_name, value in (
|
||||
("full_name", profile.full_name),
|
||||
("job_title", profile.job_title),
|
||||
):
|
||||
if value and pattern.search(value):
|
||||
highlight[field_name] = [pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", value)]
|
||||
matched_skills = [skill for skill in skills if pattern.search(skill)]
|
||||
if matched_skills:
|
||||
highlight["skills"] = [
|
||||
pattern.sub(lambda m: f"<mark>{m.group(0)}</mark>", skill)
|
||||
for skill in matched_skills[:3]
|
||||
]
|
||||
return highlight
|
||||
@@ -0,0 +1,288 @@
|
||||
"""
|
||||
Elasticsearch index definition and (re)build helpers.
|
||||
|
||||
Index design
|
||||
------------
|
||||
* One document per profile, denormalised: the fields a recruiter searches on
|
||||
(name, title, employer, skills, education, location) all live in the same
|
||||
document, so a query never needs a join.
|
||||
* Text fields are analysed with an English analyzer (lowercase, ASCII folding,
|
||||
stop words, stemming) so "engineering" matches "engineer".
|
||||
* Every field that is *filtered* or *aggregated* on has a ``keyword``
|
||||
counterpart with a lowercase normaliser - analysed text cannot be used for
|
||||
exact term filters or facet counts.
|
||||
* ``full_name.autocomplete`` uses an edge n-gram analyzer for the type-ahead;
|
||||
the search analyzer is deliberately different (``standard``) so the query
|
||||
term is not itself n-grammed.
|
||||
* **No personal contact data is indexed.** E-mail addresses and phone numbers
|
||||
stay in PostgreSQL, which removes any chance of confirming someone's e-mail
|
||||
through a crafted search query.
|
||||
|
||||
Rebuilds use an alias plus a timestamped physical index, so a reindex never
|
||||
leaves the API pointing at a half-populated index and can be rolled back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from elasticsearch import Elasticsearch
|
||||
from elasticsearch.helpers import bulk
|
||||
|
||||
from apps.profiles.models import Profile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
INDEX_SETTINGS: dict[str, Any] = {
|
||||
"number_of_shards": 1,
|
||||
"number_of_replicas": 0,
|
||||
"refresh_interval": "1s",
|
||||
"max_result_window": 10_000,
|
||||
"analysis": {
|
||||
"filter": {
|
||||
"english_stop": {"type": "stop", "stopwords": "_english_"},
|
||||
"english_stemmer": {"type": "stemmer", "language": "english"},
|
||||
"english_possessive": {"type": "stemmer", "language": "possessive_english"},
|
||||
"edge_ngrams": {"type": "edge_ngram", "min_gram": 2, "max_gram": 15},
|
||||
},
|
||||
"normalizer": {
|
||||
"lowercase_normalizer": {
|
||||
"type": "custom",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
}
|
||||
},
|
||||
"analyzer": {
|
||||
"english_text": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": [
|
||||
"lowercase",
|
||||
"asciifolding",
|
||||
"english_possessive",
|
||||
"english_stop",
|
||||
"english_stemmer",
|
||||
],
|
||||
},
|
||||
"autocomplete_index": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding", "edge_ngrams"],
|
||||
},
|
||||
"autocomplete_search": {
|
||||
"type": "custom",
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
KEYWORD = {"type": "keyword", "normalizer": "lowercase_normalizer"}
|
||||
|
||||
INDEX_MAPPINGS: dict[str, Any] = {
|
||||
"dynamic": "strict", # reject unexpected fields instead of guessing a type
|
||||
"properties": {
|
||||
"id": {"type": "integer"},
|
||||
"full_name": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"headline": {"type": "text", "analyzer": "english_text"},
|
||||
"job_title": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"job_title_role": KEYWORD,
|
||||
"job_title_sub_role": KEYWORD,
|
||||
"seniority_levels": KEYWORD,
|
||||
"company": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"past_companies": {"type": "text", "analyzer": "english_text"},
|
||||
"industry": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {"keyword": KEYWORD},
|
||||
},
|
||||
"skills": {
|
||||
"type": "text",
|
||||
"analyzer": "english_text",
|
||||
"fields": {
|
||||
"keyword": KEYWORD,
|
||||
"autocomplete": {
|
||||
"type": "text",
|
||||
"analyzer": "autocomplete_index",
|
||||
"search_analyzer": "autocomplete_search",
|
||||
},
|
||||
},
|
||||
},
|
||||
"education": {"type": "text", "analyzer": "english_text"},
|
||||
"degrees": KEYWORD,
|
||||
"majors": {"type": "text", "analyzer": "english_text", "fields": {"keyword": KEYWORD}},
|
||||
"summary": {"type": "text", "analyzer": "english_text"},
|
||||
"interests": {"type": "text", "analyzer": "english_text"},
|
||||
"languages": KEYWORD,
|
||||
"location_name": {"type": "text", "analyzer": "english_text"},
|
||||
"location_locality": KEYWORD,
|
||||
"location_region": KEYWORD,
|
||||
"location_country": KEYWORD,
|
||||
"location_continent": KEYWORD,
|
||||
"years_experience": {"type": "integer"},
|
||||
"connections": {"type": "integer"},
|
||||
"linkedin_url": {"type": "keyword", "index": False},
|
||||
"indexed_at": {"type": "date"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_client() -> Elasticsearch:
|
||||
"""Build an Elasticsearch client from settings (auth optional in dev)."""
|
||||
config = settings.SEARCH["ELASTICSEARCH"]
|
||||
kwargs: dict[str, Any] = {
|
||||
"hosts": config["HOSTS"],
|
||||
"request_timeout": config["TIMEOUT"],
|
||||
"max_retries": config["MAX_RETRIES"],
|
||||
"retry_on_timeout": True,
|
||||
}
|
||||
if config["USERNAME"] and config["PASSWORD"]:
|
||||
kwargs["basic_auth"] = (config["USERNAME"], config["PASSWORD"])
|
||||
return Elasticsearch(**kwargs)
|
||||
|
||||
|
||||
def alias_name() -> str:
|
||||
return settings.SEARCH["ELASTICSEARCH"]["INDEX"]
|
||||
|
||||
|
||||
def physical_index_name() -> str:
|
||||
return f"{alias_name()}-{datetime.now(UTC):%Y%m%d%H%M%S}"
|
||||
|
||||
|
||||
def build_document(profile: Profile) -> dict[str, Any]:
|
||||
"""Project a ``Profile`` row (plus its relations) into an index document."""
|
||||
skills = [link.skill.name for link in profile.profile_skills.all()]
|
||||
experiences = list(profile.experiences.all())
|
||||
educations = list(profile.educations.all())
|
||||
|
||||
degrees = sorted({degree for edu in educations for degree in edu.degrees})
|
||||
majors = sorted({major for edu in educations for major in edu.majors})
|
||||
schools = [edu.school_name for edu in educations if edu.school_name]
|
||||
|
||||
return {
|
||||
"id": profile.id,
|
||||
"full_name": profile.full_name,
|
||||
"headline": profile.headline,
|
||||
"job_title": profile.job_title,
|
||||
"job_title_role": profile.job_title_role,
|
||||
"job_title_sub_role": profile.job_title_sub_role,
|
||||
"seniority_levels": profile.seniority_levels,
|
||||
"company": profile.current_company.name if profile.current_company_id else "",
|
||||
"past_companies": sorted({exp.company_name for exp in experiences if exp.company_name}),
|
||||
"industry": profile.industry,
|
||||
"skills": skills,
|
||||
"education": " ".join([*schools, *majors]),
|
||||
"degrees": degrees,
|
||||
"majors": majors,
|
||||
"summary": profile.summary,
|
||||
"interests": profile.interests,
|
||||
"languages": profile.languages,
|
||||
"location_name": profile.location_name,
|
||||
"location_locality": profile.location_locality,
|
||||
"location_region": profile.location_region,
|
||||
"location_country": profile.location_country,
|
||||
"location_continent": profile.location_continent,
|
||||
"years_experience": profile.years_experience,
|
||||
"connections": profile.connections,
|
||||
"linkedin_url": profile.linkedin_url,
|
||||
"indexed_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def iter_documents(queryset: Iterable[Profile] | None = None) -> Iterator[dict[str, Any]]:
|
||||
profiles = queryset if queryset is not None else _default_queryset()
|
||||
for profile in profiles:
|
||||
yield {"_id": profile.id, "_source": build_document(profile)}
|
||||
|
||||
|
||||
def _default_queryset(): # noqa: ANN202
|
||||
return (
|
||||
Profile.objects.select_related("current_company")
|
||||
.prefetch_related("profile_skills__skill", "experiences", "educations")
|
||||
.order_by("id")
|
||||
.iterator(chunk_size=200)
|
||||
)
|
||||
|
||||
|
||||
def create_index(client: Elasticsearch, name: str) -> None:
|
||||
client.indices.create(index=name, settings=INDEX_SETTINGS, mappings=INDEX_MAPPINGS)
|
||||
|
||||
|
||||
def rebuild_index(client: Elasticsearch | None = None) -> dict[str, Any]:
|
||||
"""
|
||||
Zero-downtime rebuild.
|
||||
|
||||
Creates a fresh physical index, indexes every profile into it, then flips
|
||||
the alias atomically and removes the previous physical index.
|
||||
"""
|
||||
client = client or get_client()
|
||||
alias = alias_name()
|
||||
new_index = physical_index_name()
|
||||
|
||||
create_index(client, new_index)
|
||||
logger.info("index_created", extra={"event": "index_created", "index": new_index})
|
||||
|
||||
succeeded, errors = bulk(
|
||||
client,
|
||||
({"_index": new_index, **doc} for doc in iter_documents()),
|
||||
raise_on_error=False,
|
||||
stats_only=False,
|
||||
)
|
||||
client.indices.refresh(index=new_index)
|
||||
|
||||
previous = _current_indices(client, alias)
|
||||
actions: list[dict[str, Any]] = [{"add": {"index": new_index, "alias": alias}}]
|
||||
actions += [{"remove": {"index": name, "alias": alias}} for name in previous]
|
||||
client.indices.update_aliases(actions=actions)
|
||||
|
||||
for name in previous:
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
|
||||
result = {
|
||||
"alias": alias,
|
||||
"index": new_index,
|
||||
"indexed": succeeded,
|
||||
"errors": len(errors) if isinstance(errors, list) else errors,
|
||||
"replaced": previous,
|
||||
}
|
||||
logger.info("index_rebuilt", extra={"event": "index_rebuilt", **result})
|
||||
return result
|
||||
|
||||
|
||||
def _current_indices(client: Elasticsearch, alias: str) -> list[str]:
|
||||
if not client.indices.exists_alias(name=alias):
|
||||
# A leftover *index* (not alias) with the same name would block the
|
||||
# alias creation, so remove it first.
|
||||
if client.indices.exists(index=alias):
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
return []
|
||||
return list(client.indices.get_alias(name=alias).keys())
|
||||
|
||||
|
||||
def index_profile(profile: Profile, client: Elasticsearch | None = None) -> None:
|
||||
"""Index (or re-index) a single profile - used after a targeted update."""
|
||||
client = client or get_client()
|
||||
client.index(index=alias_name(), id=profile.id, document=build_document(profile))
|
||||
@@ -0,0 +1,52 @@
|
||||
"""``python manage.py search_index`` - manage the Elasticsearch index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from apps.search.indexing import alias_name, get_client, rebuild_index
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Create, rebuild, inspect or drop the Elasticsearch profile index."
|
||||
|
||||
def add_arguments(self, parser) -> None: # noqa: ANN001
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--rebuild", action="store_true", help="Rebuild and swap the alias.")
|
||||
group.add_argument("--status", action="store_true", help="Show index/alias status.")
|
||||
group.add_argument("--drop", action="store_true", help="Delete the index and its alias.")
|
||||
|
||||
def handle(self, *args: Any, **options: Any) -> None:
|
||||
client = get_client()
|
||||
if not client.ping():
|
||||
raise CommandError("Elasticsearch is not reachable. Check ELASTICSEARCH_HOSTS.")
|
||||
|
||||
if options["rebuild"]:
|
||||
started = time.perf_counter()
|
||||
result = rebuild_index(client)
|
||||
result["elapsed_seconds"] = round(time.perf_counter() - started, 2)
|
||||
self.stdout.write(json.dumps(result, indent=2))
|
||||
self.stdout.write(self.style.SUCCESS("Index rebuilt."))
|
||||
return
|
||||
|
||||
alias = alias_name()
|
||||
if options["status"]:
|
||||
exists = client.indices.exists_alias(name=alias)
|
||||
payload: dict[str, Any] = {"alias": alias, "exists": bool(exists)}
|
||||
if exists:
|
||||
indices = list(client.indices.get_alias(name=alias).keys())
|
||||
payload["indices"] = indices
|
||||
payload["documents"] = client.count(index=alias)["count"]
|
||||
self.stdout.write(json.dumps(payload, indent=2))
|
||||
return
|
||||
|
||||
if options["drop"]:
|
||||
if client.indices.exists_alias(name=alias):
|
||||
for name in client.indices.get_alias(name=alias):
|
||||
client.indices.delete(index=name, ignore_unavailable=True)
|
||||
client.indices.delete(index=alias, ignore_unavailable=True)
|
||||
self.stdout.write(self.style.WARNING(f"Dropped index/alias {alias}."))
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Request validation and response shaping for the search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
from django.conf import settings
|
||||
from rest_framework import serializers
|
||||
|
||||
from apps.search.types import SORT_OPTIONS, SORT_RELEVANCE, SearchRequest, SearchResult
|
||||
|
||||
MAX_QUERY_LENGTH = 200
|
||||
MAX_FILTER_VALUES = 20
|
||||
|
||||
|
||||
class CommaSeparatedField(serializers.CharField):
|
||||
"""
|
||||
Accepts ``?skills=python,django`` **and** repeated ``?skills=python&skills=django``.
|
||||
|
||||
Both spellings are common in the wild; normalising here keeps the views and
|
||||
both backends free of query-string trivia.
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
kwargs.setdefault("required", False)
|
||||
kwargs.setdefault("allow_blank", True)
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_value(self, dictionary: Any) -> Any:
|
||||
# ``QueryDict.get`` silently keeps only the last occurrence, which would
|
||||
# turn ``?skills=a&skills=b`` into ``b``. ``getlist`` keeps both.
|
||||
if hasattr(dictionary, "getlist"):
|
||||
values = dictionary.getlist(self.field_name)
|
||||
if values:
|
||||
return values
|
||||
return super().get_value(dictionary)
|
||||
|
||||
def to_internal_value(self, data: Any) -> tuple[str, ...]:
|
||||
raw: list[str] = []
|
||||
if isinstance(data, list | tuple):
|
||||
raw = [str(item) for item in data]
|
||||
elif data:
|
||||
raw = [str(data)]
|
||||
|
||||
values: list[str] = []
|
||||
for chunk in raw:
|
||||
for item in chunk.split(","):
|
||||
cleaned = item.strip().lower()[:128]
|
||||
if cleaned and cleaned not in values:
|
||||
values.append(cleaned)
|
||||
if len(values) > MAX_FILTER_VALUES:
|
||||
raise serializers.ValidationError(
|
||||
f"At most {MAX_FILTER_VALUES} values are allowed per filter."
|
||||
)
|
||||
return tuple(values)
|
||||
|
||||
|
||||
class SearchRequestSerializer(serializers.Serializer):
|
||||
"""Validates and normalises the query string of ``GET /api/v1/search/``."""
|
||||
|
||||
q = serializers.CharField(
|
||||
required=False, allow_blank=True, max_length=MAX_QUERY_LENGTH, trim_whitespace=True,
|
||||
help_text="Free-text keyword. Supports quoted phrases and -exclusions.",
|
||||
) # fmt: skip
|
||||
skills = CommaSeparatedField(help_text="Skill names, OR-combined.")
|
||||
roles = CommaSeparatedField(help_text="Job family, e.g. engineering, human_resources.")
|
||||
seniority = CommaSeparatedField(help_text="Seniority level, e.g. senior, manager, cxo.")
|
||||
countries = CommaSeparatedField(help_text="Country of residence.")
|
||||
industries = CommaSeparatedField(help_text="Industry of the current employer.")
|
||||
companies = CommaSeparatedField(help_text="Current employer name.")
|
||||
min_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
max_experience = serializers.IntegerField(required=False, min_value=0, max_value=70)
|
||||
sort = serializers.ChoiceField(choices=SORT_OPTIONS, required=False, default=SORT_RELEVANCE)
|
||||
page = serializers.IntegerField(required=False, min_value=1, default=1)
|
||||
page_size = serializers.IntegerField(required=False, min_value=1, max_value=100, default=20)
|
||||
facets = serializers.BooleanField(required=False, default=True)
|
||||
|
||||
def validate(self, attrs: dict[str, Any]) -> dict[str, Any]:
|
||||
low, high = attrs.get("min_experience"), attrs.get("max_experience")
|
||||
if low is not None and high is not None and low > high:
|
||||
raise serializers.ValidationError(
|
||||
{"min_experience": "min_experience cannot be greater than max_experience."}
|
||||
)
|
||||
|
||||
page, page_size = attrs.get("page", 1), attrs.get("page_size", 20)
|
||||
# Deep pagination is both useless to a human and expensive for the
|
||||
# engine, so it is rejected instead of silently truncated.
|
||||
if page * page_size > settings.SEARCH["MAX_RESULT_WINDOW"]:
|
||||
raise serializers.ValidationError(
|
||||
{"page": f"Cannot page beyond {settings.SEARCH['MAX_RESULT_WINDOW']} results."}
|
||||
)
|
||||
return attrs
|
||||
|
||||
def to_search_request(self) -> SearchRequest:
|
||||
data = self.validated_data
|
||||
return SearchRequest(
|
||||
q=data.get("q", "").strip(),
|
||||
skills=data.get("skills", ()),
|
||||
roles=data.get("roles", ()),
|
||||
seniority=data.get("seniority", ()),
|
||||
countries=data.get("countries", ()),
|
||||
industries=data.get("industries", ()),
|
||||
companies=data.get("companies", ()),
|
||||
min_experience=data.get("min_experience"),
|
||||
max_experience=data.get("max_experience"),
|
||||
sort=data.get("sort", SORT_RELEVANCE),
|
||||
page=data.get("page", 1),
|
||||
page_size=data.get("page_size", 20),
|
||||
include_facets=data.get("facets", True),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response serializers (declared explicitly so the OpenAPI schema is accurate)
|
||||
# ---------------------------------------------------------------------------
|
||||
class SearchHitSerializer(serializers.Serializer):
|
||||
id = serializers.IntegerField()
|
||||
score = serializers.FloatField()
|
||||
full_name = serializers.CharField()
|
||||
headline = serializers.CharField()
|
||||
job_title = serializers.CharField()
|
||||
job_title_role = serializers.CharField()
|
||||
seniority_levels = serializers.ListField(child=serializers.CharField())
|
||||
company = serializers.CharField()
|
||||
industry = serializers.CharField()
|
||||
location_name = serializers.CharField()
|
||||
location_country = serializers.CharField()
|
||||
years_experience = serializers.IntegerField(allow_null=True)
|
||||
linkedin_url = serializers.CharField()
|
||||
skills = serializers.ListField(child=serializers.CharField())
|
||||
highlight = serializers.DictField(child=serializers.ListField(child=serializers.CharField()))
|
||||
|
||||
|
||||
class FacetBucketSerializer(serializers.Serializer):
|
||||
key = serializers.CharField()
|
||||
count = serializers.IntegerField()
|
||||
|
||||
|
||||
class SearchResponseSerializer(serializers.Serializer):
|
||||
count = serializers.IntegerField()
|
||||
page = serializers.IntegerField()
|
||||
page_size = serializers.IntegerField()
|
||||
total_pages = serializers.IntegerField()
|
||||
took_ms = serializers.IntegerField()
|
||||
backend = serializers.CharField()
|
||||
results = SearchHitSerializer(many=True)
|
||||
facets = serializers.DictField(child=FacetBucketSerializer(many=True))
|
||||
|
||||
@staticmethod
|
||||
def from_result(result: SearchResult, request: SearchRequest) -> dict[str, Any]:
|
||||
total_pages = max(1, -(-result.total // request.page_size))
|
||||
return {
|
||||
"count": result.total,
|
||||
"page": request.page,
|
||||
"page_size": request.page_size,
|
||||
"total_pages": total_pages,
|
||||
"took_ms": result.took_ms,
|
||||
"backend": result.backend,
|
||||
"results": [asdict(hit) for hit in result.hits],
|
||||
"facets": {
|
||||
name: [asdict(bucket) for bucket in buckets]
|
||||
for name, buckets in result.facets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class SuggestResponseSerializer(serializers.Serializer):
|
||||
suggestions = serializers.ListField(child=serializers.CharField())
|
||||
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
Backend independent data structures.
|
||||
|
||||
Both search backends speak this vocabulary, which is what makes them
|
||||
interchangeable behind a single setting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
# Sort keys accepted by the API and understood by every backend.
|
||||
SORT_RELEVANCE = "relevance"
|
||||
SORT_EXPERIENCE_DESC = "-experience"
|
||||
SORT_EXPERIENCE_ASC = "experience"
|
||||
SORT_NAME = "name"
|
||||
SORT_OPTIONS = (SORT_RELEVANCE, SORT_EXPERIENCE_DESC, SORT_EXPERIENCE_ASC, SORT_NAME)
|
||||
|
||||
# Facet name -> the document field it aggregates.
|
||||
FACET_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"job_title_role": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"country": "location_country",
|
||||
"industry": "industry",
|
||||
"company": "company.keyword",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SearchRequest:
|
||||
"""A validated, normalised search request."""
|
||||
|
||||
q: str = ""
|
||||
skills: tuple[str, ...] = ()
|
||||
roles: tuple[str, ...] = ()
|
||||
seniority: tuple[str, ...] = ()
|
||||
countries: tuple[str, ...] = ()
|
||||
industries: tuple[str, ...] = ()
|
||||
companies: tuple[str, ...] = ()
|
||||
min_experience: int | None = None
|
||||
max_experience: int | None = None
|
||||
sort: str = SORT_RELEVANCE
|
||||
page: int = 1
|
||||
page_size: int = 20
|
||||
include_facets: bool = True
|
||||
|
||||
@property
|
||||
def offset(self) -> int:
|
||||
return (self.page - 1) * self.page_size
|
||||
|
||||
@property
|
||||
def has_filters(self) -> bool:
|
||||
return bool(
|
||||
self.skills
|
||||
or self.roles
|
||||
or self.seniority
|
||||
or self.countries
|
||||
or self.industries
|
||||
or self.companies
|
||||
or self.min_experience is not None
|
||||
or self.max_experience is not None
|
||||
)
|
||||
|
||||
def as_log_context(self) -> dict[str, Any]:
|
||||
"""Compact representation for the audit log (no free text explosion)."""
|
||||
return {
|
||||
"query": self.q[:120],
|
||||
"filters": {
|
||||
"skills": list(self.skills),
|
||||
"roles": list(self.roles),
|
||||
"seniority": list(self.seniority),
|
||||
"countries": list(self.countries),
|
||||
"industries": list(self.industries),
|
||||
"companies": list(self.companies),
|
||||
"min_experience": self.min_experience,
|
||||
"max_experience": self.max_experience,
|
||||
},
|
||||
"sort": self.sort,
|
||||
"page": self.page,
|
||||
"page_size": self.page_size,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SearchHit:
|
||||
"""One result card - identical shape whichever backend produced it."""
|
||||
|
||||
id: int
|
||||
score: float
|
||||
full_name: str
|
||||
headline: str
|
||||
job_title: str
|
||||
job_title_role: str
|
||||
seniority_levels: list[str]
|
||||
company: str
|
||||
industry: str
|
||||
location_name: str
|
||||
location_country: str
|
||||
years_experience: int | None
|
||||
linkedin_url: str
|
||||
skills: list[str]
|
||||
highlight: dict[str, list[str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FacetBucket:
|
||||
key: str
|
||||
count: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SearchResult:
|
||||
total: int
|
||||
hits: list[SearchHit]
|
||||
facets: dict[str, list[FacetBucket]]
|
||||
took_ms: int
|
||||
backend: str
|
||||
@@ -0,0 +1,11 @@
|
||||
from django.urls import path
|
||||
|
||||
from apps.search.views import ReindexView, SearchView, SuggestView
|
||||
|
||||
app_name = "search"
|
||||
|
||||
urlpatterns = [
|
||||
path("", SearchView.as_view(), name="search"),
|
||||
path("suggest/", SuggestView.as_view(), name="suggest"),
|
||||
path("reindex/", ReindexView.as_view(), name="reindex"),
|
||||
]
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Search endpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from drf_spectacular.utils import OpenApiParameter, extend_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.request import Request
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
from apps.core.permissions import IsAdminRole
|
||||
from apps.core.throttling import SearchRateThrottle
|
||||
from apps.search.backends import get_search_backend
|
||||
from apps.search.serializers import (
|
||||
SearchRequestSerializer,
|
||||
SearchResponseSerializer,
|
||||
SuggestResponseSerializer,
|
||||
)
|
||||
|
||||
audit = logging.getLogger("audit.search")
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
parameters=[SearchRequestSerializer],
|
||||
responses={200: SearchResponseSerializer},
|
||||
summary="Keyword search with filters and facets",
|
||||
description=(
|
||||
"Ranked keyword search across name, job title, skills, employer, education, "
|
||||
"location and summary, combined with structured filters. Filter values are "
|
||||
"OR-ed inside one filter and AND-ed across filters."
|
||||
),
|
||||
)
|
||||
class SearchView(APIView):
|
||||
"""
|
||||
``GET /api/v1/search/``
|
||||
|
||||
Thin by design: validate the query string, hand a normalised request to the
|
||||
configured backend, shape the response, emit an audit event. All ranking
|
||||
lives in the backend, all validation lives in the serializer.
|
||||
"""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
throttle_classes = [SearchRateThrottle]
|
||||
serializer_class = SearchRequestSerializer
|
||||
|
||||
def get(self, request: Request) -> Response:
|
||||
serializer = SearchRequestSerializer(data=request.query_params)
|
||||
serializer.is_valid(raise_exception=True)
|
||||
search_request = serializer.to_search_request()
|
||||
|
||||
backend = get_search_backend()
|
||||
result = backend.search(search_request)
|
||||
|
||||
audit.info(
|
||||
"search_executed",
|
||||
extra={
|
||||
"event": "search_executed",
|
||||
"user_id": request.user.id,
|
||||
"backend": result.backend,
|
||||
"result_count": result.total,
|
||||
"took_ms": result.took_ms,
|
||||
**search_request.as_log_context(),
|
||||
},
|
||||
)
|
||||
return Response(SearchResponseSerializer.from_result(result, search_request))
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
parameters=[
|
||||
OpenApiParameter("q", str, description="Partial name or skill.", required=True),
|
||||
OpenApiParameter("limit", int, description="Maximum suggestions (1-20)."),
|
||||
],
|
||||
responses={200: SuggestResponseSerializer},
|
||||
summary="Type-ahead suggestions",
|
||||
)
|
||||
class SuggestView(APIView):
|
||||
"""Edge n-gram completions used by the search box."""
|
||||
|
||||
permission_classes = [IsAuthenticated]
|
||||
throttle_classes = [SearchRateThrottle]
|
||||
|
||||
def get(self, request: Request) -> Response:
|
||||
prefix = request.query_params.get("q", "").strip()[:64]
|
||||
try:
|
||||
limit = min(max(int(request.query_params.get("limit", 8)), 1), 20)
|
||||
except ValueError:
|
||||
limit = 8
|
||||
suggestions = get_search_backend().suggest(prefix, limit=limit) if prefix else []
|
||||
return Response({"suggestions": suggestions})
|
||||
|
||||
|
||||
@extend_schema(
|
||||
tags=["search"],
|
||||
request=None,
|
||||
responses={202: dict},
|
||||
summary="Rebuild the search index (admin only)",
|
||||
)
|
||||
class ReindexView(APIView):
|
||||
"""
|
||||
Triggers a zero-downtime rebuild of the Elasticsearch index.
|
||||
|
||||
Restricted to the admin role: it is an expensive, cluster-wide operation.
|
||||
"""
|
||||
|
||||
permission_classes = [IsAuthenticated, IsAdminRole]
|
||||
|
||||
def post(self, request: Request) -> Response:
|
||||
from apps.search.indexing import rebuild_index
|
||||
|
||||
result = rebuild_index()
|
||||
audit.info(
|
||||
"index_rebuilt",
|
||||
extra={"event": "index_rebuilt", "user_id": request.user.id, **result},
|
||||
)
|
||||
return Response(result, status=status.HTTP_202_ACCEPTED)
|
||||
@@ -0,0 +1,7 @@
|
||||
import os
|
||||
|
||||
from django.core.asgi import get_asgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.prod")
|
||||
|
||||
application = get_asgi_application()
|
||||
@@ -0,0 +1,345 @@
|
||||
"""
|
||||
Base settings shared by every environment.
|
||||
|
||||
Environment specific modules (``dev``, ``prod``, ``test``) import everything
|
||||
from here and override only what differs. Nothing secret is ever hard coded:
|
||||
every credential is read from the environment through ``django-environ``.
|
||||
"""
|
||||
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import environ
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
env = environ.Env(
|
||||
DJANGO_DEBUG=(bool, False),
|
||||
DJANGO_ALLOWED_HOSTS=(list, ["localhost", "127.0.0.1"]),
|
||||
CORS_ALLOWED_ORIGINS=(list, ["http://localhost:5173", "http://localhost:8080"]),
|
||||
CSRF_TRUSTED_ORIGINS=(list, ["http://localhost:5173", "http://localhost:8080"]),
|
||||
SEARCH_BACKEND=(str, "elasticsearch"),
|
||||
ELASTICSEARCH_HOSTS=(list, ["http://elasticsearch:9200"]),
|
||||
ELASTICSEARCH_USERNAME=(str, ""),
|
||||
ELASTICSEARCH_PASSWORD=(str, ""),
|
||||
ELASTICSEARCH_INDEX=(str, "linkedin-profiles"),
|
||||
ELASTICSEARCH_TIMEOUT=(int, 10),
|
||||
LOGSTASH_ENABLED=(bool, False),
|
||||
LOGSTASH_HOST=(str, "logstash"),
|
||||
LOGSTASH_PORT=(int, 5959),
|
||||
LOG_LEVEL=(str, "INFO"),
|
||||
THROTTLE_ANON=(str, "30/min"),
|
||||
THROTTLE_USER=(str, "300/min"),
|
||||
THROTTLE_SEARCH=(str, "120/min"),
|
||||
THROTTLE_LOGIN=(str, "10/min"),
|
||||
SERVICE_NAME=(str, "linkedin-search-api"),
|
||||
ENVIRONMENT=(str, "development"),
|
||||
)
|
||||
|
||||
# Read the local .env file when present (docker-compose injects real env vars).
|
||||
env_file = BASE_DIR.parent / ".env"
|
||||
if env_file.exists():
|
||||
env.read_env(str(env_file))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core
|
||||
# ---------------------------------------------------------------------------
|
||||
SECRET_KEY = env("DJANGO_SECRET_KEY")
|
||||
DEBUG = env("DJANGO_DEBUG")
|
||||
ALLOWED_HOSTS = env("DJANGO_ALLOWED_HOSTS")
|
||||
ENVIRONMENT = env("ENVIRONMENT")
|
||||
SERVICE_NAME = env("SERVICE_NAME")
|
||||
|
||||
INSTALLED_APPS = [
|
||||
"django.contrib.admin",
|
||||
"django.contrib.auth",
|
||||
"django.contrib.contenttypes",
|
||||
"django.contrib.sessions",
|
||||
"django.contrib.messages",
|
||||
"django.contrib.staticfiles",
|
||||
"django.contrib.postgres",
|
||||
# Third party
|
||||
"rest_framework",
|
||||
"rest_framework_simplejwt.token_blacklist",
|
||||
"django_filters",
|
||||
"corsheaders",
|
||||
"drf_spectacular",
|
||||
# Local
|
||||
"apps.core",
|
||||
"apps.accounts",
|
||||
"apps.profiles",
|
||||
"apps.search",
|
||||
]
|
||||
|
||||
MIDDLEWARE = [
|
||||
"apps.core.middleware.RequestIDMiddleware",
|
||||
"django.middleware.security.SecurityMiddleware",
|
||||
"whitenoise.middleware.WhiteNoiseMiddleware",
|
||||
"corsheaders.middleware.CorsMiddleware",
|
||||
"django.contrib.sessions.middleware.SessionMiddleware",
|
||||
"django.middleware.common.CommonMiddleware",
|
||||
"django.middleware.csrf.CsrfViewMiddleware",
|
||||
"django.contrib.auth.middleware.AuthenticationMiddleware",
|
||||
"django.contrib.messages.middleware.MessageMiddleware",
|
||||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||
"apps.core.middleware.AccessLogMiddleware",
|
||||
]
|
||||
|
||||
ROOT_URLCONF = "config.urls"
|
||||
WSGI_APPLICATION = "config.wsgi.application"
|
||||
ASGI_APPLICATION = "config.asgi.application"
|
||||
|
||||
TEMPLATES = [
|
||||
{
|
||||
"BACKEND": "django.template.backends.django.DjangoTemplates",
|
||||
"DIRS": [BASE_DIR / "templates"],
|
||||
"APP_DIRS": True,
|
||||
"OPTIONS": {
|
||||
"context_processors": [
|
||||
"django.template.context_processors.debug",
|
||||
"django.template.context_processors.request",
|
||||
"django.contrib.auth.context_processors.auth",
|
||||
"django.contrib.messages.context_processors.messages",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database
|
||||
# ---------------------------------------------------------------------------
|
||||
DATABASES = {
|
||||
"default": env.db("DATABASE_URL"),
|
||||
}
|
||||
DATABASES["default"]["CONN_MAX_AGE"] = env.int("DATABASE_CONN_MAX_AGE", default=60)
|
||||
DATABASES["default"]["ATOMIC_REQUESTS"] = False
|
||||
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cache (also backs DRF throttling counters)
|
||||
# ---------------------------------------------------------------------------
|
||||
CACHES = {
|
||||
"default": {
|
||||
"BACKEND": "django_redis.cache.RedisCache",
|
||||
"LOCATION": env("REDIS_URL", default="redis://redis:6379/1"),
|
||||
"OPTIONS": {
|
||||
"CLIENT_CLASS": "django_redis.client.DefaultClient",
|
||||
"IGNORE_EXCEPTIONS": True, # a cache outage must not take the API down
|
||||
},
|
||||
"KEY_PREFIX": "lks",
|
||||
}
|
||||
}
|
||||
DJANGO_REDIS_IGNORE_EXCEPTIONS = True
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Authentication
|
||||
# ---------------------------------------------------------------------------
|
||||
AUTH_USER_MODEL = "accounts.User"
|
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [
|
||||
{"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
|
||||
{
|
||||
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
|
||||
"OPTIONS": {"min_length": 10},
|
||||
},
|
||||
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"},
|
||||
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
|
||||
]
|
||||
|
||||
# Argon2 first: modern, memory hard, recommended by OWASP.
|
||||
PASSWORD_HASHERS = [
|
||||
"django.contrib.auth.hashers.Argon2PasswordHasher",
|
||||
"django.contrib.auth.hashers.PBKDF2PasswordHasher",
|
||||
"django.contrib.auth.hashers.BCryptSHA256PasswordHasher",
|
||||
]
|
||||
|
||||
SIMPLE_JWT = {
|
||||
"ACCESS_TOKEN_LIFETIME": timedelta(minutes=env.int("JWT_ACCESS_MINUTES", default=15)),
|
||||
"REFRESH_TOKEN_LIFETIME": timedelta(days=env.int("JWT_REFRESH_DAYS", default=7)),
|
||||
"ROTATE_REFRESH_TOKENS": True,
|
||||
"BLACKLIST_AFTER_ROTATION": True,
|
||||
"UPDATE_LAST_LOGIN": True,
|
||||
"ALGORITHM": "HS256",
|
||||
"SIGNING_KEY": env("JWT_SIGNING_KEY", default=SECRET_KEY),
|
||||
"AUTH_HEADER_TYPES": ("Bearer",),
|
||||
"USER_ID_FIELD": "id",
|
||||
"USER_ID_CLAIM": "user_id",
|
||||
"TOKEN_OBTAIN_SERIALIZER": "apps.accounts.serializers.TokenObtainPairSerializer",
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Django REST Framework
|
||||
# ---------------------------------------------------------------------------
|
||||
REST_FRAMEWORK = {
|
||||
"DEFAULT_AUTHENTICATION_CLASSES": (
|
||||
"rest_framework_simplejwt.authentication.JWTAuthentication",
|
||||
),
|
||||
# Deny by default; endpoints opt in to public access explicitly.
|
||||
"DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.IsAuthenticated",),
|
||||
"DEFAULT_PAGINATION_CLASS": "apps.core.pagination.PageNumberPagination",
|
||||
"PAGE_SIZE": 20,
|
||||
"DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",),
|
||||
"DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
|
||||
"DEFAULT_THROTTLE_CLASSES": (
|
||||
"rest_framework.throttling.AnonRateThrottle",
|
||||
"rest_framework.throttling.UserRateThrottle",
|
||||
),
|
||||
"DEFAULT_THROTTLE_RATES": {
|
||||
"anon": env("THROTTLE_ANON"),
|
||||
"user": env("THROTTLE_USER"),
|
||||
"search": env("THROTTLE_SEARCH"),
|
||||
"login": env("THROTTLE_LOGIN"),
|
||||
},
|
||||
"EXCEPTION_HANDLER": "apps.core.exceptions.api_exception_handler",
|
||||
"DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",),
|
||||
# URL namespace = API version, so /api/v1/... is resolved automatically and
|
||||
# a future /api/v2/ is a routing change rather than a rewrite.
|
||||
"DEFAULT_VERSIONING_CLASS": "rest_framework.versioning.NamespaceVersioning",
|
||||
"DEFAULT_VERSION": "v1",
|
||||
"ALLOWED_VERSIONS": ["v1"],
|
||||
"UNAUTHENTICATED_USER": "django.contrib.auth.models.AnonymousUser",
|
||||
}
|
||||
|
||||
SPECTACULAR_SETTINGS = {
|
||||
"TITLE": "LinkedIn Profile Search API",
|
||||
"DESCRIPTION": (
|
||||
"Keyword search and faceted filtering over a LinkedIn profile dataset. "
|
||||
"Backed by Elasticsearch with a PostgreSQL full-text fallback."
|
||||
),
|
||||
"VERSION": "1.0.0",
|
||||
"SERVE_INCLUDE_SCHEMA": False,
|
||||
"COMPONENT_SPLIT_REQUEST": True,
|
||||
"SCHEMA_PATH_PREFIX": "/api/v1",
|
||||
"SERVE_PERMISSIONS": ["rest_framework.permissions.AllowAny"],
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CORS / CSRF
|
||||
# ---------------------------------------------------------------------------
|
||||
CORS_ALLOWED_ORIGINS = env("CORS_ALLOWED_ORIGINS")
|
||||
CORS_ALLOW_CREDENTIALS = False
|
||||
CORS_ALLOW_HEADERS = (
|
||||
"accept",
|
||||
"authorization",
|
||||
"content-type",
|
||||
"origin",
|
||||
"user-agent",
|
||||
"x-request-id",
|
||||
)
|
||||
CORS_EXPOSE_HEADERS = ("x-request-id",)
|
||||
CSRF_TRUSTED_ORIGINS = env("CSRF_TRUSTED_ORIGINS")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Baseline security headers (tightened further in prod.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
SECURE_CONTENT_TYPE_NOSNIFF = True
|
||||
SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
|
||||
SECURE_CROSS_ORIGIN_OPENER_POLICY = "same-origin"
|
||||
X_FRAME_OPTIONS = "DENY"
|
||||
SESSION_COOKIE_HTTPONLY = True
|
||||
SESSION_COOKIE_SAMESITE = "Lax"
|
||||
CSRF_COOKIE_HTTPONLY = False
|
||||
CSRF_COOKIE_SAMESITE = "Lax"
|
||||
DATA_UPLOAD_MAX_MEMORY_SIZE = 5 * 1024 * 1024 # 5 MB
|
||||
DATA_UPLOAD_MAX_NUMBER_FIELDS = 1000
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Internationalisation
|
||||
# ---------------------------------------------------------------------------
|
||||
LANGUAGE_CODE = "en-us"
|
||||
TIME_ZONE = "UTC"
|
||||
USE_I18N = True
|
||||
USE_TZ = True
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Static files
|
||||
# ---------------------------------------------------------------------------
|
||||
STATIC_URL = "/static/"
|
||||
STATIC_ROOT = BASE_DIR / "staticfiles"
|
||||
STORAGES = {
|
||||
"default": {"BACKEND": "django.core.files.storage.FileSystemStorage"},
|
||||
"staticfiles": {"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage"},
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Search
|
||||
# ---------------------------------------------------------------------------
|
||||
SEARCH = {
|
||||
# "elasticsearch" (default) or "postgres" - see apps/search/backends/
|
||||
"BACKEND": env("SEARCH_BACKEND"),
|
||||
"ELASTICSEARCH": {
|
||||
"HOSTS": env("ELASTICSEARCH_HOSTS"),
|
||||
"USERNAME": env("ELASTICSEARCH_USERNAME"),
|
||||
"PASSWORD": env("ELASTICSEARCH_PASSWORD"),
|
||||
"INDEX": env("ELASTICSEARCH_INDEX"),
|
||||
"TIMEOUT": env("ELASTICSEARCH_TIMEOUT"),
|
||||
"MAX_RETRIES": 2,
|
||||
},
|
||||
"MAX_PAGE_SIZE": 100,
|
||||
"MAX_RESULT_WINDOW": 10_000,
|
||||
"FACET_SIZE": 25,
|
||||
}
|
||||
|
||||
# Path of the dataset used by the ``import_profiles`` management command.
|
||||
DATASET_PATH = env("DATASET_PATH", default=str(BASE_DIR.parent / "data" / "linkedin_profiles.csv"))
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging - JSON on stdout, optionally mirrored to Logstash for the ELK stack
|
||||
# ---------------------------------------------------------------------------
|
||||
LOG_LEVEL = env("LOG_LEVEL")
|
||||
LOGSTASH_ENABLED = env("LOGSTASH_ENABLED")
|
||||
|
||||
LOGGING = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"filters": {
|
||||
"request_id": {"()": "apps.core.logging.RequestIDFilter"},
|
||||
},
|
||||
"formatters": {
|
||||
"json": {
|
||||
"()": "apps.core.logging.JSONFormatter",
|
||||
"service": SERVICE_NAME,
|
||||
"environment": ENVIRONMENT,
|
||||
},
|
||||
"console": {
|
||||
"format": "[{asctime}] {levelname} {name} {message}",
|
||||
"style": "{",
|
||||
},
|
||||
},
|
||||
"handlers": {
|
||||
"stdout": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "json",
|
||||
"filters": ["request_id"],
|
||||
},
|
||||
},
|
||||
"root": {"handlers": ["stdout"], "level": LOG_LEVEL},
|
||||
"loggers": {
|
||||
"django": {"handlers": ["stdout"], "level": LOG_LEVEL, "propagate": False},
|
||||
"django.db.backends": {"handlers": ["stdout"], "level": "WARNING", "propagate": False},
|
||||
# Elasticsearch client is chatty at INFO; keep the noise down.
|
||||
"elastic_transport": {"handlers": ["stdout"], "level": "WARNING", "propagate": False},
|
||||
"apps": {"handlers": ["stdout"], "level": LOG_LEVEL, "propagate": False},
|
||||
# Business level audit trail: every executed search lands here.
|
||||
"audit.search": {"handlers": ["stdout"], "level": "INFO", "propagate": False},
|
||||
"audit.auth": {"handlers": ["stdout"], "level": "INFO", "propagate": False},
|
||||
},
|
||||
}
|
||||
|
||||
if LOGSTASH_ENABLED:
|
||||
LOGGING["handlers"]["logstash"] = {
|
||||
"class": "logstash_async.handler.AsynchronousLogstashHandler",
|
||||
"formatter": "json",
|
||||
"filters": ["request_id"],
|
||||
"transport": "logstash_async.transport.TcpTransport",
|
||||
"host": env("LOGSTASH_HOST"),
|
||||
"port": env("LOGSTASH_PORT"),
|
||||
# In-memory buffer: no extra volume, and a Logstash outage never blocks
|
||||
# a request thread.
|
||||
"database_path": None,
|
||||
}
|
||||
for logger in ("root", *LOGGING["loggers"]):
|
||||
target = LOGGING if logger == "root" else LOGGING["loggers"]
|
||||
cfg = target["root"] if logger == "root" else target[logger]
|
||||
cfg["handlers"] = [*cfg["handlers"], "logstash"]
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Local development settings: verbose, permissive, never for production."""
|
||||
|
||||
from .base import * # noqa: F403
|
||||
from .base import INSTALLED_APPS, REST_FRAMEWORK, env
|
||||
|
||||
DEBUG = True
|
||||
|
||||
INSTALLED_APPS += ["django_extensions"]
|
||||
|
||||
# Browsable API is handy while developing against the endpoints by hand.
|
||||
REST_FRAMEWORK["DEFAULT_RENDERER_CLASSES"] = (
|
||||
"rest_framework.renderers.JSONRenderer",
|
||||
"rest_framework.renderers.BrowsableAPIRenderer",
|
||||
)
|
||||
|
||||
ALLOWED_HOSTS = ["*"]
|
||||
CORS_ALLOW_ALL_ORIGINS = env.bool("CORS_ALLOW_ALL_ORIGINS", default=True)
|
||||
|
||||
# Never enforce HTTPS locally.
|
||||
SECURE_SSL_REDIRECT = False
|
||||
SESSION_COOKIE_SECURE = False
|
||||
CSRF_COOKIE_SECURE = False
|
||||
@@ -0,0 +1,42 @@
|
||||
"""
|
||||
Production settings.
|
||||
|
||||
Everything that can be hardened is hardened here; the module intentionally
|
||||
fails loudly on start-up when a required secret is missing rather than falling
|
||||
back to an insecure default.
|
||||
"""
|
||||
|
||||
from .base import * # noqa: F403
|
||||
from .base import DEBUG, SECRET_KEY, env
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fail fast on unsafe configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
if DEBUG: # pragma: no cover - configuration guard
|
||||
raise RuntimeError("DJANGO_DEBUG must be false when using production settings.")
|
||||
|
||||
if len(SECRET_KEY) < 50: # pragma: no cover - configuration guard
|
||||
raise RuntimeError("DJANGO_SECRET_KEY must be at least 50 characters in production.")
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Transport security
|
||||
# ---------------------------------------------------------------------------
|
||||
SECURE_SSL_REDIRECT = env.bool("SECURE_SSL_REDIRECT", default=True)
|
||||
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
|
||||
SECURE_HSTS_SECONDS = 31_536_000 # one year
|
||||
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
|
||||
SECURE_HSTS_PRELOAD = True
|
||||
|
||||
SESSION_COOKIE_SECURE = True
|
||||
CSRF_COOKIE_SECURE = True
|
||||
SESSION_COOKIE_SAMESITE = "Strict"
|
||||
CSRF_COOKIE_SAMESITE = "Strict"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Misc hardening
|
||||
# ---------------------------------------------------------------------------
|
||||
# Admin lives behind an unguessable prefix so credential stuffing bots miss it.
|
||||
ADMIN_URL = env("DJANGO_ADMIN_URL", default="admin/")
|
||||
|
||||
# Deny by default; CORS_ALLOWED_ORIGINS must be set explicitly.
|
||||
CORS_ALLOW_ALL_ORIGINS = False
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Test settings: fast, hermetic, no external services required."""
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("DJANGO_SECRET_KEY", "test-secret-key-not-used-anywhere-else")
|
||||
os.environ.setdefault("DATABASE_URL", "postgres://postgres:postgres@localhost:5432/test_db")
|
||||
|
||||
from .base import * # noqa: E402, F403
|
||||
from .base import LOGGING, SEARCH # noqa: E402
|
||||
|
||||
DEBUG = False
|
||||
SEARCH["BACKEND"] = "postgres" # tests must not need a live Elasticsearch
|
||||
|
||||
# Cheap password hashing keeps the suite quick.
|
||||
PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] # noqa: S105
|
||||
|
||||
CACHES = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
|
||||
|
||||
# Throttling off by default; individual tests enable it explicitly.
|
||||
REST_FRAMEWORK_THROTTLE_DISABLED = True
|
||||
|
||||
EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
|
||||
|
||||
LOGGING["root"]["level"] = "CRITICAL"
|
||||
for _logger in LOGGING["loggers"].values():
|
||||
_logger["level"] = "CRITICAL"
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Root URL configuration.
|
||||
|
||||
Everything the SPA talks to lives under ``/api/v1/`` and is namespaced, which
|
||||
is what DRF's ``NamespaceVersioning`` uses to resolve ``request.version``.
|
||||
Adding a ``/api/v2/`` later is therefore a routing change, not a rewrite.
|
||||
"""
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib import admin
|
||||
from django.urls import include, path
|
||||
from drf_spectacular.views import (
|
||||
SpectacularAPIView,
|
||||
SpectacularRedocView,
|
||||
SpectacularSwaggerView,
|
||||
)
|
||||
|
||||
api_v1 = [
|
||||
path("auth/", include("apps.accounts.urls", namespace="accounts")),
|
||||
path("profiles/", include("apps.profiles.urls", namespace="profiles")),
|
||||
path("search/", include("apps.search.urls", namespace="search")),
|
||||
path("health/", include("apps.core.urls", namespace="core")),
|
||||
]
|
||||
|
||||
urlpatterns = [
|
||||
path(getattr(settings, "ADMIN_URL", "admin/"), admin.site.urls),
|
||||
path("api/v1/", include((api_v1, "api"), namespace="v1")),
|
||||
# OpenAPI schema and the two documentation renderers.
|
||||
path("api/schema/", SpectacularAPIView.as_view(), name="schema"),
|
||||
path(
|
||||
"api/docs/",
|
||||
SpectacularSwaggerView.as_view(url_name="schema"),
|
||||
name="swagger-ui",
|
||||
),
|
||||
path("api/redoc/", SpectacularRedocView.as_view(url_name="schema"), name="redoc"),
|
||||
]
|
||||
|
||||
admin.site.site_header = "LinkedIn Profile Search"
|
||||
admin.site.site_title = "LinkedIn Profile Search"
|
||||
admin.site.index_title = "Operations"
|
||||
@@ -0,0 +1,7 @@
|
||||
import os
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.prod")
|
||||
|
||||
application = get_wsgi_application()
|
||||
Executable
+71
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# Container entrypoint.
|
||||
#
|
||||
# Waits for the dependencies the API cannot start without, applies migrations,
|
||||
# and optionally seeds the dataset - all guarded by environment flags so a
|
||||
# production deploy can opt out of every side effect.
|
||||
# =============================================================================
|
||||
set -Eeuo pipefail
|
||||
|
||||
log() { printf '{"level":"INFO","logger":"entrypoint","message":"%s"}\n' "$1"; }
|
||||
fail() { printf '{"level":"ERROR","logger":"entrypoint","message":"%s"}\n' "$1" >&2; exit 1; }
|
||||
|
||||
wait_for() {
|
||||
local name="$1" host="$2" port="$3" attempts="${4:-60}"
|
||||
log "waiting for ${name} at ${host}:${port}"
|
||||
for _ in $(seq 1 "${attempts}"); do
|
||||
if (echo > "/dev/tcp/${host}/${port}") >/dev/null 2>&1; then
|
||||
log "${name} is reachable"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
fail "${name} did not become reachable at ${host}:${port}"
|
||||
}
|
||||
|
||||
: "${POSTGRES_HOST:=postgres}"
|
||||
: "${POSTGRES_PORT:=5432}"
|
||||
: "${RUN_MIGRATIONS:=true}"
|
||||
: "${SEED_DATASET:=false}"
|
||||
: "${SEED_USERS:=false}"
|
||||
: "${WAIT_FOR_ELASTICSEARCH:=true}"
|
||||
|
||||
wait_for "postgres" "${POSTGRES_HOST}" "${POSTGRES_PORT}"
|
||||
|
||||
if [[ "${SEARCH_BACKEND:-elasticsearch}" == "elasticsearch" && "${WAIT_FOR_ELASTICSEARCH}" == "true" ]]; then
|
||||
wait_for "elasticsearch" "${ELASTICSEARCH_HOST:-elasticsearch}" "${ELASTICSEARCH_PORT:-9200}" 90
|
||||
fi
|
||||
|
||||
if [[ "${RUN_MIGRATIONS}" == "true" ]]; then
|
||||
log "applying database migrations"
|
||||
python manage.py migrate --noinput
|
||||
fi
|
||||
|
||||
if [[ "${SEED_USERS}" == "true" ]]; then
|
||||
log "seeding demo users"
|
||||
python manage.py seed_users
|
||||
fi
|
||||
|
||||
# Importing ~300 profiles takes a few seconds and is idempotent, but it is still
|
||||
# opt-in so restarting a production container never touches the data.
|
||||
if [[ "${SEED_DATASET}" == "true" ]]; then
|
||||
if python manage.py shell -c "
|
||||
import sys
|
||||
from apps.profiles.models import Profile
|
||||
sys.exit(0 if Profile.objects.exists() else 1)
|
||||
" >/dev/null 2>&1; then
|
||||
log "dataset already present, skipping import"
|
||||
else
|
||||
log "importing dataset"
|
||||
python manage.py import_profiles
|
||||
fi
|
||||
|
||||
if [[ "${SEARCH_BACKEND:-elasticsearch}" == "elasticsearch" ]]; then
|
||||
log "building the search index"
|
||||
python manage.py search_index --rebuild
|
||||
fi
|
||||
fi
|
||||
|
||||
log "starting: $*"
|
||||
exec "$@"
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Gunicorn configuration.
|
||||
|
||||
Sized for a container: worker count follows the CPU quota, requests are
|
||||
recycled to bound memory growth, and access logs are emitted as JSON so they
|
||||
land in Elasticsearch in the same shape as the application logs.
|
||||
"""
|
||||
|
||||
import multiprocessing
|
||||
import os
|
||||
|
||||
bind = os.getenv("GUNICORN_BIND", "0.0.0.0:8000")
|
||||
workers = int(os.getenv("GUNICORN_WORKERS", str(min(multiprocessing.cpu_count() * 2 + 1, 8))))
|
||||
threads = int(os.getenv("GUNICORN_THREADS", "2"))
|
||||
worker_class = "gthread"
|
||||
timeout = int(os.getenv("GUNICORN_TIMEOUT", "45"))
|
||||
graceful_timeout = 30
|
||||
keepalive = 5
|
||||
|
||||
# Recycle workers to keep any slow leak from growing without bound.
|
||||
max_requests = 1000
|
||||
max_requests_jitter = 100
|
||||
|
||||
preload_app = False
|
||||
forwarded_allow_ips = os.getenv("GUNICORN_FORWARDED_ALLOW_IPS", "*")
|
||||
|
||||
# Refuse absurd request lines/headers early - cheap protection against a class
|
||||
# of denial-of-service probes.
|
||||
limit_request_line = 8190
|
||||
limit_request_fields = 100
|
||||
limit_request_field_size = 8190
|
||||
|
||||
accesslog = "-"
|
||||
errorlog = "-"
|
||||
loglevel = os.getenv("GUNICORN_LOG_LEVEL", "info")
|
||||
access_log_format = (
|
||||
'{"level":"INFO","logger":"gunicorn.access","message":"%(r)s",'
|
||||
'"http_status":%(s)s,"duration_ms":%(D)s,"client_ip":"%(h)s",'
|
||||
'"request_id":"%({x-request-id}i)s"}'
|
||||
)
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/env python
|
||||
"""Django's command-line utility for administrative tasks."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.dev")
|
||||
try:
|
||||
from django.core.management import execute_from_command_line
|
||||
except ImportError as exc: # pragma: no cover - defensive
|
||||
raise ImportError(
|
||||
"Couldn't import Django. Are you sure it's installed and available "
|
||||
"on your PYTHONPATH environment variable? Did you forget to "
|
||||
"activate a virtual environment?"
|
||||
) from exc
|
||||
execute_from_command_line(sys.argv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
[project]
|
||||
name = "linkedin-search-backend"
|
||||
version = "1.0.0"
|
||||
description = "LinkedIn dataset search engine - Django/DRF backend"
|
||||
requires-python = ">=3.12"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Ruff: linting + import sorting (replaces flake8/isort)
|
||||
# ---------------------------------------------------------------------------
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py312"
|
||||
exclude = ["migrations", ".venv", "staticfiles"]
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = [
|
||||
"E", # pycodestyle errors
|
||||
"W", # pycodestyle warnings
|
||||
"F", # pyflakes
|
||||
"I", # isort
|
||||
"B", # flake8-bugbear
|
||||
"C4", # flake8-comprehensions
|
||||
"DJ", # flake8-django
|
||||
"UP", # pyupgrade
|
||||
"S", # bandit (security)
|
||||
"SIM", # flake8-simplify
|
||||
"RET", # flake8-return
|
||||
"ARG", # unused arguments
|
||||
]
|
||||
ignore = [
|
||||
"S101", # assert is fine inside tests
|
||||
"B008", # function call in argument defaults (common in DRF)
|
||||
]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"tests/*" = ["S105", "S106", "ARG001", "ARG002", "ARG005"]
|
||||
# The repair fixtures are raw CSV rows; wrapping them would change the input.
|
||||
"tests/test_dataset_repair.py" = ["E501"]
|
||||
"*/settings/*" = ["F403", "F405"]
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
known-first-party = ["apps", "config"]
|
||||
|
||||
[tool.ruff.lint.flake8-unused-arguments]
|
||||
# Django and DRF hand every hook `*args`/`**kwargs` whether it needs them or not.
|
||||
ignore-variadic-names = true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Black: formatting
|
||||
# ---------------------------------------------------------------------------
|
||||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ["py312"]
|
||||
extend-exclude = "migrations"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pytest
|
||||
# ---------------------------------------------------------------------------
|
||||
[tool.pytest.ini_options]
|
||||
DJANGO_SETTINGS_MODULE = "config.settings.test"
|
||||
python_files = ["test_*.py"]
|
||||
addopts = "-ra --strict-markers --cov=apps --cov-report=term-missing"
|
||||
testpaths = ["tests"]
|
||||
filterwarnings = ["ignore::DeprecationWarning"]
|
||||
|
||||
[tool.coverage.run]
|
||||
omit = ["*/migrations/*", "*/tests/*", "manage.py", "config/asgi.py", "config/wsgi.py"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bandit (security linting)
|
||||
# ---------------------------------------------------------------------------
|
||||
[tool.bandit]
|
||||
exclude_dirs = ["tests", "migrations", ".venv", "staticfiles"]
|
||||
skips = ["B101"] # asserts are only used in tests
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mypy
|
||||
# ---------------------------------------------------------------------------
|
||||
[tool.mypy]
|
||||
python_version = "3.12"
|
||||
plugins = ["mypy_django_plugin.main", "mypy_drf_plugin.main"]
|
||||
ignore_missing_imports = true
|
||||
warn_unused_ignores = true
|
||||
warn_redundant_casts = true
|
||||
exclude = ["migrations"]
|
||||
|
||||
[tool.django-stubs]
|
||||
django_settings_module = "config.settings.dev"
|
||||
@@ -0,0 +1,40 @@
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runtime dependencies.
|
||||
# Pinned with compatible-release specifiers so patch level security fixes are
|
||||
# picked up while the API surface stays stable.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Core framework
|
||||
Django~=5.1.0
|
||||
djangorestframework~=3.15.2
|
||||
django-filter~=24.3
|
||||
|
||||
# Configuration & environment
|
||||
django-environ~=0.11.2
|
||||
|
||||
# Database
|
||||
psycopg[binary]~=3.2.3
|
||||
|
||||
# Authentication / authorisation
|
||||
djangorestframework-simplejwt~=5.3.1
|
||||
argon2-cffi~=23.1.0
|
||||
|
||||
# API schema & interactive documentation
|
||||
drf-spectacular~=0.27.2
|
||||
|
||||
# Cross origin access for the SPA
|
||||
django-cors-headers~=4.4.0
|
||||
|
||||
# Cache / throttle backend
|
||||
redis~=5.1.1
|
||||
django-redis~=5.4.0
|
||||
|
||||
# Search engine client
|
||||
elasticsearch~=8.15.1
|
||||
|
||||
# Observability: ship structured logs to Logstash without blocking requests
|
||||
python-logstash-async~=4.0
|
||||
|
||||
# Serving
|
||||
gunicorn~=23.0.0
|
||||
whitenoise~=6.7.0
|
||||
@@ -0,0 +1,20 @@
|
||||
-r base.txt
|
||||
|
||||
# Testing
|
||||
pytest~=8.3.3
|
||||
pytest-django~=4.9.0
|
||||
pytest-cov~=5.0.0
|
||||
factory-boy~=3.3.1
|
||||
|
||||
# Quality gates
|
||||
ruff~=0.6.9
|
||||
black~=24.10.0
|
||||
mypy~=1.11.2
|
||||
django-stubs[compatible-mypy]~=5.1.0
|
||||
djangorestframework-stubs~=3.15.1
|
||||
bandit~=1.7.10
|
||||
pip-audit~=2.7.3
|
||||
|
||||
# Developer experience
|
||||
django-extensions~=3.2.3
|
||||
ipython~=8.28.0
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Shared pytest fixtures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from apps.accounts.models import Role, User
|
||||
from apps.profiles.models import Company, Profile, ProfileSkill, Skill
|
||||
|
||||
PASSWORD = "Str0ng-Passw0rd!" # noqa: S105 - test credential only
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def api_client() -> APIClient:
|
||||
return APIClient()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_user(db): # noqa: ANN001, ANN201
|
||||
def _make(email: str = "user@example.com", role: str = Role.VIEWER, **extra) -> User:
|
||||
return User.objects.create_user(email=email, password=PASSWORD, role=role, **extra)
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def viewer(make_user) -> User: # noqa: ANN001
|
||||
return make_user("viewer@example.com", Role.VIEWER)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def analyst(make_user) -> User: # noqa: ANN001
|
||||
return make_user("analyst@example.com", Role.ANALYST)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def admin_user(make_user) -> User: # noqa: ANN001
|
||||
return make_user("admin@example.com", Role.ADMIN, is_staff=True, is_superuser=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def auth_client(api_client): # noqa: ANN001, ANN201
|
||||
def _login(user: User) -> APIClient:
|
||||
api_client.force_authenticate(user=user)
|
||||
return api_client
|
||||
|
||||
return _login
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_profiles(db) -> list[Profile]: # noqa: ANN001
|
||||
"""Three profiles with enough variety to exercise every filter."""
|
||||
from apps.profiles.importers import ProfileImporter
|
||||
|
||||
acme = Company.objects.create(name="acme corp", industry="software", size="51-200")
|
||||
globex = Company.objects.create(name="globex", industry="banking", size="1001-5000")
|
||||
|
||||
people = [
|
||||
{
|
||||
"full_name": "sara python",
|
||||
"job_title": "senior backend engineer",
|
||||
"job_title_role": "engineering",
|
||||
"seniority_levels": ["senior"],
|
||||
"current_company": acme,
|
||||
"industry": "software",
|
||||
"location_country": "germany",
|
||||
"location_name": "berlin, germany",
|
||||
"years_experience": 9,
|
||||
"summary": "Builds distributed systems with django and postgres.",
|
||||
"skills": ["python", "django", "postgresql"],
|
||||
},
|
||||
{
|
||||
"full_name": "john recruiter",
|
||||
"job_title": "recruiting manager",
|
||||
"job_title_role": "human_resources",
|
||||
"seniority_levels": ["manager"],
|
||||
"current_company": globex,
|
||||
"industry": "banking",
|
||||
"location_country": "united states",
|
||||
"location_name": "denton, texas, united states",
|
||||
"years_experience": 12,
|
||||
"summary": "Talent acquisition and employer branding.",
|
||||
"skills": ["recruiting", "hiring", "leadership"],
|
||||
},
|
||||
{
|
||||
"full_name": "maria frontend",
|
||||
"job_title": "frontend developer",
|
||||
"job_title_role": "engineering",
|
||||
"seniority_levels": [],
|
||||
"current_company": acme,
|
||||
"industry": "software",
|
||||
"location_country": "spain",
|
||||
"location_name": "madrid, spain",
|
||||
"years_experience": 3,
|
||||
"summary": "React and typescript specialist.",
|
||||
"skills": ["react", "typescript", "javascript"],
|
||||
},
|
||||
]
|
||||
|
||||
created: list[Profile] = []
|
||||
for index, payload in enumerate(people, start=1):
|
||||
skills = payload.pop("skills")
|
||||
profile = Profile.objects.create(
|
||||
external_id=str(1000 + index),
|
||||
linkedin_url=f"linkedin.com/in/person-{index}",
|
||||
skills_text=" ".join(skills),
|
||||
companies_text=payload["current_company"].name,
|
||||
**payload,
|
||||
)
|
||||
for position, name in enumerate(skills):
|
||||
skill, _ = Skill.objects.get_or_create(
|
||||
slug=Skill.make_slug(name), defaults={"name": name}
|
||||
)
|
||||
ProfileSkill.objects.create(profile=profile, skill=skill, position=position)
|
||||
created.append(profile)
|
||||
|
||||
ProfileImporter._refresh_skill_counters()
|
||||
ProfileImporter._refresh_search_vectors()
|
||||
return created
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Authentication, authorisation and PII masking at the API boundary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from django.urls import reverse
|
||||
|
||||
from apps.accounts.models import Role, User
|
||||
from tests.conftest import PASSWORD
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_register_creates_a_viewer(api_client) -> None: # noqa: ANN001
|
||||
response = api_client.post(
|
||||
reverse("v1:accounts:register"),
|
||||
{
|
||||
"email": "New.User@Example.com",
|
||||
"full_name": "New User",
|
||||
"password": PASSWORD,
|
||||
"password_confirm": PASSWORD,
|
||||
},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 201
|
||||
assert response.data["role"] == Role.VIEWER
|
||||
# E-mails are normalised so "A@x.com" and "a@x.com" cannot both exist.
|
||||
assert User.objects.get(email="new.user@example.com")
|
||||
|
||||
|
||||
def test_register_rejects_mismatched_and_weak_passwords(api_client) -> None: # noqa: ANN001
|
||||
url = reverse("v1:accounts:register")
|
||||
mismatch = api_client.post(
|
||||
url,
|
||||
{"email": "a@example.com", "password": PASSWORD, "password_confirm": "other"},
|
||||
format="json",
|
||||
)
|
||||
assert mismatch.status_code == 400
|
||||
assert mismatch.data["error"]["code"] == "validation_error"
|
||||
|
||||
weak = api_client.post(
|
||||
url,
|
||||
{"email": "b@example.com", "password": "12345678", "password_confirm": "12345678"},
|
||||
format="json",
|
||||
)
|
||||
assert weak.status_code == 400
|
||||
|
||||
|
||||
def test_login_returns_tokens_and_identity(api_client, viewer) -> None: # noqa: ANN001
|
||||
response = api_client.post(
|
||||
reverse("v1:accounts:login"),
|
||||
{"email": viewer.email, "password": PASSWORD},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert {"access", "refresh", "user"} <= set(response.data)
|
||||
assert response.data["user"]["role"] == Role.VIEWER
|
||||
|
||||
|
||||
def test_search_requires_authentication(api_client) -> None: # noqa: ANN001
|
||||
response = api_client.get(reverse("v1:search:search"))
|
||||
assert response.status_code == 401
|
||||
assert response.data["error"]["code"] in {"authentication_failed", "not_authenticated"}
|
||||
|
||||
|
||||
def test_reindex_is_admin_only(
|
||||
auth_client, viewer, admin_user, monkeypatch
|
||||
) -> None: # noqa: ANN001
|
||||
url = reverse("v1:search:reindex")
|
||||
assert auth_client(viewer).post(url).status_code == 403
|
||||
|
||||
monkeypatch.setattr(
|
||||
"apps.search.indexing.rebuild_index", lambda *_args, **_kwargs: {"indexed": 0}
|
||||
)
|
||||
response = auth_client(admin_user).post(url)
|
||||
assert response.status_code in {202, 503}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("role", "should_be_masked"),
|
||||
[(Role.VIEWER, True), (Role.ANALYST, False), (Role.ADMIN, False)],
|
||||
)
|
||||
def test_contact_details_are_masked_for_unprivileged_roles(
|
||||
auth_client, make_user, sample_profiles, role: str, should_be_masked: bool
|
||||
) -> None: # noqa: ANN001
|
||||
profile = sample_profiles[0]
|
||||
profile.emails = [{"address": "sara.python@acme.com", "type": "professional"}]
|
||||
profile.phone_numbers = ["+19402058928"]
|
||||
profile.save(update_fields=["emails", "phone_numbers"])
|
||||
|
||||
user = make_user(f"{role}@example.com", role)
|
||||
response = auth_client(user).get(reverse("v1:profiles:detail", args=[profile.id]))
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.data["pii_masked"] is should_be_masked
|
||||
address = response.data["emails"][0]["address"]
|
||||
assert (address != "sara.python@acme.com") is should_be_masked
|
||||
|
||||
|
||||
def test_response_carries_a_request_id_header(api_client) -> None: # noqa: ANN001
|
||||
response = api_client.get(reverse("v1:core:liveness"))
|
||||
assert response.status_code == 200
|
||||
assert response["X-Request-ID"]
|
||||
|
||||
|
||||
def test_client_supplied_request_id_is_echoed(api_client) -> None: # noqa: ANN001
|
||||
response = api_client.get(reverse("v1:core:liveness"), HTTP_X_REQUEST_ID="abc-123")
|
||||
assert response["X-Request-ID"] == "abc-123"
|
||||
|
||||
|
||||
def test_malicious_request_id_is_replaced(api_client) -> None: # noqa: ANN001
|
||||
response = api_client.get(reverse("v1:core:liveness"), HTTP_X_REQUEST_ID="<script>x</script>")
|
||||
assert "<script>" not in response["X-Request-ID"]
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Tests for the self-healing CSV reader.
|
||||
|
||||
Each test reproduces one defect that is actually present in the supplied export
|
||||
so a regression in the repair logic fails loudly instead of quietly importing
|
||||
garbage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from apps.profiles.dataset import DatasetReader
|
||||
|
||||
HEADER = (
|
||||
"full_name,first_name,last_name,gender,linkedin_url,linkedin_username,linkedin_id,"
|
||||
"industry,job_title,job_title_role,job_title_levels,job_company_name,location_name,"
|
||||
"location_country,skills,experience,summary"
|
||||
)
|
||||
|
||||
|
||||
def _write(tmp_path: Path, body: str) -> Path:
|
||||
path = tmp_path / "sample.csv"
|
||||
path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def read(tmp_path): # noqa: ANN001, ANN201
|
||||
def _read(body: str): # noqa: ANN202
|
||||
return DatasetReader(_write(tmp_path, body)).read()
|
||||
|
||||
return _read
|
||||
|
||||
|
||||
def test_clean_row_is_read_verbatim(read) -> None: # noqa: ANN001
|
||||
records, report = read(
|
||||
f"""
|
||||
{HEADER}
|
||||
jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python', 'django']","[{{'company': {{'name': 'acme'}}, 'title': {{'name': 'backend engineer'}}}}]",Loves databases.
|
||||
"""
|
||||
)
|
||||
assert report.records == 1
|
||||
record = records[0]
|
||||
assert record["full_name"] == "jane doe"
|
||||
assert record["job_title_role"] == "engineering"
|
||||
assert record["skills"] == "['python', 'django']"
|
||||
|
||||
|
||||
def test_leading_watermark_column_is_stripped(read) -> None: # noqa: ANN001
|
||||
records, _ = read(
|
||||
f"""
|
||||
{HEADER}
|
||||
H:\\New folder\\leak\\linkedin\\@BreachedData1,jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi.
|
||||
"""
|
||||
)
|
||||
assert records[0]["full_name"] == "jane doe"
|
||||
assert records[0]["linkedin_url"] == "linkedin.com/in/janedoe"
|
||||
|
||||
|
||||
def test_missing_column_does_not_shift_the_rest_of_the_row(read) -> None: # noqa: ANN001
|
||||
"""``job_title_role`` is absent - every later value must still land right."""
|
||||
records, _ = read(
|
||||
f"""
|
||||
{HEADER}
|
||||
jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi.
|
||||
"""
|
||||
)
|
||||
record = records[0]
|
||||
assert record["job_title_role"] == ""
|
||||
assert record["job_title_levels"] == "['senior']"
|
||||
assert record["job_company_name"] == "acme"
|
||||
assert record["location_country"] == "germany"
|
||||
assert record["skills"] == "['python']"
|
||||
|
||||
|
||||
def test_values_that_fail_their_signature_are_discarded(read) -> None: # noqa: ANN001
|
||||
"""A non-URL in the URL column is dropped rather than persisted."""
|
||||
records, report = read(
|
||||
f"""
|
||||
{HEADER}
|
||||
jane doe,jane,doe,female,not-a-linkedin-url,janedoe,123456,software,engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi.
|
||||
"""
|
||||
)
|
||||
assert records[0]["linkedin_url"] == ""
|
||||
assert report.discarded_values >= 1
|
||||
|
||||
|
||||
def test_repeated_header_and_duplicate_person_are_removed(read) -> None: # noqa: ANN001
|
||||
row = (
|
||||
"jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,engineer,"
|
||||
'engineering,"[\'senior\']",acme,"berlin, germany",germany,"[\'python\']","[]",Hi.'
|
||||
)
|
||||
records, report = read(f"{HEADER}\n{row}\n{HEADER}\n{row}\n")
|
||||
assert report.duplicate_headers == 1
|
||||
assert report.duplicate_records == 1
|
||||
assert len(records) == 1
|
||||
|
||||
|
||||
def test_row_split_by_a_stray_quote_is_reassembled(read) -> None: # noqa: ANN001
|
||||
"""A record cut in half by broken quoting is stitched back together."""
|
||||
records, report = read(
|
||||
f"""
|
||||
{HEADER}
|
||||
jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,engineer
|
||||
engineering,"['senior']",acme,"berlin, germany",germany,"['python', 'sql']","[]",Hi.
|
||||
"""
|
||||
)
|
||||
assert len(records) == 1
|
||||
assert report.repaired_rows >= 1
|
||||
assert records[0]["skills"] == "['python', 'sql']"
|
||||
assert records[0]["location_country"] == "germany"
|
||||
|
||||
|
||||
def test_empty_file_yields_no_records(tmp_path) -> None: # noqa: ANN001
|
||||
path = tmp_path / "empty.csv"
|
||||
path.write_text("", encoding="utf-8")
|
||||
records, report = DatasetReader(path).read()
|
||||
assert records == []
|
||||
assert report.records == 0
|
||||
@@ -0,0 +1,128 @@
|
||||
"""The importer must normalise, enrich and stay idempotent."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from apps.profiles.importers import ProfileImporter
|
||||
from apps.profiles.models import Company, Education, Experience, Profile, Skill
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
HEADER = (
|
||||
"full_name,first_name,last_name,gender,linkedin_url,linkedin_username,linkedin_id,"
|
||||
"industry,job_title,job_title_role,job_title_levels,job_company_name,job_company_size,"
|
||||
"location_name,location_country,inferred_years_experience,linkedin_connections,"
|
||||
"skills,experience,education,emails,phone_numbers,summary"
|
||||
)
|
||||
|
||||
EXPERIENCE = (
|
||||
"[{'company': {'name': 'acme corp', 'size': '51-200', 'industry': 'software', "
|
||||
"'location': {'name': 'berlin, germany', 'country': 'germany'}}, "
|
||||
"'title': {'name': 'backend engineer', 'role': 'engineering', 'sub_role': 'backend', "
|
||||
"'levels': ['senior']}, 'start_date': '2019-04', 'end_date': None, 'is_primary': True}]"
|
||||
)
|
||||
EDUCATION = (
|
||||
"[{'school': {'name': 'tu berlin', 'type': 'post-secondary institution', "
|
||||
"'location': {'name': 'berlin, germany', 'country': 'germany'}}, "
|
||||
"'degrees': ['bachelors'], 'majors': ['computer science'], 'minors': [], "
|
||||
"'start_date': '2012', 'end_date': '2016', 'gpa': None}]"
|
||||
)
|
||||
|
||||
|
||||
def _dataset(tmp_path: Path, row: str) -> str:
|
||||
path = tmp_path / "dataset.csv"
|
||||
path.write_text(f"{HEADER}\n{row}\n", encoding="utf-8")
|
||||
return str(path)
|
||||
|
||||
|
||||
def _row(*, with_top_level_role: bool = True) -> str:
|
||||
role = "engineering," if with_top_level_role else ""
|
||||
return (
|
||||
f"jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,55501,software,"
|
||||
f'backend engineer,{role}"[\'senior\']",acme corp,51-200,"berlin, germany",germany,'
|
||||
f"8.0,540.0,\"['Python', 'django', 'python']\",\"{EXPERIENCE}\",\"{EDUCATION}\","
|
||||
f"\"[{{'address': 'jane@acme.com', 'type': 'professional'}}]\",\"['+491701234567']\","
|
||||
f"Backend engineer."
|
||||
)
|
||||
|
||||
|
||||
def test_import_populates_the_relational_model(tmp_path) -> None: # noqa: ANN001
|
||||
stats = ProfileImporter(_dataset(tmp_path, _row())).run()
|
||||
|
||||
assert stats.created == 1
|
||||
profile = Profile.objects.get()
|
||||
assert profile.full_name == "jane doe"
|
||||
assert profile.years_experience == 8
|
||||
assert profile.connections == 540
|
||||
assert profile.current_company.name == "acme corp"
|
||||
assert Experience.objects.filter(profile=profile).count() == 1
|
||||
assert Education.objects.get(profile=profile).majors == ["computer science"]
|
||||
assert profile.emails == [{"address": "jane@acme.com", "type": "professional"}]
|
||||
|
||||
|
||||
def test_skills_are_deduplicated_and_normalised(tmp_path) -> None: # noqa: ANN001
|
||||
ProfileImporter(_dataset(tmp_path, _row())).run()
|
||||
profile = Profile.objects.get()
|
||||
names = [link.skill.name for link in profile.profile_skills.all()]
|
||||
# "Python" and "python" collapse into a single normalised skill.
|
||||
assert names == ["python", "django"]
|
||||
assert Skill.objects.get(slug="python").profile_count == 1
|
||||
|
||||
|
||||
def test_missing_top_level_fields_are_derived_from_the_experience_blob(
|
||||
tmp_path,
|
||||
) -> None: # noqa: ANN001
|
||||
"""The export frequently omits ``job_title_role``; the nested data has it."""
|
||||
ProfileImporter(_dataset(tmp_path, _row(with_top_level_role=False))).run()
|
||||
profile = Profile.objects.get()
|
||||
assert profile.job_title_role == "engineering"
|
||||
assert profile.job_title_sub_role == "backend"
|
||||
assert profile.current_company.name == "acme corp"
|
||||
|
||||
|
||||
def test_import_is_idempotent(tmp_path) -> None: # noqa: ANN001
|
||||
path = _dataset(tmp_path, _row())
|
||||
first = ProfileImporter(path).run()
|
||||
second = ProfileImporter(path).run()
|
||||
|
||||
assert first.created == 1
|
||||
assert second.created == 0
|
||||
assert second.updated == 1
|
||||
assert Profile.objects.count() == 1
|
||||
assert Company.objects.filter(name="acme corp").count() == 1
|
||||
# Child rows are replaced, never duplicated.
|
||||
assert Experience.objects.count() == 1
|
||||
|
||||
|
||||
def test_search_vector_is_populated(tmp_path) -> None: # noqa: ANN001
|
||||
ProfileImporter(_dataset(tmp_path, _row())).run()
|
||||
profile = Profile.objects.get()
|
||||
assert profile.search_vector is not None
|
||||
assert "django" in profile.skills_text
|
||||
|
||||
|
||||
def test_management_command_runs(tmp_path, capsys) -> None: # noqa: ANN001
|
||||
from django.core.management import call_command
|
||||
|
||||
call_command("import_profiles", "--path", _dataset(tmp_path, _row()), "--truncate")
|
||||
assert Profile.objects.count() == 1
|
||||
assert "profiles_created" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_broken_dataset_reports_but_does_not_crash(tmp_path) -> None: # noqa: ANN001
|
||||
path = tmp_path / "broken.csv"
|
||||
path.write_text(
|
||||
textwrap.dedent(
|
||||
f"""
|
||||
{HEADER}
|
||||
,,,,,,,,,,,,,,,,,,,,,,
|
||||
"""
|
||||
).lstrip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
stats = ProfileImporter(str(path)).run()
|
||||
assert stats.report.records >= 0
|
||||
@@ -0,0 +1,40 @@
|
||||
"""The index document must stay in sync with the mapping - and hide PII."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from apps.search.indexing import INDEX_MAPPINGS, build_document
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def test_document_only_contains_mapped_fields(sample_profiles) -> None: # noqa: ANN001
|
||||
"""
|
||||
The mapping is ``dynamic: strict``: an unmapped field makes Elasticsearch
|
||||
reject the whole document, so this assertion protects every reindex.
|
||||
"""
|
||||
document = build_document(sample_profiles[0])
|
||||
assert set(document) <= set(INDEX_MAPPINGS["properties"])
|
||||
|
||||
|
||||
def test_document_never_carries_contact_details(sample_profiles) -> None: # noqa: ANN001
|
||||
profile = sample_profiles[0]
|
||||
profile.emails = [{"address": "sara@acme.com", "type": "professional"}]
|
||||
profile.phone_numbers = ["+491701234567"]
|
||||
profile.save(update_fields=["emails", "phone_numbers"])
|
||||
|
||||
document = build_document(profile)
|
||||
serialised = str(document)
|
||||
assert "emails" not in document
|
||||
assert "phone_numbers" not in document
|
||||
assert "sara@acme.com" not in serialised
|
||||
assert "+491701234567" not in serialised
|
||||
|
||||
|
||||
def test_document_denormalises_the_related_rows(sample_profiles) -> None: # noqa: ANN001
|
||||
document = build_document(sample_profiles[0])
|
||||
assert document["full_name"] == "sara python"
|
||||
assert document["company"] == "acme corp"
|
||||
assert "python" in document["skills"]
|
||||
assert document["years_experience"] == 9
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Masking helpers must never leak the original value."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from apps.profiles.privacy import mask_email, mask_emails, mask_phone, mask_phones
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("raw", "expected"),
|
||||
[
|
||||
("joseph.holland@garverusa.com", "jo************@garverusa.com"),
|
||||
# A single character local part is hidden completely - keeping it would
|
||||
# reveal the whole address.
|
||||
("a@b.com", "*@b.com"),
|
||||
],
|
||||
)
|
||||
def test_mask_email_keeps_domain_and_hides_local_part(raw: str, expected: str) -> None:
|
||||
masked = mask_email(raw)
|
||||
assert masked == expected
|
||||
assert masked.endswith("@" + raw.split("@")[1])
|
||||
|
||||
|
||||
def test_mask_email_handles_garbage() -> None:
|
||||
assert mask_email("not-an-email") == "***"
|
||||
assert mask_email("") == "***"
|
||||
|
||||
|
||||
def test_mask_phone_hides_the_middle() -> None:
|
||||
masked = mask_phone("+19402058928")
|
||||
assert masked.startswith("+1940")
|
||||
assert masked.endswith("28")
|
||||
assert "*" in masked
|
||||
assert "20589" not in masked
|
||||
|
||||
|
||||
def test_mask_phone_rejects_short_input() -> None:
|
||||
assert mask_phone("+123") == "***"
|
||||
|
||||
|
||||
def test_list_helpers_preserve_structure() -> None:
|
||||
emails = [{"address": "a.person@corp.com", "type": "professional"}]
|
||||
masked = mask_emails(emails)
|
||||
assert masked[0]["type"] == "professional"
|
||||
assert masked[0]["address"] != emails[0]["address"]
|
||||
assert mask_phones(["+19402058928"])[0] != "+19402058928"
|
||||
@@ -0,0 +1,76 @@
|
||||
"""
|
||||
Unit tests for the Elasticsearch query builder.
|
||||
|
||||
``build_query`` is a pure function, so the ranking and filtering rules can be
|
||||
asserted without a running cluster - the fastest possible feedback loop for the
|
||||
part of the system that is easiest to get subtly wrong.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from apps.search.backends.elastic import build_query
|
||||
from apps.search.types import SORT_EXPERIENCE_DESC, SORT_NAME, SearchRequest
|
||||
|
||||
|
||||
def _bool(request: SearchRequest) -> dict:
|
||||
return build_query(request)["query"]["bool"]
|
||||
|
||||
|
||||
def test_empty_query_matches_everything() -> None:
|
||||
clause = _bool(SearchRequest())
|
||||
assert clause["must"] == [{"match_all": {}}]
|
||||
assert clause["filter"] == []
|
||||
|
||||
|
||||
def test_keyword_query_uses_fuzzy_multi_match_and_phrase_boost() -> None:
|
||||
clause = _bool(SearchRequest(q="python developer"))
|
||||
multi_match = clause["must"][0]["multi_match"]
|
||||
assert multi_match["query"] == "python developer"
|
||||
assert multi_match["fuzziness"] == "AUTO"
|
||||
assert any(field.startswith("full_name^") for field in multi_match["fields"])
|
||||
# The phrase clause only boosts; it must never be required.
|
||||
assert clause["should"][0]["multi_match"]["type"] == "phrase"
|
||||
|
||||
|
||||
def test_filters_are_terms_clauses_in_filter_context() -> None:
|
||||
clause = _bool(
|
||||
SearchRequest(skills=("python", "django"), countries=("germany",), seniority=("senior",))
|
||||
)
|
||||
assert {"terms": {"skills.keyword": ["python", "django"]}} in clause["filter"]
|
||||
assert {"terms": {"location_country": ["germany"]}} in clause["filter"]
|
||||
assert {"terms": {"seniority_levels": ["senior"]}} in clause["filter"]
|
||||
# Filters must not leak into the scoring clauses.
|
||||
assert clause["must"] == [{"match_all": {}}]
|
||||
|
||||
|
||||
def test_experience_range_is_translated_to_a_range_filter() -> None:
|
||||
clause = _bool(SearchRequest(min_experience=5, max_experience=10))
|
||||
assert {"range": {"years_experience": {"gte": 5, "lte": 10}}} in clause["filter"]
|
||||
|
||||
open_ended = _bool(SearchRequest(min_experience=5))
|
||||
assert {"range": {"years_experience": {"gte": 5}}} in open_ended["filter"]
|
||||
|
||||
|
||||
def test_pagination_maps_to_from_and_size() -> None:
|
||||
body = build_query(SearchRequest(page=3, page_size=25))
|
||||
assert body["from"] == 50
|
||||
assert body["size"] == 25
|
||||
assert body["track_total_hits"] is True
|
||||
|
||||
|
||||
def test_sort_options() -> None:
|
||||
assert build_query(SearchRequest(sort=SORT_NAME))["sort"][0] == {"full_name.keyword": "asc"}
|
||||
assert build_query(SearchRequest(sort=SORT_EXPERIENCE_DESC))["sort"][0] == {
|
||||
"years_experience": {"order": "desc", "missing": "_last"}
|
||||
}
|
||||
assert build_query(SearchRequest())["sort"][0] == "_score"
|
||||
|
||||
|
||||
def test_highlight_only_requested_with_a_keyword() -> None:
|
||||
assert "highlight" not in build_query(SearchRequest())
|
||||
assert "highlight" in build_query(SearchRequest(q="python"))
|
||||
|
||||
|
||||
def test_facets_can_be_switched_off() -> None:
|
||||
assert "aggs" in build_query(SearchRequest())
|
||||
assert "aggs" not in build_query(SearchRequest(include_facets=False))
|
||||
@@ -0,0 +1,145 @@
|
||||
"""
|
||||
End-to-end tests of the search endpoint against the PostgreSQL backend.
|
||||
|
||||
They pin down the behaviour the task description asks for: keyword search plus
|
||||
at least two filters, and a correct contract between frontend and backend.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from django.urls import reverse
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def search(client, **params): # noqa: ANN001, ANN201
|
||||
return client.get(reverse("v1:search:search"), params)
|
||||
|
||||
|
||||
def names(response) -> list[str]: # noqa: ANN001
|
||||
return [hit["full_name"] for hit in response.data["results"]]
|
||||
|
||||
|
||||
def test_empty_query_returns_every_profile(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst))
|
||||
assert response.status_code == 200
|
||||
assert response.data["count"] == len(sample_profiles)
|
||||
assert response.data["backend"] == "postgres"
|
||||
|
||||
|
||||
def test_keyword_matches_skills_and_summary(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), q="django")
|
||||
assert names(response) == ["sara python"]
|
||||
|
||||
|
||||
def test_keyword_matches_a_person_name(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
assert names(search(auth_client(analyst), q="maria")) == ["maria frontend"]
|
||||
|
||||
|
||||
def test_keyword_tolerates_a_typo_in_the_name(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
"""Trigram similarity is what makes ``jhon`` still find ``john``."""
|
||||
assert "john recruiter" in names(search(auth_client(analyst), q="jhon recruiter"))
|
||||
|
||||
|
||||
def test_skill_filter(auth_client, analyst, sample_profiles) -> None: # noqa: ANN001
|
||||
assert names(search(auth_client(analyst), skills="react")) == ["maria frontend"]
|
||||
|
||||
|
||||
def test_multiple_values_inside_one_filter_are_or_combined(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), skills="react,django")
|
||||
assert sorted(names(response)) == ["maria frontend", "sara python"]
|
||||
|
||||
|
||||
def test_different_filters_are_and_combined(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), skills="react,django", countries="spain")
|
||||
assert names(response) == ["maria frontend"]
|
||||
|
||||
|
||||
def test_role_and_experience_filters(auth_client, analyst, sample_profiles) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), roles="engineering", min_experience=5)
|
||||
assert names(response) == ["sara python"]
|
||||
|
||||
|
||||
def test_repeated_query_parameters_are_supported(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
client = auth_client(analyst)
|
||||
response = client.get(f"{reverse('v1:search:search')}?skills=react&skills=django")
|
||||
assert sorted(names(response)) == ["maria frontend", "sara python"]
|
||||
|
||||
|
||||
def test_facets_reflect_the_filtered_result_set(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), roles="engineering")
|
||||
countries = {bucket["key"] for bucket in response.data["facets"]["country"]}
|
||||
assert countries == {"germany", "spain"}
|
||||
assert response.data["facets"]["skills"]
|
||||
|
||||
|
||||
def test_sorting_by_experience(auth_client, analyst, sample_profiles) -> None: # noqa: ANN001
|
||||
assert names(search(auth_client(analyst), sort="-experience"))[0] == "john recruiter"
|
||||
assert names(search(auth_client(analyst), sort="experience"))[0] == "maria frontend"
|
||||
assert names(search(auth_client(analyst), sort="name"))[0] == "john recruiter"
|
||||
|
||||
|
||||
def test_pagination_metadata(auth_client, analyst, sample_profiles) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), page_size=2, page=2, sort="name")
|
||||
assert response.data["page"] == 2
|
||||
assert response.data["page_size"] == 2
|
||||
assert response.data["total_pages"] == 2
|
||||
assert len(response.data["results"]) == 1
|
||||
|
||||
|
||||
def test_highlight_is_returned_for_keyword_searches(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
hit = search(auth_client(analyst), q="django").data["results"][0]
|
||||
assert "<mark>" in " ".join(hit["highlight"].get("skills", []))
|
||||
|
||||
|
||||
def test_invalid_experience_range_is_rejected(auth_client, analyst) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), min_experience=10, max_experience=2)
|
||||
assert response.status_code == 400
|
||||
assert response.data["error"]["code"] == "validation_error"
|
||||
|
||||
|
||||
def test_page_size_is_capped(auth_client, analyst) -> None: # noqa: ANN001
|
||||
assert search(auth_client(analyst), page_size=5000).status_code == 400
|
||||
|
||||
|
||||
def test_deep_pagination_is_refused(auth_client, analyst) -> None: # noqa: ANN001
|
||||
assert search(auth_client(analyst), page=99999, page_size=100).status_code == 400
|
||||
|
||||
|
||||
def test_unknown_filter_values_return_an_empty_page_not_an_error(
|
||||
auth_client, analyst, sample_profiles
|
||||
) -> None: # noqa: ANN001
|
||||
response = search(auth_client(analyst), skills="cobol")
|
||||
assert response.status_code == 200
|
||||
assert response.data["count"] == 0
|
||||
|
||||
|
||||
def test_suggest_endpoint(auth_client, analyst, sample_profiles) -> None: # noqa: ANN001
|
||||
response = auth_client(analyst).get(reverse("v1:search:suggest"), {"q": "mar"})
|
||||
assert response.status_code == 200
|
||||
assert "maria frontend" in response.data["suggestions"]
|
||||
|
||||
|
||||
def test_skill_dictionary_endpoint(auth_client, analyst, sample_profiles) -> None: # noqa: ANN001
|
||||
response = auth_client(analyst).get(reverse("v1:profiles:skills"), {"q": "py"})
|
||||
assert response.status_code == 200
|
||||
assert any(item["name"] == "python" for item in response.data)
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,49 @@
|
||||
# =============================================================================
|
||||
# Development overlay: hot reload for both halves of the stack.
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build
|
||||
# → SPA (Vite dev server) http://localhost:5173
|
||||
# → API http://localhost:8000/api/docs/
|
||||
#
|
||||
# Sources are bind-mounted, so an edit is live without rebuilding an image.
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
backend:
|
||||
build:
|
||||
target: runtime
|
||||
command:
|
||||
- python
|
||||
- manage.py
|
||||
- runserver
|
||||
- 0.0.0.0:8000
|
||||
environment:
|
||||
DJANGO_SETTINGS_MODULE: config.settings.dev
|
||||
DJANGO_DEBUG: "true"
|
||||
LOG_LEVEL: DEBUG
|
||||
volumes:
|
||||
- ./backend:/app
|
||||
- ./data:/data:ro
|
||||
ports:
|
||||
- "127.0.0.1:${BACKEND_PORT:-8000}:8000"
|
||||
# runserver reloads on save; gunicorn's healthcheck curl is still valid.
|
||||
user: "0:0"
|
||||
|
||||
frontend:
|
||||
image: node:22-alpine
|
||||
build: !reset null
|
||||
working_dir: /app
|
||||
command: sh -c "npm install --no-audit --no-fund && npm run dev -- --host 0.0.0.0"
|
||||
environment:
|
||||
VITE_PROXY_TARGET: http://backend:8000
|
||||
volumes:
|
||||
- ./frontend:/app
|
||||
- frontend-node-modules:/app/node_modules
|
||||
ports:
|
||||
- "5173:5173"
|
||||
read_only: false
|
||||
tmpfs: !reset []
|
||||
networks: [frontend-net]
|
||||
|
||||
volumes:
|
||||
frontend-node-modules:
|
||||
@@ -0,0 +1,74 @@
|
||||
# =============================================================================
|
||||
# Production overlay.
|
||||
#
|
||||
# docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
#
|
||||
# What it changes:
|
||||
# * Elasticsearch authentication is switched on and every client is given
|
||||
# credentials.
|
||||
# * Kibana is no longer published; reach it through an SSH tunnel or an
|
||||
# authenticating reverse proxy.
|
||||
# * Containers get CPU/memory limits and no longer seed the dataset on boot.
|
||||
# * The API is fronted by HTTPS (SECURE_SSL_REDIRECT + HSTS come from
|
||||
# config/settings/prod.py).
|
||||
#
|
||||
# TLS between the containers is intentionally out of scope here: in a real
|
||||
# deployment the cluster sits behind a service mesh or an internal load
|
||||
# balancer that terminates it. See docs/SECURITY.md.
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
elasticsearch:
|
||||
environment:
|
||||
xpack.security.enabled: "true"
|
||||
ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:?ELASTIC_PASSWORD is required in production}
|
||||
healthcheck:
|
||||
test:
|
||||
- "CMD-SHELL"
|
||||
- "curl -fsS -u elastic:${ELASTIC_PASSWORD} 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s' >/dev/null || exit 1"
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 2g }
|
||||
|
||||
logstash:
|
||||
environment:
|
||||
# The Elasticsearch output reads its credentials from the URL.
|
||||
ELASTICSEARCH_HOSTS: http://elastic:${ELASTIC_PASSWORD}@elasticsearch:9200
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 1g }
|
||||
|
||||
kibana:
|
||||
environment:
|
||||
ELASTICSEARCH_USERNAME: elastic
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTIC_PASSWORD}
|
||||
# Remove the published port: Kibana is reached through a tunnel or proxy.
|
||||
ports: !reset []
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 1g }
|
||||
|
||||
backend:
|
||||
environment:
|
||||
DJANGO_SETTINGS_MODULE: config.settings.prod
|
||||
DJANGO_DEBUG: "false"
|
||||
SECURE_SSL_REDIRECT: "true"
|
||||
ELASTICSEARCH_USERNAME: elastic
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTIC_PASSWORD}
|
||||
# A production container must never mutate data on boot.
|
||||
SEED_DATASET: "false"
|
||||
SEED_USERS: "false"
|
||||
deploy:
|
||||
replicas: 2
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 768m }
|
||||
|
||||
frontend:
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128m }
|
||||
|
||||
postgres:
|
||||
deploy:
|
||||
resources:
|
||||
limits: { memory: 1g }
|
||||
@@ -0,0 +1,232 @@
|
||||
# =============================================================================
|
||||
# Full stack: PostgreSQL + Redis + Elasticsearch + Logstash + Kibana + API + SPA
|
||||
#
|
||||
# cp .env.example .env && docker compose up --build
|
||||
# → SPA http://localhost:8080
|
||||
# → API docs http://localhost:8080/api/docs/
|
||||
# → Kibana http://localhost:5601
|
||||
#
|
||||
# Only the SPA and Kibana are published to the host; the database, cache and
|
||||
# the whole ELK stack stay on private Docker networks.
|
||||
# =============================================================================
|
||||
|
||||
name: linkedin-search
|
||||
|
||||
x-service-defaults: &service-defaults
|
||||
restart: unless-stopped
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "10m"
|
||||
max-file: "3"
|
||||
|
||||
services:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data stores
|
||||
# ---------------------------------------------------------------------------
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
<<: *service-defaults
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-linkedin}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-linkedin}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
|
||||
# Slightly stricter than the default md5 authentication.
|
||||
POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256"
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-linkedin} -d ${POSTGRES_DB:-linkedin}"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
start_period: 20s
|
||||
networks: [backend-net]
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
<<: *service-defaults
|
||||
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
networks: [backend-net]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ELK - search engine and log pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
elasticsearch:
|
||||
image: docker.elastic.co/elasticsearch/elasticsearch:8.15.3
|
||||
<<: *service-defaults
|
||||
environment:
|
||||
node.name: elasticsearch
|
||||
discovery.type: single-node
|
||||
bootstrap.memory_lock: "true"
|
||||
ES_JAVA_OPTS: ${ES_JAVA_OPTS:--Xms512m -Xmx512m}
|
||||
# Development defaults: no TLS, no authentication, and the cluster is not
|
||||
# reachable from outside the Docker network. docker-compose.prod.yml
|
||||
# turns authentication on - see docs/SECURITY.md.
|
||||
xpack.security.enabled: "false"
|
||||
xpack.security.enrollment.enabled: "false"
|
||||
xpack.ml.enabled: "false"
|
||||
ingest.geoip.downloader.enabled: "false"
|
||||
ulimits:
|
||||
memlock: { soft: -1, hard: -1 }
|
||||
nofile: { soft: 65536, hard: 65536 }
|
||||
volumes:
|
||||
- elasticsearch-data:/usr/share/elasticsearch/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s' >/dev/null || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 12
|
||||
start_period: 60s
|
||||
networks: [backend-net, elk-net]
|
||||
|
||||
logstash:
|
||||
image: docker.elastic.co/logstash/logstash:8.15.3
|
||||
<<: *service-defaults
|
||||
environment:
|
||||
LS_JAVA_OPTS: ${LS_JAVA_OPTS:--Xms256m -Xmx256m}
|
||||
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
|
||||
ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-}
|
||||
xpack.monitoring.enabled: "false"
|
||||
volumes:
|
||||
- ./elk/logstash/pipeline:/usr/share/logstash/pipeline:ro
|
||||
- ./elk/logstash/templates:/usr/share/logstash/templates:ro
|
||||
- ./elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
|
||||
- logstash-data:/usr/share/logstash/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:9600/_node/pipelines || exit 1"]
|
||||
interval: 20s
|
||||
timeout: 10s
|
||||
retries: 10
|
||||
start_period: 60s
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
networks: [backend-net, elk-net]
|
||||
|
||||
kibana:
|
||||
image: docker.elastic.co/kibana/kibana:8.15.3
|
||||
<<: *service-defaults
|
||||
environment:
|
||||
SERVER_NAME: kibana
|
||||
SERVER_PUBLICBASEURL: http://localhost:${KIBANA_PORT:-5601}
|
||||
ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]'
|
||||
# A stable key keeps saved objects readable across restarts.
|
||||
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: ${KIBANA_ENCRYPTION_KEY:-a7f3c1d9e5b28046a7f3c1d9e5b28046}
|
||||
TELEMETRY_ENABLED: "false"
|
||||
ports:
|
||||
# Bound to the loopback interface: the dashboard is a local tool, not a
|
||||
# public endpoint.
|
||||
- "127.0.0.1:${KIBANA_PORT:-5601}:5601"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:5601/api/status | grep -q '\"level\":\"available\"' || exit 1"]
|
||||
interval: 20s
|
||||
timeout: 10s
|
||||
retries: 15
|
||||
start_period: 90s
|
||||
depends_on:
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
networks: [elk-net]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Application
|
||||
# ---------------------------------------------------------------------------
|
||||
backend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: backend/Dockerfile
|
||||
<<: *service-defaults
|
||||
environment:
|
||||
DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE:-config.settings.prod}
|
||||
DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:?DJANGO_SECRET_KEY is required}
|
||||
DJANGO_DEBUG: ${DJANGO_DEBUG:-false}
|
||||
DJANGO_ALLOWED_HOSTS: ${DJANGO_ALLOWED_HOSTS:-localhost,127.0.0.1,backend}
|
||||
DJANGO_ADMIN_URL: ${DJANGO_ADMIN_URL:-admin/}
|
||||
SECURE_SSL_REDIRECT: ${SECURE_SSL_REDIRECT:-false}
|
||||
ENVIRONMENT: ${ENVIRONMENT:-development}
|
||||
SERVICE_NAME: ${SERVICE_NAME:-linkedin-search-api}
|
||||
CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:8080}
|
||||
CSRF_TRUSTED_ORIGINS: ${CSRF_TRUSTED_ORIGINS:-http://localhost:8080}
|
||||
DATABASE_URL: postgres://${POSTGRES_USER:-linkedin}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-linkedin}
|
||||
POSTGRES_HOST: postgres
|
||||
POSTGRES_PORT: "5432"
|
||||
REDIS_URL: ${REDIS_URL:-redis://redis:6379/1}
|
||||
SEARCH_BACKEND: ${SEARCH_BACKEND:-elasticsearch}
|
||||
ELASTICSEARCH_HOST: elasticsearch
|
||||
ELASTICSEARCH_PORT: "9200"
|
||||
ELASTICSEARCH_HOSTS: ${ELASTICSEARCH_HOSTS:-http://elasticsearch:9200}
|
||||
ELASTICSEARCH_INDEX: ${ELASTICSEARCH_INDEX:-linkedin-profiles}
|
||||
ELASTICSEARCH_USERNAME: ${ELASTICSEARCH_USERNAME:-}
|
||||
ELASTICSEARCH_PASSWORD: ${ELASTICSEARCH_PASSWORD:-}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
LOGSTASH_ENABLED: ${LOGSTASH_ENABLED:-true}
|
||||
LOGSTASH_HOST: logstash
|
||||
LOGSTASH_PORT: "5959"
|
||||
THROTTLE_ANON: ${THROTTLE_ANON:-30/min}
|
||||
THROTTLE_USER: ${THROTTLE_USER:-300/min}
|
||||
THROTTLE_SEARCH: ${THROTTLE_SEARCH:-120/min}
|
||||
THROTTLE_LOGIN: ${THROTTLE_LOGIN:-10/min}
|
||||
JWT_ACCESS_MINUTES: ${JWT_ACCESS_MINUTES:-15}
|
||||
JWT_REFRESH_DAYS: ${JWT_REFRESH_DAYS:-7}
|
||||
DATASET_PATH: ${DATASET_PATH:-/data/linkedin_profiles.csv}
|
||||
RUN_MIGRATIONS: ${RUN_MIGRATIONS:-true}
|
||||
SEED_DATASET: ${SEED_DATASET:-true}
|
||||
SEED_USERS: ${SEED_USERS:-true}
|
||||
GUNICORN_WORKERS: ${GUNICORN_WORKERS:-3}
|
||||
DEMO_ADMIN_EMAIL: ${DEMO_ADMIN_EMAIL:-}
|
||||
DEMO_ADMIN_PASSWORD: ${DEMO_ADMIN_PASSWORD:-}
|
||||
DEMO_ANALYST_EMAIL: ${DEMO_ANALYST_EMAIL:-}
|
||||
DEMO_ANALYST_PASSWORD: ${DEMO_ANALYST_PASSWORD:-}
|
||||
DEMO_VIEWER_EMAIL: ${DEMO_VIEWER_EMAIL:-}
|
||||
DEMO_VIEWER_PASSWORD: ${DEMO_VIEWER_PASSWORD:-}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
elasticsearch:
|
||||
condition: service_healthy
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
networks: [backend-net, frontend-net]
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: frontend/Dockerfile
|
||||
<<: *service-defaults
|
||||
ports:
|
||||
- "${FRONTEND_PORT:-8080}:8080"
|
||||
depends_on:
|
||||
- backend
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
- /var/run
|
||||
- /var/cache/nginx
|
||||
networks: [frontend-net]
|
||||
|
||||
networks:
|
||||
# The SPA container may talk to the API and nothing else.
|
||||
frontend-net:
|
||||
driver: bridge
|
||||
# Databases and the search cluster - no route to the host.
|
||||
backend-net:
|
||||
driver: bridge
|
||||
internal: true
|
||||
elk-net:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
postgres-data:
|
||||
elasticsearch-data:
|
||||
logstash-data:
|
||||
+412
@@ -0,0 +1,412 @@
|
||||
# مرجع API
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
پایه: `/api/v1/` · نسخهبندی از طریق namespace آدرسها · اسکیمای ماشینخوان: [`openapi.yaml`](openapi.yaml)
|
||||
|
||||
مستندات تعاملی (همیشه بهروز، از روی کد تولید میشود):
|
||||
|
||||
- Swagger UI — <http://localhost:8080/api/docs/>
|
||||
- ReDoc — <http://localhost:8080/api/redoc/>
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱. قراردادهای عمومی
|
||||
|
||||
### ۱.۱ احراز هویت
|
||||
|
||||
```http
|
||||
Authorization: Bearer <access_token>
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
سیاست پیشفرض **deny** است: هر endpoint که صریحاً عمومی اعلام نشده، لاگین میخواهد.
|
||||
|
||||
</div>
|
||||
|
||||
### ۱.۲ قالب خطا
|
||||
|
||||
```json
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": "The request payload failed validation.",
|
||||
"details": { "min_experience": ["min_experience cannot be greater than max_experience."] },
|
||||
"request_id": "9f2c1a7e8b4d4c0e"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
| `code` | HTTP | معنی |
|
||||
|---|---|---|
|
||||
| `validation_error` | 400 | ورودی نامعتبر (`details` فیلد به فیلد) |
|
||||
| `authentication_failed` / `not_authenticated` | 401 | توکن نبود یا منقضی شده |
|
||||
| `permission_denied` | 403 | نقش کافی نیست |
|
||||
| `not_found` | 404 | منبع وجود ندارد |
|
||||
| `method_not_allowed` | 405 | متد پشتیبانی نمیشود |
|
||||
| `throttled` | 429 | سقف نرخ رد شده |
|
||||
| `search_unavailable` | 503 | موتور سرچ در دسترس نیست |
|
||||
| `internal_error` | 500 | خطای پیشبینینشده (جزئیات فقط در لاگ) |
|
||||
|
||||
### ۱.۳ هدرها
|
||||
|
||||
| هدر | جهت | توضیح |
|
||||
|---|---|---|
|
||||
| `X-Request-ID` | ورودی (اختیاری) | اگر بفرستید همان برمیگردد، اگر نه تولید میشود |
|
||||
| `X-Request-ID` | خروجی | کلید ردیابی درخواست در Kibana |
|
||||
|
||||
### ۱.۴ سقف نرخ
|
||||
|
||||
| گروه | پیشفرض | کلید |
|
||||
|---|---|---|
|
||||
| ناشناس | ۳۰/دقیقه | IP |
|
||||
| کاربر لاگینکرده | ۳۰۰/دقیقه | user id |
|
||||
| سرچ | ۱۲۰/دقیقه | user id |
|
||||
| لاگین/ثبتنام | ۱۰/دقیقه | **IP** (چون نام کاربری را مهاجم انتخاب میکند) |
|
||||
|
||||
---
|
||||
|
||||
## ۲. احراز هویت
|
||||
|
||||
### `POST /api/v1/auth/register/` — عمومی
|
||||
|
||||
```jsonc
|
||||
// درخواست
|
||||
{ "email": "me@example.com", "full_name": "Me",
|
||||
"password": "Str0ng-Passw0rd!", "password_confirm": "Str0ng-Passw0rd!" }
|
||||
|
||||
// 201
|
||||
{ "id": 4, "email": "me@example.com", "full_name": "Me",
|
||||
"role": "viewer", "can_view_pii": false, "date_joined": "2026-09-03T04:00:00Z" }
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ثبتنام همیشه نقش `viewer` میدهد؛ ارتقا یک اقدام صریح ادمین است. ایمیل نرمال (lowercase) میشود. رمز از اعتبارسنجهای جنگو رد میشود (حداقل ۱۰ کاراکتر، غیرعددی، غیرمعمول، غیرمشابه ایمیل).
|
||||
|
||||
</div>
|
||||
|
||||
### `POST /api/v1/auth/login/` — عمومی
|
||||
|
||||
```jsonc
|
||||
// درخواست
|
||||
{ "email": "analyst@example.com", "password": "Analyst-Passw0rd!" }
|
||||
|
||||
// 200
|
||||
{ "access": "eyJhbGciOi...", // عمر ۱۵ دقیقه
|
||||
"refresh": "eyJhbGciOi...", // عمر ۷ روز، چرخشی
|
||||
"user": { "id": 2, "email": "analyst@example.com", "role": "analyst",
|
||||
"can_view_pii": true, ... } }
|
||||
```
|
||||
|
||||
### `POST /api/v1/auth/refresh/` — عمومی
|
||||
|
||||
```jsonc
|
||||
{ "refresh": "..." } → { "access": "...", "refresh": "..." }
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
توکن refresh **چرخشی** است: هر بار یک جفت جدید میگیرید و توکن قبلی بلاکلیست میشود. اگر یک توکن دزدیده و استفاده شود، استفاده بعدی کاربر واقعی رد میشود — یعنی نشت قابل تشخیص است.
|
||||
|
||||
</div>
|
||||
|
||||
### `POST /api/v1/auth/logout/` — لاگین لازم
|
||||
|
||||
```jsonc
|
||||
{ "refresh": "..." } → 205 No Content
|
||||
```
|
||||
|
||||
### `GET /api/v1/auth/me/` — لاگین لازم
|
||||
|
||||
```jsonc
|
||||
{ "id": 2, "email": "analyst@example.com", "full_name": "Demo Analyst",
|
||||
"role": "analyst", "can_view_pii": true, "date_joined": "..." }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ۳. سرچ
|
||||
|
||||
### `GET /api/v1/search/` — لاگین لازم
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**پارامترها**
|
||||
|
||||
| نام | نوع | پیشفرض | توضیح |
|
||||
|---|---|---|---|
|
||||
| `q` | string | `""` | کلیدواژه؛ حداکثر ۲۰۰ کاراکتر. از عبارت داخل گیومه و `-حذف` پشتیبانی میکند |
|
||||
| `skills` | لیست | — | مهارتها (OR داخلی) |
|
||||
| `roles` | لیست | — | خانواده شغلی، مثل `engineering` |
|
||||
| `seniority` | لیست | — | `senior`, `manager`, `director`, `vp`, `cxo`, ... |
|
||||
| `countries` | لیست | — | کشور محل سکونت |
|
||||
| `industries` | لیست | — | صنعت کارفرمای فعلی |
|
||||
| `companies` | لیست | — | نام کارفرمای فعلی |
|
||||
| `min_experience` | int | — | ۰ تا ۷۰ |
|
||||
| `max_experience` | int | — | ۰ تا ۷۰ |
|
||||
| `sort` | enum | `relevance` | `relevance` \| `-experience` \| `experience` \| `name` |
|
||||
| `page` | int | `1` | |
|
||||
| `page_size` | int | `20` | حداکثر ۱۰۰ |
|
||||
| `facets` | bool | `true` | `false` یعنی aggregation اجرا نشود |
|
||||
|
||||
لیستها هر دو نگارش را میپذیرند: `?skills=a,b` یا `?skills=a&skills=b`. حداکثر ۲۰ مقدار در هر فیلتر.
|
||||
|
||||
**پاسخ ۲۰۰**
|
||||
|
||||
</div>
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"count": 41,
|
||||
"page": 1,
|
||||
"page_size": 20,
|
||||
"total_pages": 3,
|
||||
"took_ms": 12,
|
||||
"backend": "elasticsearch", // یا "postgres"
|
||||
"results": [
|
||||
{
|
||||
"id": 118,
|
||||
"score": 14.82,
|
||||
"full_name": "joseph holland",
|
||||
"headline": "recruiting manager at garver",
|
||||
"job_title": "recruiting manager",
|
||||
"job_title_role": "human_resources",
|
||||
"seniority_levels": ["manager"],
|
||||
"company": "garver",
|
||||
"industry": "civil engineering",
|
||||
"location_name": "denton, texas, united states",
|
||||
"location_country": "united states",
|
||||
"years_experience": 12,
|
||||
"linkedin_url": "linkedin.com/in/joeyholland",
|
||||
"skills": ["recruiting", "leadership", "human resources", "training"],
|
||||
"highlight": {
|
||||
"skills": ["<mark>leadership</mark>"],
|
||||
"job_title": ["recruiting <mark>manager</mark>"]
|
||||
}
|
||||
}
|
||||
],
|
||||
"facets": {
|
||||
"skills": [{ "key": "leadership", "count": 41 }, { "key": "management", "count": 33 }],
|
||||
"job_title_role": [{ "key": "human_resources", "count": 12 }],
|
||||
"seniority": [{ "key": "manager", "count": 29 }],
|
||||
"country": [{ "key": "united states", "count": 41 }],
|
||||
"industry": [{ "key": "banking", "count": 6 }],
|
||||
"company": [{ "key": "garver", "count": 2 }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**نکتهها**
|
||||
|
||||
- `score` فقط با `sort=relevance` معنادار است؛ در مرتبسازیهای دیگر نمایش داده نمیشود.
|
||||
- `highlight` فقط وقتی `q` داده شده باشد پر میشود و شامل تگ `<mark>` است. آن را بهعنوان HTML خام رندر نکنید — کامپوننت `Highlight` در فرانتاند نشان میدهد چطور امن رندر شود.
|
||||
- `facets` روی مجموعه فیلترشده فعلی محاسبه میشود، نه کل دیتاست.
|
||||
- `backend` برای شفافیت عملیاتی است: مشخص میکند کدام موتور جواب داده.
|
||||
|
||||
**خطاها**
|
||||
|
||||
</div>
|
||||
|
||||
```jsonc
|
||||
// 400 - بازه نامعتبر
|
||||
{"error":{"code":"validation_error","message":"The request payload failed validation.",
|
||||
"details":{"min_experience":["min_experience cannot be greater than max_experience."]},
|
||||
"request_id":"..."}}
|
||||
|
||||
// 400 - صفحهبندی عمیق
|
||||
{"error":{"code":"validation_error","details":{"page":["Cannot page beyond 10000 results."]}, ...}}
|
||||
|
||||
// 429
|
||||
{"error":{"code":"throttled","message":"Request was throttled. Expected available in 42 seconds.", ...}}
|
||||
|
||||
// 503 - موتور سرچ در دسترس نیست
|
||||
{"error":{"code":"search_unavailable","message":"The search backend is currently unavailable.", ...}}
|
||||
```
|
||||
|
||||
### `GET /api/v1/search/suggest/` — لاگین لازم
|
||||
|
||||
```
|
||||
?q=mar&limit=8 → { "suggestions": ["maria frontend", "mark heinrich", ...] }
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
از فیلد edge n-gram روی نام و مهارت استفاده میکند. `limit` بین ۱ تا ۲۰.
|
||||
|
||||
</div>
|
||||
|
||||
### `POST /api/v1/search/reindex/` — فقط نقش admin
|
||||
|
||||
```jsonc
|
||||
// 202
|
||||
{ "alias": "linkedin-profiles",
|
||||
"index": "linkedin-profiles-20260903040012",
|
||||
"indexed": 284, "errors": 0,
|
||||
"replaced": ["linkedin-profiles-20260901101500"] }
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
بازسازی بدون قطعی: ایندکس فیزیکی جدید ساخته میشود، پر میشود، و بعد alias بهصورت اتمیک جابهجا میشود. کاربر هیچوقت ایندکس نیمهپر نمیبیند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۴. پروفایلها
|
||||
|
||||
### `GET /api/v1/profiles/` — لاگین لازم
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
لیست ساده و بدون رتبهبندی، با صفحهبندی استاندارد. برای مرور و بررسی دیتاست مفید است؛ سرچ رتبهبندیشده در `/search/` است.
|
||||
|
||||
</div>
|
||||
|
||||
```jsonc
|
||||
{ "count": 284, "page": 1, "page_size": 20, "total_pages": 15,
|
||||
"next": "http://.../profiles/?page=2", "previous": null,
|
||||
"results": [ /* همان شکل کارت نتیجه */ ] }
|
||||
```
|
||||
|
||||
### `GET /api/v1/profiles/{id}/` — لاگین لازم
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"id": 118,
|
||||
"full_name": "joseph holland",
|
||||
"headline": "recruiting manager at garver",
|
||||
"current_company": { "id": 42, "name": "garver", "industry": "civil engineering",
|
||||
"size": "501-1000", "founded": 1919, "location_name": "..." },
|
||||
"summary": "Celebrating its 100th year, Garver is an employee-owned ...",
|
||||
"years_experience": 12,
|
||||
"connections": 3761,
|
||||
"interests": ["guitar", "aviation", "hiking"],
|
||||
"languages": ["english"],
|
||||
"skills": [{ "name": "recruiting", "slug": "recruiting" }, ...],
|
||||
"experiences": [
|
||||
{ "id": 903, "title": "recruiting manager", "company_name": "garver",
|
||||
"start_date": "2019-10", "end_date": "", "is_primary": true,
|
||||
"company": { "id": 42, "name": "garver", ... } }
|
||||
],
|
||||
"educations": [
|
||||
{ "id": 210, "school_name": "university of north texas",
|
||||
"degrees": ["bachelors", "bachelor of arts"],
|
||||
"majors": ["politics", "international studies"],
|
||||
"start_date": "2007", "end_date": "2008" }
|
||||
],
|
||||
"emails": [{ "address": "j3*******@yahoo.com", "type": "personal" }],
|
||||
"phone_numbers": ["+1940*****28"],
|
||||
"pii_masked": true
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**`pii_masked`** به فرانتاند میگوید آیا مقادیر تماس ماسک شدهاند یا نه، تا بتواند یک پیام روشن نشان دهد بهجای اینکه کاربر فکر کند داده خراب است.
|
||||
|
||||
| نقش | `emails` / `phone_numbers` |
|
||||
|---|---|
|
||||
| `viewer` | ماسکشده (`j3*******@yahoo.com`) |
|
||||
| `analyst` | کامل |
|
||||
| `admin` | کامل |
|
||||
|
||||
</div>
|
||||
|
||||
### `GET /api/v1/profiles/skills/` — لاگین لازم
|
||||
|
||||
```
|
||||
?q=py&limit=50 → [{ "id": 12, "name": "python", "slug": "python", "profile_count": 8 }, ...]
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
دیکشنری مهارتها، مرتب بر اساس محبوبیت. فرانتاند از آن برای type-ahead فیلتر مهارت استفاده میکند بدون اینکه برای هر کلید به Elasticsearch بزند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۵. سلامت
|
||||
|
||||
### `GET /api/v1/health/live/` — عمومی
|
||||
|
||||
```jsonc
|
||||
{ "status": "ok", "service": "linkedin-search-api" }
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
فقط میگوید پروسه زنده است. مناسب liveness probe.
|
||||
|
||||
</div>
|
||||
|
||||
### `GET /api/v1/health/ready/` — عمومی
|
||||
|
||||
```jsonc
|
||||
// 200
|
||||
{ "status": "ok",
|
||||
"checks": { "database": { "healthy": true, "detail": "reachable" },
|
||||
"search": { "healthy": true, "detail": "elasticsearch reachable" } } }
|
||||
|
||||
// 503
|
||||
{ "status": "degraded",
|
||||
"checks": { "database": { "healthy": true, "detail": "reachable" },
|
||||
"search": { "healthy": false, "detail": "Elasticsearch did not answer a ping." } } }
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
وقتی هر وابستگیای خراب باشد کد ۵۰۳ برمیگرداند تا orchestrator ترافیک را به این نمونه نفرستد.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۶. یک جریان کامل با curl
|
||||
|
||||
```bash
|
||||
BASE=http://localhost:8080/api/v1
|
||||
|
||||
# ۱) ثبتنام (اختیاری — کاربران نمونه از قبل هستند)
|
||||
curl -s -X POST $BASE/auth/register/ -H 'Content-Type: application/json' \
|
||||
-d '{"email":"me@example.com","full_name":"Me",
|
||||
"password":"Str0ng-Passw0rd!","password_confirm":"Str0ng-Passw0rd!"}' | jq
|
||||
|
||||
# ۲) لاگین
|
||||
TOKENS=$(curl -s -X POST $BASE/auth/login/ -H 'Content-Type: application/json' \
|
||||
-d '{"email":"analyst@example.com","password":"Analyst-Passw0rd!"}')
|
||||
ACCESS=$(echo "$TOKENS" | jq -r .access)
|
||||
REFRESH=$(echo "$TOKENS" | jq -r .refresh)
|
||||
|
||||
# ۳) هویت
|
||||
curl -s $BASE/auth/me/ -H "Authorization: Bearer $ACCESS" | jq
|
||||
|
||||
# ۴) سرچ با کلیدواژه و دو فیلتر
|
||||
curl -s -G $BASE/search/ -H "Authorization: Bearer $ACCESS" \
|
||||
--data-urlencode 'q=leadership' \
|
||||
--data-urlencode 'skills=management,training' \
|
||||
--data-urlencode 'countries=united states' \
|
||||
--data-urlencode 'sort=-experience' | jq '{count, backend, took_ms,
|
||||
top: [.results[:3][] | {full_name, headline, years_experience}]}'
|
||||
|
||||
# ۵) جزئیات اولین نتیجه
|
||||
ID=$(curl -s -G $BASE/search/ -H "Authorization: Bearer $ACCESS" \
|
||||
--data-urlencode 'q=leadership' | jq -r '.results[0].id')
|
||||
curl -s $BASE/profiles/$ID/ -H "Authorization: Bearer $ACCESS" | jq '{full_name, pii_masked, emails}'
|
||||
|
||||
# ۶) چرخش توکن
|
||||
curl -s -X POST $BASE/auth/refresh/ -H 'Content-Type: application/json' \
|
||||
-d "{\"refresh\":\"$REFRESH\"}" | jq
|
||||
|
||||
# ۷) خروج
|
||||
curl -s -o /dev/null -w '%{http_code}\n' -X POST $BASE/auth/logout/ \
|
||||
-H "Authorization: Bearer $ACCESS" -H 'Content-Type: application/json' \
|
||||
-d "{\"refresh\":\"$REFRESH\"}"
|
||||
```
|
||||
@@ -0,0 +1,289 @@
|
||||
# معماری
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
این سند تصمیمهای معماری پروژه و دلیل هر کدام را توضیح میدهد. برای اجرا و مرور سریع به [README](../README.md) مراجعه کنید.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱. نمای کلی
|
||||
|
||||
```
|
||||
┌────────────┐ HTTPS ┌─────────────────┐ HTTP ┌──────────────────┐
|
||||
│ مرورگر │───────────▶│ nginx (frontend)│───────────▶│ gunicorn + Django│
|
||||
│ React SPA │◀───────────│ :8080 │◀───────────│ :8000 │
|
||||
└────────────┘ └─────────────────┘ └───────┬──────────┘
|
||||
سرو باندل + پراکسی /api │
|
||||
│
|
||||
┌────────────────────┬──────────────────────┬─────────────┴────────┐
|
||||
▼ ▼ ▼ ▼
|
||||
┌──────────────┐ ┌──────────────┐ ┌─────────────────┐ ┌──────────────┐
|
||||
│ PostgreSQL │ │ Redis │ │ Elasticsearch │ │ Logstash │
|
||||
│ منبع حقیقت │ │ cache/throttle│ │ ایندکس سرچ │ │ خط لوله لاگ │
|
||||
└──────────────┘ └──────────────┘ └────────┬────────┘ └──────┬───────┘
|
||||
│ │
|
||||
└────────┬────────────┘
|
||||
▼
|
||||
┌───────────┐
|
||||
│ Kibana │
|
||||
└───────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ۲. مسیر یک درخواست سرچ
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
از لحظهای که کاربر حرفی تایپ میکند تا لحظهای که کارت نتیجه رندر میشود:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
۱. کاربر تایپ میکند
|
||||
│ useDebouncedValue(300ms) → یک درخواست بهجای شش تا
|
||||
▼
|
||||
۲. useSearchFilters وضعیت را در URL مینویسد (?q=python&skills=django)
|
||||
│ URL منبع حقیقت UI است ⇒ نتیجه قابل اشتراک و مقاوم به refresh
|
||||
▼
|
||||
۳. TanStack Query با کلید ['search', filters] کش را چک میکند
|
||||
│ اگر تازه بود، اصلاً درخواستی نمیرود
|
||||
▼
|
||||
۴. axios: هدر Bearer + رهگیر ۴۰۱ (چرخش خودکار توکن)
|
||||
▼
|
||||
۵. nginx: /api/... → backend:8000 (هممبدأ ⇒ بدون CORS)
|
||||
▼
|
||||
۶. RequestIDMiddleware: X-Request-ID میسازد/میپذیرد
|
||||
▼
|
||||
۷. SearchRequestSerializer: اعتبارسنجی و نرمالسازی کوئریاسترینگ
|
||||
│ خروجی: یک SearchRequest تغییرناپذیر (frozen dataclass)
|
||||
▼
|
||||
۸. get_search_backend() → ElasticsearchBackend یا PostgresBackend
|
||||
▼
|
||||
۹. build_query(request) → بدنه کوئری → اجرا → SearchResult
|
||||
▼
|
||||
۱۰. SearchResponseSerializer: قالب یکسان پاسخ
|
||||
▼
|
||||
۱۱. audit.search: یک رویداد JSON → Logstash → Elasticsearch → Kibana
|
||||
▼
|
||||
۱۲. AccessLogMiddleware: خط لاگ دسترسی با duration_ms و user_id
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ۳. تصمیمهای کلیدی
|
||||
|
||||
### ۳.۱ PostgreSQL منبع حقیقت، Elasticsearch یک projection
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**تصمیم:** داده بهصورت نرمالشده در Postgres ذخیره میشود و Elasticsearch فقط یک نسخه denormalizeشده برای سرچ نگه میدارد.
|
||||
|
||||
**چرا:**
|
||||
|
||||
- ایندکس همیشه با `search_index --rebuild` از دیتابیس بازسازی میشود ⇒ از دست رفتن ایندکس یک حادثه عملیاتی است، نه از دست رفتن داده.
|
||||
- تغییر mapping (که در Elasticsearch نیازمند reindex است) بدون ترس انجام میشود.
|
||||
- محدودیتهای یکتایی و ارجاعی (FK, unique) را میشود در جایی گذاشت که واقعاً تضمینشان میکند.
|
||||
|
||||
**هزینه:** دو نسخه از داده. با ۲۸۴ رکورد ناچیز است؛ در مقیاس بزرگتر باید ایندکسکردن به یک صف پسزمینه (Celery) منتقل شود — که بهخاطر interface تمیز `SearchBackend` تغییر کوچکی است.
|
||||
|
||||
</div>
|
||||
|
||||
### ۳.۲ لایه سرچ بهعنوان Strategy
|
||||
|
||||
```python
|
||||
class SearchBackend(ABC):
|
||||
name: str
|
||||
def search(self, request: SearchRequest) -> SearchResult: ...
|
||||
def suggest(self, prefix: str, limit: int = 8) -> list[str]: ...
|
||||
def ping(self) -> bool: ...
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
دو پیادهسازی: `ElasticsearchBackend` و `PostgresBackend`. انتخاب با یک متغیر محیطی (`SEARCH_BACKEND`) انجام میشود و `get_search_backend()` با `functools.cache` نتیجه را نگه میدارد (استخر اتصال یک بار در هر پروسه ساخته شود، نه در هر درخواست).
|
||||
|
||||
**چه چیزی به دست میآید:**
|
||||
|
||||
| مزیت | توضیح |
|
||||
|---|---|
|
||||
| تستپذیری | کل تستسوییت بدون کلاستر Elasticsearch اجرا میشود |
|
||||
| قابلیت مرور | ارزیاب میتواند پروژه را با یک Postgres تنها اجرا کند |
|
||||
| تابآوری | قطعی ES ⇒ سوییچ به حالت کمتوان، نه خواب کامل محصول |
|
||||
| فشار طراحی | interface مجبورمان میکند منطق سرچ از منطق HTTP جدا بماند |
|
||||
|
||||
هیچجای ویو، سریالایزر یا فرانتاند نمیداند کدام موتور جواب داده — فقط فیلد `backend` در پاسخ آن را گزارش میکند (برای شفافیت عملیاتی).
|
||||
|
||||
</div>
|
||||
|
||||
### ۳.۳ ویوهای نازک، لایههای مشخص
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| مسئولیت | کجا |
|
||||
|---|---|
|
||||
| اعتبارسنجی و نرمالسازی ورودی | `apps/search/serializers.py` |
|
||||
| منطق رتبهبندی و فیلتر | `apps/search/backends/*.py` |
|
||||
| قالب پاسخ | `SearchResponseSerializer` |
|
||||
| ماسککردن داده حساس | `apps/profiles/serializers.py` + `privacy.py` |
|
||||
| قالب خطا | `apps/core/exceptions.py` |
|
||||
| لاگ و همبستگی درخواست | `apps/core/middleware.py` + `logging.py` |
|
||||
|
||||
کل `SearchView.get()` کمتر از ۲۰ خط است. این عمدی است: ویو فقط ارکستراتور است.
|
||||
|
||||
</div>
|
||||
|
||||
### ۳.۴ dataclass تغییرناپذیر بهجای dict
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`SearchRequest` یک `@dataclass(frozen=True, slots=True)` است، نه یک دیکشنری. سه سود:
|
||||
|
||||
1. type checker اشتباه نام فیلد را در زمان کامپایل میگیرد، نه در زمان اجرا.
|
||||
2. تغییرناپذیری یعنی هیچ لایهای نمیتواند کوئری را وسط راه عوض کند.
|
||||
3. `as_log_context()` روی همان شیء زندگی میکند ⇒ لاگ و رفتار هیچوقت از هم دور نمیافتند.
|
||||
|
||||
</div>
|
||||
|
||||
### ۳.۵ چرا nginx جلوی هر دو؟
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
SPA و API از یک مبدأ (`localhost:8080`) سرو میشوند. نتیجه:
|
||||
|
||||
- **بدون CORS**: نه pre-flight، نه لیست مبدأ در پروداکشن، نه یک کلاس کامل از باگها.
|
||||
- **CSP سختگیرانه**: چون همهچیز self است، `default-src 'self'` کافی است.
|
||||
- **یک نقطه برای هدرهای امنیتی** و کشکردن assetهای هشدار.
|
||||
|
||||
در حالت توسعه (`make dev`) بهجای nginx، پراکسی خود Vite همین کار را میکند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۴. مدل داده
|
||||
|
||||
```
|
||||
┌─────────────┐ ┌──────────────┐ ┌─────────────┐
|
||||
│ Company │◀───────│ Experience │───────▶│ Profile │
|
||||
│ (unique │ FK │ (سوابق شغلی)│ FK │ │
|
||||
│ name) │ └──────────────┘ │ search_ │
|
||||
└─────────────┘ │ vector │
|
||||
▲ │ (tsvector) │
|
||||
│ FK (current_company) └──────┬──────┘
|
||||
└──────────────────────────────────────────────┤
|
||||
│
|
||||
┌─────────────┐ ┌──────────────┐ │
|
||||
│ School │◀───────│ Education │───────────────┤
|
||||
└─────────────┘ FK └──────────────┘ FK │
|
||||
│
|
||||
┌─────────────┐ ┌──────────────┐ │
|
||||
│ Skill │◀───────│ ProfileSkill │───────────────┘
|
||||
│ (slug uniq) │ FK │ (+position) │ FK
|
||||
└─────────────┘ └──────────────┘
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**ایندکسها و دلیلشان:**
|
||||
|
||||
| ایندکس | برای چه کوئریای |
|
||||
|---|---|
|
||||
| `GIN(search_vector)` | سرچ full-text در بکاند Postgres |
|
||||
| `GIN(seniority_levels)` | فیلتر `seniority_levels__overlap` |
|
||||
| `(location_country, job_title_role)` | ترکیب پرتکرار «کشور + خانواده شغلی» |
|
||||
| `GIN(full_name gin_trgm_ops)` | شباهت trigram برای تحمل غلط تایپی |
|
||||
| `GIN(name gin_trgm_ops)` روی Skill | `icontains` در type-ahead مهارتها |
|
||||
| `profile_count` روی Skill | مرتبسازی پیشنهاد فیلترها |
|
||||
|
||||
**ستونهای denormalizeشده:** `skills_text`، `companies_text`، `education_text`.
|
||||
دلیل: `SearchVector` جنگو نمیتواند از رابطه many-to-many عبور کند. با تختکردن این متنها هنگام ایمپورت، بهروزرسانی کل tsvector یک `UPDATE` تکی میشود و زمان کوئری از O(join) به O(ایندکس) میرسد.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۵. ساختار اپلیکیشنهای جنگو
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| اپ | مسئولیت | وابسته به |
|
||||
|---|---|---|
|
||||
| `core` | لاگ ساختیافته، middleware، قالب خطا، صفحهبندی، throttle، health | — |
|
||||
| `accounts` | مدل کاربر، نقشها، JWT | `core` |
|
||||
| `profiles` | مدل دامنه، خواننده CSV، ایمپورتر، ماسک PII، API پروفایل | `core` |
|
||||
| `search` | ایندکس ES، بکاندهای سرچ، API سرچ | `core`, `profiles` |
|
||||
|
||||
جهت وابستگی یکطرفه است: `search → profiles → core`. هیچ import حلقویای وجود ندارد (تنها استثنا `core.views` است که برای health-check بکاند سرچ را import میکند — بهصورت lazy داخل تابع).
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۶. پیکربندی
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
تنظیمات به چهار ماژول تقسیم شدهاند:
|
||||
|
||||
```
|
||||
config/settings/
|
||||
├── base.py مشترک؛ همه اسرار از متغیر محیطی
|
||||
├── dev.py DEBUG، Browsable API، بدون HTTPS
|
||||
├── prod.py HSTS، کوکی امن، و «fail fast» روی پیکربندی ناامن
|
||||
└── test.py سریع، بدون سرویس خارجی، بکاند Postgres
|
||||
```
|
||||
|
||||
`prod.py` عمداً روی این دو حالت **بالا نمیآید**:
|
||||
|
||||
</div>
|
||||
|
||||
```python
|
||||
if DEBUG:
|
||||
raise RuntimeError("DJANGO_DEBUG must be false when using production settings.")
|
||||
if len(SECRET_KEY) < 50:
|
||||
raise RuntimeError("DJANGO_SECRET_KEY must be at least 50 characters in production.")
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
یک deploy که شکست میخورد بینهایت بهتر از یک deploy است که با `DEBUG=True` بالا میآید.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۷. قابلیت اتکا
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| سناریو | رفتار سیستم |
|
||||
|---|---|
|
||||
| Redis از دسترس خارج شود | `IGNORE_EXCEPTIONS=True` ⇒ کش بیصدا رد میشود، API سالم میماند (throttle موقتاً سختگیر نیست) |
|
||||
| Logstash از دسترس خارج شود | ارسال لاگ async است ⇒ فقط یک warning، هیچ درخواستی بلاک نمیشود |
|
||||
| Elasticsearch از دسترس خارج شود | `SearchBackendError` ⇒ خطای ۵۰۳ با پیام روشن؛ `/health/ready/` هم ۵۰۳ میدهد تا orchestrator ترافیک نفرستد. با `SEARCH_BACKEND=postgres` سرویس ادامه میدهد |
|
||||
| یک ردیف دیتاست خراب باشد | ایمپورتر آن رکورد را لاگ و رد میکند، بقیه ایمپورت میشوند |
|
||||
| reindex نیمهکاره بماند | ایندکس فیزیکی جدید ساخته و **بعد** alias جابهجا میشود ⇒ کاربر هیچوقت ایندکس نیمهپر نمیبیند |
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۸. اگر قرار بود مقیاس بگیرد
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
با ۲۸۴ رکورد بسیاری از اینها لازم نیست، ولی مسیر رشد از قبل باز گذاشته شده:
|
||||
|
||||
| محدودیت فعلی | قدم بعدی |
|
||||
|---|---|
|
||||
| ایندکسگذاری همزمان با ایمپورت | انتقال به Celery + صف؛ `SearchBackend` تغییری نمیکند |
|
||||
| facetها در بکاند Postgres در پایتون شمرده میشوند | جایگزینی با `unnest` + `GROUP BY` یا فقط اتکا به ES |
|
||||
| صفحهبندی `from/size` | `search_after` برای اسکرول عمیق |
|
||||
| یک نود Elasticsearch | کلاستر چندنودی + replica؛ `number_of_replicas` در `INDEX_SETTINGS` |
|
||||
| بهروزرسانی کامل ایندکس | ایندکس افزایشی با سیگنالهای مدل (`index_profile` از قبل نوشته شده) |
|
||||
| بدون CDN | سرو assetهای هشدار از CDN؛ هدرهای cache از قبل درستاند |
|
||||
|
||||
</div>
|
||||
+326
@@ -0,0 +1,326 @@
|
||||
# دیتاست: تحلیل کیفیت و ترمیم
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
فایل `data/linkedin_profiles.csv` یک export واقعی و خراب است. این سند نشان میدهد دقیقاً چه مشکلی دارد، چرا خواندن ساده جواب نمیدهد، و ماژول [`apps/profiles/dataset.py`](../backend/apps/profiles/dataset.py) چطور آن را ترمیم میکند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱. آنچه در فایل هست
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| سنجه | مقدار |
|
||||
|---|---|
|
||||
| خطوط فیزیکی (بدون هدر) | ۳۳۶ |
|
||||
| ستونهای هدر | ۷۷ |
|
||||
| رکوردهای منطقی بعد از بازسازی | ۳۲۳ |
|
||||
| رکوردهای یکتا بعد از حذف تکراری | **۲۸۴** |
|
||||
| ردیفهایی که با خواندن ساده کاملاً سالماند | **۱۰** |
|
||||
|
||||
آن عدد آخر مسئله را خلاصه میکند: `csv.DictReader` روی این فایل **کار میکند** — بدون خطا، بدون هشدار — و ۹۷٪ رکوردها را غلط تحویل میدهد.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۲. چهار نقص
|
||||
|
||||
### ۲.۱ واترمارک تزریقشده
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
بسیاری از ردیفها یک ستون اضافه در ابتدا دارند:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
H:\New folder\leak\international\linkedin\@BreachedData1 Lin...,joseph holland,joseph,...
|
||||
└──────────────────── ستون شبح ────────────────────────────┘ └─ داده واقعی از اینجا شروع میشود
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
نتیجه: کل ردیف یک خانه به راست میلغزد. `full_name` مقدار واترمارک را میگیرد، `first_name` نام کامل را، و به همین ترتیب تا آخر.
|
||||
|
||||
**ترمیم:** هر ستون ابتدایی که شامل `@BreachedData` یا `H:\New folder` باشد حذف میشود.
|
||||
|
||||
### ۲.۲ ستونهای غایب (لغزش به چپ)
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
هدر: ..., job_title, job_title_role, job_title_levels, job_company_id, ...
|
||||
ردیف: ..., "assistant process engineer", "[]", "aerospace-alu...", ...
|
||||
▲
|
||||
مقدار job_title_levels است، نه job_title_role
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
رکورد `job_title_role` را ندارد و بهجای نوشتن مقدار خالی، آن را کلاً ننوشته است. همهچیز بعدش یک خانه به چپ میلغزد. این نقص در **بیشتر** ردیفها وجود دارد و گاهی چند فیلد پشت هم غایباند.
|
||||
|
||||
اثرش با یک نمونه واقعی:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
naive: linkedin_connections = "55,000-70,000" ← این بازه حقوق است!
|
||||
inferred_salary = "18.0" ← این سال تجربه است!
|
||||
repaired:linkedin_connections = 32
|
||||
inferred_salary = "55,000-70,000"
|
||||
inferred_years_exp = 18
|
||||
```
|
||||
|
||||
### ۲.۳ کوتیشن شکسته (یک رکورد در چند خط)
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
چند مقدار `summary` یک `"` فرارنشده دارند (`... Leadership and Management."`). پارسر CSV آن را پایان فیلد میفهمد و بقیه رکورد به خط بعد میریزد. نتیجه: ردیفهایی با ۶، ۹، ۳۴ یا ۳۶ ستون که ادامه رکورد قبلیاند.
|
||||
|
||||
### ۲.۴ دو export به هم چسبیده
|
||||
|
||||
خط هدر یک بار دیگر در خط ۲۵۳ تکرار شده و ۳۹ نفر دقیقاً تکراریاند. یعنی فایل حاصل `cat export1.csv export2.csv` است.
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
توزیع تعداد ستون در ردیفهای فیزیکی:
|
||||
77 ستون → ۲۸۳ ردیف (ولی اکثرشان با لغزش داخلی)
|
||||
78 ستون → ۲۵ ردیف (واترمارکدار)
|
||||
100-133 → ۱۵ ردیف (summary شکسته + واترمارک)
|
||||
6-47 → ۱۳ ردیف (تکههای ادامه رکورد)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ۳. الگوریتم ترمیم
|
||||
|
||||
### ۳.۱ ایده
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
هر ستون یک **امضا** دارد — یک قاعده ساده و قابل بررسی درباره اینکه مقدار معتبرش چه شکلی است:
|
||||
|
||||
</div>
|
||||
|
||||
```python
|
||||
FIELD_VALIDATORS = {
|
||||
"linkedin_url": _contains("linkedin.com/in/"),
|
||||
"gender": _one_of({"male", "female"}),
|
||||
"linkedin_id": _matches(r"\d{3,20}"),
|
||||
"location_geo": _matches(r"-?\d+\.\d+,-?\d+\.\d+"),
|
||||
"job_company_size": _matches(r"(1-10|11-50|51-200|201-500|...)"),
|
||||
"skills": _list_of_str, # literal لیست پایتون
|
||||
"experience": _list_of_dicts("company"), # لیست دیکشنری با کلید company
|
||||
"version_status": _dict_with("status"),
|
||||
... # ۴۶ ستون امضا دارند
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ستونهایی که امضا ندارند (نام، عنوان شغلی، صنعت، خلاصه) هر متنی را میپذیرند.
|
||||
|
||||
سپس مسئله این میشود: **بهترین انطباق بین دنباله مقادیر ردیف و دنباله ستونهای هدر چیست؟** این دقیقاً مسئله همترازی دنباله است.
|
||||
|
||||
### ۳.۲ همترازی بهسبک Needleman–Wunsch
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
ستونهای هدر →
|
||||
full_name first_name gender linkedin_url job_title_role ...
|
||||
مقادیر "jane doe" ↘
|
||||
ردیف "jane" ↘
|
||||
↓ "female" ↘
|
||||
"linkedin.com/in/j" ↘
|
||||
"['senior']" ⤵ (skip column)
|
||||
job_title_levels ↘
|
||||
```
|
||||
|
||||
```python
|
||||
# سه حرکت مجاز در هر خانه جدول برنامهنویسی پویا
|
||||
match = dp[i-1][j-1] + score(column[j], value[i]) # این مقدار به این ستون
|
||||
skip_column = dp[i][j-1] - 0.5 # رکورد این فیلد را ندارد
|
||||
drop_value = dp[i-1][j] - 3.5 (یا -0.3 اگر مقدار خالی) # تکه سرگردان
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**وزنها و منطقشان:**
|
||||
|
||||
| رویداد | امتیاز | چرا |
|
||||
|---|---|---|
|
||||
| تطابق امضا | **+۳** | قویترین شاهد؛ باید بر هر ترکیبی از ویرایشها بچربد |
|
||||
| نقض امضا | **−۴** | «شماره تلفن در ستون ایمیل» باید گرانتر از حذف باشد |
|
||||
| متن آزاد (بدون امضا) | +۰٫۳ | پاداش کوچک برای مصرف کردن مقدار |
|
||||
| مقدار خالی | +۰٫۱۵ | خنثی؛ هر جایی مینشیند |
|
||||
| رد کردن ستون | −۰٫۵ | ارزان: در این فایل خیلی رایج است |
|
||||
| دور انداختن مقدار غیرخالی | **−۳٫۵** | گران: از دست دادن داده آخرین راهحل است |
|
||||
| دور انداختن مقدار خالی | −۰٫۳ | padding انتهای ردیف بیارزش است |
|
||||
|
||||
پیچیدگی O(n×m) با n≈۷۷ و m=۷۷ ⇒ حدود ۶۰۰۰ خانه در هر ردیف. کل فایل در حدود ۱۵ ثانیه (که بیشترش صرف نوشتن در دیتابیس میشود، نه همترازی).
|
||||
|
||||
### ۳.۳ خط لوله کامل
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
۱. حذف ستون واترمارک از ابتدای هر ردیف
|
||||
▼
|
||||
۲. رد کردن خط هدرِ تکراری وسط فایل
|
||||
▼
|
||||
۳. تشخیص شروع رکورد:
|
||||
«full_name و first_name وجود دارند و full_name با first_name شروع میشود»
|
||||
ردیفی که شروع رکورد نیست ⇒ ادامه رکورد قبلی است
|
||||
▼
|
||||
۴. الحاق تکههای ادامه به رکورد قبلی — فقط اگر رکورد قبلی خودش ناقص باشد.
|
||||
وگرنه بهعنوان «تکه یتیم» شمرده و کنار گذاشته میشود (۱۲ مورد).
|
||||
دلیل: چسباندن یک تکه به رکوردی که کامل است، داده یک نفر را به نام
|
||||
نفر دیگر ثبت میکند — بدترین حالت ممکن.
|
||||
▼
|
||||
۵. همترازی DP
|
||||
▼
|
||||
۶. اعتبارسنجی دوم: هر مقداری که هنوز امضایش را رد نکند حذف میشود
|
||||
(۱۷۵ مقدار در کل فایل)
|
||||
▼
|
||||
۷. حذف تکراری بر اساس linkedin_id → linkedin_url → full_name (۳۹ مورد)
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**چرا مرحله ۶ لازم است؟** همترازی بهترین انطباق ممکن را پیدا میکند، ولی «بهترین» همیشه «درست» نیست. مثال واقعی: در یک رکورد، ستون `work_email` مقدار `"* We are focussed on providing consumer nutritional products"` گرفته بود. اعتبارسنجی دوم (`باید شامل @ باشد`) آن را دور میاندازد. **ذخیره نکردن بهتر از ذخیره کردن داده اشتباه است.**
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۴. گزارش ایمپورت
|
||||
|
||||
```bash
|
||||
$ docker compose exec backend python manage.py import_profiles --truncate
|
||||
```
|
||||
|
||||
```json
|
||||
{
|
||||
"profiles_created": 284,
|
||||
"profiles_updated": 0,
|
||||
"skills": 2376,
|
||||
"companies": 1371,
|
||||
"schools": 610,
|
||||
"experiences": 2128,
|
||||
"educations": 870,
|
||||
"physical_rows": 336,
|
||||
"duplicate_headers": 1,
|
||||
"repaired_rows": 16,
|
||||
"orphan_fragments": 12,
|
||||
"duplicate_records": 39,
|
||||
"discarded_values": 175,
|
||||
"records": 284,
|
||||
"elapsed_seconds": 15.41
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
هر عدد قابل ردیابی است. یک ایمپورت که «۲۸۴ رکورد» میگوید ولی نمیگوید چند مقدار دور ریخته، قابل اعتماد نیست.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۵. غنیسازی از داده تودرتو
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ستونهای سطحبالا اغلب غایباند، ولی همان اطلاعات معمولاً داخل بلوب `experience` هست:
|
||||
|
||||
</div>
|
||||
|
||||
```python
|
||||
[{'company': {'name': 'garver', 'size': '501-1000', 'industry': 'civil engineering', ...},
|
||||
'title': {'name': 'recruiting manager', 'role': 'human_resources',
|
||||
'sub_role': 'recruiting', 'levels': ['manager']},
|
||||
'is_primary': True, 'start_date': '2019-10', 'end_date': None}]
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`ProfileImporter._derive_current_position()` و `_resolve_current_company()` وقتی ستون سطحبالا خالی است، از رکورد `is_primary` استفاده میکنند:
|
||||
|
||||
| فیلد | فقط ستون سطحبالا | با غنیسازی | بهبود |
|
||||
|---|---|---|---|
|
||||
| `job_title_role` | ۱۰۸ | **۱۸۴** | +۷۰٪ |
|
||||
| کارفرمای فعلی | ۲۱۹ | **۲۶۵** | +۲۱٪ |
|
||||
| `job_title` | ۲۴۸ | **۲۵۹** | +۴٪ |
|
||||
|
||||
این تفاوت مستقیماً روی کیفیت فیلترها اثر میگذارد: فیلتر «خانواده شغلی» با ۱۰۸ رکورد تقریباً بیفایده است، با ۱۸۴ رکورد کاربردی میشود.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۶. نرمالسازی
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| مورد | قاعده |
|
||||
|---|---|
|
||||
| مهارتها | slug (`slugify`) کلید یکتاست ⇒ «Team Building» و «team building» یک رکورد |
|
||||
| مهارتهای تکراری در یک پروفایل | حذف، با حفظ ترتیب اصلی در `ProfileSkill.position` |
|
||||
| سقف مهارت | ۶۰ مهارت اول (بلندترین لیست ۵۰+ مورد دارد) |
|
||||
| اعداد | `"18.0"` → `18` با بازه معتبر (`0 ≤ years ≤ 70`)؛ خارج از بازه ⇒ `None` |
|
||||
| تاریخ | `"2019-10"` و `"2008"` هر دو نگه داشته میشوند + سال استخراجشده در `start_year`/`end_year` برای فیلتر |
|
||||
| شرکت/دانشگاه | یکتا بر اساس نام، با کش درونحافظهای در ایمپورتر ⇒ بدون N+1 |
|
||||
| GPA | فقط اگر بین ۰ و ۱۰ باشد |
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۷. امنیت پارس کردن
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ستونهای تودرتو با نگارش `repr` پایتون نوشته شدهاند (`'` بهجای `"` و `None` بهجای `null`)، پس `json.loads` کار نمیکند.
|
||||
|
||||
از `ast.literal_eval` استفاده شده، **نه** `eval`. تفاوت حیاتی است: `literal_eval` فقط ساختارهای داده ثابت (عدد، رشته، لیست، دیکشنری، `None`، `True`/`False`) را میپذیرد و هیچ کدی اجرا نمیکند. یک فایل CSV مخرب نمیتواند از این مسیر کد اجرا کند.
|
||||
|
||||
همه فراخوانیها در `_safe_literal()` بستهبندی شدهاند که `ValueError`، `SyntaxError`، `MemoryError` و `RecursionError` را میگیرد و `None` برمیگرداند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۸. تستها
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`backend/tests/test_dataset_repair.py` هفت تست دارد که هر کدام یکی از نقصهای واقعی را در کوچکترین شکل ممکن بازتولید میکنند:
|
||||
|
||||
| تست | تضمین |
|
||||
|---|---|
|
||||
| `test_clean_row_is_read_verbatim` | ردیف سالم دستنخورده میماند |
|
||||
| `test_leading_watermark_column_is_stripped` | ستون شبح حذف میشود |
|
||||
| `test_missing_column_does_not_shift_the_rest_of_the_row` | ستون غایب باعث لغزش بقیه نمیشود |
|
||||
| `test_values_that_fail_their_signature_are_discarded` | مقدار نامعتبر ذخیره نمیشود |
|
||||
| `test_repeated_header_and_duplicate_person_are_removed` | هدر و رکورد تکراری حذف میشوند |
|
||||
| `test_row_split_by_a_stray_quote_is_reassembled` | رکورد شکسته دوباره ساخته میشود |
|
||||
| `test_empty_file_yields_no_records` | فایل خالی کرش نمیکند |
|
||||
|
||||
بدون این تستها، یک تغییر کوچک در وزنهای همترازی میتواند بیصدا کیفیت داده را خراب کند — و هیچکس تا ماهها متوجه نشود.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۹. مسیر جایگزین
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
اگر فردا یک export تمیز برسد، هیچ چیز خراب نمیشود: ردیف سالم امتیاز کامل میگیرد و همترازی دقیقاً همان `dict(zip(header, row))` را تولید میکند. الگوریتم ترمیم روی داده سالم بیاثر است — فقط روی داده خراب فعال میشود.
|
||||
|
||||
</div>
|
||||
@@ -0,0 +1,420 @@
|
||||
# عملیات: اجرا، مانیتورینگ و عیبیابی
|
||||
|
||||
---
|
||||
|
||||
## ۱. اجرا
|
||||
|
||||
### ۱.۱ حالت کامل (پیشفرض)
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
docker compose up --build # یا: make up
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ترتیب بالا آمدن سرویسها با `depends_on: condition: service_healthy` کنترل میشود:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
postgres ──┐
|
||||
redis ─────┼──▶ backend ──▶ frontend
|
||||
elasticsearch ─┴──▶ logstash
|
||||
└──▶ kibana
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
اولین اجرا ۳ تا ۵ دقیقه طول میکشد. `entrypoint.sh` در کانتینر بکاند:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
منتظر postgres (حداکثر ۱۲۰ ثانیه)
|
||||
منتظر elasticsearch (حداکثر ۱۸۰ ثانیه)
|
||||
python manage.py migrate --noinput
|
||||
python manage.py seed_users # اگر SEED_USERS=true
|
||||
python manage.py import_profiles # اگر SEED_DATASET=true و دیتابیس خالی باشد
|
||||
python manage.py search_index --rebuild
|
||||
exec gunicorn ...
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ایمپورت **idempotent** است و فقط وقتی اجرا میشود که جدول پروفایل خالی باشد؛ ریاستارت کانتینر داده را دوباره وارد نمیکند.
|
||||
|
||||
### ۱.۲ حالت توسعه (hot reload)
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
make dev
|
||||
# SPA (Vite) → http://localhost:5173
|
||||
# API → http://localhost:8000/api/docs/
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
سورسها bind-mount میشوند: تغییر در پایتون باعث ریاستارت `runserver` و تغییر در TSX باعث HMR میشود.
|
||||
|
||||
### ۱.۳ حالت پروداکشن
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
make prod
|
||||
# = docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
تفاوتها: احراز هویت Elasticsearch روشن، Kibana منتشر نمیشود، seed خاموش، محدودیت CPU/RAM، دو نمونه بکاند.
|
||||
|
||||
### ۱.۴ اجرا بدون Elasticsearch
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
echo "SEARCH_BACKEND=postgres" >> .env
|
||||
docker compose up -d postgres redis backend frontend
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
کل API و فرانتاند بدون تغییر کار میکنند؛ فیلد `backend` در پاسخ سرچ `"postgres"` میشود.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۲. لاگ ساختیافته
|
||||
|
||||
### ۲.۱ شکل رکوردها
|
||||
|
||||
```json
|
||||
{"@timestamp":"2026-09-03T04:00:08.574Z","level":"INFO","logger":"audit.search",
|
||||
"message":"search_executed","service":"linkedin-search-api","environment":"production",
|
||||
"request_id":"bb3c21c555a54ae0a6dafcd12ea1a98c","event":"search_executed",
|
||||
"user_id":2,"backend":"elasticsearch","result_count":80,"took_ms":31,
|
||||
"query":"leadership","filters":{"skills":["management"],"countries":["united states"],
|
||||
"roles":[],"seniority":[],"industries":[],"companies":[],
|
||||
"min_experience":null,"max_experience":null},
|
||||
"sort":"relevance","page":1,"page_size":3}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| لاگر | رویدادها |
|
||||
|---|---|
|
||||
| `apps.access` | `http_request` — متد، مسیر، وضعیت، `duration_ms`، IP، user id |
|
||||
| `audit.search` | `search_executed` — کوئری، فیلترها، تعداد نتیجه، زمان، موتور |
|
||||
| `audit.auth` | `login_succeeded`، `logout`، `user_registered` |
|
||||
| `apps.errors` | `unhandled_exception` با stack trace |
|
||||
| `gunicorn.access` | خط دسترسی وبسرور (JSON) |
|
||||
|
||||
### ۲.۲ مسیر انتقال
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
Django logger
|
||||
│ JSONFormatter → یک شیء JSON در هر رکورد
|
||||
│ RequestIDFilter → افزودن request_id از ContextVar
|
||||
├──▶ stdout (همیشه؛ قابل دیدن با docker compose logs)
|
||||
└──▶ AsynchronousLogstashHandler
|
||||
│ بافر درونحافظهای + ترد جدا
|
||||
▼
|
||||
Logstash :5959 (codec: json_lines)
|
||||
│ غنیسازی، برچسبگذاری، حذف فیلدهای حساس
|
||||
▼
|
||||
Elasticsearch app-logs-{stream}-{YYYY.MM.dd}
|
||||
▼
|
||||
Kibana
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**چرا async؟** اگر Logstash از دسترس خارج شود، یک هندلر همگام ترد درخواست را بلاک میکند و کل API کند میشود. با `python-logstash-async` رکوردها در حافظه صف میشوند و یک ترد جدا آنها را میفرستد. این رفتار عملاً آزموده شده: با Logstash خاموش، تنها اثرش یک warning در stdout است.
|
||||
|
||||
### ۲.۳ مسیردهی به ایندکسها
|
||||
|
||||
</div>
|
||||
|
||||
| ایندکس | محتوا |
|
||||
|---|---|
|
||||
| `app-logs-search-*` | رویدادهای `search_executed` |
|
||||
| `app-logs-audit-*` | ورود، خروج، ثبتنام |
|
||||
| `app-logs-error-*` | `ERROR` و `CRITICAL` |
|
||||
| `app-logs-app-*` | بقیه |
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
یک ایندکس در روز برای هر جریان. نتیجه: «همه لاگهای قدیمیتر از ۳۰ روز را پاک کن» یک حذف ایندکس است، نه یک `delete_by_query` سنگین.
|
||||
|
||||
**برچسبهای خودکار:** `slow_search` (بیش از ۵۰۰ms) و `zero_results`.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۳. Kibana
|
||||
|
||||
```bash
|
||||
make kibana-setup # ساخت data view «Application logs»
|
||||
open http://localhost:5601
|
||||
```
|
||||
|
||||
### کوئریهای مفید در Discover
|
||||
|
||||
```
|
||||
event: "search_executed" and result_count: 0
|
||||
→ کاربران چه چیزی میخواستند که پیدا نکردند؟ (بهترین سیگنال برای بهبود سرچ)
|
||||
|
||||
event: "search_executed" and took_ms > 200
|
||||
→ کدام ترکیب فیلترها کند است؟
|
||||
|
||||
event: "search_executed" and filters.skills: *
|
||||
→ کدام مهارتها بیشترین جستجو را دارند؟
|
||||
|
||||
level: "ERROR"
|
||||
→ خطاها با stack trace کامل
|
||||
|
||||
request_id: "bb3c21c555a54ae0a6dafcd12ea1a98c"
|
||||
→ کل ردپای یک درخواست خاص (شامل خط دسترسی و رویداد ممیزی)
|
||||
|
||||
http_status >= 400 and http_path: "/api/v1/search/"
|
||||
→ درخواستهای ناموفق سرچ
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**چطور `request_id` را پیدا کنم؟** هدر `X-Request-ID` در تب Network مرورگر، یا فیلد `request_id` داخل بدنه هر پاسخ خطا.
|
||||
|
||||
### داشبوردی که ارزش ساختن دارد
|
||||
|
||||
| ویجت | کوئری |
|
||||
|---|---|
|
||||
| نرخ سرچ | `event: "search_executed"` — Date histogram |
|
||||
| صدک ۹۵ زمان پاسخ | `took_ms` — Percentiles |
|
||||
| نرخ نتیجهی صفر | `result_count: 0` تقسیم بر کل |
|
||||
| پرجستجوترین مهارتها | `filters.skills` — Terms |
|
||||
| نرخ خطا | `http_status >= 500` |
|
||||
| کاربران فعال | `user_id` — Unique count |
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۴. عملیات روزمره
|
||||
|
||||
```bash
|
||||
make ps # وضعیت کانتینرها + healthcheck
|
||||
make logs # لاگ زنده همه سرویسها
|
||||
docker compose logs -f backend # فقط API
|
||||
|
||||
make seed # ایمپورت مجدد کامل + بازسازی ایندکس
|
||||
make reindex # فقط بازسازی ایندکس از روی Postgres
|
||||
make index-status # وضعیت alias و تعداد داکیومنت
|
||||
make users # ساخت/بهروزرسانی کاربران نمونه
|
||||
make superuser # ساخت superuser تعاملی
|
||||
make shell # شل جنگو
|
||||
|
||||
make test # تستها داخل کانتینر
|
||||
make lint # lint + type-check
|
||||
make audit # اسکن امنیتی وابستگیها
|
||||
make schema # نوشتن docs/openapi.yaml
|
||||
```
|
||||
|
||||
### بررسی سلامت
|
||||
|
||||
```bash
|
||||
curl -s localhost:8080/api/v1/health/live/ | jq
|
||||
curl -s localhost:8080/api/v1/health/ready/ | jq
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ۵. عیبیابی
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### Elasticsearch بالا نمیآید
|
||||
|
||||
**علامت:** کانتینر `elasticsearch` مدام ریاستارت میشود؛ در لاگ: `max virtual memory areas vm.max_map_count [65530] is too low`.
|
||||
|
||||
**علت:** Elasticsearch به `vm.max_map_count ≥ 262144` نیاز دارد.
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
sudo sysctl -w vm.max_map_count=262144 # موقت
|
||||
echo 'vm.max_map_count=262144' | sudo tee -a /etc/sysctl.conf # دائمی
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
روی Docker Desktop (مک/ویندوز) معمولاً از قبل تنظیم است.
|
||||
|
||||
### Elasticsearch بهخاطر حافظه کشته میشود (exit 137)
|
||||
|
||||
`ES_JAVA_OPTS` را در `.env` کم کنید:
|
||||
|
||||
</div>
|
||||
|
||||
```env
|
||||
ES_JAVA_OPTS=-Xms256m -Xmx256m
|
||||
LS_JAVA_OPTS=-Xms192m -Xmx192m
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
یا اگر رم کم است، Kibana و Logstash را بالا نیاورید:
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
docker compose up -d postgres redis elasticsearch backend frontend
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### بکاند با «DJANGO_SECRET_KEY is required» بالا نمیآید
|
||||
|
||||
`.env` ساخته نشده یا کلید خالی است:
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
python -c "import secrets; print(secrets.token_urlsafe(64))" # مقدار را در .env بگذارید
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### سرچ ۵۰۳ میدهد
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
curl -s localhost:8080/api/v1/health/ready/ | jq # کدام وابستگی خراب است؟
|
||||
docker compose logs elasticsearch | tail -50
|
||||
make index-status # آیا ایندکس اصلاً وجود دارد؟
|
||||
make reindex # بازسازی
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
راهحل موقت: `SEARCH_BACKEND=postgres` در `.env` و `docker compose restart backend`.
|
||||
|
||||
### سرچ نتیجه نمیدهد ولی داده هست
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
make index-status
|
||||
# {"alias": "linkedin-profiles", "exists": true, "documents": 284}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
اگر `documents: 0` بود، ایمپورت انجام شده ولی ایندکس ساخته نشده: `make reindex`.
|
||||
|
||||
### لاگها در Kibana دیده نمیشوند
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
docker compose logs logstash | tail -30 # آیا pipeline بالا آمده؟
|
||||
docker compose exec backend env | grep LOGSTASH # LOGSTASH_ENABLED=true ؟
|
||||
curl -s localhost:5601/api/status | jq .status.overall
|
||||
./scripts/kibana_setup.sh # آیا data view ساخته شده؟
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
Logstash حدود ۶۰ ثانیه طول میکشد تا آماده شود؛ لاگهای قبل از آن در بافر میمانند و بعد ارسال میشوند.
|
||||
|
||||
### فرانتاند «Network Error» میدهد
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
docker compose ps backend # سالم است؟
|
||||
docker compose exec frontend wget -qO- http://backend:8000/api/v1/health/live/
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
اگر با `npm run dev` بیرون از داکر کار میکنید، مطمئن شوید مبدأ `http://localhost:5173` در `CORS_ALLOWED_ORIGINS` هست.
|
||||
|
||||
### مایگریشن روی `pg_trgm` شکست میخورد
|
||||
|
||||
مایگریشن `0002_postgres_extensions` نیاز به دسترسی ساخت extension دارد. در ایمیج رسمی Postgres کاربر پیشفرض superuser است؛ در دیتابیس مدیریتشده باید ادمین یک بار اجرا کند:
|
||||
|
||||
</div>
|
||||
|
||||
```sql
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
CREATE EXTENSION IF NOT EXISTS unaccent;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ۶. پشتیبانگیری و بازیابی
|
||||
|
||||
```bash
|
||||
# پشتیبان
|
||||
docker compose exec -T postgres pg_dump -U linkedin linkedin | gzip > backup-$(date +%F).sql.gz
|
||||
|
||||
# بازیابی
|
||||
gunzip -c backup-2026-09-03.sql.gz | docker compose exec -T postgres psql -U linkedin linkedin
|
||||
make reindex # ایندکس از Postgres بازسازی میشود
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
از Elasticsearch **پشتیبان نمیگیریم**. ایندکس یک projection دورریختنی است و همیشه از Postgres قابل بازسازی است — این دقیقاً دلیل انتخاب آن معماری بود.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۷. سنجههای عملکرد (دیتاست ۲۸۴ رکوردی)
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| عملیات | زمان |
|
||||
|---|---|
|
||||
| ایمپورت کامل (خواندن + ترمیم + نوشتن) | ~۱۵ ثانیه |
|
||||
| بازسازی ایندکس | ~۲ ثانیه |
|
||||
| سرچ با Elasticsearch | ~۵–۱۵ms |
|
||||
| سرچ با PostgreSQL | ~۱۵–۳۵ms |
|
||||
| جزئیات پروفایل | ~۲۰ms |
|
||||
| بیلد ایمیج بکاند (بدون کش) | ~۹۰ ثانیه |
|
||||
| بیلد ایمیج فرانتاند (بدون کش) | ~۶۰ ثانیه |
|
||||
|
||||
اعداد سرچ روی همین دیتاست و ماشین توسعه اندازهگیری شدهاند؛ فیلد `took_ms` در هر پاسخ همین را گزارش میکند و در Kibana قابل نمودار کردن است.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۸. چکلیست پیش از استقرار
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
- [ ] `DJANGO_SECRET_KEY` تصادفی و ≥۵۰ کاراکتر
|
||||
- [ ] `POSTGRES_PASSWORD` و `ELASTIC_PASSWORD` عوض شده
|
||||
- [ ] رمزهای کاربران نمونه عوض یا `SEED_USERS=false`
|
||||
- [ ] `DJANGO_DEBUG=false` و `DJANGO_SETTINGS_MODULE=config.settings.prod`
|
||||
- [ ] `DJANGO_ALLOWED_HOSTS` روی دامنه واقعی
|
||||
- [ ] `DJANGO_ADMIN_URL` روی یک مسیر غیرقابلحدس
|
||||
- [ ] HTTPS در لبه + `SECURE_SSL_REDIRECT=true`
|
||||
- [ ] `SEED_DATASET=false`
|
||||
- [ ] overlay پروداکشن فعال (احراز هویت ES)
|
||||
- [ ] پشتیبانگیری خودکار Postgres
|
||||
- [ ] سیاست نگهداری لاگ (ILM) روی `app-logs-*`
|
||||
- [ ] `docker compose exec backend python manage.py check --deploy` بدون خطا
|
||||
|
||||
</div>
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
# منطق سرچ و فیلترها
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
این سند دقیقاً توضیح میدهد یک کوئری چطور به نتیجه تبدیل میشود: چه چیزی امتیاز میگیرد، چه چیزی فقط حذف میکند، و چرا.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱. آناتومی یک درخواست
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| بخش | نقش | امتیاز میدهد؟ |
|
||||
|---|---|---|
|
||||
| `q` | تعیین میکند چه کسی **مرتبط** است | ✅ |
|
||||
| فیلترها | تعیین میکنند چه کسی **واجد شرایط** است | ❌ |
|
||||
| `sort` | ترتیب نهایی | ❌ |
|
||||
| `page` / `page_size` | برش | ❌ |
|
||||
|
||||
این تفکیک در Elasticsearch بهصورت مستقیم به `must`/`should` (بخش امتیازدهی) در برابر `filter` (بخش حذف) نگاشت میشود. مزیت عملی: کلازهای `filter` توسط ES کش میشوند و هزینه محاسبه امتیاز ندارند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۲. ایندکس Elasticsearch
|
||||
|
||||
### ۲.۱ آنالایزرها
|
||||
|
||||
```jsonc
|
||||
"english_text": { // برای همه فیلدهای متنی
|
||||
"tokenizer": "standard",
|
||||
"filter": ["lowercase", "asciifolding",
|
||||
"english_possessive", "english_stop", "english_stemmer"]
|
||||
}
|
||||
"autocomplete_index": { "filter": [..., "edge_ngram(2,15)"] } // زمان ایندکس
|
||||
"autocomplete_search": { "filter": ["lowercase", "asciifolding"] } // زمان کوئری
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
سه نکته:
|
||||
|
||||
1. **`asciifolding`** باعث میشود «José» با «jose» پیدا شود.
|
||||
2. **`english_stemmer`** «engineering»، «engineer» و «engineers» را یکی میکند.
|
||||
3. **آنالایزر متفاوت برای ایندکس و سرچ در autocomplete** حیاتی است: اگر عبارت کاربر هم n-gram شود، «jo» با هر چیزی که حرف «j» دارد مچ میشود. با `search_analyzer` جدا، فقط پیشوندها مچ میشوند.
|
||||
|
||||
### ۲.۲ چرا هر فیلد دو نسخه دارد
|
||||
|
||||
</div>
|
||||
|
||||
```jsonc
|
||||
"skills": {
|
||||
"type": "text", "analyzer": "english_text", // برای سرچ متنی
|
||||
"fields": {
|
||||
"keyword": { "type": "keyword",
|
||||
"normalizer": "lowercase_normalizer" }, // برای فیلتر و facet
|
||||
"autocomplete": { ... } // برای type-ahead
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
متن آنالایزشده برای فیلتر دقیق و شمارش aggregation قابل استفاده نیست — «project management» به دو توکن شکسته میشود و در facet دو ردیف جدا میسازد. زیرفیلد `keyword` مقدار خام (فقط lowercase و ascii-folded) را نگه میدارد.
|
||||
|
||||
### ۲.۳ `dynamic: strict`
|
||||
|
||||
mapping عمداً strict است. اگر داکیومنتی فیلدی داشته باشد که در mapping نیست، Elasticsearch کل داکیومنت را رد میکند — بهجای اینکه بیصدا نوعش را حدس بزند و بعداً یک facet را خراب کند. یک تست (`test_document_only_contains_mapped_fields`) این قرارداد را قفل کرده است.
|
||||
|
||||
### ۲.۴ چه چیزی ایندکس **نمیشود**
|
||||
|
||||
ایمیل، شماره تلفن و آدرس خیابان **هرگز** وارد ایندکس نمیشوند. دلیل صرفاً «حریم خصوصی بهطور کلی» نیست: اگر ایمیل ایندکس شود، هر کاربری میتواند با یک سرچ حدسی وجود یک آدرس ایمیل را تأیید کند (oracle attack). این دادهها فقط در Postgres میمانند و در لایه سریالایزر ماسک میشوند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۳. کوئری کلیدواژه
|
||||
|
||||
```python
|
||||
# apps/search/backends/elastic.py
|
||||
SEARCH_FIELDS = [
|
||||
"full_name^6", # نام مهمترین سیگنال است
|
||||
"job_title^4",
|
||||
"skills^3.5",
|
||||
"company^2.5",
|
||||
"headline^2",
|
||||
"past_companies^1.5",
|
||||
"industry^1.5",
|
||||
"majors^1.5",
|
||||
"education",
|
||||
"location_name",
|
||||
"summary", # پرحجمترین و کمسیگنالترین
|
||||
"interests",
|
||||
]
|
||||
PHRASE_FIELDS = ["full_name^8", "job_title^6", "skills^5", "company^4", "headline^3"]
|
||||
```
|
||||
|
||||
### ۳.۱ چهار پارامتر کلیدی
|
||||
|
||||
| پارامتر | مقدار | چرا |
|
||||
|---|---|---|
|
||||
| `type` | `best_fields` | یک پروفایل که «python» را قوی در `skills` دارد، بهتر از پروفایلی است که آن را ضعیف در چند فیلد پخش کرده |
|
||||
| `minimum_should_match` | `2<70%` | کوئری یککلمهای: مچ اجباری. کوئری سهکلمهای: ۷۰٪ کافی است، پس «senior python django» کسی را که فقط دوتایش را دارد هم برمیگرداند |
|
||||
| `fuzziness` | `AUTO` | فاصله ویرایشی متناسب با طول کلمه؛ «pyhton» → «python» |
|
||||
| `prefix_length` | `1` | حرف اول باید دقیق باشد؛ جلوی انفجار کاندیدها و نتایج بیربط را میگیرد |
|
||||
|
||||
### ۳.۲ بوست عبارت
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
کلاز `should` (با `boost: 4`) هیچوقت نتیجهای را حذف نمیکند، فقط امتیاز اضافه میکند. اثرش:
|
||||
|
||||
| کوئری | بدون بوست عبارت | با بوست عبارت |
|
||||
|---|---|---|
|
||||
| `product manager` | کسی که «product» در عنوان و «manager» در خلاصه دارد ممکن است اول بیاید | کسی که عنوانش دقیقاً «product manager» است اول میآید |
|
||||
|
||||
`slop: 1` اجازه میدهد یک کلمه فاصله بیفتد، پس «product marketing manager» هم بوست میگیرد ولی کمتر.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۴. فیلترها
|
||||
|
||||
### ۴.۱ نگاشت پارامتر به فیلد ایندکس
|
||||
|
||||
```python
|
||||
FILTER_FIELDS = {
|
||||
"skills": "skills.keyword",
|
||||
"roles": "job_title_role",
|
||||
"seniority": "seniority_levels",
|
||||
"countries": "location_country",
|
||||
"industries": "industry.keyword",
|
||||
"companies": "company.keyword",
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
هر کدام به یک کلاز `terms` تبدیل میشوند و بازه تجربه به یک `range`.
|
||||
|
||||
### ۴.۲ قاعده OR/AND
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
?skills=python,django&countries=germany&min_experience=5
|
||||
|
||||
⇒ filter: [
|
||||
{ terms: { "skills.keyword": ["python", "django"] } }, ← OR داخلی
|
||||
{ terms: { "location_country": ["germany"] } }, ← AND با قبلی
|
||||
{ range: { "years_experience": { "gte": 5 } } } ← AND با قبلی
|
||||
]
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**چرا این قاعده؟** چون با مدل ذهنی کاربر از یک پنل فیلتر میخواند:
|
||||
|
||||
- تیک زدن مهارت دومِ **همان دسته** = «یا این یا آن» ⇒ نتایج بیشتر میشوند.
|
||||
- تیک زدن یک کشور در دسته **دیگر** = «و همچنین» ⇒ نتایج کمتر میشوند.
|
||||
|
||||
اگر مهارتها هم AND میشدند، کاربر با تیک زدن مهارت دوم معمولاً به صفر نتیجه میرسید و فکر میکرد سیستم خراب است.
|
||||
|
||||
### ۴.۳ هر دو نگارش کوئریاسترینگ پشتیبانی میشوند
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
?skills=python,django # جدا با کاما
|
||||
?skills=python&skills=django # پارامتر تکراری
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`CommaSeparatedField.get_value()` در حالت دوم از `QueryDict.getlist` استفاده میکند (متد `get` فقط آخرین مقدار را برمیگرداند — یک باگ کلاسیک). هر دو نگارش نرمال، lowercase، یکتا و به حداکثر ۲۰ مقدار محدود میشوند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۵. Facetها
|
||||
|
||||
```jsonc
|
||||
"aggs": {
|
||||
"skills": { "terms": { "field": "skills.keyword", "size": 25 } },
|
||||
"job_title_role": { "terms": { "field": "job_title_role", "size": 25 } },
|
||||
"seniority": { "terms": { "field": "seniority_levels","size": 25 } },
|
||||
"country": { "terms": { "field": "location_country","size": 25 } },
|
||||
"industry": { "terms": { "field": "industry.keyword","size": 25 } },
|
||||
"company": { "terms": { "field": "company.keyword", "size": 25 } }
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
facetها روی **مجموعه نتایج فیلترشده فعلی** محاسبه میشوند، نه روی کل دیتاست. یعنی وقتی کاربر «engineering» را انتخاب کرده، شمارش کشورها فقط مهندسها را میشمارد.
|
||||
|
||||
دو جزئیات UX که در فرانتاند رعایت شدهاند:
|
||||
|
||||
1. **مقدار انتخابشده هیچوقت ناپدید نمیشود** — حتی اگر شمارشش صفر شود، وگرنه کاربر نمیتواند تیکش را بردارد.
|
||||
2. **رشته خالی از facet حذف میشود** — دیتاست پر از فیلدهای خالی است و یک باکت بینام فقط سردرگمی میسازد.
|
||||
|
||||
`facets=false` را میشود فرستاد تا aggregation اجرا نشود (برای صفحات بعدی که پنل فیلتر تغییری نکرده).
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۶. مرتبسازی
|
||||
|
||||
| `sort` | Elasticsearch | PostgreSQL |
|
||||
|---|---|---|
|
||||
| `relevance` (پیشفرض) | `["_score", {"id": "asc"}]` | `-rank, full_name, id` |
|
||||
| `-experience` | `years_experience desc, missing _last` | `years_experience desc nulls last` |
|
||||
| `experience` | `years_experience asc, missing _last` | `years_experience asc nulls last` |
|
||||
| `name` | `full_name.keyword asc` | `full_name asc` |
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**نکته `missing: _last`:** ۶۴ پروفایل از ۲۸۴ سال تجربه ندارند. بدون این تنظیم، مرتبسازی صعودی همهشان را اول میگذارد و صفحه اول پر از داده ناقص میشود.
|
||||
|
||||
**نکته tie-break:** همهجا `id` بهعنوان معیار آخر اضافه شده تا صفحهبندی پایدار باشد؛ بدون آن، دو رکورد با امتیاز برابر میتوانند بین صفحه ۱ و ۲ جابهجا شوند و یکی دوبار دیده و دیگری اصلاً دیده نشود.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۷. Highlight
|
||||
|
||||
```jsonc
|
||||
"highlight": {
|
||||
"pre_tags": ["<mark>"], "post_tags": ["</mark>"],
|
||||
"fields": {
|
||||
"full_name": { "number_of_fragments": 0 }, // کل مقدار
|
||||
"job_title": { "number_of_fragments": 0 },
|
||||
"skills": { "number_of_fragments": 3 },
|
||||
"summary": { "fragment_size": 160, "number_of_fragments": 1 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`number_of_fragments: 0` یعنی «کل فیلد را برگردان، نه یک تکه» — برای نام و عنوان که کوتاهاند درست است. برای `summary` یک قطعه ۱۶۰ کاراکتری کافی است.
|
||||
|
||||
**نکته امنیتی:** snippet برگشتی شامل تگ HTML است ولی محتوایش داده کاربر است. فرانتاند آن را با `dangerouslySetInnerHTML` رندر **نمیکند**؛ کامپوننت `Highlight` رشته را روی تگهای `<mark>` میشکند و بقیه را بهعنوان text node رندر میکند. نتیجه یکسان، بدون سطح حمله XSS.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۸. بکاند PostgreSQL
|
||||
|
||||
### ۸.۱ tsvector وزندار
|
||||
|
||||
```python
|
||||
Profile.objects.update(search_vector=(
|
||||
SearchVector("full_name", weight="A", config="english")
|
||||
+ SearchVector("job_title", weight="B", config="english")
|
||||
+ SearchVector("skills_text", weight="B", config="english")
|
||||
+ SearchVector("companies_text", weight="B", config="english")
|
||||
+ SearchVector("industry", weight="C", config="english")
|
||||
+ SearchVector("education_text", weight="C", config="english")
|
||||
+ SearchVector("location_name", weight="C", config="english")
|
||||
+ SearchVector("summary", weight="D", config="english")
|
||||
))
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
وزنهای پیشفرض PostgreSQL: A=1.0، B=0.4، C=0.2، D=0.1 — همان سلسلهمراتبی که در ES با بوست فیلدها ساخته شد.
|
||||
|
||||
### ۸.۲ تحمل غلط تایپی
|
||||
|
||||
</div>
|
||||
|
||||
```python
|
||||
.annotate(
|
||||
text_rank = SearchRank(F("search_vector"), query),
|
||||
name_similarity= TrigramSimilarity("full_name", term),
|
||||
)
|
||||
.annotate(rank=Greatest(F("text_rank"), <name_similarity اگر ≥ 0.3>))
|
||||
.filter(Q(search_vector=query) | Q(name_similarity__gte=0.3))
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`Greatest` یعنی یک تطابق قوی نام بهاندازه یک تطابق قوی متنی ارزش دارد. آستانه ۰٫۳ تجربی است: پایینتر نویز میآورد، بالاتر «jhon» را رد میکند.
|
||||
|
||||
### ۸.۳ نگارش کوئری
|
||||
|
||||
`search_type="websearch"` استفاده شده، پس کاربر همان نحوی را دارد که از موتورهای جستجو میشناسد:
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
python django دو کلمه، هر دو لازم (AND)
|
||||
"product manager" عبارت دقیق
|
||||
python or ruby یکی از دو
|
||||
python -junior شامل python، بدون junior
|
||||
```
|
||||
|
||||
### ۸.۴ تفاوتهای آگاهانه با Elasticsearch
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| موضوع | Elasticsearch | PostgreSQL |
|
||||
|---|---|---|
|
||||
| چند کلمه | ۷۰٪ کافی است (بازیابی بالاتر) | همه لازماند (دقت بالاتر) |
|
||||
| غلط تایپی | روی همه فیلدها | فقط روی نام |
|
||||
| highlight | highlighter بومی | جایگزینی توکن سمت پایتون |
|
||||
| facet | aggregation | شمارش در پایتون روی مجموعه فیلترشده |
|
||||
| مقیاس | میلیونها رکورد | تا حدود دهها هزار رکورد راحت |
|
||||
|
||||
این تفاوتها پنهان نشدهاند: پاسخ API فیلد `backend` دارد و UI هم موتور فعال را نشان میدهد. برای دیتاستی با ۲۸۴ رکورد، تفاوت عملاً محسوس نیست — که دقیقاً چیزی است که یک fallback خوب را ممکن میکند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۹. محافظها
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| محدودیت | مقدار | چرا |
|
||||
|---|---|---|
|
||||
| طول `q` | ۲۰۰ کاراکتر | کوئری غولپیکر = مصرف CPU |
|
||||
| تعداد مقدار در هر فیلتر | ۲۰ | جلوگیری از `terms` با هزاران مقدار |
|
||||
| `page_size` | حداکثر ۱۰۰ | جلوگیری از پاسخ چندمگابایتی |
|
||||
| `page × page_size` | حداکثر ۱۰٬۰۰۰ | صفحهبندی عمیق برای انسان بیفایده و برای موتور گران است — رد میشود، نه اینکه بیصدا کوتاه شود |
|
||||
| rate limit سرچ | ۱۲۰/دقیقه | سرچ گرانترین endpoint است |
|
||||
|
||||
همه در `SearchRequestSerializer` اعمال میشوند، یعنی **قبل** از اینکه چیزی به موتور برسد.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱۰. نمونههای عملی
|
||||
|
||||
```bash
|
||||
# سادهترین حالت: همه، مرتب بر اساس نام
|
||||
GET /api/v1/search/?sort=name
|
||||
|
||||
# کلیدواژه
|
||||
GET /api/v1/search/?q=recruiting
|
||||
|
||||
# کلیدواژه + دو فیلتر (خواسته حداقلی صورت تسک)
|
||||
GET /api/v1/search/?q=leadership&skills=management&roles=human_resources
|
||||
|
||||
# چند مقدار در یک فیلتر → OR
|
||||
GET /api/v1/search/?skills=python,django,react
|
||||
|
||||
# بازه تجربه + مرتبسازی
|
||||
GET /api/v1/search/?min_experience=10&max_experience=20&sort=-experience
|
||||
|
||||
# صفحه دوم، بدون محاسبه دوباره facetها
|
||||
GET /api/v1/search/?q=engineer&page=2&facets=false
|
||||
|
||||
# تحمل غلط تایپی
|
||||
GET /api/v1/search/?q=jhon # john را پیدا میکند
|
||||
|
||||
# type-ahead
|
||||
GET /api/v1/search/suggest/?q=mar&limit=5
|
||||
```
|
||||
@@ -0,0 +1,309 @@
|
||||
# امنیت
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
این سند مدل تهدید، لایههای دفاعی و — مهمتر — **trade-offهایی که آگاهانه پذیرفته شدهاند** را توضیح میدهد. یک سند امنیتی که فقط فهرست کارهای خوب باشد، نصف حقیقت را میگوید.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱. مدل تهدید
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| دارایی | چرا ارزشمند است |
|
||||
|---|---|
|
||||
| اطلاعات تماس ۲۸۴ نفر واقعی | ایمیل، موبایل، آدرس خیابان — داده شخصی قابل سوءاستفاده |
|
||||
| کل دیتاست | ارزش تجاری؛ استخراج انبوه (scraping) تهدید اصلی است |
|
||||
| حسابهای کاربری | دسترسی به داده بالا |
|
||||
| زیرساخت | Postgres، Elasticsearch، Kibana |
|
||||
|
||||
| مهاجم | توان | دفاع اصلی |
|
||||
|---|---|---|
|
||||
| ناشناس از اینترنت | درخواست HTTP | احراز هویت اجباری، rate limit، هدرهای امنیتی |
|
||||
| کاربر لاگینکرده با نیت بد | API معتبر | ماسک PII، سقف نرخ، صفحهبندی محدود، لاگ ممیزی |
|
||||
| مهاجم روی شبکه | شنود | HTTPS در لبه، شبکه داخلی داکر |
|
||||
| فایل ورودی مخرب | محتوای CSV | `literal_eval` بهجای `eval`، اعتبارسنجی امضا |
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۲. احراز هویت
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| مورد | پیادهسازی | دلیل |
|
||||
|---|---|---|
|
||||
| هش رمز | **Argon2id** (اول در `PASSWORD_HASHERS`) | توصیه OWASP؛ مقاوم در برابر GPU |
|
||||
| سیاست رمز | ≥۱۰ کاراکتر + ۴ اعتبارسنج جنگو | جلوی رمزهای فهرستشده را میگیرد |
|
||||
| عمر access token | ۱۵ دقیقه | پنجره سوءاستفاده از توکن دزدیدهشده کوتاه |
|
||||
| عمر refresh token | ۷ روز، **چرخشی** | هر refresh یک جفت جدید میدهد |
|
||||
| بلاکلیست | `ROTATE_REFRESH_TOKENS` + `BLACKLIST_AFTER_ROTATION` | استفاده دوباره از توکن قدیمی رد میشود ⇒ نشت **قابل تشخیص** است |
|
||||
| الگوریتم | HS256 با کلید جدا (`JWT_SIGNING_KEY`) | چرخاندن کلید JWT بدون بیاعتبار کردن session جنگو |
|
||||
| کلیدهای داخل توکن | فقط `user_id`، `role`، `email` | payload کوچک؛ هیچ داده حساسی در توکنی که کلاینت میبیند نیست |
|
||||
|
||||
### چرا `sessionStorage` و نه کوکی؟
|
||||
|
||||
توکنها در `sessionStorage` نگه داشته میشوند:
|
||||
|
||||
- ✅ بهصورت خودکار به هیچ درخواست cross-site پیوست نمیشوند ⇒ **کل کلاس حملات CSRF منتفی است** و API به محافظت CSRF برای JSON نیاز ندارد.
|
||||
- ✅ با بستن تب پاک میشوند.
|
||||
- ⚠️ در برابر XSS آسیبپذیرند — بههمین دلیل CSP سختگیرانه است، هیچجا `dangerouslySetInnerHTML` استفاده نشده، و React بهصورت پیشفرض خروجی را escape میکند.
|
||||
|
||||
جایگزین «کوکی `HttpOnly` + `SameSite=Strict`» در برابر XSS بهتر است ولی CSRF را برمیگرداند و برای SPA روی دامنه دیگر پیچیده میشود. برای این معماری (SPA و API هممبدأ، بدون کوکی) انتخاب فعلی سازگارتر است. در یک سیستم با ریسک بالاتر، کوکی `HttpOnly` + محافظت CSRF گزینه درستتر است.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۳. مجوزدهی
|
||||
|
||||
```python
|
||||
class Role(models.TextChoices):
|
||||
ADMIN = "admin" # همهچیز + بازسازی ایندکس + PII
|
||||
ANALYST = "analyst" # سرچ + PII
|
||||
VIEWER = "viewer" # سرچ، PII ماسکشده
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| قاعده | جای اعمال |
|
||||
|---|---|
|
||||
| **deny by default** — هر endpoint لاگین میخواهد مگر صریحاً باز باشد | `DEFAULT_PERMISSION_CLASSES = (IsAuthenticated,)` |
|
||||
| ثبتنام همیشه `viewer` میدهد | `RegisterSerializer.create()` |
|
||||
| بازسازی ایندکس فقط `admin` | `IsAdminRole` روی `ReindexView` |
|
||||
| دیدن PII فقط `analyst`/`admin` | `User.can_view_pii` + `PIIMixin` |
|
||||
|
||||
`role` یک فیلد ساده است نه گروه جنگو. با سه نقش، این انتخاب باعث میشود مجوزها را بشود در یک نگاه خواند و در یک خط تست کرد — و payload توکن کوچک بماند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۴. حریم خصوصی داده
|
||||
|
||||
### ۴.۱ ماسککردن
|
||||
|
||||
```
|
||||
joseph.holland@garverusa.com → jo************@garverusa.com
|
||||
+19402058928 → +1940*****28
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ماسک در لایه **سریالایز** انجام میشود، پس مقدار خام هیچوقت از دیتابیس برای کاربر بیمجوز خارج نمیشود. تست `test_privacy.py` تضمین میکند بخش محلی ایمیل و میانه شماره واقعاً پنهان میمانند (و برای بخش محلی یککاراکتری، کل آن پنهان میشود).
|
||||
|
||||
### ۴.۲ PII در Elasticsearch نیست
|
||||
|
||||
این تصمیم مهمتر از ماسککردن است. اگر ایمیل ایندکس شود، هر کاربر لاگینکرده میتواند با سرچ حدسی (`q=john.smith@acme.com`) وجود یک آدرس را **تأیید** کند، بدون اینکه هیچوقت آن را ببیند. این یک oracle است و ماسککردن جلویش را نمیگیرد.
|
||||
|
||||
راهحل: `build_document()` اصلاً این فیلدها را نمیسازد و `test_document_never_carries_contact_details` آن را قفل کرده است.
|
||||
|
||||
### ۴.۳ لاگها
|
||||
|
||||
pipeline لاگاستش فیلدهای `password`، `authorization`، `emails` و `phone_numbers` را حذف میکند — یک شبکه ایمنی برای وقتی که کسی بهاشتباه چیزی را لاگ کند. لاگ ممیزی سرچ، **کوئری و فیلترها** را ثبت میکند (که برای تحلیل رفتار لازم است) ولی هیچ داده تماسی را.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۵. اعتبارسنجی ورودی
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| ورودی | محافظ |
|
||||
|---|---|
|
||||
| `q` | حداکثر ۲۰۰ کاراکتر |
|
||||
| هر فیلتر | حداکثر ۲۰ مقدار، هر مقدار ۱۲۸ کاراکتر |
|
||||
| `page_size` | حداکثر ۱۰۰ |
|
||||
| `page × page_size` | حداکثر ۱۰٬۰۰۰ (رد میشود، نه اینکه بیصدا کوتاه شود) |
|
||||
| بدنه درخواست | `DATA_UPLOAD_MAX_MEMORY_SIZE = 5MB`، `MAX_NUMBER_FIELDS = 1000` |
|
||||
| `X-Request-ID` | حداکثر ۶۴ کاراکتر و فقط alphanumeric/`-`/`_`؛ وگرنه دور ریخته و تولید میشود |
|
||||
| خط درخواست HTTP | `limit_request_line = 8190` در gunicorn |
|
||||
|
||||
**تزریق SQL:** همه کوئریها از ORM جنگو میگذرند. تنها SQL خام پروژه دو `UPDATE` بدون پارامتر کاربر در `_refresh_skill_counters()` است.
|
||||
|
||||
**تزریق کوئری Elasticsearch:** ورودی کاربر همیشه بهعنوان **مقدار** در ساختار کوئری قرار میگیرد، هیچوقت بهعنوان JSON الحاقشده. `query_string` (که نحو قابل سوءاستفاده دارد) استفاده نشده — فقط `multi_match` و `terms`.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۶. هدرها و سیاستها
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**Django (تنظیمات پروداکشن)**
|
||||
|
||||
</div>
|
||||
|
||||
```python
|
||||
SECURE_HSTS_SECONDS = 31_536_000 # یک سال
|
||||
SECURE_HSTS_INCLUDE_SUBDOMAINS = True
|
||||
SECURE_HSTS_PRELOAD = True
|
||||
SECURE_SSL_REDIRECT = True
|
||||
SECURE_CONTENT_TYPE_NOSNIFF = True
|
||||
SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin"
|
||||
SECURE_CROSS_ORIGIN_OPENER_POLICY = "same-origin"
|
||||
X_FRAME_OPTIONS = "DENY"
|
||||
SESSION_COOKIE_SECURE = CSRF_COOKIE_SECURE = True
|
||||
SESSION_COOKIE_SAMESITE = CSRF_COOKIE_SAMESITE = "Strict"
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
**nginx**
|
||||
|
||||
</div>
|
||||
|
||||
```nginx
|
||||
Content-Security-Policy: default-src 'self'; script-src 'self';
|
||||
style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self';
|
||||
font-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'
|
||||
X-Content-Type-Options: nosniff
|
||||
X-Frame-Options: DENY
|
||||
Referrer-Policy: strict-origin-when-cross-origin
|
||||
Permissions-Policy: camera=(), microphone=(), geolocation=(), interest-cohort=()
|
||||
server_tokens off;
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`'unsafe-inline'` فقط برای `style-src` باقی مانده (React برای چند استایل inline در JSX از آن استفاده میکند). `script-src` کاملاً `'self'` است، یعنی هیچ اسکریپت inline یا خارجی اجرا نمیشود.
|
||||
|
||||
**CORS:** چون SPA و API هممبدأ هستند، در پروداکشن `CORS_ALLOW_ALL_ORIGINS = False` و لیست مبدأ خالی میماند. CORS فقط برای حالت توسعه (`npm run dev` روی پورت ۵۱۷۳) لازم است.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۷. کانتینرها و شبکه
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
| اقدام | جزئیات |
|
||||
|---|---|
|
||||
| بیلد چندمرحلهای | کامپایلر و ابزار بیلد در ایمیج نهایی نیستند |
|
||||
| کاربر غیرroot | `app` (uid 10001) در بکاند، `nginx` در فرانتاند |
|
||||
| `no-new-privileges` | روی هر دو سرویس اپلیکیشن |
|
||||
| فایلسیستم read-only | فرانتاند، با tmpfs برای `/tmp`، `/var/run`، `/var/cache/nginx` |
|
||||
| بدون shell در مسیر حساس | `useradd --shell /usr/sbin/nologin` |
|
||||
| healthcheck | هر سرویس؛ `depends_on: condition: service_healthy` |
|
||||
| سقف لاگ | `max-size: 10m`, `max-file: 3` ⇒ لاگ دیسک را پر نمیکند |
|
||||
|
||||
**توپولوژی شبکه**
|
||||
|
||||
</div>
|
||||
|
||||
```
|
||||
┌─ frontend-net ──────────────┐ ┌─ backend-net (internal: true) ─────────┐
|
||||
│ frontend ⟷ backend │ │ postgres redis elasticsearch │
|
||||
│ ▲ │ │ logstash backend │
|
||||
└─────┼───────────────────────┘ └────────────────────────────────────────┘
|
||||
│ :8080 منتشرشده ▲ هیچ مسیری به میزبان ندارد
|
||||
میزبان │
|
||||
┌─ elk-net ─────────────┐
|
||||
│ elasticsearch │
|
||||
│ logstash kibana │
|
||||
└───────────────────────┘
|
||||
▲ kibana فقط روی 127.0.0.1:5601
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
`backend-net` با `internal: true` تعریف شده: کانتینرهای روی آن هیچ مسیری به بیرون ندارند. Postgres و Redis هیچ پورتی منتشر نمیکنند. Kibana فقط به `127.0.0.1` بایند شده، نه `0.0.0.0`.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۸. مدیریت اسرار
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
- هیچ رمزی در کد نیست؛ همه از متغیر محیطی خوانده میشوند.
|
||||
- `.env` در `.gitignore` و `.dockerignore` است؛ فقط `.env.example` نگهداری میشود.
|
||||
- `docker-compose.yml` برای متغیرهای حیاتی از نگارش «اجباری» استفاده میکند:
|
||||
`${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}` — بدون آن، استک اصلاً بالا نمیآید.
|
||||
- `settings/prod.py` روی کلید کوتاهتر از ۵۰ کاراکتر یا `DEBUG=True` **کرش میکند**.
|
||||
- hook `detect-private-key` در pre-commit جلوی کامیت کلید خصوصی را میگیرد.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۹. Trade-offهای پذیرفتهشده
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
### ۹.۱ احراز هویت Elasticsearch در حالت پیشفرض خاموش است
|
||||
|
||||
**واقعیت:** در `docker-compose.yml` مقدار `xpack.security.enabled: "false"` است.
|
||||
|
||||
**چرا:** روشن کردنش نیازمند مدیریت گواهی، تنظیم رمز کاربر `kibana_system` و یک کانتینر setup است. برای پروژهای که هدفش «با یک دستور اجرا شود» است، این چهار نقطه شکست اضافه میکند — و رایجترین دلیلی است که یک تسک آزمایشی روی ماشین ارزیاب بالا نمیآید.
|
||||
|
||||
**چه چیزی جبرانش میکند:** کلاستر روی شبکه `internal` داکر است و هیچ پورتی منتشر نمیکند. از خارج کانتینرها بههیچوجه در دسترس نیست.
|
||||
|
||||
**راه درست برای پروداکشن:** overlay آماده است —
|
||||
|
||||
</div>
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml -f docker-compose.prod.yml up -d
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
که `xpack.security.enabled=true` میکند، به هر سه کلاینت (بکاند، Logstash، Kibana) اعتبارنامه میدهد، پورت Kibana را برمیدارد و محدودیت منابع اعمال میکند.
|
||||
|
||||
### ۹.۲ TLS بین کانتینرها نیست
|
||||
|
||||
ترافیک داخلی شبکه داکر رمز نشده است. در یک استقرار واقعی این کار به service mesh یا load balancer داخلی سپرده میشود، نه به هر سرویس بهصورت جداگانه. HTTPS در **لبه** (nginx یا load balancer بالادست) الزامی است و `SECURE_SSL_REDIRECT` + HSTS در تنظیمات پروداکشن آن را تحمیل میکنند.
|
||||
|
||||
### ۹.۳ دیتاست داخل ایمیج است
|
||||
|
||||
فایل CSV در ایمیج بکاند کپی میشود تا `docker compose up` بدون هیچ مرحله دستی داده داشته باشد. در پروداکشن، داده باید از منبع خارجی (S3، دیتابیس مبدأ) بیاید و ایمیج فقط کد باشد. برای یک تسک ارزیابی، سادگی اجرا مهمتر است — و `SEED_DATASET=false` در overlay پروداکشن این رفتار را خاموش میکند.
|
||||
|
||||
### ۹.۴ رمزهای نمونه در `.env.example`
|
||||
|
||||
رمزهای کاربران دمو در فایل نمونه هستند تا ارزیاب بتواند بلافاصله لاگین کند. این فایل **نمونه** است، نه `.env` واقعی، و README صریحاً هشدار میدهد که قبل از هر استقرار عوض شوند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱۰. امنیت در CI
|
||||
|
||||
```yaml
|
||||
- bandit -q -c pyproject.toml -r apps config # تحلیل ایستای امنیتی پایتون
|
||||
- pip-audit -r requirements/base.txt # CVE در وابستگیهای پایتون
|
||||
- npm audit --omit=dev # CVE در وابستگیهای فرانتاند
|
||||
- python manage.py check --deploy # چکلیست استقرار خود جنگو
|
||||
```
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
ruff هم قواعد `S` (bandit) را در lint معمولی اجرا میکند، پس مشکلات رایج قبل از commit گرفته میشوند.
|
||||
|
||||
</div>
|
||||
|
||||
---
|
||||
|
||||
## ۱۱. اگر فردا مسئولش میشدم
|
||||
|
||||
<div dir="rtl">
|
||||
|
||||
فهرست کارهایی که برای یک استقرار واقعی لازماند و اینجا انجام نشدهاند — چون صادق بودن دربارهشان بهتر از وانمود کردن است:
|
||||
|
||||
1. احراز هویت Elasticsearch را روشن و TLS داخلی را اضافه کن (overlay آماده است).
|
||||
2. اسرار را به یک مدیر راز (Vault / AWS Secrets Manager) منتقل کن؛ متغیر محیطی برای اسرار بلندمدت کافی نیست.
|
||||
3. احراز هویت دومرحلهای برای نقش `admin`.
|
||||
4. قفل حساب بعد از N تلاش ناموفق (الان فقط rate limit روی IP هست).
|
||||
5. سیاست نگهداری داده: حذف خودکار لاگهای قدیمیتر از ۳۰ روز با ILM.
|
||||
6. اسکن ایمیجها (Trivy) در CI.
|
||||
7. امضای ایمیج و SBOM.
|
||||
8. هشدار روی الگوهای مشکوک: یک کاربر که در یک ساعت ۱۰٬۰۰۰ پروفایل باز میکند (دادهاش در لاگ ممیزی هست، فقط قاعده هشدار لازم است).
|
||||
|
||||
</div>
|
||||
+1055
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
# Logstash node configuration.
|
||||
# The pipeline itself lives in pipeline/logstash.conf.
|
||||
|
||||
http.host: "0.0.0.0"
|
||||
node.name: logstash
|
||||
|
||||
# One worker per CPU is plenty for an application log stream; batching keeps
|
||||
# the number of bulk requests to Elasticsearch low.
|
||||
pipeline.workers: 2
|
||||
pipeline.batch.size: 250
|
||||
pipeline.batch.delay: 50
|
||||
pipeline.ordered: false
|
||||
|
||||
# Persisted queue: a short Elasticsearch outage does not lose log events.
|
||||
queue.type: persisted
|
||||
queue.max_bytes: 256mb
|
||||
|
||||
log.level: info
|
||||
xpack.monitoring.enabled: false
|
||||
@@ -0,0 +1,85 @@
|
||||
# =============================================================================
|
||||
# Application log pipeline.
|
||||
#
|
||||
# Django emits one JSON object per log record and ships it over TCP with
|
||||
# `python-logstash-async` (asynchronous, buffered - a Logstash outage never
|
||||
# blocks a request thread). Because the payload is already structured there is
|
||||
# no grok parsing here: the pipeline only enriches, classifies and routes.
|
||||
# =============================================================================
|
||||
|
||||
input {
|
||||
tcp {
|
||||
port => 5959
|
||||
codec => json_lines
|
||||
tags => ["django"]
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
# Fields Logstash adds itself and that carry no information for us.
|
||||
mutate {
|
||||
remove_field => ["port", "@version"]
|
||||
}
|
||||
|
||||
# The application already stamps an ISO-8601 timestamp; trust it so the log
|
||||
# line is filed under the moment it happened, not the moment it arrived.
|
||||
date {
|
||||
match => ["@timestamp", "ISO8601"]
|
||||
target => "@timestamp"
|
||||
tag_on_failure => ["_timestamp_parse_failure"]
|
||||
}
|
||||
|
||||
# Route by event type. `event` is set by the application through `extra=`.
|
||||
if [event] == "search_executed" {
|
||||
mutate { add_field => { "[@metadata][stream]" => "search" } }
|
||||
|
||||
# A slow search is worth alerting on; pre-computing the flag keeps the
|
||||
# Kibana query simple.
|
||||
if [took_ms] and [took_ms] > 500 {
|
||||
mutate { add_tag => ["slow_search"] }
|
||||
}
|
||||
if [result_count] == 0 {
|
||||
mutate { add_tag => ["zero_results"] }
|
||||
}
|
||||
} else if [event] in ["login_succeeded", "login_failed", "logout", "user_registered"] {
|
||||
mutate { add_field => { "[@metadata][stream]" => "audit" } }
|
||||
} else if [level] in ["ERROR", "CRITICAL"] {
|
||||
mutate { add_field => { "[@metadata][stream]" => "error" } }
|
||||
} else {
|
||||
mutate { add_field => { "[@metadata][stream]" => "app" } }
|
||||
}
|
||||
|
||||
# Never let a stray contact detail reach the log store.
|
||||
mutate {
|
||||
remove_field => ["password", "authorization", "emails", "phone_numbers"]
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
elasticsearch {
|
||||
# Credentials, when the cluster has security enabled, travel inside the
|
||||
# URL (see docker-compose.prod.yml) so this single pipeline file works
|
||||
# both with and without authentication.
|
||||
hosts => ["${ELASTICSEARCH_HOSTS:http://elasticsearch:9200}"]
|
||||
|
||||
# One index per stream per day keeps retention policies simple and makes
|
||||
# "delete everything older than 30 days" a single index deletion.
|
||||
index => "app-logs-%{[@metadata][stream]}-%{+YYYY.MM.dd}"
|
||||
|
||||
# Composable index template (Elasticsearch 8 style): pins the types of the
|
||||
# fields Kibana filters and aggregates on instead of letting dynamic
|
||||
# mapping guess them.
|
||||
manage_template => true
|
||||
template_api => "composable"
|
||||
template_name => "app-logs"
|
||||
template => "/usr/share/logstash/templates/app-logs.json"
|
||||
template_overwrite => true
|
||||
|
||||
action => "index"
|
||||
}
|
||||
|
||||
# Anything that fails to parse is still visible in `docker compose logs`.
|
||||
if "_jsonparsefailure" in [tags] or "_timestamp_parse_failure" in [tags] {
|
||||
stdout { codec => rubydebug }
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user