cyberyan_linkedin_search/backend/tests/test_dataset_repair.py

"""
Tests for the self-healing CSV reader.

Each test reproduces one defect that is actually present in the supplied export
so a regression in the repair logic fails loudly instead of quietly importing
garbage.
"""

from __future__ import annotations

import textwrap
from pathlib import Path

import pytest

from apps.profiles.dataset import DatasetReader

HEADER = (
    "full_name,first_name,last_name,gender,linkedin_url,linkedin_username,linkedin_id,"
    "industry,job_title,job_title_role,job_title_levels,job_company_name,location_name,"
    "location_country,skills,experience,summary"
)


def _write(tmp_path: Path, body: str) -> Path:
    path = tmp_path / "sample.csv"
    path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8")
    return path


@pytest.fixture
def read(tmp_path):  # noqa: ANN001, ANN201
    def _read(body: str):  # noqa: ANN202
        return DatasetReader(_write(tmp_path, body)).read()

    return _read


def test_clean_row_is_read_verbatim(read) -> None:  # noqa: ANN001
    records, report = read(
        f"""
        {HEADER}
        jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python', 'django']","[{{'company': {{'name': 'acme'}}, 'title': {{'name': 'backend engineer'}}}}]",Loves databases.
        """
    )
    assert report.records == 1
    record = records[0]
    assert record["full_name"] == "jane doe"
    assert record["job_title_role"] == "engineering"
    assert record["skills"] == "['python', 'django']"


def test_leading_watermark_column_is_stripped(read) -> None:  # noqa: ANN001
    records, _ = read(
        f"""
        {HEADER}
        H:\\New folder\\leak\\linkedin\\@BreachedData1,jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi.
        """
    )
    assert records[0]["full_name"] == "jane doe"
    assert records[0]["linkedin_url"] == "linkedin.com/in/janedoe"


def test_missing_column_does_not_shift_the_rest_of_the_row(read) -> None:  # noqa: ANN001
    """``job_title_role`` is absent - every later value must still land right."""
    records, _ = read(
        f"""
        {HEADER}
        jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi.
        """
    )
    record = records[0]
    assert record["job_title_role"] == ""
    assert record["job_title_levels"] == "['senior']"
    assert record["job_company_name"] == "acme"
    assert record["location_country"] == "germany"
    assert record["skills"] == "['python']"


def test_values_that_fail_their_signature_are_discarded(read) -> None:  # noqa: ANN001
    """A non-URL in the URL column is dropped rather than persisted."""
    records, report = read(
        f"""
        {HEADER}
        jane doe,jane,doe,female,not-a-linkedin-url,janedoe,123456,software,engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi.
        """
    )
    assert records[0]["linkedin_url"] == ""
    assert report.discarded_values >= 1


def test_repeated_header_and_duplicate_person_are_removed(read) -> None:  # noqa: ANN001
    row = (
        "jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,engineer,"
        'engineering,"[\'senior\']",acme,"berlin, germany",germany,"[\'python\']","[]",Hi.'
    )
    records, report = read(f"{HEADER}\n{row}\n{HEADER}\n{row}\n")
    assert report.duplicate_headers == 1
    assert report.duplicate_records == 1
    assert len(records) == 1


def test_row_split_by_a_stray_quote_is_reassembled(read) -> None:  # noqa: ANN001
    """A record cut in half by broken quoting is stitched back together."""
    records, report = read(
        f"""
        {HEADER}
        jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,engineer
        engineering,"['senior']",acme,"berlin, germany",germany,"['python', 'sql']","[]",Hi.
        """
    )
    assert len(records) == 1
    assert report.repaired_rows >= 1
    assert records[0]["skills"] == "['python', 'sql']"
    assert records[0]["location_country"] == "germany"


def test_empty_file_yields_no_records(tmp_path) -> None:  # noqa: ANN001
    path = tmp_path / "empty.csv"
    path.write_text("", encoding="utf-8")
    records, report = DatasetReader(path).read()
    assert records == []
    assert report.records == 0