""" Tests for the self-healing CSV reader. Each test reproduces one defect that is actually present in the supplied export so a regression in the repair logic fails loudly instead of quietly importing garbage. """ from __future__ import annotations import textwrap from pathlib import Path import pytest from apps.profiles.dataset import DatasetReader HEADER = ( "full_name,first_name,last_name,gender,linkedin_url,linkedin_username,linkedin_id," "industry,job_title,job_title_role,job_title_levels,job_company_name,location_name," "location_country,skills,experience,summary" ) def _write(tmp_path: Path, body: str) -> Path: path = tmp_path / "sample.csv" path.write_text(textwrap.dedent(body).lstrip(), encoding="utf-8") return path @pytest.fixture def read(tmp_path): # noqa: ANN001, ANN201 def _read(body: str): # noqa: ANN202 return DatasetReader(_write(tmp_path, body)).read() return _read def test_clean_row_is_read_verbatim(read) -> None: # noqa: ANN001 records, report = read( f""" {HEADER} jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python', 'django']","[{{'company': {{'name': 'acme'}}, 'title': {{'name': 'backend engineer'}}}}]",Loves databases. """ ) assert report.records == 1 record = records[0] assert record["full_name"] == "jane doe" assert record["job_title_role"] == "engineering" assert record["skills"] == "['python', 'django']" def test_leading_watermark_column_is_stripped(read) -> None: # noqa: ANN001 records, _ = read( f""" {HEADER} H:\\New folder\\leak\\linkedin\\@BreachedData1,jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi. """ ) assert records[0]["full_name"] == "jane doe" assert records[0]["linkedin_url"] == "linkedin.com/in/janedoe" def test_missing_column_does_not_shift_the_rest_of_the_row(read) -> None: # noqa: ANN001 """``job_title_role`` is absent - every later value must still land right.""" records, _ = read( f""" {HEADER} jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,backend engineer,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi. """ ) record = records[0] assert record["job_title_role"] == "" assert record["job_title_levels"] == "['senior']" assert record["job_company_name"] == "acme" assert record["location_country"] == "germany" assert record["skills"] == "['python']" def test_values_that_fail_their_signature_are_discarded(read) -> None: # noqa: ANN001 """A non-URL in the URL column is dropped rather than persisted.""" records, report = read( f""" {HEADER} jane doe,jane,doe,female,not-a-linkedin-url,janedoe,123456,software,engineer,engineering,"['senior']",acme,"berlin, germany",germany,"['python']","[]",Hi. """ ) assert records[0]["linkedin_url"] == "" assert report.discarded_values >= 1 def test_repeated_header_and_duplicate_person_are_removed(read) -> None: # noqa: ANN001 row = ( "jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,engineer," 'engineering,"[\'senior\']",acme,"berlin, germany",germany,"[\'python\']","[]",Hi.' ) records, report = read(f"{HEADER}\n{row}\n{HEADER}\n{row}\n") assert report.duplicate_headers == 1 assert report.duplicate_records == 1 assert len(records) == 1 def test_row_split_by_a_stray_quote_is_reassembled(read) -> None: # noqa: ANN001 """A record cut in half by broken quoting is stitched back together.""" records, report = read( f""" {HEADER} jane doe,jane,doe,female,linkedin.com/in/janedoe,janedoe,123456,software,engineer engineering,"['senior']",acme,"berlin, germany",germany,"['python', 'sql']","[]",Hi. """ ) assert len(records) == 1 assert report.repaired_rows >= 1 assert records[0]["skills"] == "['python', 'sql']" assert records[0]["location_country"] == "germany" def test_empty_file_yields_no_records(tmp_path) -> None: # noqa: ANN001 path = tmp_path / "empty.csv" path.write_text("", encoding="utf-8") records, report = DatasetReader(path).read() assert records == [] assert report.records == 0