leocrm/tests/test_import_export.py

"""Import/export tests — helpers, service partial-failure, preview/validate routes, exports."""

from __future__ import annotations

import json
import pytest
from httpx import AsyncClient

from app.services.import_export_helpers import (
    parse_csv,
    parse_json,
    write_csv,
    write_json,
    map_fields,
    validate_row,
    build_error_report,
    build_import_result,
    suggest_mapping,
    detect_format,
)
from app.services import import_export_service
from tests.conftest import ORIGIN_HEADER, login_client, seed_tenant_and_users

# ─── Test data ───────────────────────────────────────────────────────────────

CSV_COMPANIES = """name,industry,phone,email,website
ImportCorp,IT,123456,import@example.com,https://import.example
TechImport,Finance,654321,tech@example.com,https://tech.example
"""

CSV_COMPANIES_INVALID = """name,industry
,IT
ValidCorp,Finance
"""

CSV_CONTACTS = """first_name,last_name,email,phone,mobile,position,department
Alice,Wonderland,alice@example.com,123,456,Manager,Sales
Bob,Builder,bob@example.com,789,012,Developer,Tech
"""

CSV_CONTACTS_PARTIAL = """firstname,surname,email,phone
Alice,Wonderland,alice@example.com,123
,,bad@example.com,456
Bob,Builder,bob@example.com,789
,,not-an-email,999
Charlie,Chaplin,charlie@example.com,555
"""

CSV_COMPANIES_PARTIAL = """name,email,phone
ValidCorp1,corp1@example.com,123
,corp2@example.com,456
ValidCorp2,corp2@example.com,789
,bad-email,999
ValidCorp3,corp3@example.com,111
"""

JSON_CONTACTS = json.dumps([
    {"firstname": "Json", "surname": "User", "email": "json@example.com"},
    {"firstname": "Another", "surname": "Person", "email": "another@example.com"},
])


# ─── Helper unit tests ───────────────────────────────────────────────────────

class TestHelpers:
    """Unit tests for import_export_helpers functions."""

    def test_parse_csv_basic(self):
        """parse_csv parses CSV bytes into list of dicts."""
        content = b"name,email\nAlice,alice@example.com\nBob,bob@example.com\n"
        rows = parse_csv(content)
        assert len(rows) == 2
        assert rows[0]["name"] == "Alice"
        assert rows[0]["email"] == "alice@example.com"
        assert rows[1]["name"] == "Bob"

    def test_parse_csv_with_bom(self):
        """parse_csv handles UTF-8 BOM."""
        content = b"\xef\xbb\xbfname,email\nAlice,alice@example.com\n"
        rows = parse_csv(content)
        assert len(rows) == 1
        assert rows[0]["name"] == "Alice"

    def test_parse_json_array(self):
        """parse_json parses JSON array into list of dicts."""
        content = json.dumps([{"a": "1"}, {"a": "2"}]).encode()
        rows = parse_json(content)
        assert len(rows) == 2
        assert rows[0]["a"] == "1"
        assert rows[1]["a"] == "2"

    def test_parse_json_single_object(self):
        """parse_json wraps single JSON object into list."""
        content = json.dumps({"a": "1"}).encode()
        rows = parse_json(content)
        assert len(rows) == 1
        assert rows[0]["a"] == "1"

    def test_write_csv_basic(self):
        """write_csv produces correct CSV bytes."""
        rows = [{"name": "Alice", "email": "alice@example.com"}]
        result = write_csv(rows, ["name", "email"])
        text = result.decode("utf-8")
        assert "name,email" in text
        assert "Alice,alice@example.com" in text

    def test_write_csv_missing_fields(self):
        """write_csv fills missing fields with empty string."""
        rows = [{"name": "Alice"}]
        result = write_csv(rows, ["name", "email", "phone"])
        text = result.decode("utf-8")
        assert "Alice," in text

    def test_write_json_basic(self):
        """write_json produces correct JSON bytes."""
        rows = [{"name": "Alice"}, {"name": "Bob"}]
        result = write_json(rows)
        data = json.loads(result)
        assert len(data) == 2
        assert data[0]["name"] == "Alice"
        assert data[1]["name"] == "Bob"

    def test_map_fields_basic(self):
        """map_fields maps source columns to target fields."""
        row = {"first_name": "Alice", "last_name": "Wonder", "email": "alice@example.com"}
        mapping = {"first_name": "firstname", "last_name": "surname", "email": "email"}
        result = map_fields(row, mapping)
        assert result["firstname"] == "Alice"
        assert result["surname"] == "Wonder"
        assert result["email"] == "alice@example.com"

    def test_map_fields_skip_missing(self):
        """map_fields skips source columns not in row."""
        row = {"first_name": "Alice"}
        mapping = {"first_name": "firstname", "last_name": "surname"}
        result = map_fields(row, mapping)
        assert "firstname" in result
        assert "surname" not in result

    def test_validate_row_required(self):
        """validate_row detects missing required fields."""
        row = {"name": "", "email": "test@example.com"}
        errors = validate_row(row, ["name"])
        assert len(errors) == 1
        assert "name" in errors[0]

    def test_validate_row_valid(self):
        """validate_row returns empty list for valid row."""
        row = {"name": "Alice", "email": "alice@example.com"}
        errors = validate_row(row, ["name"], {"email": {"type": "email"}})
        assert len(errors) == 0

    def test_validate_row_email_validator(self):
        """validate_row detects invalid email."""
        row = {"name": "Alice", "email": "not-an-email"}
        errors = validate_row(row, ["name"], {"email": {"type": "email"}})
        assert len(errors) == 1
        assert "email" in errors[0]

    def test_validate_row_url_validator(self):
        """validate_row detects invalid URL."""
        row = {"website": "not-a-url"}
        errors = validate_row(row, [], {"website": {"type": "url"}})
        assert len(errors) == 1
        assert "URL" in errors[0]

    def test_validate_row_max_length(self):
        """validate_row enforces max_length."""
        row = {"name": "A" * 100}
        errors = validate_row(row, [], {"name": {"max_length": 50}})
        assert len(errors) == 1
        assert "max length" in errors[0]

    def test_build_error_report_structure(self):
        """build_error_report returns correct structure."""
        errors = [
            {"row": 1, "field": "name", "message": "Missing required"},
            {"row": 2, "field": "email", "message": "Invalid email"},
        ]
        report = build_error_report(errors)
        assert report["total_errors"] == 2
        assert len(report["errors"]) == 2
        assert report["errors"][0]["row"] == 1
        assert report["errors"][0]["message"] == "Missing required"

    def test_build_error_report_empty(self):
        """build_error_report handles empty list."""
        report = build_error_report([])
        assert report["total_errors"] == 0
        assert report["errors"] == []

    def test_build_import_result_success(self):
        """build_import_result returns 'success' status when all succeed."""
        result = build_import_result(total=10, succeeded=10, failed=0)
        assert result["status"] == "success"
        assert result["succeeded"] == 10
        assert result["failed"] == 0

    def test_build_import_result_partial(self):
        """build_import_result returns 'partial_success' when some fail."""
        result = build_import_result(total=10, succeeded=7, failed=3)
        assert result["status"] == "partial_success"

    def test_build_import_result_all_failed(self):
        """build_import_result returns 'failed' when all fail."""
        result = build_import_result(total=10, succeeded=0, failed=10)
        assert result["status"] == "failed"

    def test_suggest_mapping_exact_match(self):
        """suggest_mapping matches exact column names."""
        mapping = suggest_mapping(["firstname", "surname", "email"], ["firstname", "surname", "email"])
        assert mapping["firstname"] == "firstname"
        assert mapping["surname"] == "surname"
        assert mapping["email"] == "email"

    def test_suggest_mapping_alias(self):
        """suggest_mapping matches common aliases."""
        mapping = suggest_mapping(["first_name", "last_name"], ["firstname", "surname"])
        assert mapping["first_name"] == "firstname"
        assert mapping["last_name"] == "surname"

    def test_detect_format_csv(self):
        """detect_format identifies CSV from filename."""
        assert detect_format("data.csv", b"name,email\n") == "csv"

    def test_detect_format_json(self):
        """detect_format identifies JSON from filename."""
        assert detect_format("data.json", b"[]") == "json"

    def test_detect_format_xlsx(self):
        """detect_format identifies XLSX from filename."""
        assert detect_format("data.xlsx", b"PK\x03\x04") == "xlsx"


# ─── Service partial-failure tests ───────────────────────────────────────────

@pytest.mark.asyncio
class TestImportCompaniesPartialFailure:
    """Import companies with partial-failure semantics."""

    async def test_import_companies_partial_failure(self, client: AsyncClient, db_session):
        """Import companies: 3 success, 2 failed (empty name + invalid email)."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES_PARTIAL.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 5
        assert result["succeeded"] == 3
        assert result["failed"] == 2
        assert result["status"] == "partial_success"
        assert len(result["created"]) == 3
        # 2 failed rows, but 3 total error messages (row 4 has 2 errors)
        assert result["error_report"]["total_errors"] == 3

    async def test_import_companies_all_valid(self, client: AsyncClient, db_session):
        """Import companies: all rows valid."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2
        assert result["failed"] == 0
        assert result["status"] == "success"


@pytest.mark.asyncio
class TestImportContactsPartialFailure:
    """Import contacts with partial-failure semantics."""

    async def test_import_contacts_partial_failure(self, client: AsyncClient, db_session):
        """Import contacts: 3 success, 2 failed (empty names + invalid email)."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.csv", CSV_CONTACTS_PARTIAL.encode(), "text/csv")}
        data = {"entity_type": "contacts"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 5
        assert result["succeeded"] == 3
        assert result["failed"] == 2
        assert result["status"] == "partial_success"
        assert len(result["created"]) == 3

    async def test_import_contacts_all_valid(self, client: AsyncClient, db_session):
        """Import contacts: all rows valid."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
        data = {"entity_type": "contacts"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2
        assert result["status"] == "success"


# ─── Preview & Validate route tests ──────────────────────────────────────────

@pytest.mark.asyncio
class TestImportPreview:
    """Preview endpoint returns first 10 rows + mapping suggestion."""

    async def test_preview_returns_columns_and_rows(self, client: AsyncClient, db_session):
        """POST /import/preview returns columns, preview_rows, mapping_suggestion."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import/preview",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total_rows"] == 2
        assert "name" in result["columns"]
        assert "email" in result["columns"]
        assert len(result["preview_rows"]) == 2
        assert "mapping_suggestion" in result
        assert "target_fields" in result

    async def test_preview_contacts_mapping_suggestion(self, client: AsyncClient, db_session):
        """Preview contacts: mapping suggestion maps first_name→firstname."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
        data = {"entity_type": "contacts"}
        resp = await client.post(
            "/api/v1/import/preview",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        mapping = result["mapping_suggestion"]
        # first_name should map to firstname
        assert mapping.get("first_name") == "firstname"
        # last_name should map to surname
        assert mapping.get("last_name") == "surname"

    async def test_preview_no_db_changes(self, client: AsyncClient, db_session):
        """Preview does not create any records."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import/preview",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        # Verify no companies were created
        list_resp = await client.get("/api/v1/companies", headers=ORIGIN_HEADER)
        names = [item["name"] for item in list_resp.json()["items"]]
        assert "ImportCorp" not in names
        assert "TechImport" not in names


@pytest.mark.asyncio
class TestImportValidate:
    """Validate endpoint returns error report without importing."""

    async def test_validate_returns_error_report(self, client: AsyncClient, db_session):
        """POST /import/validate returns error report for invalid rows."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES_PARTIAL.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import/validate",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 5
        assert result["succeeded"] == 3
        assert result["failed"] == 2
        assert result["dry_run"] is True
        # 2 failed rows, but 3 total error messages (row 4 has 2 errors)
        assert result["error_report"]["total_errors"] == 3

    async def test_validate_no_db_changes(self, client: AsyncClient, db_session):
        """Validate does not create any records."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import/validate",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        list_resp = await client.get("/api/v1/companies", headers=ORIGIN_HEADER)
        names = [item["name"] for item in list_resp.json()["items"]]
        assert "ImportCorp" not in names

    async def test_validate_with_field_mapping(self, client: AsyncClient, db_session):
        """Validate with explicit field mapping."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
        mapping = json.dumps({
            "first_name": "firstname",
            "last_name": "surname",
            "email": "email",
            "phone": "phone",
            "mobile": "mobile",
            "position": "function",
            "department": "department",
        })
        data = {"entity_type": "contacts", "field_mapping": mapping}
        resp = await client.post(
            "/api/v1/import/validate",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2
        assert result["failed"] == 0


# ─── Job status route tests ──────────────────────────────────────────────────

@pytest.mark.asyncio
class TestImportJobStatus:
    """Job status endpoint."""

    async def test_job_status_not_found(self, client: AsyncClient, db_session):
        """GET /import/status/{nonexistent_id} returns 404."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        resp = await client.get(
            "/api/v1/import/status/nonexistent-job-id",
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 404


# ─── Export route tests ──────────────────────────────────────────────────────

@pytest.mark.asyncio
class TestExport:
    """Export endpoints."""

    async def test_export_contacts_csv(self, client: AsyncClient, db_session):
        """GET /export?entity_type=contacts&format=csv returns CSV file."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        resp = await client.get(
            "/api/v1/export?entity_type=contacts&format=csv",
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        assert "text/csv" in resp.headers.get("content-type", "")

    async def test_export_companies_csv(self, client: AsyncClient, db_session):
        """GET /export?entity_type=companies&format=csv returns CSV file."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        resp = await client.get(
            "/api/v1/export?entity_type=companies&format=csv",
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        assert "text/csv" in resp.headers.get("content-type", "")

    async def test_export_contacts_xlsx(self, client: AsyncClient, db_session):
        """GET /export?entity_type=contacts&format=xlsx returns XLSX file."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        resp = await client.get(
            "/api/v1/export?entity_type=contacts&format=xlsx",
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        ct = resp.headers.get("content-type", "")
        # Could be xlsx or csv fallback if openpyxl missing (but it's installed)
        assert "spreadsheet" in ct or "text/csv" in ct

    async def test_export_contacts_json(self, client: AsyncClient, db_session):
        """GET /export?entity_type=contacts&format=json returns JSON file."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        resp = await client.get(
            "/api/v1/export?entity_type=contacts&format=json",
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        assert "application/json" in resp.headers.get("content-type", "")


# ─── Direct import route tests (legacy compat) ───────────────────────────────

@pytest.mark.asyncio
class TestImportCompanies:
    """AC 20: CSV import for companies (backward compat)."""

    async def test_import_companies_csv_returns_200(self, client: AsyncClient, db_session):
        """POST /api/v1/import CSV + entity_type=companies -> 200 + result."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2
        assert result["failed"] == 0
        assert len(result["created"]) == 2

    async def test_import_companies_with_invalid_rows(self, client: AsyncClient, db_session):
        """Import CSV with some invalid rows — partial success."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES_INVALID.encode(), "text/csv")}
        data = {"entity_type": "companies"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 1
        assert result["failed"] == 1
        assert result["status"] == "partial_success"
        assert len(result["created"]) == 1


@pytest.mark.asyncio
class TestImportContacts:
    """Import contacts via CSV (backward compat)."""

    async def test_import_contacts_csv_returns_200(self, client: AsyncClient, db_session):
        """Import contacts via CSV."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
        data = {"entity_type": "contacts"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2
        assert result["failed"] == 0
        assert len(result["created"]) == 2
        # Verify contacts appear in list
        list_resp = await client.get("/api/v1/contacts", headers=ORIGIN_HEADER)
        assert list_resp.json()["total"] >= 2


# ─── JSON import test ────────────────────────────────────────────────────────

@pytest.mark.asyncio
class TestImportJson:
    """Import from JSON format."""

    async def test_import_contacts_json(self, client: AsyncClient, db_session):
        """Import contacts from JSON file."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.json", JSON_CONTACTS.encode(), "application/json")}
        data = {"entity_type": "contacts"}
        resp = await client.post(
            "/api/v1/import",
            files=files,
            data=data,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2
        assert len(result["created"]) == 2


# ─── Dedicated import routes ─────────────────────────────────────────────────

@pytest.mark.asyncio
class TestImportContactsRoute:
    """POST /import/contacts dedicated route."""

    async def test_import_contacts_route(self, client: AsyncClient, db_session):
        """POST /import/contacts imports contacts."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("contacts.csv", CSV_CONTACTS.encode(), "text/csv")}
        resp = await client.post(
            "/api/v1/import/contacts",
            files=files,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2


@pytest.mark.asyncio
class TestImportCompaniesRoute:
    """POST /import/companies dedicated route."""

    async def test_import_companies_route(self, client: AsyncClient, db_session):
        """POST /import/companies imports companies."""
        await seed_tenant_and_users(db_session)
        await login_client(client, "admin@tenanta.com")
        files = {"file": ("companies.csv", CSV_COMPANIES.encode(), "text/csv")}
        resp = await client.post(
            "/api/v1/import/companies",
            files=files,
            headers=ORIGIN_HEADER,
        )
        assert resp.status_code == 200
        result = resp.json()
        assert result["total"] == 2
        assert result["succeeded"] == 2