# Copyright (C) 2026 Wilhelmus Dingenus de Pooter
# SPDX-License-Identifier: GPL-3.0-or-later
"""
enc_scanner.py
Scans a folder or zip for S-57 ENC (.000) files, with a two-stage flow:
  1) scan_for_charts(source)        -> chart winners (dedup by latest EDTN/ISDT)
  2) scan_for_object_layers(files)  -> object-layer index across chosen charts
Sources may be a plain folder or a .zip exchange set. Before scanning, a
staging pass normalises the structure so GDAL can apply update files
(.001/.002/…) correctly: each base .000 and its sequential updates must sit
together in one directory. Zips are extracted to a temp staging dir.
Deduplicates by dataset name, keeping the latest issue date (ISDT) / highest
edition (EDTN) from the DSID record.

The Auditor uses a separate, leaner entry point — stage_for_audit(source) — that
gates on the single-exchange-set policy and collects cells fast (no per-cell DSID
dedup); see its docstring.
"""

import os
import re
import shutil
import zipfile
import tempfile
from datetime import datetime
from qgis.core import QgsMessageLog, Qgis

# Fully native: this module reads S-57/S-101 cells via the in-house ISO 8211
# engine (iso8211 / s57 / s101 + the bundled catalogue) — no GDAL/OGR S-57 driver.


def _log(msg, level=Qgis.Info):
    QgsMessageLog.logMessage(msg, "ENCore GIS", level)


# Match S-57 cell files: an 8-character cell name + 3-digit numeric extension.
# The extension is the update sequence: .000 = base, .001/.002/… = updates.
# Requiring exactly 8 name chars (the S-57 cell-name length) excludes the
# exchange-set catalog (CATALOG.031) and ancillary files (README.TXT, PIC/INFO
# files, etc.), which would otherwise be mis-read as cells with broken chains.
_CELL_RE = re.compile(r"^(?P<base>[A-Za-z0-9]{8})\.(?P<seq>\d{3})$")

# Match S-101 cell files: the NW-Europe S-101 file-naming convention (NL-led,
# adopted by BE/DE/FR/GB) — "101" + 2-letter producer + tile id, e.g.
# 101BE004_514N028E.000. Longer than 8 chars and underscore-bearing, so the S-57
# regex above (deliberately) rejects them; this one accepts them so the same
# folder workflow recognises S-101 sources instead of reporting "no charts".
_S101_CELL_RE = re.compile(r"^(?P<base>101[A-Za-z]{2}[A-Za-z0-9_]+)\.(?P<seq>\d{3})$")


def _match_cell(fname):
    """Return (base, seq, product) for an S-57 or S-101 cell file, else None.
    Product is 'S-57' or 'S-101'. Name-based (cheap) — used while gathering."""
    m = _CELL_RE.match(fname)
    if m:
        return m.group("base"), int(m.group("seq")), "S-57"
    m = _S101_CELL_RE.match(fname)
    if m:
        return m.group("base"), int(m.group("seq")), "S-101"
    return None


def classify_cell(path):
    """Authoritative product of a staged .000 cell — 'S-101' or 'S-57' — read
    from the DSID natively (GDAL-free). S-101 cells declare the S-100 Part 10a
    encoding / an S-101 product spec in DSID; S-57 cells do not carry ENSP. Falls
    back to the filename convention if the DSID can't be read. Never raises."""
    try:
        try:
            from .iso8211.reader import Iso8211Reader
        except (ImportError, ValueError):
            from iso8211.reader import Iso8211Reader
        for rec in Iso8211Reader.open(path).records():
            dsid = rec.first("DSID")
            if dsid:
                ensp = str(dsid.get("ENSP") or "")
                prsp = str(dsid.get("PRSP") or "")
                if "Part 10a" in ensp or "S-100" in ensp or "S-101" in prsp:
                    return "S-101"
                return "S-57"
    except Exception:                                     # noqa: BLE001
        pass
    return "S-101" if _S101_CELL_RE.match(os.path.basename(path)) else "S-57"

# Zip-bomb guard: cap decompressed bytes while extracting a zip source. A real
# S-57 cell file is normally well under 5 MB, so these are generous — they only
# stop a malicious or corrupt archive from filling the temp disk. Bytes are
# counted as they stream (the zip's declared sizes are not trusted). Tune freely.
MAX_MEMBER_BYTES = 75 * 1024 * 1024      # per extracted file
MAX_STAGE_BYTES = 512 * 1024 * 1024      # cumulative across one staging pass

# S-57 / Inland-ENC navigational-purpose (usage) bands. The band is the 3rd
# character of the cell name. 1–6 are standard S-57; 7–8 are Inland-ENC.
USAGE_LABELS = {
    1: "Overview",
    2: "General",
    3: "Coastal",
    4: "Approach",
    5: "Harbour",
    6: "Berthing",
    7: "Inland overview",
    8: "Inland berthing",
}


def usage_band(cell_name):
    """Return the integer usage band (1–8) from an 8-char S-57 cell name, or 0
    if it can't be determined. The band is the 3rd character of the name."""
    if not cell_name or len(cell_name) < 3:
        return 0
    ch = cell_name[2]
    if ch.isdigit():
        b = int(ch)
        if 1 <= b <= 8:
            return b
    return 0


def usage_label(band):
    """Human label for a usage band integer; falls back to a generic string."""
    return USAGE_LABELS.get(band, f"Usage {band}" if band else "Unknown usage")


def _is_zip(path):
    return os.path.isfile(path) and path.lower().endswith(".zip")


def _inplace_check(source):
    """Decide whether a FOLDER source can be loaded in place, with no copy.

    A clean exchange-set folder already has each base (.000) sitting beside its
    own update files, and scan_for_charts both walks for the .000s and
    deduplicates editions itself — so GDAL can apply the updates straight from
    the source and the copy-staging step is pure overhead. We only fall back to
    the full copy-staging when an update sequence is broken (a gap): that is the
    one case worth isolating for handling/repair.

    Returns (can_load_in_place, validation). Validation events use the same
    shape as stage_source, so the Process log is identical either way. A broken
    update sequence anywhere sets can_load_in_place=False (the caller then
    copy-stages as before). Zips never reach here.
    """
    validation = []
    # { cell_name: { directory: set(seqs) } }
    cells = {}
    for root, _dirs, files in os.walk(source):
        for fname in files:
            m = _match_cell(fname)
            if m:
                base, seq, _product = m
                cells.setdefault(base, {}).setdefault(root, set()).add(seq)

    if not cells:
        # Nothing recognisable — let the normal path report the empty result.
        return False, validation

    broken = False
    for cell in sorted(cells):
        locations = cells[cell]
        base_locs = [loc for loc, seqs in locations.items() if 0 in seqs]
        if not base_locs:
            validation.append(
                {"cell": cell, "level": "error", "status": "orphan_update",
                 "message": f"Update file(s) for '{cell}' but no .000 base — "
                            f"cell cannot be loaded."})
            continue
        for loc in base_locs:
            updates = sorted(s for s in locations[loc] if s > 0)
            if not updates:
                validation.append(
                    {"cell": cell, "level": "info", "status": "ok",
                     "message": f"'{cell}': base only (no updates)."})
                continue
            expected = list(range(1, max(updates) + 1))
            missing = sorted(set(expected) - set(updates))
            if missing:
                broken = True
                last_good = 0
                for s in expected:
                    if s in updates:
                        last_good = s
                    else:
                        break
                validation.append(
                    {"cell": cell, "level": "warning", "status": "sequence_gap",
                     "message": f"'{cell}': update sequence broken — missing "
                                f"{', '.join(f'.{x:03d}' for x in missing)}. "
                                f"Loading only up to .{last_good:03d}."})
            else:
                validation.append(
                    {"cell": cell, "level": "info", "status": "ok",
                     "message": f"'{cell}': base + {len(updates)} update(s), "
                                f"sequence complete."})
    return (not broken), validation


def stage_source(source, log=_log):
    """Normalise a source (folder or .zip) into a clean staging directory ready
    for GDAL, applying these corrections:

      • Zips are extracted to a temp dir.
      • Each base cell (.000) and its update files (.001, .002, …) are grouped
        into one directory per cell name, so GDAL's S-57 driver finds and
        applies the updates in sequence (it only does so when they sit beside
        the base). Files scattered by a zip or a flat folder are moved together.

    Returns (staging_dir, validation) where validation is a list of structured
    events: {"cell": name, "level": "info|warning|error", "status": str,
    "message": str}. Detected-but-not-fixed problems (sequence gaps, orphan
    updates) are reported here, not silently corrected.

    The staging dir is a fresh temp dir we own; callers may delete it when done.
    """
    # ── Fast path: a plain folder whose update sequences are all valid is read
    #    in place — no copy. Only zips (which must be extracted) and folders with
    #    a broken update sequence fall through to the full copy-staging below.
    if not _is_zip(source):
        can_inplace, validation = _inplace_check(source)
        if can_inplace:
            log(f"[stage] clean folder — loading in place, no copy needed: "
                f"{source}")
            return source, validation

    validation = []
    staging = tempfile.mkdtemp(prefix="enc_explorer_stage_")
    log(f"[stage] staging into {staging}")

    # ── 1) Gather candidate files, keyed by (cell_name, source_location).
    #    A "location" is the directory (folder source) or the in-zip parent path
    #    (zip source) a file lives in. Files that share a cell name AND a
    #    location are ONE edition's fileset (base + its own update sequence).
    #    The same cell name in two different locations = two editions, which we
    #    must keep separate so they don't collide — then pick the winner.
    #    candidates: { cell_name: { location_key: {seq: ref} } }
    candidates = {}

    def _add(cell, location_key, seq, ref):
        candidates.setdefault(cell, {}).setdefault(location_key, {})[seq] = ref

    zf = None
    if _is_zip(source):
        log(f"[stage] source is a zip: {os.path.basename(source)}")
        try:
            zf = zipfile.ZipFile(source)
        except Exception as e:
            log(f"[stage] zip open failed: {e}", Qgis.Critical)
            validation.append({"cell": "", "level": "error",
                               "status": "zip_error",
                               "message": f"Could not open zip: {e}. Try "
                                          f"extracting it and selecting the "
                                          f"folder instead."})
            return staging, validation
        for member in zf.namelist():
            if member.endswith("/"):
                continue
            m = _match_cell(os.path.basename(member))
            if m:
                base, seq, _product = m
                loc = os.path.dirname(member)  # in-zip folder = the location
                _add(base, loc, seq, ("zip", member))
    else:
        log(f"[stage] source is a folder: {source}")
        for root, _dirs, files in os.walk(source):
            for fname in files:
                m = _match_cell(fname)
                if m:
                    base, seq, _product = m
                    _add(base, root, seq, ("file", os.path.join(root, fname)))

    if not candidates:
        log("[stage] no S-57 or S-101 cell files (.000) found in source.",
            Qgis.Warning)
        if zf:
            zf.close()
        return staging, validation

    # ── 2) For each cell, pick the WINNING edition (location), then stage only
    #    that edition's base + its update sequence. Edition winner = the base
    #    (.000) with the latest EDTN/ISDT; if a location has no .000 it can't
    #    win (orphan). Reading EDTN/ISDT needs the base extracted, so we
    #    extract each candidate base to a scratch area to read its DSID.
    scratch = tempfile.mkdtemp(prefix="enc_explorer_scratch_")
    unsupported = False
    oversized = False
    staged_bytes = 0   # cumulative decompressed bytes this pass (zip-bomb guard)

    def _materialise(ref, dest_path):
        """Copy a file (folder ref) or extract a zip member to dest_path.

        Returns True on success. Sets `unsupported` if the zip uses a
        compression method we can't read, and `oversized` if extracting would
        breach the zip-bomb guard — a single member past MAX_MEMBER_BYTES or the
        staging total past MAX_STAGE_BYTES. Bytes are counted as they stream, so
        a member that under-reports its size in its header is still caught; any
        partial file written before the trip is removed.
        """
        nonlocal unsupported, oversized, staged_bytes
        kind, val = ref
        try:
            if kind == "zip":
                written = 0
                with zf.open(val) as s, open(dest_path, "wb") as d:
                    while True:
                        chunk = s.read(65536)
                        if not chunk:
                            break
                        written += len(chunk)
                        if (written > MAX_MEMBER_BYTES
                                or staged_bytes + written > MAX_STAGE_BYTES):
                            oversized = True
                            break
                        d.write(chunk)
                if oversized:
                    try:
                        os.remove(dest_path)
                    except OSError:
                        pass
                    return False
                staged_bytes += written
            else:
                shutil.copy2(val, dest_path)
            return True
        except NotImplementedError:
            unsupported = True
            return False
        except Exception as e:
            log(f"[stage] materialise failed for {val}: {e}", Qgis.Warning)
            return False

    staged_count = 0
    for cell, locations in sorted(candidates.items()):
        # Evaluate each location that HAS a base (.000) to find the winner.
        best_loc = None
        best_isdt = None
        best_edtn = None
        for loc_key, seqmap in locations.items():
            if 0 not in seqmap:
                continue  # no base in this location — can't be a winner
            # Extract just the base to read its edition.
            tmp_base = os.path.join(scratch, f"{cell}_{abs(hash(loc_key))}.000")
            if not _materialise(seqmap[0], tmp_base):
                if unsupported or oversized:
                    break
                continue
            try:
                dsid = _read_dsid(tmp_base)
                isdt = dsid.get("ISDT")
                edtn = dsid.get("EDTN", 0)
            except Exception:
                isdt, edtn = None, 0
            if best_loc is None or _is_newer(isdt, edtn, best_isdt, best_edtn):
                best_loc, best_isdt, best_edtn = loc_key, isdt, edtn

        if unsupported or oversized:
            break

        if best_loc is None:
            # No location had a base — every fileset for this cell is orphan
            # updates. Report and skip.
            validation.append(
                {"cell": cell, "level": "error", "status": "orphan_update",
                 "message": f"Update file(s) for '{cell}' but no .000 base — "
                            f"cell cannot be loaded."})
            log(f"[stage] ORPHAN: {cell} (no base in any location)", Qgis.Warning)
            continue

        # Stage the winning location's base + its update sequence.
        winner_seqs = locations[best_loc]
        cell_dir = os.path.join(staging, cell)
        os.makedirs(cell_dir, exist_ok=True)
        for seq in sorted(winner_seqs):
            dest = os.path.join(cell_dir, f"{cell}.{seq:03d}")
            _materialise(winner_seqs[seq], dest)
            if oversized:
                break
        if oversized:
            break
        staged_count += 1

        n_other = len(locations) - 1
        if n_other > 0:
            log(f"[stage] {cell}: {len(locations)} editions found, kept "
                f"EDTN={best_edtn} ISDT={best_isdt}, dropped {n_other} older")

        # ── Validate the winner's update sequence.
        seqs = set(winner_seqs)
        updates = sorted(s for s in seqs if s > 0)
        if updates:
            expected = list(range(1, max(updates) + 1))
            missing = sorted(set(expected) - set(updates))
            if missing:
                last_good = 0
                for s in expected:
                    if s in updates:
                        last_good = s
                    else:
                        break
                validation.append(
                    {"cell": cell, "level": "warning",
                     "status": "sequence_gap",
                     "message": f"'{cell}': update sequence broken — missing "
                                f"{', '.join(f'.{m:03d}' for m in missing)}. "
                                f"Loading only up to .{last_good:03d}."})
                log(f"[stage] GAP: {cell} missing {missing}; "
                    f"effective up to .{last_good:03d}", Qgis.Warning)
            else:
                validation.append(
                    {"cell": cell, "level": "info", "status": "ok",
                     "message": f"'{cell}': base + {len(updates)} update(s), "
                                f"sequence complete."})
        else:
            validation.append({"cell": cell, "level": "info", "status": "ok",
                               "message": f"'{cell}': base only (no updates)."})

    if zf:
        zf.close()
    shutil.rmtree(scratch, ignore_errors=True)

    if oversized:
        log("[stage] zip exceeds the decompression size guard — aborting.",
            Qgis.Critical)
        validation.append(
            {"cell": "", "level": "error", "status": "zip_too_large",
             "message": "This zip decompresses past the safety size limit "
                        "(possible zip bomb, or an unusually large archive). "
                        "Please extract it with your file manager and select "
                        "the folder instead."})
        return staging, validation

    if unsupported:
        log("[stage] zip uses an unsupported compression method.", Qgis.Critical)
        validation.append(
            {"cell": "", "level": "error", "status": "zip_compression",
             "message": "This zip uses a compression method that can't be read "
                        "(often Deflate64 in large archives). Please extract it "
                        "with your file manager and select the folder instead."})
        return staging, validation

    log(f"[stage] staged {staged_count} cell(s); "
        f"{sum(1 for v in validation if v['level'] != 'info')} issue(s).")
    return staging, validation


def scan_for_charts(source):
    """STAGE 1 — find chart cells in a source (folder or .zip), stage them so
    GDAL can apply updates, deduplicate to winners (latest EDTN/ISDT), and
    return the winners plus validation events.

    Returns:
        best_files : dict  { dataset_name -> staged .000 path }  (deduplicated)
        skipped    : list  of (path, reason) for files that failed to open
        validation : list  of structured staging events (see stage_source)
        staging    : str   path to the staging dir we created (caller may clean)
    """
    staging, validation = stage_source(source)

    candidates = {}
    skipped = []
    found_files = []

    # After staging, base cells live at <staging>/<cell>/<cell>.000
    for root, _dirs, files in os.walk(staging):
        for fname in files:
            if fname.lower().endswith(".000"):
                found_files.append(os.path.join(root, fname))

    _log(f"Total base .000 files after staging: {len(found_files)}")

    if not found_files:
        _log("No .000 files found after staging.", Qgis.Warning)
        return {}, skipped, validation, staging

    for fpath in found_files:
        fname = os.path.basename(fpath)
        try:
            dsid = _read_dsid(fpath)
            _log(f"  OK: {fname} → DSNM={dsid.get('DSNM')} "
                 f"ISDT={dsid.get('ISDT')} EDTN={dsid.get('EDTN')}")
        except Exception as e:
            _log(f"  SKIP: {fname} → {e}", Qgis.Warning)
            skipped.append((fpath, str(e)))
            dsnm = os.path.splitext(fname)[0].upper()
            dsid = {"DSNM": dsnm, "ISDT": None, "EDTN": 0}

        dsnm = dsid.get("DSNM") or os.path.splitext(fname)[0].upper()
        isdt = dsid.get("ISDT")
        edtn = dsid.get("EDTN", 0)

        if dsnm not in candidates:
            candidates[dsnm] = {"path": fpath, "isdt": isdt, "edtn": edtn}
        else:
            existing = candidates[dsnm]
            if _is_newer(isdt, edtn, existing["isdt"], existing["edtn"]):
                _log(f"  Replacing {existing['path']} with {fpath} (newer)")
                candidates[dsnm] = {"path": fpath, "isdt": isdt, "edtn": edtn}

    best_files = {name: info["path"] for name, info in candidates.items()}
    _log(f"Unique charts after deduplication: {len(best_files)}")
    return best_files, skipped, validation, staging


# ── Lean staging for the Auditor ─────────────────────────────────────────────
class AuditStaging:
    """Result of :func:`stage_for_audit`.

    mode           -- 'single' (one exchange set), 'loose' (cells but no
                      CATALOG.0xx — exchange checks skipped + noted) or 'multi'
                      (more than one exchange set — the Auditor refuses).
    cell_paths     -- base .000 paths to audit (empty for 'multi').
    staging_dir    -- temp dir we own (zip case) or None (folder, read in place).
    catalog_count  -- number of CATALOG.0xx files found in the source.
    message        -- user-facing note (the refusal text for 'multi').
    """

    __slots__ = ("mode", "cell_paths", "staging_dir", "catalog_count",
                 "message")

    def __init__(self, mode, cell_paths, staging_dir, catalog_count, message=""):
        self.mode = mode
        self.cell_paths = cell_paths
        self.staging_dir = staging_dir
        self.catalog_count = catalog_count
        self.message = message


def stage_for_audit(source, log=_log):
    """Minimal, fast staging for the Auditor — no edition dedup, no in-place
    heuristics, no structured staging-validation pass (those belong to the
    loader). It does only what the audit needs:

      1. Gate on the exchange-set count (CATALOG.0xx files), via ``ExchangeSet``:
           >= 2  -> 'multi'  : the Auditor is single-exchange-set only; refuse.
           == 1  -> 'single' : full audit, including the Exchange-set checks.
           == 0  -> 'loose'  : audit the cells; Exchange-set checks are skipped
                               and the report notes CATALOG.031/exchange-set were
                               not tested.
      2. Collect the base ``.000`` cell paths, keeping update files (.00n) beside
         their base so GDAL and the native reader both apply them (the
         as-delivered, update-merged state). A folder is read IN PLACE (no copy);
         a zip is extracted PRESERVING its structure (zip-slip + zip-bomb
         guarded), cell files only.

    Reads only the (small) catalogue to gate — it does NOT read every cell's
    DSID the way the loader's dedup does, which is what made the old path slow to
    start. Returns an :class:`AuditStaging`.
    """
    try:
        from .s57 import ExchangeSet
    except (ImportError, ValueError):
        from s57 import ExchangeSet

    es = ExchangeSet.from_source(source)
    cc = es.catalog_count
    try:
        es.close()
    except Exception:                                     # noqa: BLE001
        pass

    if cc >= 2:
        log(f"[audit-stage] {cc} exchange-set catalogues found — refusing "
            f"(single exchange sets only).", Qgis.Warning)
        return AuditStaging(
            "multi", [], None, cc,
            "Auditor is only available for single exchange sets.")

    mode = "single" if cc == 1 else "loose"

    if _is_zip(source):
        staging_dir, cell_paths = _extract_cells_preserving(source, log)
    else:
        staging_dir = None
        cell_paths = []
        for root, _dirs, files in os.walk(source):
            for fn in files:
                if fn.lower().endswith(".000") and _match_cell(fn):
                    cell_paths.append(os.path.join(root, fn))

    cell_paths = sorted(cell_paths)

    # No edition dedup is done here (a conformant single exchange set has one
    # edition per cell). If a cell name nonetheless appears more than once
    # (non-conformant), audit BOTH and say so — rather than silently pick a
    # "winner" the way the loader's staging would.
    from collections import Counter
    dup = [n for n, k in Counter(
        os.path.splitext(os.path.basename(p))[0].upper()
        for p in cell_paths).items() if k > 1]
    if dup:
        log(f"[audit-stage] duplicate cell name(s) in this set — auditing all "
            f"copies (no dedup): {', '.join(sorted(dup))}", Qgis.Warning)

    log(f"[audit-stage] mode={mode}, {len(cell_paths)} base cell(s) "
        f"(catalogues={cc}).")
    return AuditStaging(mode, cell_paths, staging_dir, cc, "")


def _extract_cells_preserving(source, log=_log):
    """Extract a zip's cell files (.000 + .00n updates) to a fresh temp dir,
    PRESERVING the in-zip directory structure so each base keeps its updates
    beside it (GDAL applies updates only when they sit next to the base).
    Zip-slip and zip-bomb guarded; non-cell members (README, PIC/INFO, the
    catalogue itself) are not extracted. Returns (staging_dir, base_000_paths).
    """
    staging = os.path.normpath(tempfile.mkdtemp(prefix="enc_audit_stage_"))
    base_paths = []
    staged_bytes = 0
    try:
        zf = zipfile.ZipFile(source)
    except Exception as e:                                # noqa: BLE001
        log(f"[audit-stage] zip open failed: {e}", Qgis.Critical)
        return staging, base_paths

    with zf:
        for member in zf.namelist():
            if member.endswith("/"):
                continue
            bn = os.path.basename(member)
            if not _match_cell(bn):
                continue
            # Zip-slip guard: the resolved destination must stay inside staging.
            rel = member.replace("\\", "/")
            dest = os.path.normpath(os.path.join(staging, rel))
            if dest != staging and not dest.startswith(staging + os.sep):
                log(f"[audit-stage] skipped unsafe zip path: {member}",
                    Qgis.Warning)
                continue
            os.makedirs(os.path.dirname(dest), exist_ok=True)
            written = 0
            tripped = False
            try:
                with zf.open(member) as s, open(dest, "wb") as d:
                    while True:
                        chunk = s.read(65536)
                        if not chunk:
                            break
                        written += len(chunk)
                        if (written > MAX_MEMBER_BYTES or
                                staged_bytes + written > MAX_STAGE_BYTES):
                            tripped = True
                            break
                        d.write(chunk)
            except Exception as e:                        # noqa: BLE001
                log(f"[audit-stage] could not extract {member}: {e}",
                    Qgis.Warning)
                continue
            if tripped:
                log("[audit-stage] size guard tripped — aborting extract.",
                    Qgis.Critical)
                try:
                    os.remove(dest)
                except OSError:
                    pass
                return staging, base_paths
            staged_bytes += written
            if bn.lower().endswith(".000"):
                base_paths.append(dest)
    return staging, base_paths


def scan_for_object_layers(best_files):
    """STAGE 2 — build the object-layer index across a set of chosen chart files
    (the user's checked subset of stage-1 winners).

    Args:
        best_files : dict { dataset_name -> .000 path }  (already staged)
    Returns:
        layer_index : dict { layer_name -> [file_path, ...] }
    """
    layer_index = {}
    for dsnm, fpath in best_files.items():
        try:
            layers = _get_layer_names(fpath)
            _log(f"  Layers in {os.path.basename(fpath)}: {layers}")
        except Exception as e:
            _log(f"  Could not read layers from {fpath}: {e}", Qgis.Warning)
            layers = []
        for lname in layers:
            layer_index.setdefault(lname, []).append(fpath)

    _log(f"Total unique object layers found: {len(layer_index)}")
    return layer_index


def scan_for_object_layers_s101(best_files):
    """STAGE 2, S-101 variant — native object-layer index for S-101 cells, by
    feature type. GDAL-free (S-101 is read by the native engine). Mirrors the
    shape of ``scan_for_object_layers`` so the same UI populates uniformly.

    Args:
        best_files : dict { dataset_name -> .000 path }  (S-101 cells)
    Returns:
        layer_index : dict { feature_type_name -> [file_path, ...] }
    """
    try:
        from .iso8211 import Iso8211Reader
        from .s101.records import S101Cell
    except (ImportError, ValueError):
        from iso8211 import Iso8211Reader
        from s101.records import S101Cell
    layer_index = {}
    for dsnm, fpath in best_files.items():
        try:
            recs = list(Iso8211Reader.open(fpath).records())
            cell = S101Cell.from_records(recs, name=dsnm)
            types = sorted(cell.feature_type_histogram().keys(), key=str)
            _log(f"  S-101 feature types in {os.path.basename(fpath)}: "
                 f"{len(types)}")
        except Exception as e:                            # noqa: BLE001
            _log(f"  Could not read S-101 layers from {fpath}: {e}", Qgis.Warning)
            types = []
        for tname in types:
            layer_index.setdefault(str(tname), []).append(fpath)
    _log(f"Total unique S-101 feature types found: {len(layer_index)}")
    return layer_index


def scan_enc_folder(folder_path):
    """Back-compat one-shot scan (charts + all object layers in one call).
    Kept for any existing callers; the GUI now uses the two-stage functions.
    Returns (best_files, layer_index, skipped) as before.
    """
    best_files, skipped, _validation, _staging = scan_for_charts(folder_path)
    if not best_files:
        return {}, {}, skipped
    layer_index = scan_for_object_layers(best_files)
    return best_files, layer_index, skipped



def read_dsid_full(fpath):
    """Thoroughly read the DSID (and DSSI/DSPM) metadata record from an S-57
    file, returning the fields used by the ENC coverage hybrid layer:

        EDTN UPDN ISDT DSNM PRSP PRED AGEN CSCL COMT

    Any field the file doesn't provide comes back as "" (blank), so the caller
    always gets a complete, uniform attribute set. Returns a dict keyed by the
    field names above, all values strings.

    GDAL's S-57 driver only fully populates the DSID layer when opened with the
    right options; the default open used elsewhere can return blanks. We open
    here with explicit S-57 options to maximise what we recover, and we read by
    discovering the DSID layer's actual fields rather than assuming names.
    """
    wanted = ["EDTN", "UPDN", "ISDT", "DSNM", "PRSP", "PRED", "AGEN", "CSCL", "COMT"]
    out = {k: "" for k in wanted}

    # Native (GDAL-free): read the DSID + DSPM records straight from the ISO 8211
    # bytes. EDTN/UPDN/ISDT/DSNM/PRSP/PRED/AGEN/COMT are DSID subfields; CSCL is a
    # DSPM subfield. Prefer DSID, fall back to DSPM for shared/Spm-only keys. This
    # also recovers the values GDAL's default open leaves blank (the bytes carry
    # them) without the OGR_S57_OPTIONS dance.
    try:
        try:
            from .iso8211.reader import Iso8211Reader
        except (ImportError, ValueError):
            from iso8211.reader import Iso8211Reader
        dsid = dspm = None
        for rec in Iso8211Reader.open(fpath).records():
            if dsid is None:
                dsid = rec.first("DSID")
            if dspm is None:
                dspm = rec.first("DSPM")
            if dsid is not None and dspm is not None:
                break
        for key in wanted:
            val = (dsid or {}).get(key)
            if (val is None or str(val).strip() == "") and dspm:
                val = dspm.get(key)
            if val is not None and str(val).strip() != "":
                out[key] = str(val).strip()
    except Exception as e:                                # noqa: BLE001
        _log(f"  DSID full read failed for {os.path.basename(fpath)}: {e}",
             Qgis.Warning)

    # ISDT is an S-57 date, stored as YYYYMMDD. Normalise to ISO YYYY-MM-DD for
    # readable maptips/labels and correct sorting, but only when it's exactly
    # the expected 8 digits — otherwise leave whatever was read untouched.
    isdt = out.get("ISDT", "")
    if len(isdt) == 8 and isdt.isdigit():
        out["ISDT"] = f"{isdt[0:4]}-{isdt[4:6]}-{isdt[6:8]}"

    return out


def _read_dsid_native(fpath):
    """Read DSID (DSNM/EDTN/ISDT) directly from the ISO 8211 bytes with the
    native reader. GDAL's default S-57 open returns these blank on many cells
    (DSNM=None/EDTN=0), but the bytes carry them — the native reader recovers
    them. Returns a dict {DSNM, ISDT, EDTN} matching the GDAL path's shape, or
    None if no DSID could be read. GDAL-free and never raises."""
    try:
        try:
            from .iso8211.reader import Iso8211Reader
        except (ImportError, ValueError):
            from iso8211.reader import Iso8211Reader
        for rec in Iso8211Reader.open(fpath).records():
            dsid = rec.first("DSID")
            if not dsid:
                continue
            dsnm = dsid.get("DSNM")
            edtn_raw = dsid.get("EDTN")
            try:
                edtn = int(edtn_raw) if edtn_raw not in (None, "") else 0
            except (ValueError, TypeError):
                edtn = 0
            return {
                "DSNM": (str(dsnm).strip() or None) if dsnm not in (None, "")
                        else None,
                "ISDT": _parse_date(dsid.get("ISDT")),
                "EDTN": edtn,
            }
    except Exception:
        return None
    return None


def _read_dsid(fpath):
    """Extract DSID fields (DSNM/ISDT/EDTN) from an S-57 *or* S-101 cell, GDAL-
    free, via the native ISO 8211 reader (it recovers values GDAL's default open
    leaves blank — the bytes carry them). Returns defaults if no DSID is read."""
    native = _read_dsid_native(fpath)
    if native is not None:
        return native
    _log(f"  No DSID readable in {os.path.basename(fpath)}, using filename as key")
    return {"DSNM": None, "ISDT": None, "EDTN": 0}


def _parse_date(raw):
    """Parse S-57 date strings: YYYYMMDD or DD-MMM-YYYY."""
    if not raw:
        return None
    raw = str(raw).strip()
    for fmt in ("%Y%m%d", "%d-%b-%Y", "%Y-%m-%d"):
        try:
            return datetime.strptime(raw, fmt).date()
        except ValueError:
            continue
    return None


def _is_newer(isdt_new, edtn_new, isdt_old, edtn_old):
    """Return True if the new candidate is preferred over the existing one."""
    if isdt_new is not None and isdt_old is not None:
        if isdt_new != isdt_old:
            return isdt_new > isdt_old
        return edtn_new > edtn_old
    if isdt_new is not None:
        return True
    return edtn_new > edtn_old


_CATALOGUE = None


def _catalogue():
    """The bundled S-57/Inland catalogue (code↔acronym), built once. GDAL-free."""
    global _CATALOGUE
    if _CATALOGUE is None:
        try:
            from .validation import Catalogue
        except (ImportError, ValueError):
            from validation import Catalogue
        _CATALOGUE = Catalogue(os.path.join(os.path.dirname(__file__),
                                            "s57catalogue"))
    return _CATALOGUE


def _get_layer_names(fpath):
    """Native (GDAL-free) list of the S-57 object classes present in a cell, as
    catalogue acronyms — so Inland-ENC classes (boylat, gatcon, …) decode to
    their real acronyms instead of GDAL's catch-all 'Generic' layer.

    Light single-pass read: it pulls OBJL straight from the base cell's FRID
    records via the raw ISO 8211 reader, WITHOUT building the full record model /
    geometry. That keeps the object-layer scan fast across many cells (the class
    list only needs which classes are present). Updates are not merged here —
    they modify/insert features but practically never introduce a brand-new class
    to a cell's tree entry; the load path reads the full merged set."""
    try:
        from .iso8211.reader import Iso8211Reader
    except (ImportError, ValueError):
        from iso8211.reader import Iso8211Reader
    code_to_acr = _catalogue().code_to_acronym
    names, seen = [], set()
    try:
        for rec in Iso8211Reader.open(fpath).records(only_tags={"FRID"}):
            frid = rec.first("FRID")
            if not frid:
                continue
            try:
                objl = int(frid.get("OBJL"))
            except (TypeError, ValueError):
                continue
            acr = code_to_acr.get(objl)
            if acr and acr not in seen:
                seen.add(acr)
                names.append(acr)
    except Exception as e:                                # noqa: BLE001
        _log(f"  Could not read S-57 layers from {os.path.basename(fpath)}: {e}",
             Qgis.Warning)
    return names
