import json
import logging
import math
import os
import re
import shutil
from typing import List, Optional, Dict
import typing
import uuid
import tempfile
from datetime import datetime
from dateutil.tz import tzlocal


from .editor import prevent_conflicted_copy

from .common import UPLOAD_CHUNK_SIZE, DeltaChangeType, InvalidProject, ClientError, PullActionType
from .models import ProjectDelta, ProjectDeltaChange, ProjectDeltaItemDiff, PullAction
from .utils import (
    generate_checksum,
    is_versioned_file,
    int_version,
    do_sqlite_checkpoint,
    unique_path_name,
    conflicted_copy_file_name,
    edit_conflict_file_name,
)
from .local_changes import FileChange

this_dir = os.path.dirname(os.path.realpath(__file__))


# Try to import pygeodiff from "deps" sub-directory (which should be present e.g. when
# used within QGIS plugin), if that's not available then try to import it from standard
# python paths.
try:
    from .deps import pygeodiff
except (ImportError, ModuleNotFoundError):
    import pygeodiff


class MerginProject:
    """Base class for Mergin Maps local projects.

    Linked to existing local directory, with project metadata (mergin.json) and backups located in .mergin directory.
    """

    def __init__(self, directory):
        self.dir = os.path.abspath(directory)
        if not os.path.exists(self.dir):
            raise InvalidProject("Project directory does not exist")

        self.meta_dir = os.path.join(self.dir, ".mergin")
        if not os.path.exists(self.meta_dir):
            os.mkdir(self.meta_dir)

        # location for files from unfinished pull
        self.unfinished_pull_dir = os.path.join(self.meta_dir, "unfinished_pull")

        self.cache_dir = os.path.join(self.meta_dir, ".cache")
        if not os.path.exists(self.cache_dir):
            os.mkdir(self.cache_dir)

        # metadata from JSON are lazy loaded
        self._metadata = None
        self.is_old_metadata = False

        self.setup_logging(directory)

        # make sure we can load correct pygeodiff
        try:
            self.geodiff = pygeodiff.GeoDiff()
        except pygeodiff.geodifflib.GeoDiffLibVersionError:
            # this is a fatal error, we can't live without geodiff
            self.log.error("Unable to load geodiff! (lib version error)")
            raise ClientError("Unable to load geodiff library!")

        # redirect any geodiff output to our log file
        def _logger_callback(level, text_bytes):
            text = text_bytes.decode()  # convert bytes to str
            if level == pygeodiff.GeoDiff.LevelError:
                self.log.error("GEODIFF: " + text)
            elif level == pygeodiff.GeoDiff.LevelWarning:
                self.log.warning("GEODIFF: " + text)
            else:
                self.log.info("GEODIFF: " + text)

        self.geodiff.set_logger_callback(_logger_callback)
        self.geodiff.set_maximum_logger_level(pygeodiff.GeoDiff.LevelDebug)

    def setup_logging(self, logger_name):
        """Setup logging into project directory's .mergin/client-log.txt file."""
        self.log = logging.getLogger("mergin.project." + str(logger_name))
        self.log.setLevel(logging.DEBUG)  # log everything (it would otherwise log just warnings+errors)
        if not self.log.handlers:
            # we only need to set the handler once
            # (otherwise we would get things logged multiple times as loggers are cached)
            log_handler = logging.FileHandler(os.path.join(self.meta_dir, "client-log.txt"), encoding="utf-8")
            log_handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
            self.log.addHandler(log_handler)

    def remove_logging_handler(self):
        """Check if there is a logger handler defined for the project and remove it. There should be only one."""
        if len(self.log.handlers) > 0:
            handler = self.log.handlers[0]
            self.log.removeHandler(handler)
            handler.flush()
            handler.close()

    def fpath(self, file, other_dir=None):
        """
        Helper function to get absolute path of project file. Defaults to project dir but
        alternative dir get be provided (mostly meta or temp). Also making sure that parent dirs to file exist.

        :param file: relative file path in project (posix)
        :type file: str
        :param other_dir: alternative base directory for file, defaults to None
        :type other_dir: str
        :returns: file's absolute path
        :rtype: str
        """
        root = other_dir or self.dir
        abs_path = os.path.abspath(os.path.join(root, file))
        f_dir = os.path.dirname(abs_path)
        os.makedirs(f_dir, exist_ok=True)
        return abs_path

    def fpath_meta(self, file):
        """Helper function to get absolute path of file in meta dir."""
        return self.fpath(file, self.meta_dir)

    def fpath_unfinished_pull(self, file):
        """Helper function to get absolute path of file in unfinished_pull dir."""
        return self.fpath(file, self.unfinished_pull_dir)

    def fpath_cache(self, file, version=None):
        """Helper function to get absolute path of file in cache dir.
        It can be either in root cache directory (.mergin/.cache/) or in some version's subfolder
        """
        if version:
            return self.fpath(file, os.path.join(self.cache_dir, version))
        return self.fpath(file, self.cache_dir)

    def project_full_name(self) -> str:
        """Returns fully qualified project name: <workspace>/<name>"""
        self._read_metadata()
        if self.is_old_metadata:
            return self._metadata.get("name")
        workspace = self._metadata.get("workspace", {})
        if workspace:
            return f"{workspace.get('name')}/{self._metadata['name']}"
        return f"{self._metadata.get('namespace')}/{self._metadata['name']}"

    def project_name(self) -> str:
        """Returns only project name, without its workspace name"""
        full_name = self.project_full_name()
        slash_index = full_name.index("/")
        return full_name[slash_index + 1 :]

    def workspace_name(self) -> str:
        """Returns name of the workspace where the project belongs"""
        full_name = self.project_full_name()
        slash_index = full_name.index("/")
        return full_name[:slash_index]

    def project_role(self) -> str:
        self._read_metadata()
        return self._metadata.get("role")

    def project_id(self) -> str:
        """Returns ID of the project (UUID using 8-4-4-4-12 formatting without braces)

        Raises ClientError if project id is not present in the project metadata. This should
        only happen with projects downloaded with old client, before February 2023,
        see https://github.com/MerginMaps/python-api-client/pull/154
        """
        self._read_metadata()

        # "id" or "project_id" may not exist in projects downloaded with old client version
        if self.is_old_metadata:
            if "project_id" not in self._metadata:
                raise ClientError(
                    "The project directory has been created with an old version of the Mergin Maps client. "
                    "Please delete the project directory and re-download the project."
                )
            return self._metadata["project_id"]
        else:
            return self._metadata["id"]

    def workspace_id(self) -> int:
        """Returns ID of the workspace where the project belongs"""
        # unfortunately we currently do not have information about workspace ID
        # in project's metadata...
        self._read_metadata()

        # "workspace_id" does not exist in projects downloaded with old client version
        if self.is_old_metadata:
            raise ClientError(
                "The project directory has been created with an old version of the Mergin Maps client. "
                "Please delete the project directory and re-download the project."
            )

        workspace = self._metadata.get("workspace", {})
        if workspace:
            return workspace.get("id")
        return self._metadata.get("workspace_id")

    def version(self) -> str:
        """Returns project version (e.g. "v123")"""
        self._read_metadata()
        return self._metadata["version"]

    def files(self) -> list:
        """Returns project's list of files (each file being a dictionary)"""
        self._read_metadata()
        return self._metadata["files"]

    @property
    def metadata(self) -> dict:
        """Gets raw access to metadata. Kept only for backwards compatibility and will be removed."""
        # as we will change what is written in mergin.json, we do not want
        # client code to use this getter, and rather use project_full_name(), version() etc.
        from warnings import warn

        warn("MerginProject.metadata getter should not be used anymore", DeprecationWarning)
        self._read_metadata()
        return self._metadata

    def _read_metadata(self) -> None:
        """Loads the project's metadata from JSON"""
        if self._metadata is not None:
            return
        if not os.path.exists(self.fpath_meta("mergin.json")):
            raise InvalidProject("Project metadata has not been created yet")
        with open(self.fpath_meta("mergin.json"), "r") as file:
            self._metadata = json.load(file)

        self.is_old_metadata = "/" in self._metadata["name"]

    def update_metadata(self, data: dict):
        """Writes project metadata and updates cached metadata."""
        self._metadata = data
        MerginProject.write_metadata(self.dir, data)

    @staticmethod
    def write_metadata(project_directory: str, data: dict):
        """Writes project metadata to <project_directory>/.mergin/mergin.json

        In most cases it is better to call update_metadata() as that will also
        update in-memory cache of metadata in MerginProject - this static method is
        useful for cases when this is the first time metadata are being written
        (and therefore creating MerginProject would fail).
        """
        meta_dir = os.path.join(project_directory, ".mergin")
        os.makedirs(meta_dir, exist_ok=True)
        metadata_json_file = os.path.abspath(os.path.join(meta_dir, "mergin.json"))
        with open(metadata_json_file, "w") as file:
            json.dump(data, file, indent=2)

    def is_versioned_file(self, file):
        """Check if file is compatible with geodiff lib and hence suitable for versioning.

        :param file: file path
        :type file: str
        :returns: if file is compatible with geodiff lib
        :rtype: bool
        """
        return is_versioned_file(file)

    def is_gpkg_open(self, path):
        """
        Check whether geopackage file is open (and wal file exists)

        :param path: absolute path of file on disk
        :type path: str
        :returns: whether file is open
        :rtype: bool
        """
        f_extension = os.path.splitext(path)[1]
        if f_extension != ".gpkg":
            return False
        if os.path.exists(f"{path}-wal"):
            return True
        return False

    def ignore_file(self, file):
        """
        Helper function for blacklisting certain types of files.

        :param file: file path in project
        :type file: str
        :returns: whether file should be ignored
        :rtype: bool
        """
        ignore_ext = re.compile(r"({})$".format("|".join(re.escape(x) for x in ["-shm", "-wal", "~", "pyc", "swap"])))
        ignore_files = [".DS_Store", ".directory"]
        name, ext = os.path.splitext(file)
        if ext and ignore_ext.search(ext):
            return True
        if file in ignore_files:
            return True
        return False

    def inspect_files(self):
        """
        Inspect files in project directory and return metadata.

        :returns: metadata for files in project directory in server required format
        :rtype: list[dict]
        """
        files_meta = []
        for root, dirs, files in os.walk(self.dir, topdown=True):
            dirs[:] = [d for d in dirs if d not in [".mergin"]]
            for file in files:
                if self.ignore_file(file):
                    continue

                abs_path = os.path.abspath(os.path.join(root, file))
                rel_path = os.path.relpath(abs_path, start=self.dir)
                proj_path = "/".join(rel_path.split(os.path.sep))  # we need posix path
                files_meta.append(
                    {
                        "path": proj_path,
                        "checksum": generate_checksum(abs_path),
                        "size": os.path.getsize(abs_path),
                        "mtime": datetime.fromtimestamp(os.path.getmtime(abs_path), tzlocal()),
                    }
                )
        return files_meta

    def compare_file_sets(self, origin: List[Dict], current: List[Dict]):
        """
        Helper function to calculate difference between two sets of files metadata using file names and checksums.

        :Example:

        >>> origin = [{'checksum': '1111111111111111111111111111111111111111', 'path': 'base.gpkg', 'size': 2793, 'mtime': '2019-08-26T11:08:34.051221+02:00'}]
        >>> current = [{'checksum': '2222222222222222222222222222222222222222', 'path': 'test.qgs', 'size': 31980, 'mtime': '2019-08-26T11:09:30.051221+02:00'}]
        >>> self.compare_file_sets(origin, current)
        {"added": [{'checksum': '2222222222222222222222222222222222222222', 'path': 'test.qgs', 'size': 31980, 'mtime': '2019-08-26T11:09:30.051221+02:00'}], "removed": [[{'checksum': '1111111111111111111111111111111111111111', 'path': 'base.gpkg', 'size': 2793, 'mtime': '2019-08-26T11:08:34.051221+02:00'}]], "renamed": [], "updated": []}

        :param origin: origin set of files metadata
        :type origin: list[dict]
        :param current: current set of files metadata to be compared against origin
        :type current: list[dict]
        :returns: changes between two sets with change type
        :rtype: dict[str, list[dict]]'
        """
        origin_map = {f["path"]: f for f in origin}
        current_map = {f["path"]: f for f in current}
        removed = [f for f in origin if f["path"] not in current_map]
        added = [f for f in current if f["path"] not in origin_map]
        updated = []
        for f in current:
            path = f["path"]
            if path not in origin_map:
                continue
            # with open WAL files we don't know yet, better to mark file as updated
            if not self.is_gpkg_open(self.fpath(path)) and f["checksum"] == origin_map[path]["checksum"]:
                continue
            f["origin_checksum"] = origin_map[path]["checksum"]
            updated.append(f)

        return {"renamed": [], "added": added, "removed": removed, "updated": updated}

    def get_pull_changes(self, server_files: List[dict], version: str) -> Dict:
        """
        Calculate changes needed to be pulled from server.

        Calculate diffs between local files metadata and server's ones. Because simple metadata like file size or
        checksum are not enough to determine geodiff files changes, evaluate also their history (provided by server).

        .. seealso:: self.compare_file_sets

        ::deprecated

        :param server_files: list of server files' metadata with mandatory 'history' field (see also self.inspect_files(),
        self.project_info(project_path, since=v1))
        :type server_files: list[dict]
        :returns: changes metadata for files to be pulled from server
        :rtype: dict
        """

        delta = self.get_pull_delta(server_files, version)

        changes = {"added": [], "removed": [], "updated": []}
        for item in delta.changes:
            change_dict = {
                "path": item.path,
                "size": item.size,
                "checksum": item.checksum,
                "version": item.version,
            }
            if item.diffs:
                change_dict["diffs"] = [diff.id for diff in item.diffs]

            if item.type == DeltaChangeType.CREATE:
                changes["added"].append(change_dict)
            elif item.type == DeltaChangeType.DELETE:
                changes["removed"].append(change_dict)
            elif item.type in [DeltaChangeType.UPDATE, DeltaChangeType.UPDATE_DIFF]:
                changes["updated"].append(change_dict)

        return changes

    def get_pull_delta(self, server_files: List[Dict], server_version: str) -> ProjectDelta:
        """
        Calculate delta needed to be pulled from server.

        This is to support legacy servers without /delta endpoint (server feature v2_pull_enabled=True).

        Calculate diffs between local files metadata and server's ones. Because simple metadata like file size or
        checksum are not enough to determine geodiff files changes, evaluate also their history (provided by server).

        This method is similar to get_pull_changes but returns ProjectDelta object instead of raw dict.

        .. seealso:: self.compare_file_sets
        .. seealso:: ProjectDelta
        .. seealso:: self.get_pull_changes

        :param server_files: list of server files' metadata with mandatory 'history' field (see also self.inspect_files(),
        self.project_info(project_path, since=v1))
        :type server_files: list[dict]
        :param server_version: version of the server
        :type server_version: str
        """

        # first let's have a look at the added/updated/removed files
        result = []
        changes = self.compare_file_sets(self.files(), server_files)

        added = changes.get("added", [])
        removed = changes.get("removed", [])
        updated = changes.get("updated", [])
        for change in added:
            result.append(
                ProjectDeltaChange(
                    type=DeltaChangeType.CREATE,
                    path=change["path"],
                    size=change["size"],
                    checksum=change["checksum"],
                    version=server_version,
                )
            )
        for change in removed:
            result.append(
                ProjectDeltaChange(
                    type=DeltaChangeType.DELETE,
                    path=change["path"],
                    size=change["size"],
                    checksum=change["checksum"],
                    version=server_version,
                )
            )

        # then let's inspect our versioned files (geopackages) if there are any relevant changes
        for change in updated:
            path = change.get("path")
            if not self.is_versioned_file(path):
                result.append(
                    ProjectDeltaChange(
                        type=DeltaChangeType.UPDATE,
                        path=path,
                        size=change["size"],
                        checksum=change["checksum"],
                        version=server_version,
                    )
                )
                continue

            diffs = []
            is_updated = False

            # get sorted list of the history (they may not be sorted or using lexical sorting - "v10", "v11", "v5", "v6", ...)
            history_list = []
            for version_str, version_info in change["history"].items():
                history_list.append((int_version(version_str), version_info))
            history_list = sorted(history_list, key=lambda item: item[0])  # sort tuples based on version numbers

            # need to track geodiff file history to see if there were any changes
            for version, version_info in history_list:
                if version <= int_version(self.version()):
                    continue  # ignore history of no interest
                is_updated = True
                diff = version_info.get("diff")
                if diff:
                    size = diff.get("size")
                    diffs.append(ProjectDeltaItemDiff(id=diff.get("path"), version=f"v{version}", size=size))
                else:
                    diffs = []
                    break  # we found force update in history, does not make sense to download diffs

            if is_updated:
                result.append(
                    ProjectDeltaChange(
                        type=DeltaChangeType.UPDATE_DIFF if diffs else DeltaChangeType.UPDATE,
                        path=path,
                        size=change["size"],
                        checksum=change["checksum"],
                        version=server_version,
                        diffs=diffs,
                    )
                )
        return ProjectDelta(to_version=server_version, changes=result)

    def get_pull_action(
        self,
        server_change: DeltaChangeType,
        local_change: Optional[DeltaChangeType] = None,
        server_checksum: Optional[str] = None,
        local_checksum: Optional[str] = None,
    ) -> Optional[PullActionType]:
        """
        Determine pull actions for files by comparing server_change and local_change.
        """
        # FATAL combinations
        if (server_change, local_change) in [
            (DeltaChangeType.CREATE, DeltaChangeType.UPDATE),
            (DeltaChangeType.CREATE, DeltaChangeType.DELETE),
            (DeltaChangeType.CREATE, DeltaChangeType.UPDATE_DIFF),
            (DeltaChangeType.UPDATE_DIFF, DeltaChangeType.CREATE),
            (DeltaChangeType.DELETE, DeltaChangeType.CREATE),
        ]:
            self.log.critical(f"Invalid combination of changes: server {server_change}, local {local_change}")
            raise ClientError(f"Invalid combination of changes: server {server_change}, local {local_change}")

        checksum_conflict = server_checksum != local_checksum if server_checksum and local_checksum else False

        pull_action_map = {
            (DeltaChangeType.CREATE, None): PullActionType.COPY,
            (DeltaChangeType.CREATE, DeltaChangeType.CREATE): (
                PullActionType.COPY_CONFLICT if checksum_conflict else None
            ),
            (DeltaChangeType.UPDATE, None): PullActionType.COPY,
            (DeltaChangeType.UPDATE, DeltaChangeType.UPDATE): (
                PullActionType.COPY_CONFLICT if checksum_conflict else None
            ),
            (DeltaChangeType.UPDATE, DeltaChangeType.DELETE): PullActionType.COPY,
            (DeltaChangeType.UPDATE, DeltaChangeType.UPDATE_DIFF): PullActionType.COPY_CONFLICT,
            (DeltaChangeType.UPDATE, DeltaChangeType.CREATE): PullActionType.COPY_CONFLICT,
            (DeltaChangeType.UPDATE_DIFF, None): PullActionType.APPLY_DIFF_NO_REBASE,  # without rebase
            (DeltaChangeType.UPDATE_DIFF, DeltaChangeType.UPDATE): PullActionType.COPY_CONFLICT,
            (DeltaChangeType.UPDATE_DIFF, DeltaChangeType.DELETE): PullActionType.COPY,
            (DeltaChangeType.UPDATE_DIFF, DeltaChangeType.UPDATE_DIFF): PullActionType.APPLY_DIFF_REBASE,  # rebase
            (DeltaChangeType.DELETE, None): PullActionType.DELETE,
            # TODO: Need to discuss If we want files to have reloaded if they are updated but removed
            (DeltaChangeType.DELETE, DeltaChangeType.UPDATE): None,
            (DeltaChangeType.DELETE, DeltaChangeType.DELETE): None,
            (DeltaChangeType.DELETE, DeltaChangeType.UPDATE_DIFF): None,
        }

        return pull_action_map.get((server_change, local_change))

    def get_local_delta(self, diff_directory: str) -> List[ProjectDeltaChange]:
        """
        Calculate local delta needed to be pushed to server.

        Calculate diffs between local files metadata and actual files in project directory. Because simple metadata like
        file size or checksum are not enough to determine geodiff files changes, geodiff tool is used to determine change
        of file content and update corresponding metadata.

        .. seealso:: self.compare_file_sets

        :param diff_directory: directory where temporary diff files can be stored
        :returns: delta items needed for files needed to be applied on a server
        :rtype: List[ProjectDeltaItem]
        """
        result = []
        changes = self.compare_file_sets(self.files(), self.inspect_files())
        added = changes.get("added", [])
        removed = changes.get("removed", [])
        updated = changes.get("updated", [])

        for change in added:
            result.append(
                ProjectDeltaChange(
                    type=DeltaChangeType.CREATE,
                    path=change.get("path"),
                    size=change.get("size"),
                    checksum=change.get("checksum"),
                    version="",
                )
            )
        for change in removed:
            result.append(
                ProjectDeltaChange(
                    type=DeltaChangeType.DELETE,
                    path=change.get("path"),
                    size=change.get("size"),
                    checksum=change.get("checksum"),
                    version="",
                )
            )

        for change in updated:
            path = change.get("path")
            if not self.is_versioned_file(path):
                result.append(
                    ProjectDeltaChange(
                        type=DeltaChangeType.UPDATE,
                        path=change.get("path"),
                        size=change.get("size"),
                        checksum=change.get("checksum"),
                        version="",
                    )
                )
                continue
            # we use geodiff to check if we geopackage is diffable
            current_file = self.fpath(path)
            origin_file = self.fpath_meta(path)
            diff_id = str(uuid.uuid4())
            diff_name = path + "-diff-" + diff_id
            diff_file = os.path.join(diff_name)
            delta_item = ProjectDeltaChange(
                type=DeltaChangeType.UPDATE,
                path=path,
                size=change["size"],
                checksum=change["checksum"],
                version="",
            )
            checkpoint_size, checkpoint_checksum = do_sqlite_checkpoint(path, self.log)
            if checkpoint_size and checkpoint_checksum:
                delta_item.size = checkpoint_size
                delta_item.checksum = checkpoint_checksum

            try:
                diff_location = self.fpath(diff_file, diff_directory)
                self.geodiff.create_changeset(origin_file, current_file, diff_location)
                if not self.geodiff.has_changes(diff_location):
                    os.remove(diff_location)
                    continue

                delta_item.checksum = change.get("origin_checksum")
                delta_item.type = DeltaChangeType.UPDATE_DIFF
                os.remove(diff_location)
            except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e:
                self.log.warning("failed to create changeset for " + path)
                # probably the database schema has been modified if geodiff cannot create changeset.
                # we will need to do full upload of the file

            result.append(delta_item)

        return result

    def get_push_changes(self):
        """
        Calculate changes needed to be pushed to server.

        Calculate diffs between local files metadata and actual files in project directory. Because simple metadata like
        file size or checksum are not enough to determine geodiff files changes, geodiff tool is used to determine change
        of file content and update corresponding metadata.

        .. seealso:: self.compare_file_sets

        :returns: changes metadata for files to be pushed to server
        :rtype: dict
        """
        changes = self.compare_file_sets(self.files(), self.inspect_files())
        # do checkpoint to push changes from wal file to gpkg
        for file in changes["added"] + changes["updated"]:
            size, checksum = do_sqlite_checkpoint(self.fpath(file["path"]), self.log)
            if size and checksum:
                file["size"] = size
                file["checksum"] = checksum
            file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(file["size"] / UPLOAD_CHUNK_SIZE))]

        # need to check for real changes in geodiff files using geodiff tool (comparing checksum is not enough)
        not_updated = []
        for file in changes["updated"]:
            path = file["path"]
            if not self.is_versioned_file(path):
                continue

            # we use geodiff to check if we can push only diff files
            current_file = self.fpath(path)
            origin_file = self.fpath_meta(path)
            diff_id = str(uuid.uuid4())
            diff_name = path + "-diff-" + diff_id
            diff_file = self.fpath_meta(diff_name)
            try:
                self.geodiff.create_changeset(origin_file, current_file, diff_file)
                if self.geodiff.has_changes(diff_file):
                    diff_size = os.path.getsize(diff_file)
                    file["checksum"] = file["origin_checksum"]  # need to match basefile on server
                    file["chunks"] = [str(uuid.uuid4()) for i in range(math.ceil(diff_size / UPLOAD_CHUNK_SIZE))]
                    file["mtime"] = datetime.fromtimestamp(os.path.getmtime(current_file), tzlocal())
                    file["diff"] = {
                        "path": diff_name,
                        "checksum": generate_checksum(diff_file),
                        "size": diff_size,
                        "mtime": datetime.fromtimestamp(os.path.getmtime(diff_file), tzlocal()),
                    }
                else:
                    if os.path.exists(diff_file):
                        os.remove(diff_file)
                    not_updated.append(file)
            except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e:
                self.log.warning("failed to create changeset for " + path)
                # probably the database schema has been modified if geodiff cannot create changeset.
                # we will need to do full upload of the file
                pass

        changes["updated"] = [f for f in changes["updated"] if f not in not_updated]
        return changes

    def copy_versioned_file_for_upload(self, f: FileChange, tmp_dir: str) -> str:
        """
        Make a temporary copy of the versioned file using geodiff, to make sure that we have full
        content in a single file (nothing left in WAL journal)
        """
        path = f.path
        self.log.info("Making a temporary copy (full upload): " + path)
        tmp_file = os.path.join(tmp_dir, path)
        os.makedirs(os.path.dirname(tmp_file), exist_ok=True)
        self.geodiff.make_copy_sqlite(self.fpath(path), tmp_file)
        f.size = os.path.getsize(tmp_file)
        f.checksum = generate_checksum(tmp_file)
        f.chunks = [str(uuid.uuid4()) for i in range(math.ceil(f.size / UPLOAD_CHUNK_SIZE))]
        f.upload_file = tmp_file
        return tmp_file

    def get_list_of_push_changes(self, push_changes):
        changes = {}
        for idx, file in enumerate(push_changes["updated"]):
            if "diff" in file:
                changeset_path = file["diff"]["path"]
                changeset = self.fpath_meta(changeset_path)
                result_file = self.fpath("change_list" + str(idx), self.meta_dir)
                try:
                    self.geodiff.list_changes_summary(changeset, result_file)
                    with open(result_file, "r") as f:
                        change = f.read()
                        changes[file["path"]] = json.loads(change)
                    os.remove(result_file)
                except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError):
                    pass
        return changes

    def apply_pull_actions(self, actions: List[PullAction], download_dir: str, server_project: dict, mc) -> List[str]:
        """
        Apply pull actions for files.

        Update project files according to pull action. Apply changes to geodiff basefiles as well
        so they are up to date with server. In case of conflicts create backups from locally modified versions.

        :param actions: list of pull actions
        :type actions: List[PullAction]
        :param download_dir: directory with downloaded files from server
        :type download_dir: str
        :param server_project: project metadata from the server
        :type server_project: dict
        :param mc: mergin client
        :type mc: mergin.client.MerginClient
        :returns: list of files with conflicts
        :rtype: list[str]
        """
        conflicts = []

        for action in actions:
            path = action.path
            server_file = self.fpath(path, download_dir)
            live_file = self.fpath(path)
            basefile = self.fpath_meta(path)
            action_type = action.type
            if action_type == PullActionType.COPY:
                # simply copy the file from server
                if is_versioned_file(path):
                    self.geodiff.make_copy_sqlite(server_file, live_file)
                    self.geodiff.make_copy_sqlite(server_file, basefile)
                else:
                    shutil.copy(server_file, live_file)
            elif action_type == PullActionType.APPLY_DIFF_NO_REBASE:
                # simply apply the diff without rebase (no local changes or non-conflicting local changes)
                self.update_without_rebase(path, server_file, live_file, basefile, download_dir)
            elif action_type == PullActionType.APPLY_DIFF_REBASE:
                # rebase needed only if both server and local changes are diffs
                conflict = self.update_with_rebase(path, server_file, live_file, basefile, download_dir, mc.username())
                if conflict:
                    conflicts.append(conflict)

            elif action_type == PullActionType.COPY_CONFLICT:
                if prevent_conflicted_copy(path, mc, server_project.get("role")):
                    continue

                conflict = self.create_conflicted_copy(path, mc.username())
                conflicts.append(conflict)
                if self.is_versioned_file(path):
                    try:
                        self.geodiff.make_copy_sqlite(server_file, live_file)
                        self.geodiff.make_copy_sqlite(server_file, basefile)
                    except pygeodiff.GeoDiffLibError:
                        self.log.info("failed to create SQLite copy for file: " + path)
                        # create unfinished pull copy instead
                        f_server_unfinished = self.fpath_unfinished_pull(path)
                        self.geodiff.make_copy_sqlite(server_file, f_server_unfinished)
                else:
                    shutil.copy(server_file, live_file)

            elif action_type == PullActionType.DELETE:
                # remove local file
                if os.path.exists(live_file):
                    os.remove(live_file)
                    if self.is_versioned_file(path) and os.path.exists(basefile):
                        os.remove(basefile)

        return conflicts

    def update_with_rebase(self, path, src, dest, basefile, temp_dir, user_name):
        """
        Update a versioned file with rebase.

        Try to peform automatic rebase, create conflict file if failed.pyt

        .. seealso:: self.update_without_rebase

        :param path: path to geodiff file
        :type path: str
        :param src: path to the server version of the file
        :type src: str
        :param dest: path to the local version of the file
        :type dest: str
        :param basefile: path to a file in meta dir
        :type basefile: str
        :param temp_dir: directory with downloaded files from server
        :type temp_dir: str
        :returns: path to conflict file if rebase fails, empty string on success
        :rtype: str
        """
        self.log.info("updating file with rebase: " + path)

        server_diff = self.fpath(f"{path}-server_diff", temp_dir)  # diff between server file and local basefile
        local_diff = self.fpath(f"{path}-local_diff", temp_dir)

        # temporary backup of file pulled from server for recovery
        f_server_backup = self.fpath(f"{path}-server_backup", temp_dir)
        self.geodiff.make_copy_sqlite(src, f_server_backup)

        # create temp backup (ideally with geodiff) of locally modified file if needed later
        f_conflict_file = self.fpath(f"{path}-local_backup", temp_dir)

        try:
            self.geodiff.create_changeset(basefile, dest, local_diff)
            self.geodiff.make_copy_sqlite(basefile, f_conflict_file)
            self.geodiff.apply_changeset(f_conflict_file, local_diff)
        except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError):
            self.log.info("backup of local file with geodiff failed - need to do hard copy")
            self.geodiff.make_copy_sqlite(dest, f_conflict_file)

        # in case there will be any conflicting operations found during rebase,
        # they will be stored in a JSON file - if there are no conflicts, the file
        # won't even be created
        rebase_conflicts = unique_path_name(
            edit_conflict_file_name(self.fpath(path), user_name, int_version(self.version()))
        )

        # try to do rebase magic
        try:
            self.geodiff.create_changeset(basefile, src, server_diff)
            self.geodiff.rebase(basefile, src, dest, rebase_conflicts)
            # make sure basefile is in the same state as remote server file (for calc of push changes)
            self.geodiff.apply_changeset(basefile, server_diff)
            self.log.info("rebase successful!")
        except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as err:
            self.log.warning("rebase failed! going to create conflict file")
            try:
                # it would not be possible to commit local changes, they need to end up in new conflict file
                self.geodiff.make_copy_sqlite(f_conflict_file, dest)
                conflict = self.create_conflicted_copy(path, user_name)
                # original file synced with server
                self.geodiff.make_copy_sqlite(f_server_backup, basefile)
                self.geodiff.make_copy_sqlite(f_server_backup, dest)
                return conflict
            except pygeodiff.GeoDiffLibError as err:
                self.log.warning("creation of conflicted copy failed! going to create an unfinished pull")
                f_server_unfinished = self.fpath_unfinished_pull(path)
                self.geodiff.make_copy_sqlite(f_server_backup, f_server_unfinished)

        return ""

    def update_without_rebase(self, path, src, dest, basefile, temp_dir):
        """
        Update a versioned file without rebase.

        Apply the diff between local copy and server to both the local
        copy and its basefile.

        .. seealso:: self.update_with_rebase

        :param path: path to geodiff file
        :type path: str
        :param src: path to the server version of the file
        :type src: str
        :param dest: path to the local version of the file
        :type dest: str
        :param basefile: path to a file in meta dir
        :type basefile: str
        :param temp_dir: directory with downloaded files from server
        :type temp_dir: str
        """
        self.log.info("updating file without rebase: " + path)
        try:
            server_diff = self.fpath(f"{path}-server_diff", temp_dir)  # diff between server file and local basefile
            # TODO: it could happen that basefile does not exist.
            # It was either never created (e.g. when pushing without geodiff)
            # or it was deleted by mistake(?) by the user. We should detect that
            # when starting pull and download it as well
            self.geodiff.create_changeset(basefile, src, server_diff)
            self.geodiff.apply_changeset(dest, server_diff)
            self.geodiff.apply_changeset(basefile, server_diff)
            self.log.info("update successful")
        except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError):
            self.log.warning("update failed! going to copy file")
            # something bad happened and we have failed to patch our local files - this should not happen if there
            # wasn't a schema change or something similar that geodiff can't handle.
            self.geodiff.make_copy_sqlite(src, dest)
            self.geodiff.make_copy_sqlite(src, basefile)

    def apply_push_changes(self, changes):
        """
        For geodiff files update basefiles according to changes pushed to server.

        :param changes: metadata for pulled files
        :type changes: dict[str, list[dict]]
        """
        for k, v in changes.items():
            for item in v:
                path = item["path"]
                if not self.is_versioned_file(path):
                    continue

                basefile = self.fpath_meta(path)
                if k == "removed":
                    os.remove(basefile)
                elif k == "added":
                    self.geodiff.make_copy_sqlite(self.fpath(path), basefile)
                elif k == "updated":
                    # in case for geopackage cannot be created diff (e.g. forced update with committed changes from wal file)
                    diff = item.get("diff")
                    if not diff:
                        self.log.info("updating basefile (copy) for: " + path)
                        self.geodiff.make_copy_sqlite(self.fpath(path), basefile)
                    else:
                        self.log.info("updating basefile (diff) for: " + path)
                        # better to apply diff to previous basefile to avoid issues with geodiff tmp files
                        changeset = self.fpath_meta(diff["path"])
                        patch_error = self.apply_diffs(basefile, [changeset])
                        if patch_error:
                            # in case of local sync issues it is safier to remove basefile, next time it will be downloaded from server
                            self.log.warning("removing basefile (because of apply diff error) for: " + path)
                            os.remove(basefile)
                else:
                    pass

    def create_conflicted_copy(self, file: str, user_name: str):
        """
        Create conflicted copy file next to its origin.

        :param file: path of file in project
        :type file: str
        :returns: path to conflicted copy
        :rtype: str
        """
        src = self.fpath(file)
        if not os.path.exists(src):
            return

        backup_path = unique_path_name(
            conflicted_copy_file_name(self.fpath(file), user_name, int_version(self.version()))
        )

        if self.is_versioned_file(file):
            self.geodiff.make_copy_sqlite(src, backup_path)
        else:
            shutil.copy(src, backup_path)
        return backup_path

    def apply_diffs(self, basefile, diffs):
        """
        Helper function to update content of geodiff file using list of diffs.
        Input file will be overwritten (make sure to create backup if needed).

        :param basefile: abs path to file to be updated
        :type basefile: str
        :param diffs: list of abs paths to geodiff changeset files
        :type diffs: list[str]
        :returns: error message if diffs were not successfully applied or None
        :rtype: str
        """
        error = None
        if not self.is_versioned_file(basefile):
            return error

        for index, diff in enumerate(diffs):
            try:
                self.geodiff.apply_changeset(basefile, diff)
            except (pygeodiff.GeoDiffLibError, pygeodiff.GeoDiffLibConflictError) as e:
                self.log.warning("failed to apply changeset " + diff + " to " + basefile)
                error = str(e)
                break
        return error

    def has_unfinished_pull(self):
        """Check if there is an unfinished pull for this project.

        Unfinished pull means that a previous pull_project() call has
        failed in the final stage due to some files being in read-only
        mode. When a project has unfinished pull, it has to be resolved
        before allowing further pulls or pushes.

        .. seealso:: self.resolve_unfinished_pull

        :returns: whether there is an unfinished pull
        :rtype: bool
        """
        return os.path.exists(self.unfinished_pull_dir)

    def resolve_unfinished_pull(self, user_name):
        """
        Try to resolve unfinished pull.

        Unfinished pull means that a previous pull_project() call has
        failed in the final stage due to some files being in read-only
        mode. When a project has unfinished pull, it has to be resolved
        before allowing further pulls or pushes.

        Resolving unfinihed pull means creation of the conflicted copy
        and replacement of the original file by the new version from the
        server.

        .. seealso:: self.has_unfinished_pull

        :param user_name: name of the user
        :type user_name: str
        :returns: files where conflicts were found
        :rtype: list[str]
        """
        conflicts = []

        if not self.has_unfinished_pull():
            self.log.warning("no unfinished pulls found")
            return

        self.log.info("resolving unfinished pull")

        for root, dirs, files in os.walk(self.unfinished_pull_dir):
            for file_name in files:
                src = os.path.join(root, file_name)
                file_path = os.path.relpath(src, self.unfinished_pull_dir)
                dest = self.fpath(file_path)
                basefile = self.fpath_meta(file_path)

                self.log.info("trying to resolve unfinished pull for: " + file_path)

                # 'src' here is a server version of the file from unfinished
                # pull and 'dest' is a local version of the same file.
                # to resolve unfinished pull we create a conflicted copy and
                # replace local file with the changes from the server.
                try:
                    # conflicted copy
                    conflict = self.create_conflicted_copy(file_path, user_name)
                    conflicts.append(conflict)
                    # original file synced with server
                    self.geodiff.make_copy_sqlite(src, basefile)
                    self.geodiff.make_copy_sqlite(src, dest)
                except pygeodiff.GeoDiffLibError as err:
                    self.log.error("unable to apply changes from previous unfinished pull!")
                    raise ClientError("Unable to resolve unfinished pull!")

        shutil.rmtree(self.unfinished_pull_dir)
        self.log.info("unfinished pull resolved successfuly!")
        return conflicts

    def set_tables_to_skip(self, tables):
        """
        Set list of tables to exclude from geodiff operations. Once defined, these
        tables will be excluded from the following operations: create changeset,
        apply changeset, rebase, get database schema, dump database contents, copy
        database between different drivers.

        If empty list is passed, list will be reset.

        :param tables: list of table names to ignore
        :type tables: list[str]
        """
        self.geodiff.set_tables_to_skip(tables)

    def get_geodiff_changes_count(self, diff_rel_path: str):
        """
        Best-effort: return number of changes in the .gpkg diff (int) or None.
        Never raises – diagnostics/logging must not fail.
        """

        diff_abs = self.fpath_meta(diff_rel_path)
        try:
            return pygeodiff.GeoDiff().changes_count(diff_abs)
        except (
            pygeodiff.GeoDiffLibError,
            pygeodiff.GeoDiffLibConflictError,
            pygeodiff.GeoDiffLibUnsupportedChangeError,
            pygeodiff.GeoDiffLibVersionError,
            FileNotFoundError,
        ):
            return None
