import json
import logging
import os
import tempfile
import time
import zipfile
from pathlib import Path

import requests

from ..api.object import (
    NdffResult,
)
from ..exceptions import NdffLibError
from ..utils import is_uri

log = logging.getLogger(__name__)


class NdffApi:
    """
    An NdffApi is the actual NDFF-api CLIENT for user/client applications.
    It has several methods to communicate with the NDFF-api (sent or receive information)

    By design, a client application should create only NdffApi instance and use it.
    Most times the client app will use the NdffConnector class to work with: that one can initialize itself with one
    or more sets of configuration files (needed for mapping of fields or data etc.)

    An NdffApi instance is either created for the Acceptance environment, or for the Production environment.
    Based on the credentials given, it will try to create a connection and fetch a connection-token.
    The connection token (with some extra info) is 'pickled' into the user's Temp folder as 'ndff_tokens' file.

    When the NdffApi set's up a connection, it first checks the ndff_tokens file to see if there is a valid token,
    else one is to be retrieved first.

    Having a valid NdffApi instance (using 'test_connection' if wanted/needed) most methods can be used to sent or
    retrieve json.
    By design, it is chosen to sent json (and not NdffObservation instances) so this class could be used from within
    other applications (as using R to create the json and only use NdffConnector/NdffApi instances to do the
    communication).
    """

    # tokens are in OS-tempdir (so user can inspect them, OR remove them in case of issues)
    TOKEN_SETS_FILE = f'{tempfile.gettempdir()}{os.sep}{"ndff_tokens"}_{os.getlogin()}'

    # Tried 15 first, but got ERROR 2022-12-22 15:31:31,121 - HTTPSConnectionPool(host='accapi.ndff.nl', port=443): Read timed out. (read timeout=15)
    NETWORK_TIME_OUT = 50  # SECONDS

    ACC_ENVIRONMENT = "ACC"
    PRD_ENVIRONMENT = "PRD"

    CONTENT_TYPE_HAL_JSON = "application/hal+json"
    CONTENT_TYPE_JSON = "application/json"
    CONTENT_TYPE_ZIP = "application/zip"

    # current used environment
    environment = ACC_ENVIRONMENT  # let's default to ACC

    def __init__(self, api_config: dict = None, fresh_one=False):
        """
        Constructor to create a NdffApi instance. It is given a dict with the needed key value pairs.
        Usually the NdffConnector instance is responsible for reading a 'ndff_api.csv' settings file
        and will create the needed dict (AND the NdffAPi instance).

        The 'ndff_api.csv' will need the following keys:

            # the NDFF environment to connect to, ether ACC or PRD
            api_environment,ACC
            domain,000
            domain_key,32_char_domain_key
            user,api_user
            password,api_password
            client_id,40_char_client_id
            client_secret,126_char_client_secret

        :param fresh_one: instruct to create a fresh api, that is do NOT use the access token maybe available in saved ndff_tokens file
        :param api_config: a dict with the needed information above
        """
        if api_config is None or api_config == {} or not isinstance(api_config, dict):
            raise NdffLibError(
                "Trying to create an Api connection without any configuration!",
                errors=[1],
            )
        self.config = api_config

        self.user = api_config.get("user", "user_from_ndff_api.csv_settings")
        self.password = api_config.get(
            "password", "password_from_ndff_api.csv_settings"
        )
        if (
            self.user is None
            or len(self.user) < 2
            or self.password is None
            or len(self.password) < 5
        ):
            raise NdffLibError(
                "Trying to create an Api connection without user or password, is your configuration OK?",
                errors=[2],
            )

        # defaulting to ACC just to be sure we do not mess up
        # NOTE earlier versions the user was responsible for the api URL's, but that was error-prone
        # NOTE ! ONLY token_url needs to end with '/'
        self.environment = api_config.get(
            "api_environment", '"ACC" (acceptatie) or "PRD" (productie)'
        )
        if self.environment == "PRD":
            self.environment = self.PRD_ENVIRONMENT
            self.api_url = "https://api.ndff.nl/api/v2/domains"
            self.codes_url = "https://api.ndff.nl/codes/v2"
            self.token_url = "https://api.ndff.nl/o/v2/token/"  # SLASH! # NOQA
        else:
            self.environment = self.ACC_ENVIRONMENT
            self.api_url = "https://accapi.ndff.nl/api/v2/domains"
            self.codes_url = "https://accapi.ndff.nl/codes/v2"
            self.token_url = "https://accapi.ndff.nl/o/v2/token/"  # SLASH! # NOQA

        self.domain = api_config.get("domain", "domain_from_ndff_api.csv_settings")
        # either there is a domain key OR it is None/empty
        self.domain_key = api_config.get(
            "domain_key", "domain_key_from_ndff_api.csv_settings"
        )

        self.client_id = api_config.get(
            "client_id", "client_id_from_ndff_api.csv_settings"
        )
        self.client_secret = api_config.get(
            "client_secret", "client_secret_from_ndff_api.csv_settings"
        )

        # The idea is that this is only used by the QGIS plugin (so the plugin will set this, and save in QSettings)
        self.verify_ssl_certificate = api_config.get("verify_ssl_certificate", True)

        if (
            self.domain is None
            or len(self.domain) < 1  # there are domains like '3'
            or self.client_id is None
            or len(self.client_id) < 2
            or self.client_secret is None
            or len(self.client_secret) < 2
        ):
            raise NdffLibError(
                "Trying to create an Api connection without domain, client_id or client_secret, "
                "is your configuration OK?",
                errors=[4],
            )

        """
        - IF the users requested a 'fresh_one', we will deliberately remove the key from the saved tokens file first.
        - look for a saved token-set in the users temp dir
        - if there: try the 'access_token':
            if it works: done,
               just use it, but keep testing it before every call ??
            if not try the 'refresh-token' to get a fresh 'access_token'
                and pickle it to a token-set in the users temp dir
        - if still here that is either No pickles,
            OR no working access or refresh:
            do a token call to get fresh tokens and pickle them
        """
        self.access_token = False

        tokens = {}

        token_set_key = self.get_token_set_key()

        if fresh_one:
            self.remove_token_set(token_set_key)

        token_sets = self.token_sets_from_disk()

        # here we get the right token set from the token_setS
        if isinstance(token_sets, dict) and token_set_key in token_sets:
            tokens = token_sets[token_set_key]

        if self.token_ok(tokens):
            return  # done

        # Try to fresh at api
        tokens = self.refresh_access_token(tokens)
        if self.token_ok(tokens):
            return  # done

        # no pickled token found AND refresh failed: just get new ones
        tokens = self.get_new_tokens()
        if self.token_ok(tokens):
            return  # done

        # if still not OK, we did all we can to either get or refresh tokens, throw an exception
        if tokens:
            raise NdffLibError(
                "Failed to get or refresh NDFF Api tokens with given credential, please check logs..."
            )
        else:
            raise NdffLibError(
                "Failed to get or refresh NDFF Api tokens, but further information not available..."
            )

    def post_save_tokens(self, payload) -> dict:
        """
        Internal work method, to POST a payload like:
        {
            'client_id': <client_id>,
            'client_secret': <client_secret>,
            'grant_type': 'password',
            'username': <user>,
            'password': <password>
        }
        To the server, and then either return a valid token dict (see below), OR raise a connection Exception

        A valid token dict, containing an access_token to be used as valid key for authentication...
        AND is pickled to disk, looks like:
        {
            'access_token': 'gbBSEQBz0lb21CbJcHzYt2YPB1W35V',
            'expires_in': 86400,
            'token_type': 'Bearer',
            'scope': 'read write groups',
            'refresh_token': 'f1C3iLmVC2n6Mg2FaQwK8yNiqUN'
        }

        (old idea was to return an empty dict, but now we are throwing exceptions...)

        :raise: a NdffLibError in case something went wrong
        :return: a dict with the actual access token (and scope/expires info)
        """
        log.debug(
            f'POST to "{self.token_url}", SSL-validation: {self.verify_ssl_certificate}, DATA: {payload}'
        )
        try:
            r = requests.post(
                self.token_url,
                data=payload,
                timeout=self.NETWORK_TIME_OUT,
                verify=self.verify_ssl_certificate,
            )
        except Exception as e:
            raise NdffLibError(e)
        if r.status_code == 200:
            tokens = r.json()
            # adding some human-readable/check-able stuff, so we know easier if token is still valid
            t = time.time()
            tokens["expires_at_utc"] = t + tokens["expires_in"]
            tokens["expires_at_local"] = time.asctime(
                time.localtime(t + tokens["expires_in"])
            )

            token_sets = self.token_sets_from_disk()
            token_sets[self.get_token_set_key()] = tokens
            # write full token_sets to disk again
            self._write_token_sets_to_disk(token_sets)
            log.debug(f"Status = {r.status_code}, tokens seem OK and saved: {tokens}")
            return tokens
        elif r.status_code == 400:
            message = f"Status = {r.status_code}, wrong credentials, or check if you are maybe using an old or wrong saved api token. Remove {self.TOKEN_SETS_FILE} file ??"
            log.warning(message)
        elif r.status_code == 401:  # 401 == Unauthorized
            message = f'Status = {r.status_code} Unauthorized, did you offer the right credentials? Error message = {r.json()["error"]}'
            log.warning(message)
        else:
            message = f"Status = {r.status_code}, r.text={r.text} r = {r}"
            log.info(message)
        # https://stackoverflow.com/questions/2052390/manually-raising-throwing-an-exception-in-python
        raise NdffLibError(message)

    def _write_token_sets_to_disk(self, token_sets):
        """
        Write the token_sets to disk in the user's TEMP dir as 'ndff_tokens_loginname'

        Note that the token_sets is a dict with more than one access token for this user.
        """
        try:
            with open(self.TOKEN_SETS_FILE, "w") as f:
                # write to disk
                json.dump(token_sets, f, indent=2)
        except Exception as e:
            raise NdffLibError(f"Error writing token sets to file: {e}")

    def token_sets_from_disk(self) -> dict:
        """
        Load the TOKEN_SETS_FILE from disk and return it as dict

        :return: the dict of all token_sets
        """
        all_token_sets = {}
        if os.path.isfile(self.TOKEN_SETS_FILE):
            try:
                log.debug("Found saved tokens on disk, checking if working...")
                if not os.access(self.TOKEN_SETS_FILE, os.W_OK):
                    msg = f'Tokens on disk "{self.TOKEN_SETS_FILE}" are not writable for current user {os.getlogin()}, please check permissions'
                    log.debug(msg)
                    raise NdffLibError(msg)
                with open(self.TOKEN_SETS_FILE, "rb") as f:
                    all_token_sets = json.load(f)
            except ValueError:  # includes json.decoder.JSONDecodeError
                msg = f'Tokens on disk "{self.TOKEN_SETS_FILE}" seem to be corrupt (should be valid json), better remove them, and start over.'
                # log.info(msg)
                raise NdffLibError(msg)
            except KeyError:
                msg = f'Key error: tokens on disk "{self.TOKEN_SETS_FILE}" seem to have missing keys, please check logs or remove the file'
                # log.info(msg)
                raise NdffLibError(msg)
        return all_token_sets

    def get_token_set_key(self) -> str:
        """
        Because we want a user to be able to handle different environments/domains at a time, we now create a
        dict with tokens per 'api_environment_domain_user'-key, the so called 'token_set_key'
        :return str: token like: 'ACC_233_username' (where ACC = ndff environment, 233 = domain id, username = username)
        """
        return f"{self.environment}_{self.domain}_{self.user}"

    def get_new_tokens(self) -> dict:
        """
        User method to get a NEW token (when no refresh token is available)

        :raise: a NdffLibError in case something went wrong
        :return: a dict with the actual access token (and scope/expires info)
        """
        log.debug("Calling api to get NEW tokens...")
        payload = {
            "client_id": self.client_id,
            "client_secret": self.client_secret,
            "grant_type": "password",
            "username": self.user,
            "password": self.password,
        }
        return self.post_save_tokens(payload)

    def refresh_access_token(self, tokens) -> dict:
        """
        User method to refresh the access token, by sending a refresh_token

        :raise: a NdffLibError in case something went wrong
        :return: a dict with the actual access token (and scope/expires info)
        """
        if isinstance(tokens, dict) and "refresh_token" in tokens:
            log.debug("Calling api to REFRESH tokens...")
            refresh_token = tokens["refresh_token"]
            payload = {
                "client_id": self.client_id,
                "client_secret": self.client_secret,
                "grant_type": "refresh_token",
                "refresh_token": refresh_token,
            }
            return self.post_save_tokens(payload)
        else:
            return {}

    @staticmethod
    def remove_all_saved_tokens() -> bool:
        """
        Remove the whole so called 'self.TOKEN_SETS_FILE' file, which is json file PER (OS)-USER with all
        access_tokens this user has used and could be used to connect to the NDFF api.

        It is actually a json/dict like

        {
          "ACC_708_api-testngb": {
            "access_token": "9ZAUW0RFjnEv7wNBUadYi2kBzDgWw3",
            "expires_in": 86400,
            "token_type": "Bearer",
            "scope": "read write groups",
            "refresh_token": "C0Lds5BqRNxtPOZaAOm03tWE1F1LF6",
            "expires_at_utc": 1697877770.3800373,
            "expires_at_local": "Sat Oct 21 10:42:50 2023"
          },
          "ACC_249_vakantieganger": {
            "access_token": "sgsmoaUC2LrtLL9DYPE4s9rJAcYztJ",
            "expires_in": 86400,
            "token_type": "Bearer",
            "scope": "read write groups",
            "refresh_token": "cFLuViByS7O0IA2lWFlppPSF15cgCP",
            "expires_at_utc": 1697877922.3283129,
            "expires_at_local": "Sat Oct 21 10:45:22 2023"
          }
        }

        containing access_tokens for every domain/environment this user has connected (and authorized) to

        :return: True in case something is successfully removed OR there was nothing to remove
        or False in case there WAS a file, but we were unable to remove...
        """
        token_sets_file = Path(NdffApi.TOKEN_SETS_FILE)
        if token_sets_file.exists() and token_sets_file.is_file():
            try:
                # try to remove it, return False in case of failing
                token_sets_file.unlink(missing_ok=True)
            except Exception as e:
                log.debug(
                    f"Failing to unlink/remove the ndff_token_sets ({token_sets_file}) file\n{e}"
                )
                return False
            return True
        else:
            return False  # missing or non-existing file

    def remove_token_set(self, token_set_key) -> bool:
        """
        Remove one token set from the token sets file of a user by using the token_set_key of it.

        :param token_set_key: specific key for token set to remove
        :return bool: True in case something is successfully removed OR there was nothing to remove
        or False in case there WAS a file, but we were unable to remove...
        """
        token_sets_file = Path(NdffApi.TOKEN_SETS_FILE)
        if token_sets_file.exists() and token_sets_file.is_file():
            try:
                token_sets = self.token_sets_from_disk()
                if token_set_key in token_sets:
                    del token_sets[token_set_key]
                self._write_token_sets_to_disk(token_sets)
            except Exception as e:
                log.error(
                    f"Failing to remove {token_set_key} from ndff_token_sets file {token_sets_file}:\n{e}"
                )
                return False
            return True
        else:
            return False  # missing or non-existing file

    def token_ok(self, tokens) -> bool:
        """
        Check if a (loaded from disk) token is OK or not.

        A quick way is to see IF there is a token, and if the expires_at_utc is still OK
        This way is used to not try to retrieve new tokens for every api call.

        If that is NOT the case, then get a fresh one, save it and the actually check it by checking if an api call
        to the domain/observations/ end point returns a 200.

        tokens look like:
        {
            'access_token': 'gbBSEQBz0lb21CbJcHzYt2YPB1W35V',
            'expires_in': 86400,
            'token_type': 'Bearer',
            'scope': 'read write groups',
            'refresh_token': 'f1C3iLmVC2n6Mg2FaQwK8yNiqUN'
        }
        :param tokens:
        :return: bool if token seems OK or not
        """
        # Try the quick way: only checking the expires key
        if (
            isinstance(tokens, dict)
            and "expires_at_utc" in tokens
            and tokens["expires_at_utc"] - time.time() > 100
        ):  # 100 sec slack
            log.debug(
                f'OK QUICK checked access_token, token will expire at: {tokens["expires_at_local"]}'
            )
            self.access_token = tokens["access_token"]
            return True
        else:
            return False

        # Old way: we try to get one observation from current domain by doing a GET
        # access_token = tokens['access_token']
        # log.debug("access_token/bearer: {}".format(access_token))
        # headers = {'content-type': self.CONTENT_TYPE_HAL_JSON,
        #            'authorization': 'Bearer {}'.format(access_token)}
        # # https://accapi.ndff.nl/api/v2/domains/{domain}/observations/?limit=1
        # uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/observations/')
        # params = {'limit': 1}
        # log.debug(f'GET {uri} to check Token validity. {headers=} {params=} timout={self.NETWORK_TIME_OUT} verify_ssl={self.verify_ssl_certificate}')
        # r = requests.get(uri, headers=headers, params=params, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        # if r.status_code == 200:  # and r.encoding == self.CONTENT_TYPE_JSON:
        #     log.info(f'OK checked access_token for domain {self.domain}, seems to work: {access_token}')
        #     self.access_token = access_token
        #     return True
        # else:
        #     log.warning(f'Checked access_token for domain {self.domain}, seems NOT to work. Is this a user with a "domain-key" maybe?')
        #     # detail is giving wrong info, only showing title
        #     # r.json()['detail']
        #     log.warning('{}'.format(r.json()['title']))
        #     return False

    def add_domain_key_option(self, uri: str) -> str:
        """
        Certain users have special permissions to perform actions in domains of others.
        The client then should add a query parameter such as: '&domain_key=3a02863211375b08940c9e4733f41' to the uri's

        Because we never know who is using the api (domain owner itself OR a client using a domain key), we actually
        always have to check if the client is set up with a domain-key

        :param uri: an api uri string at which the domain_key is added
        :return: uri with domain_key added IF this config has one
        """
        if self.domain_key and self.domain_key.strip() not in ("-", "None"):
            if "?" in uri:
                if uri.endswith("?"):
                    uri = f"{uri}domain_key={self.domain_key}"
                else:
                    if uri.endswith("&"):
                        uri = f"{uri}domain_key={self.domain_key}"
                    else:
                        uri = f"{uri}&domain_key={self.domain_key}"
            else:
                # no ? in the uri yet, add it
                uri = f"{uri}?domain_key={self.domain_key}"
        return uri

    def test_connection(self) -> NdffResult:
        """
        Test if a connection and credentials are actually all valid by doing a GET to /<domain>/datasets

        :return: NdffResult instance (which should return 200)
        """

        # https://accapi.ndff.nl/api/v2/domains/{domain}/datasets
        # api behaviour (May 2023):
        # using a domain key of another organisation you are NOT able to see the root / of the domain,
        # BUT you can get all datasets from the organisation...
        # THAT is why we check if the connection is OK on /<domain>/datasets?limit=1 and not /<domain>
        uri = self.add_domain_key_option(
            f"{self.api_url}/{self.domain}/datasets?limit=1"
        )
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(
            f"GET {uri}: one dataset of current domain to test the connection and permission"
        )
        response = requests.get(
            uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, "test_connection")

    def get_data(self, ndff_uri=None):
        """
        Get one or one(!) page of whatever objects from current Api user/domain.
        The idea of the 'uri' parameter is that the user/client checks(!) IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.

        IMPORTANT
        The NdffApi instance is NOT responsible for the aggregation of the pages!
        The api-user (connector or client) is!

        :param ndff_uri: potential 'next'-href to get the next page
        :raise: a NdffLibError in case called without a ndff_uri
        :return: NdffResult expecting a 200 as http status
        """
        if ndff_uri:
            # use that one
            pass
        else:
            raise NdffLibError("Trying to use get_data() without URL", errors=[1])

        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(f"GET {ndff_uri}")
        response = requests.get(
            ndff_uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, "test_connection")

    def get_waarneming(self, ndff_id_or_uri) -> NdffResult:
        """
        Fetch an Observation/Waarneming via the API, either via a ndff_id OR on full url

        :param ndff_id_or_uri:
        :return: NdffResult instance (which should return 200)
        """
        if f"{ndff_id_or_uri}".upper().startswith("HTTP"):
            uri = ndff_id_or_uri
        else:
            uri = self.add_domain_key_option(
                f"{self.api_url}/{self.domain}/observations/{ndff_id_or_uri}"
            )
        headers = {
            "content-type": self.CONTENT_TYPE_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        response = requests.get(
            uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200)

    def has_related_codes(self, ndff_uri=None) -> bool:
        """
        Only check IF this ndff_uri has related codes, by doing a GET and inspecting the results in the NdffResult

        NOTE: API searches using the NDFF URL, but connector searches/pages using the abundancy_schema URI

        :param: ndff_uri NDFF uri with potential related codes
        :return: bool if this uri has related codes or not
        """
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(f"GET {ndff_uri} to check if there are related codes")
        response = requests.get(
            ndff_uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        result = self.handle_response(response, 200)
        if (
            "related_uri" in result and result["related_uri"]
        ):  # there IS an actual related_uri in the result...
            related_uri = result["related_uri"]
            # now do a GET to actually receive (all, or just max?) the related codes
            log.debug(f"GET {related_uri} to find related codes")
            response = requests.get(
                related_uri,
                headers=headers,
                timeout=self.NETWORK_TIME_OUT,
                verify=self.verify_ssl_certificate,
            )
            result = self.handle_response(response, 200)
            if result["http_response"]["count"] > 0:
                return True
        return False

    def get_related_codes(self, ndff_uri=None, search_text=None) -> NdffResult:
        """
        Get one(!) page of related codes (url?) for given ndff_uri

        Using optional search_text to search INTO the related codes

        The idea of the 'ndff_uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.

        The api is NOT responsible for the aggregation of the pages, the api-user (connector or client) itself is.

        NOTE: API searches using the NDFF URL, but connector searches/pages using the abundancy_schema URI

        :param: ndff_uri NDFF uri with related codes
        :return:
        """
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(f"GET {ndff_uri} to check if there are related codes")
        response = requests.get(
            ndff_uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        result = self.handle_response(response, 200)
        if (
            "related_uri" in result and result["related_uri"]
        ):  # there IS an actual related_uri in the result...
            related_uri = result["related_uri"]
            if search_text:
                related_uri = f"{related_uri}?search={search_text}"
            # now do a GET to actually receive (all, or just max?) the related codes
            log.debug(f"GET {related_uri} to find the related codes")
            response = requests.get(
                related_uri,
                headers=headers,
                timeout=self.NETWORK_TIME_OUT,
                verify=self.verify_ssl_certificate,
            )
            return self.handle_response(response, 200)
        else:
            return result

    def post_dataset(
        self, ndff_dataset_json: str, dataset_id: str, epsg="EPSG:4326"
    ) -> NdffResult:
        """
        Sent/create a Dataset to the api by POST-ing Dataset json to the NDFF-api

        :param ndff_dataset_json:
        :param dataset_id: ndff object id, optionally returned in handle_response returned object
        :param epsg: epsg code for 'Content-Crs' header (EPSG:4326 or EPSG:28992)
        :return: NdffResult expecting a 201 as http status
        """
        headers = {
            "Content-Crs": f"{epsg}",
            "content-type": self.CONTENT_TYPE_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        uri = self.add_domain_key_option(f"{self.api_url}/{self.domain}/datasets/")
        log.debug(f"POST to '{uri}':\n{ndff_dataset_json}")
        response = requests.post(
            uri,
            headers=headers,
            data=ndff_dataset_json,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 201, dataset_id)

    def put_dataset(
        self, ndff_dataset_json, ndff_uri, dataset_id, epsg="EPSG:4326"
    ) -> NdffResult:
        """
        Update a Dataset by sending a PUT to it's NDFF dataset(id) uri

        :param ndff_dataset_json:
        :param ndff_uri:
        :param dataset_id: ndff object id, optionally returned in handle_response returned object
        :param epsg: epsg code for 'Content-Crs' header (EPSG:4326 or EPSG:28992)
        :return: NdffResult expecting a 200 as http status
        """
        headers = {
            "Content-Crs": f"{epsg}",
            "content-type": self.CONTENT_TYPE_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        ndff_uri = self.add_domain_key_option(ndff_uri)
        log.debug(f"PUT to '{ndff_uri}':\n{ndff_dataset_json}")
        response = requests.put(
            ndff_uri,
            headers=headers,
            data=ndff_dataset_json,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, dataset_id)

    def delete_dataset(self, ndff_uri, dataset_id) -> NdffResult:
        """
        Remove a dataset by sending the ndff Dataset uri.

        NOTE: not working in current version of NDFF API!

        :param ndff_uri:
        :param dataset_id: ndff object id, optionally returned in handle_response returned object
        :return: NdffResult expecting a 204 as http status
        """
        headers = {
            "content-type": self.CONTENT_TYPE_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        ndff_uri = self.add_domain_key_option(ndff_uri)
        log.debug(f"DELETE: '{ndff_uri}'")
        response = requests.delete(
            ndff_uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 204, dataset_id)

    def get_datasets(self, ndff_uri=None, page_size: int = 25) -> NdffResult:
        """
        Get one or one(!) page of datasets from current Api user/domain.
        The idea of the 'uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.
        The api is NOT responsible for the aggregation of the pages, the api-user itself (connector) is.

        :param ndff_uri: potential 'next'-href to get the next page
        :param page_size: pagesize for paging large datasets
        :return: NdffResult expecting a 200 as http status
        """
        if ndff_uri:
            # use that one
            pass
        else:
            ndff_uri = self.add_domain_key_option(
                f"{self.api_url}/{self.domain}/datasets/"
            )
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        parameters = {"limit": page_size}
        log.debug(f"GET {ndff_uri}")
        response = requests.get(
            ndff_uri,
            headers=headers,
            params=parameters,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, "test_connection")

    def search_dataset(self, dataset_identity) -> NdffResult:
        """
        Search for a Dataset based on it's identity

        :param dataset_identity:
        :return: NdffResult expecting a 200 as http status
        """
        headers = {
            "content-type": self.CONTENT_TYPE_HAL_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        # without the format=json param, we also get json returned, but the 'self' uri is cleaner
        uri = self.add_domain_key_option(f"{self.api_url}/{self.domain}/datasets/")
        parameters = {"identity": f"{dataset_identity}"}
        log.debug(f"Search GET {uri}")
        response = requests.get(
            uri,
            headers=headers,
            params=parameters,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, dataset_identity)

    def get_dataset_types(self, uri=None) -> NdffResult:
        """
        Get one or one(!) page of dataset types from current Api user/domain.
        The idea of the 'uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.
        The api is NOT responsible for the aggregation of the pages, the api-user itself (connector) is.

        :param uri: potential 'next'-href to get the next page
        :return: NdffResult expecting a 200 as http status
        """
        if uri:
            # use that one
            pass
        else:
            uri = self.add_domain_key_option(
                f"{self.api_url}/{self.domain}/datasettypes/"
            )
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(f"GET {uri}")
        response = requests.get(
            uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, "test_connection")

    def get_protocols(self, ndff_uri=None):
        """
        Get one or one(!) page of protocols from current Api user/domain.
        The idea of the 'uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.
        The api is NOT responsible for the aggregation of the pages, the api-user itself (connector) is.

        :param ndff_uri: potential 'next'-href to get the next page
        :return: NdffResult expecting a 200 as http status
        """
        if ndff_uri:
            # use that one
            pass
        else:
            ndff_uri = self.add_domain_key_option(
                f"{self.api_url}/{self.domain}/protocols/"
            )
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(f"GET {ndff_uri}")
        response = requests.get(
            ndff_uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, "test_connection")

    def search_waarneming(
        self, field="identity", field_value="", page_size: int = 25
    ) -> NdffResult:
        """
        One can search Observation(s) at NDFF-API using url's like:
        https://accapi.ndff.nl/api/v2/domains/708/observations/?identity=http://ecoreest.nl/test1/1

        It's also possible to search for a set of Observations using the other fields. For example the following
        would return all observations in given dataset:
        https://accapi.ndff.nl/api/v2/domains/708/observations/?dataset=http://ndff.nl/api-testngb/folders/1671554924123426753


        :param: field_value
        :param: field defaulting to
        :return: NdffResult expecting a 200 as http status
        """
        headers = {
            "content-type": self.CONTENT_TYPE_HAL_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        # without the format=json param, we also get json returned, but the 'self' uri is cleaner
        uri = self.add_domain_key_option(f"{self.api_url}/{self.domain}/observations/")
        parameters = {field: f"{field_value}", "limit": page_size}
        response = requests.get(
            uri,
            headers=headers,
            params=parameters,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, field_value)

    # DELETE kan nog niet worden gebruikt? (zie antwoord onder)
    # omdat dan wel de observation resource wordt verwijderd, maar niet de
    # bijbehorende identity uri, waarmee het onmogelijk wordt om dezelfde
    # resource NOG een keer aan te maken met dezelfde identity uri
    # NEE: zie API-handleiding:
    #     Wanneer een waarneming inhoudelijk wijzigt of wordt gekoppeld aan een andere
    #     waarnemer of in uw database wordt verwijderd, is het van belang dat deze mutatie ook
    #     in de NDFF terecht komt. Het is dan wel van belang dat u de juiste waarneming in de
    #     NDFF kunt verwijderen of wijzigen en daarvoor is de resource-URI of eventueel een door
    #     u opgegeven identity van belang!
    #     In de NDFF wordt een verwijderde waarneming nooit hard verwijderd. Er is altijd sprake
    #     van een soft-delete; de waarneming komt dan op een verwijder-status. Wanneer
    #     dezelfde waarneming toch weer van waarde blijkt kan deze door een wijziging met de
    #     API weer in de NDFF worden gezet. Datasets en personen kunnen niet worden verwijderd
    #     uit de NDFF met de API.
    # Delete gebeurt met de volledige (voor acc en prd unieke) ndff waarneming URL
    def delete_waarneming(self, ndff_waarneming_url) -> NdffResult:
        """
        Remove a Waarneming/Observation by sending the ndff Observation uri.

        NOTE: not working in current version of NDFF API!

        :param ndff_waarneming_url:
        :return: NdffResult expecting a 204 as http status
        """
        headers = {
            "content-type": self.CONTENT_TYPE_HAL_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        response = requests.delete(
            ndff_waarneming_url,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 204, ndff_waarneming_url)

    def post_waarneming(
        self, ndff_waarneming_json: str, waarneming_id: str, epsg="EPSG:4326"
    ) -> NdffResult:
        """
        Sent/create a Waarneming to the api by POST-ing Waarneming json to the NDFF-api

        :param ndff_waarneming_json: string/json version of the Waarneming
        :param waarneming_id: ndff object id, optionally returned in handle_response returned object
        :param epsg: epsg code for 'Content-Crs' header (EPSG:4326 or EPSG:28992)
        :return: NdffResult expecting a 201 as http status
        """
        headers = {
            "Content-Crs": f"{epsg}",
            "content-type": self.CONTENT_TYPE_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        uri = self.add_domain_key_option(f"{self.api_url}/{self.domain}/observations/")
        log.debug(f"POST to '{uri}':\n{ndff_waarneming_json}")
        response = requests.post(
            uri,
            headers=headers,
            data=ndff_waarneming_json,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 201, waarneming_id)

    def put_waarneming(
        self, ndff_waarneming_json, ndff_uri, waarneming_id, epsg="EPSG:4326"
    ) -> NdffResult:
        """
        Update a Waarneming by sending a PUT to it's NDFF Waarneming(id) uri

        :param ndff_waarneming_json:
        :param ndff_uri:
        :param waarneming_id: ndff object id, optionally returned in handle_response returned object
        :param epsg: epsg code for 'Content-Crs' header (EPSG:4326 or EPSG:28992)
        :return: NdffResult expecting a 200 as http status
        """
        headers = {
            "Content-Crs": f"{epsg}",
            "content-type": self.CONTENT_TYPE_JSON,
            "authorization": f"Bearer {self.access_token}",
        }
        ndff_uri = self.add_domain_key_option(ndff_uri)
        log.debug(f"PUT to '{ndff_uri}':\n{ndff_waarneming_json}")
        response = requests.put(
            ndff_uri,
            headers=headers,
            data=ndff_waarneming_json,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, waarneming_id)

    def search_codes(
        self,
        search_text: str = "",
        search_type: str = "",
        search_field: str = None,
        next_link: str = None,
        page_size: int = 25,
    ) -> NdffResult:
        """
        Search code uses the filter or search endpoint of the codes list of the API

        The search_type is the codes type to search for
        (the api endpoints, like abundancies, taxa, extrainfo etc., see https://accapi.ndff.nl/codes/v2/)

        When a 'search_field' is given, that field is used to search in.
        Possible fields: description, identity, indexvalue, name, rank, language, speciesgroup
        For example search in the name field of taxa (NOTE: name searches in both Dutch and Scientific name!!)
        https://accapi.ndff.nl/codes/v2/taxa/?name=pipi&ordering=-indexvalue
        or
        https://accapi.ndff.nl/codes/v2/taxa/?name=paardenbloem&ordering=-indexvalue

        If the 'search_field' is None, do not search over the field endpoint, but over the more general 'search' endpoint
        Either search for search-text using the 'search' endpoint
        https://accapi.ndff.nl/codes/v2/extrainfo/?search=location_id

        When 'search_text' is an uri, search for an object via the identity uri like:
        https://accapi.ndff.nl/codes/v2/extrainfo/?identity=[IDENTITYURI]&ordering=-indexvalue

        Note that it is possible that the result contains a 'next line', meaning we can page over the results.
        It is the responsibility of the api client to do this!

        For a second (or more) page, use the next_link parameter

        :param: search_text the string or url to search for
        :param: search_type the observation fields (like extrainfo, taxa, etc.)
        :param: search_field name or description
        :param: next_link link for next page if not None
        :param: page_size number of results per 'page' from api (default to ndff default 25)
        :return: NdffResult
        """
        # We order on index value?? giving best result first?
        if next_link:
            search_url = next_link
        elif is_uri(search_text):
            # we try to find a code based on identity
            # mmm, no url encoding needed???
            search_url = f"{self.codes_url}/{search_type}/?identity={search_text}&ordering=-indexvalue&limit={page_size}"
        elif search_field:
            # try to find based on a field, e.g. name or description
            search_url = f"{self.codes_url}/{search_type}/?{search_field}={search_text}&ordering=-indexvalue&limit={page_size}"
        else:
            # general search via search endpoint
            search_url = f"{self.codes_url}/{search_type}/?search={search_text}&ordering=-indexvalue&limit={page_size}"
        headers = {"content-type": self.CONTENT_TYPE_JSON}
        log.debug(f"Search code: {search_url} headers: {headers}")
        response = requests.get(
            search_url,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, ok_http_status=200)

    def post_bulkset(
        self, bulk_set_path: Path, description="bulkset", epsg="EPSG:4326"
    ) -> NdffResult:
        """
        Sent/create a bulk set of Waarnemingen to the api by POST-ing to the bulk NDFF-api-endpoint

        https://accapi.ndff.nl/docs/v2/#/api/api_v2_domains_bulksets_create

        :param bulk_set_path: is a Path to either a json bulk input file OR a ZIPPED (valid) json bulk input file
        :description: description of the bulk set (will be returned in the NDFF response)
        :param epsg: epsg code for 'Content-Crs' header (EPSG:4326 or EPSG:28992)
        :return: NdffResult expecting a 201 as http status
        """
        headers = {
            "Content-Crs": f"{epsg}",
            "authorization": f"Bearer {self.access_token}",
        }
        uri = self.add_domain_key_option(f"{self.api_url}/{self.domain}/bulksets/")
        log.debug(f"POST bulk data '{bulk_set_path}' to '{uri}'")

        if zipfile.is_zipfile(bulk_set_path):
            content_type = self.CONTENT_TYPE_ZIP
        else:
            # could be tested more...
            content_type = self.CONTENT_TYPE_JSON
        data = open(bulk_set_path, "rb")
        bulkset_data = {
            "description": (None, description),
            "dataFile": (bulk_set_path.name, data, content_type),
        }
        response = requests.post(
            uri,
            headers=headers,
            files=bulkset_data,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )

        return self.handle_response(response, 201)

    def get_bulkset_result(self, bulkset_uri=None):
        """
        TODO
        :param bulkset_uri: potential 'next'-href to get the next page
        :return: NdffResult expecting a 200 as http status
        """
        headers = {
            "content-type": "json",
            "authorization": f"Bearer {self.access_token}",
        }
        log.debug(f"GET {bulkset_uri}")
        response = requests.get(
            bulkset_uri,
            headers=headers,
            timeout=self.NETWORK_TIME_OUT,
            verify=self.verify_ssl_certificate,
        )
        return self.handle_response(response, 200, "get_bulkset_result")

    # noinspection PyMethodMayBeStatic
    def handle_response(
        self, response: requests.Response, ok_http_status: int, ndff_object_id: str = ""
    ) -> NdffResult:
        """
        Big working horse of the NdffApi class.

        Handling the response and returning a generic NdffResult instance

        Main tasks:
        - It parses the (HTTP) response
        - It checks the expected HTTP status code, and if not OK (200/201), it will try to find extra information or
        messages in the HTTP Response to put in the NdffResult object, so it can be shown by the user.
        - It puts the ndff_object_id in the NdffResult, so it can be used later (for example for POST/PUT round trips)

        :param response: raw Response object (currently requests.Response)
        :param ok_http_status: expected HTTP status number
        :param ndff_object_id:
        :return: NdffResult
        """
        json_data = {}
        ndff_uri = ""
        object_id = ""
        related_uri = None
        if (
            response.text != ""
        ):  # there is probably a text/json content (http 204/DELETE 404/Not Found has NO content)
            try:
                json_data = response.json()
            # sometimes non json is returned, will be handled later
            except (json.decoder.JSONDecodeError, requests.JSONDecodeError):
                pass
        else:
            # a DELETE does not get a json object returned, so we only have an NDFF uri to log
            ndff_object_id = "0"
            ndff_uri = response.url
            log.info(f"response.text == empty: {response=}")

        http_status = response.status_code
        headers = response.headers

        if "X-Rate-Limit-Limit" in headers.keys():
            # Willy-Bas: "Wat ik kan aanraden is om de status 429 af te handelen door een wait
            #   van X-Rate-Limit-Reset+1 uit te voeren.
            #   De +1 is omdat X-Rate-Limit-Reset al op 0 staat in de laatste seconde."
            log.debug(
                f"X-Rate-Limit-Limit:{headers['X-Rate-Limit-Limit']} X-Rate-Limit-Remaining:{headers['X-Rate-Limit-Remaining']} X-Rate-Limit-Reset:{headers['X-Rate-Limit-Reset']}"
            )
            x_rate_limit_remaining = int(headers["X-Rate-Limit-Remaining"])
            if x_rate_limit_remaining < 3:
                x_rate_limit_reset = 15 + int(
                    headers["X-Rate-Limit-Reset"]
                )  # 15 extra seconds, to be gentle
                log.info(
                    f"HTTP_STATUS: {http_status} Hitting NDFF-API 'X-Rate-Limit-Limit', waiting for {x_rate_limit_reset} seconds..."
                )
                time.sleep(x_rate_limit_reset)

        if http_status == ok_http_status:
            # in search results, the data is 'embedded'
            if "_embedded" in json_data:
                if len(json_data["_embedded"]["items"]) > 0:
                    # 20220413 RD: NOT going to only sent the items back, as
                    # then we are missing out on 'next' links etc.
                    # json_data = json_data['_embedded']['items']
                    pass
                else:
                    # resetting http_status to 404 (as we received a 200)
                    http_status = 404
            if "identity" in json_data:
                object_id = json_data["identity"]
            ndff_uri = json_data["_links"]["self"]["href"]
            if "related_codes" in json_data["_links"]:
                related_uri = json_data["_links"]["related_codes"]["href"]
        else:
            log.debug(
                f"Response NOT OK. Expecting: {ok_http_status}, received: {http_status}"
            )
            # Status codes: https://acc-web02.ndff.nl/api/statuscodes/
            if response.status_code == 404:
                # a 404 is returned when trying to GET a NON-existing object
                ndff_uri = response.request.url
            elif response.status_code in (409,):
                # 409 is returned trying to POST an already existing object
                # Try extracting the internal uri/id from the failing body:
                # try to extract the 'self link' from the json of the result
                try:
                    if (
                        "_embedded" in json_data["detail"]
                        and "observation" in json_data["detail"]["_embedded"]
                    ):
                        # not all 409's have this, conflicts do NOT hold an observation...
                        ndff_uri = json_data["detail"]["_embedded"]["observation"][
                            "_links"
                        ]["self"]["href"]
                except NdffLibError:
                    # the identity is the client identity, not so much of use...
                    # but apparently we failed to get the actual uri...
                    pass
                body = json.loads(response.request.body)
                object_id = body["identity"]
            elif response.status_code == 429:
                # a 429 means too many requests in short time
                # the affected object is in the 'instance' object of the response:
                # {
                #     'status': 429,
                #     'type': 'https://acc-web02.ndff.nl/api/statuscodes/#429',
                #     'instance': '/api/v2/domains/708/observations/162136381/',
                #     'detail': 'Het ingestelde maximaal aantal aanvragen per tijdseenheid is overschreden. Opnieuw beschikbaar in 23 seconden.',
                #     'title': 'Te veel aanvragen'
                # }
                # HACK as ndff returns the instance without domain: '/api/v2/domains/708/observations/162136381/', I try to add it myself
                # api_url is something like: 'https://api.ndff.nl/api/v2/domains'
                if "instance" in json_data:
                    ndff_uri = f'{self.api_url.replace("/api/v2/domains", "")}{json_data["instance"]}'
                else:
                    ndff_uri = f' 429 but no "instance" in json_data: "{json_data}"\nresponse: {response}'
                log.debug(
                    f"handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}"
                )
            elif response.status_code == 500:
                # this is an API issue... (probably serverside exception)
                json_data = {"SERVER ERROR": "500"}
                log.debug(
                    f"handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}"
                )
            elif response.status_code in (400, 403):
                # 400 is returned on a PUT/POST with wrong params values
                # 403 is returned when a user does not have enough privileges for a certain request
                if response.content:
                    json_data = json.loads(response.content)
                log.debug(
                    f"handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}"
                )
            else:
                # e.g. 502 is returned when a wrong url is used and is NOT returning json, but html
                json_data = {
                    "title": f"FOUT: {http_status}",
                    "detail": f"Server returned: {response.content}",
                }

        log.debug(
            f"Server response: HTTP {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}"
        )
        ndff_result = NdffResult(
            waarneming_id=ndff_object_id,
            object_type="observation",
            object_id=object_id,
            ndff_uri=ndff_uri,
            http_method=response.request.method,
            http_status=http_status,
            http_response=json_data,
            id=None,
            tstamp=None,
            related_uri=related_uri,
        )
        return ndff_result

    # noinspection PyMethodMayBeStatic
    def print_headers(self, headers_dict) -> None:
        """
        Internal method to debug sent headers

        :param headers_dict: dict with header key/value pairs
        """
        for header, value in headers_dict.items():
            print("{:20}: {}".format(header, value))


if __name__ == "__main__":
    NdffApi()
