import requests
import tempfile
import os
import pickle
import json
import time
import logging
from ..api.object import (
    NdffResult,
)
from pathlib import Path
from ..utils import is_uri
log = logging.getLogger(__name__)


class NdffApiException(Exception):
    """
    A NdffApiException is ...
    """

    def __init__(self, message, errors=None):
        # Call the base class constructor with the parameters it needs
        super().__init__(message)
        # custom code, list of errors?
        self.errors = errors


class Api:
    """
    An Api is ...

    Better to rename to NdffApi for consistency.
    """

    # pickled tokens are in OS-tempdir
    PICKLED_TOKENS = f'{tempfile.gettempdir()}{os.sep}{"ndff_tokens"}'

    NETWORK_TIME_OUT = 50  # SECONDS, tried 15 first, but got ERROR 2022-12-22 15:31:31,121 - HTTPSConnectionPool(host='accapi.ndff.nl', port=443): Read timed out. (read timeout=15)

    ACC_ENVIRONMENT = 'ACC'
    PRD_ENVIRONMENT = 'PRD'

    environment = ACC_ENVIRONMENT  # let's default to ACC

    def __init__(self, api_config: dict = None):
        if api_config is None or api_config == {} or not isinstance(api_config, dict):
            raise NdffApiException('Trying to create an Api connection without any configuration!', errors=[1])
        self.config = api_config

        self.user = api_config.get('user', 'user_from_ndff_api.csv_settings')
        self.password = api_config.get('password', 'password_from_ndff_api.csv_settings')
        if self.user is None or len(self.user) < 2 or self.password is None or len(self.password) < 5:
            raise NdffApiException('Trying to create an Api connection without user or password, is your configuration OK?', errors=[2])

        # defaulting to ACC just to be sure we do not mess up
        self.environment = api_config.get('api_environment', '"ACC" (acceptatie) or "PRD" (productie)')
        if self.environment == 'PRD':
            self.environment = self.PRD_ENVIRONMENT
        else:
            self.environment = self.ACC_ENVIRONMENT

        # NOTE ! ONLY token_url needs to end with '/'
        if self.environment == self.PRD_ENVIRONMENT:
            self.api_url = 'https://api.ndff.nl/api/v2/domains'
            self.codes_url = 'https://api.ndff.nl/codes/v2'
            self.token_url = 'https://api.ndff.nl/o/v2/token/'  # SLASH!
        else:
            self.api_url = 'https://accapi.ndff.nl/api/v2/domains'
            self.codes_url = 'https://accapi.ndff.nl/codes/v2'
            self.token_url = 'https://accapi.ndff.nl/o/v2/token/'  # SLASH!

        self.domain = api_config.get('domain', 'domain_from_ndff_api.csv_settings')
        # either there is a domain key OR it is None/empty
        self.domain_key = api_config.get('domain_key', 'domain_key_from_ndff_api.csv_settings')

        self.client_id = api_config.get('client_id', 'client_id_from_ndff_api.csv_settings')
        self.client_secret = api_config.get('client_secret', 'client_secret_from_ndff_api.csv_settings')

        # The idea is that this is only used by the QGIS plugin (so the plugin will set this, and save in QSettings)
        self.verify_ssl_certificate = api_config.get('verify_ssl_certificate', True)

        if self.domain is None or len(self.domain) < 2 or self.client_id is None or len(self.client_id) < 2 or self.client_secret is None or len(self.client_secret) < 2:
            raise NdffApiException('Trying to create an Api connection without domain, client_id or client_secret, is your configuration OK?', errors=[4])

        self.access_token = False

        """
        - look for a pickled token-set in the users temp dir
        - if there: try the 'access_token':
            if it works: done, 
               just use it, but keep testing it before every call ??
            if not try the 'refresh-token' to get a fresh 'access_token'
                and pickle it to a token-set in the users temp dir
        - if still here that is either No pickles, 
            OR no working access or refresh: 
            do a token call to get fresh tokens and pickle them
        """
        tokens = None
        try:
            if os.path.isfile(self.PICKLED_TOKENS):
                log.debug('Found pickled tokens on disk, checking if working...')
                with open(self.PICKLED_TOKENS, 'rb') as f:
                    tokens = pickle.load(f)
                #if self.token_ok(tokens['access_token']):
                if self.token_ok(tokens):
                    return  # done
                else:
                    tokens = self.refresh_access_token(tokens)
                    if tokens and 'access_token' in tokens and self.token_ok(tokens):
                        return  # done
            # no pickled token found OR refresh failed: just get new ones
            tokens = self.get_new_tokens()
            if tokens and 'access_token' in tokens and self.token_ok(tokens):
                return  # done
        except NdffApiException:
            pass

        # if still not OK, we did all we can to either get or refresh tokens, throw an exception
        if tokens:
            raise NdffApiException('Failed to get or refresh NDFF Api tokens with given credential, please check logs...')
        else:
            raise NdffApiException('Failed to get or refresh NDFF Api tokens, but further information not available...')

    def get_new_tokens(self):
        log.debug('Calling api to get NEW tokens...')
        payload = {'client_id': self.client_id,
                   'client_secret': self.client_secret,
                   'grant_type': 'password',
                   'username': self.user,
                   'password': self.password}
        return self.post_save_tokens(payload)

    def refresh_access_token(self, tokens):
        log.debug('Calling api to REFRESH tokens...')
        refresh_token = tokens['refresh_token']
        payload = {'client_id': self.client_id,
                   'client_secret': self.client_secret,
                   'grant_type': 'refresh_token',
                   'refresh_token': refresh_token}
        return self.post_save_tokens(payload)

    @staticmethod
    def remove_saved_tokens():
        """
        Remove the so called 'self.PICKLED_TOKENS' file, which is actually a
        json file like, but (python) pickled:
        {
            'access_token': 'gbBSEQBz0lb21CbJcHzYt2YPB1W35V',
            'expires_in': 86400,
            'token_type': 'Bearer',
            'scope': 'read write groups',
            'refresh_token': 'f1C3iLmVC2n6Mg2FaQwK8yNiqUN'
        }
        containing an access_token to be used as valid key for authentication...

        This should be removed in case there is trouble with it (like you are
        sending an access token for a domain you do not have access to)...

        :return: True in case something is successfully removed OR there was nothing to remove
        or False in case there WAS a file, but we were unable to remove...
        """
        token = Path(Api.PICKLED_TOKENS)
        if token.exists() and token.is_file():
            try:
                # try to remove it, return False in case of failing
                token.unlink(missing_ok=True)
            except Exception as e:
                log.debug(f'Failing to unlink/remove the ndff _tokens file {e}')
                return False
            return True
        else:
            return False  # missing or non-existing file

    def post_save_tokens(self, payload):
        """
        Either return valid tokens, json like:
        {
            'access_token': 'gbBSEQBz0lb21CbJcHzYt2YPB1W35V',
            'expires_in': 86400,
            'token_type': 'Bearer',
            'scope': 'read write groups',
            'refresh_token': 'f1C3iLmVC2n6Mg2FaQwK8yNiqUN'
        }
        containing an access_token to be used as valid key for authentication...

        OR TODO idea was to return an empty dict, but now we are throwing exceptions...
        """
        log.debug(f'POST to "{self.token_url}", SSL-validation: {self.verify_ssl_certificate}, DATA: {payload}')
        try:
            r = requests.post(self.token_url, data=payload, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        except Exception as e:
            raise NdffApiException(e)
        if r.status_code == 200:
            tokens = r.json()
            # adding some human-readable/check-able stuff, so we know easier if token is still valid
            t = time.time()
            tokens['expires_at_utc'] = t + tokens['expires_in']
            tokens['expires_at_local'] = time.asctime(time.localtime(t+tokens['expires_in']))
            with open(self.PICKLED_TOKENS, 'wb') as f:
                pickle.dump(tokens, f)
            log.debug(f'Status = {r.status_code}, tokens seem OK and saved: {tokens}')
            return tokens
        elif r.status_code == 400:
            message = f'Status = {r.status_code}, wrong credentials, or check if you are maybe using an old or wrong saved api token. Remove {self.PICKLED_TOKENS} file ??'
            log.warning(message)
        elif r.status_code == 401:  # 401 == Unauthorized
            message = f'Status = {r.status_code} Unauthorized, did you offer the right credentials? Error message = {r.json()["error"]}'
            log.warning(message)
        else:
            message = f'Status = {r.status_code}, r.text={r.text} r = {r}'
            log.info(message)
        # https://stackoverflow.com/questions/2052390/manually-raising-throwing-an-exception-in-python
        raise NdffApiException(message)

    def token_ok(self, tokens):
        """
        tokens look like:
        {
            'access_token': 'gbBSEQBz0lb21CbJcHzYt2YPB1W35V',
            'expires_in': 86400,
            'token_type': 'Bearer',
            'scope': 'read write groups',
            'refresh_token': 'f1C3iLmVC2n6Mg2FaQwK8yNiqUN'
        }
        :param tokens:
        :return:
        """
        # the quick way
        now = time.time()
        expires_at_utc = tokens['expires_at_utc']
        if expires_at_utc - now > 100:  # 100 sec slack
            log.info(f'OK QUICK checked access_token, token will expires at: {tokens["expires_at_local"]}')
            self.access_token = tokens['access_token']
            return True

        # else try to get new ones
        access_token = tokens['access_token']
        log.debug("access_token/bearer: {}".format(access_token))
        headers = {'content-type': 'application/hal+json',
                   'authorization': 'Bearer {}'.format(access_token)}
        # https://accapi.ndff.nl/api/v2/domains/{domain}/observations/?limit=1
        uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/observations/')
        params = {'limit': 1}
        log.debug(f'GET {uri} to check Token validity. {headers=} {params=} timout={self.NETWORK_TIME_OUT} verify_ssl={self.verify_ssl_certificate}')
        r = requests.get(uri, headers=headers, params=params, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        if r.status_code == 200:  # and r.encoding == 'application/hal+json':
            log.info(f'OK checked access_token for domain {self.domain}, seems to work: {access_token}')
            self.access_token = access_token
            return True
        else:
            log.warning(f'Checked access_token for domain {self.domain}, seems NOT to work. Is this a user with a "domain-key" maybe?')
            # detail is giving wrong info, only showing title
            # r.json()['detail']
            log.warning('{}'.format(r.json()['title']))
            return False

    def add_domain_key_option(self, uri):
        """
        Certain users have special permissions to perform actions in domains of others.
        The api should add a query parameter such as: '&domain_key=3a02863211375b08940c9e4733f41' to the uri's
        :param uri:
        :return: uri with domain_key added IF this config has one
        """
        if self.domain_key and self.domain_key.strip() not in ('-', 'None'):
            if '?' in uri:
                if uri.endswith('?'):
                    uri = f'{uri}domain_key={self.domain_key}'
                else:
                    if uri.endswith('&'):
                        uri = f'{uri}domain_key={self.domain_key}'
                    else:
                        uri = f'{uri}&domain_key={self.domain_key}'
            else:
                # no ? in the uri yet, add it
                uri = f'{uri}?domain_key={self.domain_key}'
        return uri

    def test_connection(self) -> NdffResult:
        # https://accapi.ndff.nl/api/v2/domains/{domain}/datasets
        # Strange api behaviour (May 2023)
        # using a domain key of another organisation you are NOT able to see the root / of the domain,
        # BUT you can get all datasets from the organisation...
        # THAT is why we check if the connection is ok on /<domain>/datasets and nog /<domain>
        uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/datasets')
        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        log.debug(f'GET {uri} the domains to test the connection')
        response = requests.get(uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, 'test_connection')

    def get_waarneming(self, ndff_id_or_uri) -> NdffResult:
        """
        get_waarneming can fetch waarneming both based on ndff_id AND on full url
        
        :param ndff_id_or_uri: 
        :return: 
        """
        if f'{ndff_id_or_uri}'.upper().startswith('HTTP'):
            uri = ndff_id_or_uri
        else:
            uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/observations/{ndff_id_or_uri}')
        headers = {'content-type': 'application/json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        response = requests.get(uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200)

    def has_related_codes(self, ndff_uri=None) -> bool:
        """
        Only check IF the ndff_uri has related codes

        NOTE: API searches using the NDFF URL, but connector searches/pages using the abundancy_schema URI

        :param: ndff_uri NDFF uri with related codes
        :return:
        """
        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        log.debug(f'GET {ndff_uri} to check if there are related codes')
        response = requests.get(ndff_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        result = self.handle_response(response, 200)
        if 'related_uri' in result and result['related_uri']:  # there IS an actual related_uri in the result...
            related_uri = result['related_uri']
            # now do a GET to actually receive (all, or just max?) the related codes
            log.debug(f'GET {related_uri} to find related codes')
            response = requests.get(related_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
            result = self.handle_response(response, 200)
            if result['http_response']['count'] > 0:
                return True
        return False

    def get_related_codes(self, ndff_uri=None, search_text=None) -> NdffResult:
        """
        Get one(!) page of related codes (url?) for given ndff_uri

        Using optional search_text to search INTO the related codes

        The idea of the 'ndff_uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.

        The api is NOT responsible for the aggregation of the pages, the api-user (connector or client) itself is.

        NOTE: API searches using the NDFF URL, but connector searches/pages using the abundancy_schema URI

        :param: ndff_uri NDFF uri with related codes
        :return:
        """
        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        log.debug(f'GET {ndff_uri} to check if there are related codes')
        response = requests.get(ndff_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        result = self.handle_response(response, 200)
        if 'related_uri' in result and result['related_uri']:  # there IS an actual related_uri in the result...
            related_uri = result['related_uri']
            if search_text:
                related_uri = f'{related_uri}?search={search_text}'
            # now do a GET to actually receive (all, or just max?) the related codes
            log.debug(f'GET {related_uri} to find the related codes')
            response = requests.get(related_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
            return self.handle_response(response, 200)
        else:
            return result

    def post_dataset(self, ndff_dataset_json, dataset_id, epsg='EPSG:4326') -> NdffResult:
        headers = {'Content-Crs': f'{epsg}',
                   'content-type': 'application/json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/datasets/')
        log.debug(f"POST to '{uri}':\n{ndff_dataset_json}")
        response = requests.post(uri, headers=headers, data=ndff_dataset_json, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 201, dataset_id)

    def put_dataset(self, ndff_dataset_json, ndff_uri, dataset_id, epsg='EPSG:4326') -> NdffResult:
        headers = {'Content-Crs': f'{epsg}',
                   'content-type': 'application/json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        ndff_uri = self.add_domain_key_option(ndff_uri)
        log.debug(f"PUT to '{ndff_uri}':\n{ndff_dataset_json}")
        response = requests.put(ndff_uri, headers=headers, data=ndff_dataset_json, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, dataset_id)

    def delete_dataset(self, ndff_uri, dataset_id) -> NdffResult:
        headers = {'content-type': 'application/json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        ndff_uri = self.add_domain_key_option(ndff_uri)
        log.debug(f"DELETE: '{ndff_uri}'")
        response = requests.delete(ndff_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, dataset_id)

    def get_datasets(self, ndff_uri=None, page_size: int = 25) -> NdffResult:
        """
        Get one or one(!) page of datasets from current Api user/domain.
        The idea of the 'uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.
        The api is NOT responsible for the aggregation of the pages, the api-user itself (connector) is.

        :param ndff_uri: potential 'next'-href to get the next page
        :param page_size: pagesize for paging large datasets
        :return:
        """
        if ndff_uri:
            # use that one
            pass
        else:
            ndff_uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/datasets/')
        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        parameters = {'limit': page_size}
        log.debug(f'GET {ndff_uri}')
        response = requests.get(ndff_uri, headers=headers, params=parameters, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, 'test_connection')

    def get_data(self, ndff_uri=None):
        """
        Get one or one(!) page of whatever objects from current Api user/domain.
        The idea of the 'uri' parameter is that the user/client checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.

        The api is NOT responsible for the aggregation of the pages, the api-user (connector or client)  itself is.

        :param ndff_uri: potential 'next'-href to get the next page
        :return:
        """
        if ndff_uri:
            # use that one
            pass
        else:
            raise NdffApiException('Trying to use get_data() without URL', errors=[1])

        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        log.debug(f'GET {ndff_uri}')
        response = requests.get(ndff_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, 'test_connection')

    def search_dataset(self, dataset_identity) -> NdffResult:
        headers = {'content-type': 'application/hal+json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        # without the format=json param, we also get json returned, but the 'self' uri is cleaner
        uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/datasets/')
        parameters = {'identity': f'{dataset_identity}'}
        response = requests.get(uri, headers=headers, params=parameters, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, dataset_identity)

    def get_dataset_types(self, uri=None) -> NdffResult:
        """
        Get one or one(!) page of dataset types from current Api user/domain.
        The idea of the 'uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.
        The api is NOT responsible for the aggregation of the pages, the api-user itself (connector) is.

        :param uri: potential 'next'-href to get the next page
        :return:
        """
        if uri:
            # use that one
            pass
        else:
            uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/datasettypes/')
        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        log.debug(f'GET {uri}')
        response = requests.get(uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, 'test_connection')

    def get_protocols(self, ndff_uri=None):
        """
        Get one or one(!) page of protocols from current Api user/domain.
        The idea of the 'uri' parameter is that the user checks IF there is a
        ['_links']['next'] object in the result, and IF so, that it can be
        used to get the next set/page of datasets.
        The api is NOT responsible for the aggregation of the pages, the api-user itself (connector) is.

        :param ndff_uri: potential 'next'-href to get the next page
        :return:
        """
        if ndff_uri:
            # use that one
            pass
        else:
            ndff_uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/protocols/')
        headers = {'content-type': 'json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        log.debug(f'GET {ndff_uri}')
        response = requests.get(ndff_uri, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, 'test_connection')

    def search_waarneming(self, field='identity', field_value='', page_size: int = 25) -> NdffResult:
        """
        One can search Observation(s) at NDFF-API using url's like:
        https://accapi.ndff.nl/api/v2/domains/708/observations/?identity=http://ecoreest.nl/test1/1

        It's also possible to search for a set of Observations using the other fields. For example the following
        would return all observations in given dataset:
        https://accapi.ndff.nl/api/v2/domains/708/observations/?dataset=http://ndff.nl/api-testngb/folders/1671554924123426753


        :param: field_value
        :param: field defaulting to
        :return:
        """
        headers = {'content-type': 'application/hal+json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        # parameters = {'identity': f'{waarneming_identity}', 'format': 'json'}
        # without the format=json param, we also get json returned, but the 'self' uri is cleaner
        uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/observations/')
        parameters = {field: f'{field_value}', 'limit': page_size}
        response = requests.get(uri, headers=headers, params=parameters, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, field_value)

    # DELETE kan nog niet worden gebruikt? (zie antwoord onder)
    # omdat dan wel de observation resource wordt verwijderd, maar niet de
    # bijbehorende identity uri, waarmee het onmogelijk wordt om dezelfde
    # resource NOG een keer aan te maken met dezelfde identity uri
    # NEE: zie API-handleiding:
    #     Wanneer een waarneming inhoudelijk wijzigt of wordt gekoppeld aan een andere
    #     waarnemer of in uw database wordt verwijderd, is het van belang dat deze mutatie ook
    #     in de NDFF terecht komt. Het is dan wel van belang dat u de juiste waarneming in de
    #     NDFF kunt verwijderen of wijzigen en daarvoor is de resource-URI of eventueel een door
    #     u opgegeven identity van belang!
    #     In de NDFF wordt een verwijderde waarneming nooit hard verwijderd. Er is altijd sprake
    #     van een soft-delete; de waarneming komt dan op een verwijder-status. Wanneer
    #     dezelfde waarneming toch weer van waarde blijkt kan deze door een wijziging met de
    #     API weer in de NDFF worden gezet. Datasets en personen kunnen niet worden verwijderd
    #     uit de NDFF met de API.
    # Delete gebeurt met de volledige (voor acc en prd unieke) ndff waarneming URL
    def delete_waarneming(self, ndff_waarneming_url) -> NdffResult:
        headers = {'content-type': 'application/hal+json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        response = requests.delete(ndff_waarneming_url, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 204, ndff_waarneming_url)

    def post_waarneming(self, ndff_waarneming_json, waarneming_id, epsg='EPSG:4326') -> NdffResult:
        headers = {'Content-Crs': f'{epsg}',
                   'content-type': 'application/json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        uri = self.add_domain_key_option(f'{self.api_url}/{self.domain}/observations/')
        log.debug(f"POST to '{uri}':\n{ndff_waarneming_json}")
        response = requests.post(uri, headers=headers, data=ndff_waarneming_json, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 201, waarneming_id)

    def put_waarneming(self, ndff_waarneming_json, ndff_uri, waarneming_id, epsg='EPSG:4326') -> NdffResult:
        headers = {'Content-Crs': f'{epsg}',
                   'content-type': 'application/json',
                   'authorization': 'Bearer {}'.format(self.access_token)}
        ndff_uri = self.add_domain_key_option(ndff_uri)
        log.debug(f"PUT to '{ndff_uri}':\n{ndff_waarneming_json}")
        response = requests.put(ndff_uri, headers=headers, data=ndff_waarneming_json, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, 200, waarneming_id)

    def search_codes(self, search_text: str = '', search_type: str = '', search_field: str = None, next_link: str = None, page_size: int = 25) -> NdffResult:
        """
        Search code uses the filter or search endpoint of the codes list of the API

        The search_type is the codes type to search for
        (the api endpoints, like abundancies, taxa, extrainfo etc, see https://accapi.ndff.nl/codes/v2/)

        When a 'search_field' is given, that field is used to search in.
        Possible fields: description, identity, indexvalue, name, rank, language, speciesgroup
        For example search in the name field of taxa (NOTE: name searches in both Dutch and Scientific name!!)
        https://accapi.ndff.nl/codes/v2/taxa/?name=pipi&ordering=-indexvalue
        or
        https://accapi.ndff.nl/codes/v2/taxa/?name=paardenbloem&ordering=-indexvalue

        If the 'search_field' is None, do not search over the field endpoint, but over the more general 'search' endpoint
        Either search for search-text using the 'search' endpoint
        https://accapi.ndff.nl/codes/v2/extrainfo/?search=location_id

        When 'search_text' is an uri, search for an object via the identity uri like:
        https://accapi.ndff.nl/codes/v2/extrainfo/?identity=[IDENTITYURI]&ordering=-indexvalue

        Note that it is possible that the result contains a 'next line', meaning we can page over the results.
        It is the responsibility of the api client to do this!

        For a second (or more) page, use the next_link parameter

        :param: search_text the string or url to search for
        :param: search_type the observation fields (like extrainfo, taxa, etc.)
        :param: search_field name or description
        :param: next_link link for next page if not None
        :param: page_size number of results per 'page' from api (default to ndff default 25)
        :return: NdffResult
        """
        # We order on index value?? giving best result first?
        if next_link:
            search_url = next_link
        elif is_uri(search_text):
            # we try to find a code based on identity
            # mmm, no url encoding needed???
            search_url = f'{self.codes_url}/{search_type}/?identity={search_text}&ordering=-indexvalue&limit={page_size}'
        elif search_field:
            # try to find based on a field, e.g. name or description
            search_url = f'{self.codes_url}/{search_type}/?{search_field}={search_text}&ordering=-indexvalue&limit={page_size}'
        else:
            # general search via search endpoint
            search_url = f'{self.codes_url}/{search_type}/?search={search_text}&ordering=-indexvalue&limit={page_size}'
        headers = {'content-type': 'application/json'}
        log.debug(f'Search code: {search_url} headers: {headers}')
        response = requests.get(search_url, headers=headers, timeout=self.NETWORK_TIME_OUT, verify=self.verify_ssl_certificate)
        return self.handle_response(response, ok_http_status=200)

    # noinspection PyMethodMayBeStatic
    def handle_response(self, response, ok_http_status, ndff_object_id='') -> NdffResult:
        json_data = {}
        ndff_uri = ''
        object_id = ''
        related_uri = None
        # TODO: find out different kind of 404 messages, some contain text/json others don't
        #if response.status_code != 404 and response.text != '':  # there is probably a text/json content (http 204/DELETE 404/Not Found has NO content)
        if response.text != '':  # there is probably a text/json content (http 204/DELETE 404/Not Found has NO content)
            try:
                json_data = response.json()
                # log.debug('handle_response:\n{}'
                #               .format(json.dumps(json_data,
                #               indent=4, sort_keys=False)))
            # sometimes searching something with a wrong url does NOT return proper json, but just a NOT FOUND html page...
            # so we return the {} as json_data
            except json.decoder.JSONDecodeError:
                pass
            except requests.JSONDecodeError:
                pass
        else:
            # a DELETE does not get a json object returned, so we only have an NDFF uri to log
            ndff_object_id = '0'
            ndff_uri = response.url
            log.info(f'response.text == empty: {response=}')
        http_status = response.status_code
        headers = response.headers

        if 'X-Rate-Limit-Limit' in headers.keys():
            # Willy-Bas: "Wat ik kan aanraden is om de status 429 af te handelen door een wait
            #   van X-Rate-Limit-Reset+1 uit te voeren.
            #   De +1 is omdat X-Rate-Limit-Reset al op 0 staat in de laatste seconde."
            log.debug(f"X-Rate-Limit-Limit:{headers['X-Rate-Limit-Limit']} X-Rate-Limit-Remaining:{headers['X-Rate-Limit-Remaining']} X-Rate-Limit-Reset:{headers['X-Rate-Limit-Reset']}")
            x_rate_limit_remaining = int(headers['X-Rate-Limit-Remaining'])
            if x_rate_limit_remaining < 3:
                x_rate_limit_reset = 15+int(headers['X-Rate-Limit-Reset'])  # 15 extra seconds, to be gentle
                log.info(f"HTTP_STATUS: {http_status} Hitting NDFF-API 'X-Rate-Limit-Limit', waiting for {x_rate_limit_reset} seconds...")
                time.sleep(x_rate_limit_reset)

        if http_status == ok_http_status:
            # in search results, the data is 'embedded'
            if '_embedded' in json_data:
                if len(json_data['_embedded']['items']) > 0:
                    # 20220413 RD: NOT going to only sent the items back, as
                    # then we are missing out on 'next' links etc.
                    #json_data = json_data['_embedded']['items']
                    pass
                else:
                    # resetting http_status to 404 (as we received a 200)
                    http_status = 404
            if 'identity' in json_data:
                object_id = json_data['identity']
            ndff_uri = json_data['_links']['self']['href']
            if 'related_codes' in json_data['_links']:
                related_uri = json_data['_links']['related_codes']['href']
        else:
            log.debug(f"Response NOT OK. Expecting: {ok_http_status}, received: {http_status}")
            # Status codes: https://acc-web02.ndff.nl/api/statuscodes/
            if response.status_code == 404:
                # a 404 is returned when trying to GET a NON-existing object
                ndff_uri = response.request.url
            elif response.status_code in (409, ):
                # 409 is returned trying to POST an already existing object
                # Try extracting the internal uri/id from the failing body:
                # try to extract the 'self link' from the json of the result
                try:
                    if '_embedded' in json_data['detail'] and 'observation' in json_data['detail']['_embedded']:
                        # not all 409's have this, conflicts do NOT hold an observation...
                        ndff_uri = json_data['detail']['_embedded']['observation']['_links']['self']['href']
                except NdffApiException:
                    # the identity is the client identity, not so much of use...
                    # but apparently we failed to get the actual uri...
                    pass
                body = json.loads(response.request.body)
                object_id = body['identity']
            elif response.status_code == 429:
                # a 429 means too many requests in short time
                # the affected object is in the 'instance' object of the response:
                # {
                #     'status': 429,
                #     'type': 'https://acc-web02.ndff.nl/api/statuscodes/#429',
                #     'instance': '/api/v2/domains/708/observations/162136381/',
                #     'detail': 'Het ingestelde maximaal aantal aanvragen per tijdseenheid is overschreden. Opnieuw beschikbaar in 23 seconden.',
                #     'title': 'Te veel aanvragen'
                # }
                # HACK as ndff returns the instance without domain: '/api/v2/domains/708/observations/162136381/', I try to add it myself
                # api_url is something like: 'https://api.ndff.nl/api/v2/domains'
                if 'instance' in json_data:
                    ndff_uri = f'{self.api_url.replace("/api/v2/domains", "")}{json_data["instance"]}'
                else:
                    ndff_uri = f' 429 but no "instance" in json_data: "{json_data}"\nresponse: {response}'
                log.debug(f'handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}')
            elif response.status_code == 500:
                # this is an API issue... (probably serverside exception)
                json_data = {"SERVER ERROR": "500"}
                log.debug(f'handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}')
            elif response.status_code in (400, 403):
                # 400 is returned on a PUT/POST with wrong params values
                # 403 is returned when a user does not have enough privileges for a certain request
                if response.content:
                    json_data = json.loads(response.content)
                log.debug(f'handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}')
            else:
                # e.g. 502 is returned when a wrong url is used and is NOT returning json, but html
                json_data = {'title': f'FOUT: {http_status}', 'detail': f'Server returned: {response.content}'}

        log.debug(f'handle_response: {http_status}\n{json.dumps(json_data, indent=4, sort_keys=False)}')
        ndff_result = NdffResult(
                    waarneming_id=ndff_object_id,
                    object_type='observation',
                    object_id=object_id,
                    ndff_uri=ndff_uri,
                    http_method=response.request.method,
                    http_status=http_status,
                    http_response=json_data,
                    id=None,
                    tstamp=None,
                    related_uri=related_uri,
        )
        return ndff_result

    # noinspection PyMethodMayBeStatic
    def print_headers(self, headers_dict):
        for header, value in headers_dict.items():
            print('{:20}: {}'.format(header, value))


if __name__ == '__main__':
    Api()
