# -*- coding: utf-8 -*-
"""
/***************************************************************************
 PluginMainDialog
 This plugin connects to the NDFF api
                             -------------------
        begin                : 2021-12-20
        git sha              : $Format:%H$
        copyright            : (C) 2021 by Zuidt
        email                : richard@zuidt.nl
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""

import os
from typing import Union
from qgis.PyQt import uic
from qgis.PyQt.QtCore import (
    QSettings,
    Qt,
    QDate,
    QTime,
    QDateTime,
    QModelIndex,
    QUrlQuery,
    QUrl,
    pyqtSignal,
)
from qgis.PyQt.QtWidgets import (
    QDialog,
    QMessageBox,
    QPushButton,
    QFileDialog,
    QApplication,
    QLabel,
    QHeaderView,
    QAbstractItemView,
    QTableView,
)
from qgis.PyQt.QtGui import (
    QStandardItemModel,
    QStandardItem,
    QDesktopServices,
)
from qgis.core import (
    QgsMapLayerProxyModel,
    QgsProject,
    QgsGeometry,
    QgsFeatureRequest,
    QgsTask,
    QgsMessageLog,
    Qgis,
    QgsApplication,
)
from qgis.gui import (
    QgisInterface,
)
from .ext.ndff.connector import (
    NdffConnector,
)
from .ext.ndff.api import (
    NdffObservation,
    NdffApiException,
)
from pathlib import Path
import datetime
import logging
import time
import tempfile

from concurrent.futures import ThreadPoolExecutor, TimeoutError

from .ndffc_plugin_geometry_dialog import PluginGeometryDialog
from .ndffc_plugin_map_or_default_dialog import PluginMapDefaultDialog
from .ndffc_plugin_search_code_dialog import PluginSearchCodeDialog
from .ndffc_plugin_api_credentials_dialog import PluginApiCredentialsDialog
from .ndffc_plugin_extra_info_dialog import PluginExtraInfoDialog
from .ndffc_plugin_dataset_dialog import PluginDatasetDialog
from .ndffc_plugin_results_dialog import PluginResultsDialog
log = logging.getLogger(__name__)

# This loads your .ui file so that PyQt can populate your plugin with the elements from Qt Designer
FORM_CLASS, _ = uic.loadUiType(os.path.join(
    os.path.dirname(__file__), 'ndffc_plugin_dialog_base.ui'))


class PluginMainDialog(QDialog, FORM_CLASS):
    """
    Main NDFF Dialog

    The Dialog tries to save some user history/information:
    - it remembers the last used layer/settings combination in users QSettings
    So it remembers when you closed/opened this dialog where you were earlier.
    ONLY called when OK is clicked/the dialog is accepted

    - it can save for given layer the last used settings directory/set IN
    the project file as a custom layer property (so will also SAVE the project,
    AND/OR ask you to save the project)

    Some things it is doing:
    - when clicked

    """

    # key to be used for the path to the last used NDFF config for given layer
    # saved as 'custom-property' of current layer in the project file (qgs):
    #
    #   <customproperties>
    #     <Option type="Map">
    #       <Option type="QString" name="last_used_ndff_config_for_layer" value="/home/you/ndff-connector/tests/data/notatio_csv/ndff_settings"/>
    #     </Option>
    #   </customproperties>
    LAST_CONFIG_FOR_LAYER = 'last_used_ndff_config_for_layer'

    # QSettings path for the last used layer AND accompanying config dir
    LAST_USED_LAYER_AND_CONFIG = '/ndffc_plugin/last_ndff_layer_and_config'

    # Title to be used for all plugin dialogs (if possible)
    PLUGIN_DIALOGS_TITLE = 'NDFF Connector Plugin'
    # Some Texts to be able to change send button texts
    SEND_DATASET_TXT = 'Verstuur de Volledige Dataset Naar De NDFF'
    STOP_SENDING_TXT = 'STOP Met Versturen Naar De NDFF'

    SENDING_STATE_STOPPED = 0
    SENDING_STATE_BUSY = 1
    SENDING_STATE_STOPPING = 2

    ndff_feedback_signal = pyqtSignal(str)

    def __init__(self, iface: QgisInterface, parent=None):
        """Constructor."""
        super(PluginMainDialog, self).__init__(parent)
        self.iface = iface
        self.setupUi(self)

        self.current_layer = None
        self.connector = None

        # record is a plain dict holding the data fields and data value (of 1/current record)
        self.record = None

        # dict with validated records (uri == key)
        # should be set to None "invalidated" as soon as the FIELD mapping changes OR the LAYER changes....
        self.validated = None
        self.uploaded = None

        self.initial_setup_layer_select()

        self.btn_next_record.clicked.connect(self.btn_next_record_clicked)
        self.btn_send_record_to_ndff.clicked.connect(self.send_one_record_to_ndff)
        self.btn_send_dataset_to_ndff.clicked.connect(self.send_dataset_to_ndff)
        self.btn_validate_dataset.clicked.connect(self.validate_dataset)
        self.btn_api_credentials.clicked.connect(self.show_change_credentials)

        self.btn_save_settings_as.clicked.connect(self.save_new_config_to_disk)
        self.btn_save_current_settings.clicked.connect(self.save_config_to_disk)
        self.btn_open_settings.clicked.connect(self.open_config_from_disk)
        self.btn_clear_ndff_settings.clicked.connect(self.reset_config)

        self.lbl_voorbeeld.setStyleSheet('text-align:left;')
        self.lbl_voorbeeld.clicked.connect(self.show_validation_results)

        self.lbl_ndff_result.setTextInteractionFlags(Qt.LinksAccessibleByMouse | Qt.TextSelectableByKeyboard | Qt.TextSelectableByMouse)  # so user can select results or links in it

        self.search_code_dlg = PluginSearchCodeDialog(parent=self)
        self.search_code_dlg.setModal(True)
        self.search_code_dlg.setWindowModality(Qt.ApplicationModal)  # https://doc.qt.io/qt-5/qt.html#WindowModality-enum
        self.search_code_dlg.finished.connect(self.change_data_mapping_finished)

        self.mapping_dlg = PluginMapDefaultDialog(parent=self)
        # TODO: determine if we want a modal dialog or not (OR only when DEV)
        self.mapping_dlg.setModal(True)
        self.mapping_dlg.setWindowModality(Qt.ApplicationModal)  # https://doc.qt.io/qt-5/qt.html#WindowModality-enum
        self.mapping_dlg.accepted.connect(self.change_field_mapping_finished)

        self.geometry_dlg = PluginGeometryDialog(parent=self)
        self.geometry_dlg.setModal(True)
        self.geometry_dlg.setWindowModality(Qt.ApplicationModal)  # https://doc.qt.io/qt-5/qt.html#WindowModality-enum
        self.geometry_dlg.accepted.connect(self.change_location_finished)

        self.extra_info_dlg = PluginExtraInfoDialog(parent=self)
        self.extra_info_dlg.setWindowModality(Qt.ApplicationModal)  # https://doc.qt.io/qt-5/qt.html#WindowModality-enum
        self.extra_info_dlg.setModal(True)
        self.extra_info_dlg.accepted.connect(self.change_extra_info_finished)

        self.dataset_dlg = PluginDatasetDialog(parent=self)
        self.dataset_dlg.setWindowModality(Qt.ApplicationModal)  # https://doc.qt.io/qt-5/qt.html#WindowModality-enum
        self.dataset_dlg.setModal(True)
        self.dataset_dlg.accepted.connect(self.change_dataset_finished)

        self.btn_delete_extra_information.clicked.connect(self.delete_extra_info_field)

        self.extra_info_model = QStandardItemModel()
        self.tbl_extra_info.setModel(self.extra_info_model)
        # select full row upon clicking on a cell / row
        self.tbl_extra_info.setSelectionBehavior(QTableView.SelectRows)
        # only one row selectable at a time
        self.tbl_extra_info.setSelectionMode(QAbstractItemView.SingleSelection)
        # make table cells non-editable:
        self.tbl_extra_info.setEditTriggers(QAbstractItemView.NoEditTriggers)
        #self.tbl_extra_info.selectionModel().selectionChanged.connect(self.extra_info_row_clicked)
        self.tbl_extra_info.doubleClicked.connect(self.extra_info_row_clicked)

        # connect click events to field buttons which contain a mappable uri
        uri_fields = list(NdffObservation.URI_FIELDS)
        uri_fields.remove('identity')
        uri_fields.remove('dataset')
        for field in uri_fields:
            mapping_button = self.findChild(QPushButton, 'mapping_' + field)
            example_button = self.findChild(QPushButton, 'example_' + field)
            # https://stackoverflow.com/questions/54927194/python-creating-buttons-in-loop-in-pyqt5/54929235
            # strange but working construct: using field as param value does NOT work...
            mapping_button.clicked.connect(lambda m, f=field: self.change_field_mapping(f))
            example_button.clicked.connect(lambda m, f=field: self.change_data_mapping(f))
        # the NON uri fields
        for field in ('identity', 'abundance_value', 'location_buffer', 'period_start', 'period_stop'):
            mapping_button = self.findChild(QPushButton, 'mapping_' + field)
            mapping_button.clicked.connect(lambda m, f=field: self.change_field_mapping(f))
        # the dataset button to show the PluginDatasetDialog
            mapping_button = self.findChild(QPushButton, 'mapping_dataset')
            mapping_button.clicked.connect(lambda m, f='dataset': self.change_dataset_mapping(f))
        # location (example) button
        self.example_location.clicked.connect(self.change_location)

        self.btn_extra_information.clicked.connect(self.change_extra_info)

        self.results_dialog = PluginResultsDialog(self)

        # stuff needed for sending the whole set
        self.skipped = {}
        self.failed = {}
        self.updated = {}
        self.busy_sending = self.SENDING_STATE_STOPPED

        self.ndff_feedback_signal.connect(self.feedback)

    def feedback(self, message: str):
        QApplication.processEvents()
        self.lbl_ndff_result.setText(message)

    def accept(self) -> None:
        super(PluginMainDialog, self).accept()
        # save last used layer and config
        log.debug(f'Saving last_used: {self.get_layer_id_and_config_dir()}')
        QSettings().setValue(self.LAST_USED_LAYER_AND_CONFIG,
                             self.get_layer_id_and_config_dir())
        self.done(QDialog.Accepted)
        log.debug(f'NDFF dialog accepted/closed')

    def reject(self) -> None:
        super(PluginMainDialog, self).reject()
        log.debug(f'NDFF dialog rejected/closed')
        self.hide()

    def initial_setup_layer_select(self):
        # You can NOT set a filter in QtDesigner, see https://github.com/qgis/QGIS/issues/38472,
        # so we do it here (filter contains what you DO want to see)
        self.qgis_layer_select.setFilters(
            QgsMapLayerProxyModel.HasGeometry |
            QgsMapLayerProxyModel.VectorLayer |
            QgsMapLayerProxyModel.NoGeometry |
            QgsMapLayerProxyModel.PluginLayer
        )
        self.qgis_layer_select.setAllowEmptyLayer(True, text='Kies een QGIS data/kaartlaag...')
        # user changing layer OR config? BOTH handled in layer_or_config_changed, ignoring argument
        self.qgis_layer_select.layerChanged.connect(self.layer_changed)

    def show_dialog(self) -> None:
        log.debug(f'NDFF Connector Plugin show_dialog')
        # self.layer_fields_and_values.setEnabled(False)  # NOT necessary as this is handled in layer_or_config_changed ??
        # check if there is a last used layer in settings, which can be found
        last_used = QSettings().value(self.LAST_USED_LAYER_AND_CONFIG, (None, None))
        log.debug(f'Last used layer/config: {last_used}')
        last_layer = last_used[0]
        last_config = last_used[1]
        log.debug(f'Trying to load from settings, layer: "{last_layer}" config: "{last_config}"')
        layer = QgsProject.instance().mapLayer(last_layer)
        log.debug(f'Layer loaded: "{layer}" ')
        if layer:
            self.qgis_layer_select.setLayer(layer)
            self.set_ndff_settings_dir(last_config)
            self.current_layer = layer
        else:
            self.qgis_layer_select.setCurrentIndex(0)
            self.set_ndff_settings_dir()
            self.current_layer = None
            self.btn_next_record.setText('Volgende Record')
        # force an update of the layer widget AND creation of a connector
        # self.layer_or_config_changed() # not needed as qgis_layer_select.setLayer (self.layer_changed()) AND set_ndff_settings_dir()/self.layer_or_config_changed() will already do that...
        self.showNormal()

    def check_connector_credentials(self):
        # try to create an api object from current (connector AND QSettings) settings,
        # in which QSettings is leading!!
        # IF it fails it will throw an exception, and you will get presented a
        # dialog in which the user can fill in all credentials and test them
        # upon OK they will be saved in QSettings of user and used later
        try:
            
            # using self.iface.mainWindow as parent, as we are not yet initialized (potentially)...
            # Creation of a (non visible) PluginApiCredentialsDialog here forces
            # the loading of settings (either from QSettings or from setting files)
            #PluginApiCredentialsDialog(self.connector, parent=self.iface.mainWindow())
            # in the end this gave threading issues/locking of ui
            PluginApiCredentialsDialog.merge_api_settings_from_qsettings_into_connector(self.connector)

            api = self.connector.get_api()  # will either show exception OR just return
            self.lbl_ndff_result.setText(f'NDFF gebruiker (eerder gebruikt): "{api.user}" domain: "{api.domain}" url: {api.api_url}')
        except NdffApiException as e:
            log.info(f'NdffApiException in check_connector_credentials(): {e}\nNow showing the "change credentials" dialog')
            self.show_change_credentials()
        except Exception as ex:
            log.info(f'Exception in check_connector_credentials(): {ex}\nNow showing the "internet issue" dialog')
            QMessageBox.information(self.iface.mainWindow(), self.PLUGIN_DIALOGS_TITLE, "Internet-Verbindingsprobleem...\n\nDeze plugin heeft een werkende internetverbinding nodig")

    def show_change_credentials(self):
        # using self.iface.mainWindow as parent, as we are not yet initialized (potentially)...
        credentials_dlg = PluginApiCredentialsDialog(self.connector, parent=self.iface.mainWindow())
        credentials_dlg.setModal(True)
        credentials_dlg.setWindowModality(Qt.ApplicationModal)  # https://doc.qt.io/qt-5/qt.html#WindowModality-enum
        if credentials_dlg.exec_():
            log.debug("Valid API credentials, closing credentials dialog, showing main dialog")
            self.show_ndff_user(changed=True)
            self.showNormal()
        # else:
        #     log.debug("NO valid API credentials, closing credentials dialog")
        #     QMessageBox.information(self.iface.mainWindow(), self.PLUGIN_DIALOGS_TITLE, "Om verder te gaan moet u echt een geldige NDFF API authenticatie regelen")
        #     #self.done(QDialog.Rejected)  # this would close the main dialog, cleaning up and throwing away user edits..

    def show_ndff_user(self, changed=False):
        try:
            api = self.connector.get_api(fresh_one=changed)
            self.lbl_ndff_result.setText(f'NDFF-omgeving: {"<b>PRODUCTIE (!)</b>" if api.environment == api.PRD_ENVIRONMENT else "Acceptatie/Test"} - - gebruiker {"(AANGEPAST)" if changed else ""}: "{api.user}" - - domain: "{api.domain}"')
            #self.lbl_ndff_result.setStyleSheet("QLabel { color : red; }")  # NO coloring...
        except Exception as e:
            self.lbl_ndff_result.setText('Nog geen geldige NDFF gegevens... Regel dit eerst via de knop "Instellingen API en Gebruiker"')

    def layer_changed(self):
        # check if there is a mapping in the custom property of the layer, so we can use that one...
        # see self.LAST_CONFIG_FOR_LAYER
        # else cleanup old settings path ?? OR NOT...
        layer = self.qgis_layer_select.currentLayer()
        if layer:
            if layer == self.current_layer:
                # nothing changed
                return
            self.current_layer = layer
            last_used_config_for_layer = layer.customProperty(self.LAST_CONFIG_FOR_LAYER, False)
            if last_used_config_for_layer and Path(last_used_config_for_layer).is_dir():
                log.debug(f'Found a "Last Used Config For Layer" for layer {self.current_layer.name()} in project file: {last_used_config_for_layer}')
                self.set_ndff_settings_dir(last_used_config_for_layer)
            else:
                self.set_ndff_settings_dir()
        else:
            # cleanup: No layer == no nothing
            self.record = None
            self.current_layer = None
            self.connector = None
            self.set_ndff_settings_dir()

        # self.layer_or_config_changed()  # not needed as set_ndff_settings_dir will handle this via signal?

    def layer_or_config_changed(self, ignored=True):
        """
        Either when a selected layer is changed, OR the user browsed to a
        new configuration directory, we have to create a new NDFF-Connector object.
        So in this case we do not use the 'ignored' param, but lookup the values in
        both widgets
        """
        log.debug(f'Layer or config changed!')
        self.validated = None
        self.uploaded = None
        layer_id, layer_config_dir = self.get_layer_id_and_config_dir()
        # enable or disable the layer_fields_and_values(fields) based on currentIndex (>0 == True)
        self.layer_fields_and_values.setEnabled(self.qgis_layer_select.currentIndex() > 0)

        # TODO make configurable (so people can have one central 'main/global' settings dir with THEIR preferred settings
        plugin_dir = Path(__file__).parent
        # main config, for now fixed as a directory IN the plugin directory
        main_settings = plugin_dir / 'settings'
        # user/project settings
        user_settings = plugin_dir / 'settings'
        if layer_config_dir in ('', None, '-'):
            log.debug('No project/user settings configured yet, using main settings only')
        else:
            log.debug(f'Using {layer_config_dir} as project/user settings')
            user_settings = layer_config_dir

        self.connector = NdffConnector.create_from_directories(main_settings, user_settings, from_within_qgis=True)
        # ok we have a connector now created from both settings dir
        # because we are in QGIS which does NOT save the ndff-api settings in the settings dir,
        # but gets them from QSettings... we have to be sure we are using real/good/current
        # credentials (and clean settings only have the default main settings...)
        # self.check_connector_credentials() will open credentials dialog ONLY if needed!
        self.check_connector_credentials()
        # this is a connector with a BaseDatasource, we are ourselves responsible for the data records:
        if layer_id:
            # IF the user has a selection in that layer, ONLY get THOSE features
            if len(self.qgis_layer_select.currentLayer().selectedFeatures()) > 0:
                self.connector.set_data_records(self.qgis_layer_select.currentLayer().getSelectedFeatures())
            else:
                # ALL features from the layer
                layer = self.qgis_layer_select.currentLayer()
                feature_iterator = layer.getFeatures(QgsFeatureRequest())
                self.connector.set_data_records(feature_iterator)
            self.next_record()  # can be None, will call show_observation() IF there is a record
            if self.current_layer:
                if len(self.current_layer.selectedFeatures()) > 0:
                    self.btn_next_record.setText(f'Volgende Record (van {len(self.current_layer.selectedFeatures())} geselecteerd)')
                else:
                    self.btn_next_record.setText(f'Volgende Record (van {self.current_layer.featureCount()})')
        else:
            # show dialog anyway
            self.show_observation()

    def get_layer_id_and_config_dir(self) -> list:
        layer = self.qgis_layer_select.currentLayer()
        if layer:
            layer = layer.id()
        config = self.get_ndff_settings_dir()
        # if config == '':
        #     config = None  # thought to save 'None' to QSettings, but that results in @Invalid into it...
        # also thought to return a tuple here, but that also gave QSettings issues...
        return [layer, config]

    def show_ndff_results(self):
        self.results_dialog.lbl_title.setText('NDFF Resultaten:')
        self.results_dialog.set_data({**self.skipped, **self.failed, **self.updated})
        self.results_dialog.show()

    def show_validation_results(self):
        observation = self.show_observation()
        valid, reasons = observation.is_valid()
        if not valid:
            self.results_dialog.lbl_title.setText('Validatie Resultaten:')
            self.results_dialog.set_data(reasons)
            self.results_dialog.show()

    def show_observation(self, show_observation=True) -> NdffObservation:
        """
        The show_observation not only shows one observation at a time, but
        also 'fixes' the location-field in case it was not mapped and should
        be (re)created from a QgsGeometry

        show_observation show the observation given current mappings and record data

        show_observation returns current observation (can be both a valid OR an invalid observation)

        """
        #log.debug(f'show_observation, record: {self.record}')
        if self.record:
            observation = self.connector.map_data_to_ndff_observation(self.record)
            # it's very much possible that we do not have a location here... maybe because we have a QgsGeometry field?
            self.check_fix_qgis_location(observation, self.record)
        else:
            # an 'empty' observation
            observation = NdffObservation()

        if show_observation:
            uri_fields = list(NdffObservation.URI_FIELDS)
            # also add the non-simple non uri fields
            uri_fields.extend(['identity', 'abundance_value', 'dataset', 'period_start', 'period_stop', 'location_buffer', 'location'])
            # here we are setting button and field labels and show the values of the chosen mappings based on current settings:
            for field in uri_fields:
                mapping_button = self.findChild(QPushButton, 'mapping_' + field)
                example_button = self.findChild(QPushButton, 'example_' + field)
                field_label = self.findChild(QLabel, 'lbl_' + field)
                if field_label:
                    field_label.setText(self.connector.get_field_text(field))
                    #field_label.setText(f'{self.connector.get_field_text(field)} ({field})')
                if mapping_button and self.record and self.connector.get_field_mapping(field) in self.record:
                    # we have mappings, data and the data contains the mapping
                    mapping_button.setText(self.connector.get_field_mapping(field))
                    # example_button is the button showing the (example/record) data AND optional mapped URI next to it.
                    # special case, an abundance value ...
                    if field == 'abundance_value':
                        schema_uri = observation.get("abundance_schema")
                        abundance_value = self.record[self.connector.get_field_mapping('abundance_value')]
                        data_mapping_key = f'{schema_uri}:{abundance_value}'
                        if self.connector.get_data_mapping('abundance_value', data_mapping_key):
                            example_button.setText(f'{abundance_value}  -->  {self.connector.get_data_mapping("abundance_value", data_mapping_key)}')
                        else:
                            example_button.setText(f'{observation.get(field)}')
                    # IF value in the record is NOT the same as in the MAPPED observation: we get a mapped value:
                    elif self.record[self.connector.get_field_mapping(field)] != observation.get(field):
                        example_button.setText(f'{self.record[self.connector.get_field_mapping(field)]}  -->  {observation.get(field)}')
                    else:
                        example_button.setText(f'{observation.get(field)}')
                # elif self.record and field in self.record:
                #     # the data contains the field itself (BUT NOT SET BY USER) <= not doing this! user has to explicitly set it!!
                #     mapping_button.setText(field)
                #     example_button.setText(f'{observation.get(field)}')
                elif self.record and self.connector.get_field_default(field):
                    # user actually SET a default for this field
                    mapping_button.setText('Standaardwaarde ingesteld')
                    example_button.setText(f'{self.connector.get_field_default(field)} *')
                elif self.record and self.connector.get_field_url2(field):
                    # there is no default set BUT there is a default available/possible
                    mapping_button.setText('?? *')
                    example_button.setText('<-- Selecteer eerst een veld of een standaardwaarde')
                elif mapping_button:
                    mapping_button.setText('??')
                    example_button.setText('<-- Selecteer eerst een veld')

            # EXTRA INFO key/value pairs...
            # cleanup
            self.extra_info_model.removeRows(0, self.extra_info_model.rowCount())

            # WORKING but... we have no clue about which row came from which field-mapping...
            # for extra in observation.extra_info:
            #     row += 1
            #     #log.debug(f'XTRA INFO: {extra}')
            #     self.extra_info_model.appendRow([QStandardItem(extra['key']), QStandardItem(extra['value'])])
            #     # hide te column with numbers:
            #     self.tbl_extra_info.verticalHeader().setVisible(False)
            #     self.tbl_extra_info.horizontalHeader().setVisible(False)
            #     self.tbl_extra_info.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)

            # because we want to be able to EDIT mappings here, we need to know which
            # field mapping belongs to a certain extra info key/value pair
            # that is why we don't just show key/values here, but go via the fieldmappings
            # AND store them in the model, so on a double click of a row we have
            # BOTH the key/value AND the original fieldmapping(s) of that set
            for field in self.connector.field_mappings.keys():
                if field.startswith('extra_info_identity'):
                    counter = int(field.replace('extra_info_identity_', ''))
                    field_map_key = f'extra_info_identity_{counter}'
                    field_map_value = f'extra_info_value_{counter}'
                    key = self.connector.get_field_mapping(field_map_key)
                    if self.connector.get_field_mapping(field_map_value) in self.record:
                        # value is a mapped field
                        value = self.record[self.connector.get_field_mapping(field_map_value)]
                        #log.debug(f'value: {value}')
                    else:
                        # value is a plain value/text
                        value = self.connector.get_field_mapping(field_map_value)
                        #log.debug(f'value: {value}')
                    #log.debug(f'XTRA INFO: {field_map_key}  -->  {key}     {field_map_value}  -->  {value}')
                    self.extra_info_model.appendRow([QStandardItem(key), QStandardItem(f'{value}'), QStandardItem(field_map_key), QStandardItem(field_map_value)])
                    # hide te column with numbers:
                    self.tbl_extra_info.verticalHeader().setVisible(False)
                    self.tbl_extra_info.horizontalHeader().setVisible(False)
                    self.tbl_extra_info.setColumnHidden(2, True)
                    self.tbl_extra_info.setColumnHidden(3, True)
                    self.tbl_extra_info.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)
                    # make the width's of the columns fit as best as possible:
                    #self.tbl_extra_info.resizeColumnToContents(0)
                    #self.tbl_extra_info.resizeColumnToContents(1)
                    #self.tbl_extra_info.horizontalHeader().setStretchLastSection(True)

            valid, reason = observation.is_valid()

            count = ''
            if self.validated:
                count = f'{len(self.validated.keys())}/{self.current_layer.featureCount()}'

            self.btn_send_dataset_to_ndff.setText(self.SEND_DATASET_TXT)
            if valid:
                self.lbl_voorbeeld.setText(f'Waarneming lijkt OK {count}')
                self.lbl_voorbeeld.setToolTip('Velden lijken OK')
                #self.group_ndff.setEnabled(True)
                self.btn_send_dataset_to_ndff.setEnabled(True)
                self.btn_send_record_to_ndff.setEnabled(True)
            else:
                self.lbl_voorbeeld.setText(f'Waarneming {count} nog onjuist, klik hier voor meer informatie')
                self.lbl_voorbeeld.setToolTip(f'{reason}')
                #self.group_ndff.setEnabled(False)
                self.btn_send_dataset_to_ndff.setEnabled(False)
                self.btn_send_record_to_ndff.setEnabled(False)

            # always show current NDFF user
            self.show_ndff_user()

        return observation

    def check_fix_qgis_location(self, observation: NdffObservation, record: dict = None, show_feedback: bool = True):
        # the location of the observation should now either be set via field mapping,
        # OR all location fields are UNSET, and we can try to create a valid geometry
        # here based on the Geometry of current Feature (IF available)
        #log.debug(f'In show_observation, location now is: {observation.location}')
        # SHOW FEEDBACK IN DIALOG
        if observation.location and isinstance(observation.location, dict):
            # mappings created a valid location: show it
            if self.connector.get_field_mapping('location'):
                feedback = f"{self.connector.get_field_mapping('location')}  -->  {observation.get('location')}"
            else:
                feedback = f"{self.connector.get_field_mapping('location_x')}, {self.connector.get_field_mapping('location_y')}  -->  {observation.get('location')}"
        elif 'geometry' in record and isinstance(record['geometry'], QgsGeometry):
            # We moved the geometry into the connector via WKT... BUT shapely wkt creation does not handle 3D geoms...
            # TODO: find out if it is better to always use WKB !
            if record['geometry'].get().is3D():
                # after asWkb the geometry is a QByteArray, let's cast that to a more pythonic bytes/bytearray for later use in connector/ndff lib
                # bytearray is mutable, bytes are not
                log.info('Geometry contains Z-value: currently DROPPING it!')
                record['geometry'].get().dropZValue()
                temp_loc_object = {'location': bytes(record['geometry'].asWkb())}
            else:
                temp_loc_object = {'location': record['geometry'].asWkt()}
            # valid QgsGeometry in the record, let's make the connector build a location here:
            buffer = self.connector.get_field_default('location_buffer')
            # check if it was mapped (probably not, users do not put that in data...)
            if self.connector.get_field_mapping('location_buffer'):
                buffer = record[self.connector.get_field_mapping('location_buffer')]
            if buffer:
                temp_loc_object['location_buffer'] = buffer
            # SHOW FEEDBACK IN DIALOG
            if self.connector.create_geom(observation, temp_loc_object) and show_feedback:
                # NOW show it
                if len(observation.get('location')) < 120:
                    feedback = f"QGIS  -->  {observation.get('location')}"
                else:
                    feedback = f"QGIS  -->  {observation.get('location')[0:120]} ..."
            else:
                feedback = f"QGIS  -->  FOUT: {observation.get('location')}"
        else:
            # no location
            feedback = ' ?? '

        if show_feedback:
            self.example_location.setText(feedback)
        log.debug('Finished checking geom')

    # TODO user or remove
    def _check_fix_qgis_location(self, observation):
        # the location of the observation should now either be set via field mapping,
        # OR all location fields are UNSET, and we can try to create a valid geometry
        # here based on the Geometry of current Feature (IF available)
        #log.debug(f'In show_observation, location now is: {observation.location}')
        if observation.location and isinstance(observation.location, dict):
            # mappings created a valid location: show it
            if self.connector.get_field_mapping('location'):
                self.example_location.setText(f"{self.connector.get_field_mapping('location')}  -->  {observation.get('location')}")
            else:
                self.example_location.setText(f"{self.connector.get_field_mapping('location_x')}, {self.connector.get_field_mapping('location_y')}  -->  {observation.get('location')}")
        elif 'geometry' in self.record and isinstance(self.record['geometry'], QgsGeometry):
            # We moved the geometry into the connector via WKT... BUT shapely wkt creation does not handle 3D geoms...
            # TODO: find out if it is better to always use WKB !
            if self.record['geometry'].get().is3D():
                # after asWkb the geometry is a QByteArray, let's cast that to a more pythonic bytes/bytearray for later use in connector/ndff lib
                # bytearray is mutable, bytes are not
                log.info('Geometry contains Z-value: currently DROPPING it!')
                self.record['geometry'].get().dropZValue()
                temp_loc_object = {'location': bytes(self.record['geometry'].asWkb())}
            else:
                temp_loc_object = {'location': self.record['geometry'].asWkt()}
            # valid QgsGeometry in the record, let's make the connector build a location here:
            buffer = self.connector.get_field_default('location_buffer')
            # check if it was mapped (probably not, users do not put that in data...)
            if self.connector.get_field_mapping('location_buffer'):
                buffer = self.record[self.connector.get_field_mapping('location_buffer')]
            if buffer:
                temp_loc_object['location_buffer'] = buffer
            if self.connector.create_geom(observation, temp_loc_object):
                # NOW show it
                if len(observation.get('location')) < 120:
                    self.example_location.setText(f"QGIS  -->  {observation.get('location')}")
                else:
                    self.example_location.setText(f"QGIS  -->  {observation.get('location')[0:120]} ...")
            else:
                self.example_location.setText(f"QGIS  -->  FOUT: {observation.get('location')}")
        else:
            # no location
            self.example_location.setText(' ?? ')
        log.debug('Finished checking geom')

    def change_location(self):
        """
        Location can come from several sources:
        - an existing field 'location' or a mapping to a 'location'  (should be wkt then)
        - an existing field 'location_xy' or a mapping to a 'locationxy' (should be floats)
        - non of above, but we can use a QgsGeometry
        """
        self.geometry_dlg.show_dialog(self.connector, self.record, self.current_layer)

    def change_location_finished(self):
        """
        Called after showing and handling the geometry_dialog.
        Either the user set the field mappings there  OR the user removed
        the field mappings and we should try to use the QGIS geometry as location...
        """
        self.show_observation()

    def change_data_mapping(self, field: str):
        # only IF there is a record... (which there always should be ...)
        if self.record is None:
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, "Mmm, dit zou niet moeten gebeuren... Geen data nog???")
            return
        if self.connector.get_field_default(field):
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, "Er is al een standaardwaarde gedefinieerd\n(er hoeft dus geen data mapping gedefinieerd te worden)...")
            return
        # below returns EITHER the original data value, OR an uri (if already mapped in connector mappings)
        observation = self.connector.map_data_to_ndff_observation(self.record)
        # so the search value (dialog) will either search for uri's based on the data value, OR search based on the uri
        # in case the dialog will show the ndff code information
        # not sure if we should actually (also?) show the records (raw) value...
        search_value = observation.get(field)
        if search_value:
            log.debug(f'Change data mapping of field: {field}, current value: {search_value}')
            # special case:
            if field == 'abundance_value':
                # OK, the user clicked on the 'abundance_value' button. it IS possible that current 'abundance_schema'
                # has 'related_codes', IF that is the case, we are going to search for them...
                # BUT in that case we do not pass the search_value ??????????? but the abundance_schema identity ????
                abundance_schema_uri = observation.get('abundance_schema')
                # NOT sending the mapped value here, but getting the actual value to show and search for in the search code dialog:
                abundance_value = observation.get('abundance_value')
                # ONLY open the search dialog if the abundance_schema has related codes...
                if self.connector.identity_has_related_codes(abundance_schema_uri):
                    self.search_code_dlg.setup_gui(self.connector, field, search_value=abundance_value, abundance_schema_uri=abundance_schema_uri)
                else:
                    # NOT going to show the search dialog for values like 10 for exact count...
                    log.debug(f'Mapping: {field}, {abundance_schema_uri} does not seem to have related codes...')
                    return
            else:
                self.search_code_dlg.setup_gui(self.connector, field, search_value=search_value)
            self.search_code_dlg.open()
        else:
            log.debug(f'Value to search mapping for still None? {search_value}')

    def change_data_mapping_finished(self, accepted):
        if not accepted:
            # user Cancelled dialog
            return
        ndff_field = self.search_code_dlg.current_ndff_field
        #field_key = self.search_code_dlg.le_search_value.text()
        # NOT USING self.search_code_dlg.le_search_value.text() as that can be edited by the user!!
        # instead, use the actual value, by remapping it... NOPE wrong, observation can already hold an uri..
        #observation = self.connector.map_data_to_ndff_observation(self.record)
        #field_key = observation.get(field)
        # based on the field (e.g. taxon), we have to get the fields original VALUE in the original record
        field_in_record = ndff_field
        if ndff_field in self.connector.field_mappings:
            if self.connector.get_field_mapping(ndff_field):
                field_in_record = self.connector.get_field_mapping(ndff_field)
            elif self.connector.get_field_default(ndff_field):
                field_in_record = self.connector.get_field_default(ndff_field)
        field_key = self.record[field_in_record]
        field_uri = self.search_code_dlg.le_selected_uri.text()
        # SPECIAL CASE...  when we are mapping the abundance_value... THEN we are using longer keys in the mapping file... as key
        if ndff_field == 'abundance_value':
            schema = 'abundance_schema'
            if 'abundance_schema' in self.connector.field_mappings:
                if self.connector.get_field_mapping('abundance_schema'):
                    schema = self.connector.get_field_mapping('abundance_schema')
                elif self.connector.get_field_default('abundance_schema'):
                    schema = self.connector.get_field_default('abundance_schema')
            field_key = f'{schema}:{field_key}'
        log.debug(f'change_data_mapping_finished; result: {accepted} field: {ndff_field} field_key: {field_key} field_uri: {field_uri}')
        if field_key == field_uri:
            log.debug(f'IGNORING "{field_uri}" as "{field_key}" in the mappings for "{ndff_field}"')
        elif self.connector.set_data_mapping(ndff_field, field_key, field_uri):
            log.debug(f'Added "{field_uri}" as "{field_key}" in the mappings for "{ndff_field}"')
        else:
            log.debug(f'NOT ADDING "{field_uri}" as "{field_key}" in the mappings for "{ndff_field}"')
        # now reload current NdffObservation view
        self.show_observation()

    def change_dataset_mapping(self, field: str):
        """
        Function to let the user change ONE field via the PluginMapDefaultDialog dialog.
        """
        # only IF there is a record... (which there always should be ...)
        if self.record is None:
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, "Mmm, dit zou niet moeten gebeuren... Geen data nog???")
            return
        self.dataset_dlg.show(self.current_layer, self.connector, self.record, field)

    def change_dataset_finished(self):
        self.validated = None
        self.uploaded = None
        self.show_observation()

    def change_field_mapping(self, field: str):
        """
        Function to let the user change ONE field via the PluginMapDefaultDialog dialog.
        """
        # only IF there is a record... (which there always should be ...)
        if self.record is None:
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, "Mmm, dit zou niet moeten gebeuren... Geen data nog???")
            return
        self.mapping_dlg.show(self.current_layer, self.connector, self.record, field)

    def change_field_mapping_finished(self):
        self.validated = None
        self.uploaded = None
        self.show_observation()

    def change_extra_info(self, field_map_key: str = None, field_map_value: str = None):
        """
        This will open the 'extra info' dialog, either for a new one (both arguments
        will be None) OR to EDIT a set, both values will be filled
        """
        self.extra_info_dlg.show(self.current_layer, self.connector, self.record, field_map_key, field_map_value)

    def change_extra_info_finished(self):
        self.show_observation()

    def delete_extra_info_field(self):
        log.debug(f'Selected: {self.tbl_extra_info.selectionModel().selectedRows()}')
        selected_row_indexes = self.tbl_extra_info.selectionModel().selectedRows()
        if len(selected_row_indexes) == 0:
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, "Niets geselecteerd\nSelecteer eerst een rij uit onderstaande Extra Informatie tabel")
        else:
            #log.debug(selected_row_indexes[0].row())
            #log.debug(self.extra_info_model.data(selected_row_indexes[0]))
            #log.debug(self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(0)))
            #log.debug(self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(1)))
            #log.debug(self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(2)))
            #log.debug(self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(3)))
            key = self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(0))
            value = self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(1))
            field_map_key = self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(2))
            field_map_value = self.extra_info_model.data(selected_row_indexes[0].siblingAtColumn(3))
            reply = QMessageBox.question(self, self.PLUGIN_DIALOGS_TITLE, f'Zeker weten dat deze Extra Informatie regel (voor de hele dataset) moet worden verwijderd:\n"{key}" ({field_map_key})\n"{value}" ({field_map_value})')
            if reply == QMessageBox.Yes:
                self.connector.delete_extra_info_field_mapping(field_map_key, field_map_value)
        self.show_observation()

    def extra_info_row_clicked(self, extra_info_row_idx: QModelIndex):
        """
        An extra info row represents one of the 'extra_info'-field mappings...

        So we should not look at the values in that table, but to the 'index' of
        the mapping of that row (like extra_info_identity_2 etc).
        Because we are going to edit that mapping (changing either the uri of
        the key or the fieldname of the value, or removing it...).
        """
        #log.debug(extra_info_row_idx)  # QItemSelection
        if extra_info_row_idx.isValid():  # and extra_info_row_idx.indexes()[0].siblingAtColumn(2).isValid():
            #key = self.extra_info_model.data(extra_info_row_idx.siblingAtColumn(0))
            #value = self.extra_info_model.data(extra_info_row_idx.siblingAtColumn(1))
            field_map_key = self.extra_info_model.data(extra_info_row_idx.siblingAtColumn(2))
            field_map_value = self.extra_info_model.data(extra_info_row_idx.siblingAtColumn(3))
            #log.debug(f'key: {key}  value: {value} fieldmap_key: {field_map_key}  fieldmap_value: {field_map_value}')
            self.change_extra_info(field_map_key, field_map_value)
        else:
            log.debug('Row clicked but non valid index...')  # should never happen

    def btn_next_record_clicked(self):
        """
        Click handler for 'next_record' button
        Note that using that button we always want to actually see/show the observation in the dialog.
        That is why 'show_observation=True'
        """
        self.next_record(show_observation=True)

    def next_record(self, show_observation=True) -> Union[NdffObservation, bool]:
        """
        Method to go to the next (data) record. call next on the data iterator

        When there is a selection (in the QGIS layer), then it will go only over the selected features.

        After receiving the record, it will fix/create a geometry and fix time strings in the data.
        THEN (depending on the show_observation flag) it will update all widgets in the main dialog to reflect the
        current status and record.

        It either returns an NdffObservation or False in case there are no more records.
        """
        #log.debug(f'next_record called, show_observation={show_observation}')
        self.lbl_ndff_result.setText('')  # clean up
        if self.connector:
            #log.debug(f'NEXT: self.connector = {self.connector}')
            try:
                feature = self.connector.next_record()
            except StopIteration as s:
                if show_observation:
                    reply = QMessageBox.question(self, self.PLUGIN_DIALOGS_TITLE,
                                                 'Einde van dataset. Terug naar begin...?')
                    if reply == QMessageBox.Yes:
                        # create new iterator to be able to do next()
                        if len(self.qgis_layer_select.currentLayer().selectedFeatures()) > 0:  # selectedFeatures returns a List
                            iterator = self.qgis_layer_select.currentLayer().getSelectedFeatures()  # getSelectedFeaures returns an Iterator
                        else:
                            iterator = self.qgis_layer_select.currentLayer().getFeatures()
                        self.connector.set_data_records(iterator)
                        feature = self.connector.next_record()
                    else:
                        return False
                else:
                    return False
            self.record = feature.attributeMap()
            # hack to be able to move around a simple dict instead of a QgsFeature:
            # we move the geometry IF available in the field 'geometry'
            if feature.hasGeometry():
                self.record['geometry'] = feature.geometry()
            # FIX special treatment of QDate en QDatetime and QTime!!
            for field in self.record:
                if type(self.record[field]) == QDateTime:
                    self.record[field] = self.record[field].toString('yyyy-MM-ddThh:mm:ss')
                elif type(self.record[field]) == QDate:
                    self.record[field] = self.record[field].toString('yyyy-MM-dd')
                elif type(self.record[field]) == QTime:
                    self.record[field] = self.record[field].toString('hh:mm:ss')
            return self.show_observation(show_observation)
        else:
            log.debug(f'NEXT No connector (yet?)...')
            self.record = None
            return False

    def set_ndff_settings_dir(self, path_dir: str = '', layer_or_config_changed=True):
        """
        Method to either set or unset the visible path in the dialog.
        This can be a QGIS-filechooser OR a simple label.
        Using an empty path_dir or None will clear it.
        """
        if path_dir is None:
            path_dir = ''
        #self.fd_mappings_csv.setFilePath(path_dir)
        self.lbl_ndff_settings_dir.setText(path_dir)
        if layer_or_config_changed:
            self.layer_or_config_changed()

    def get_ndff_settings_dir(self) -> str:
        #return str(self.fd_mappings_csv.filePath()).strip()
        return self.lbl_ndff_settings_dir.text()

    def reset_config(self):
        # will clear all buttons/mappings, and finally create a fresh connector... are you sure?
        reply = QMessageBox.question(self, self.PLUGIN_DIALOGS_TITLE, 'Dit verwijdert al uw ingestelde mappings. Zeker weten?')
        if reply == QMessageBox.Yes:
            log.debug(f'Reset current settings')
        else:
            log.debug(f'NOT Resetting current settings')
            return
        self.set_ndff_settings_dir('', layer_or_config_changed=True)

    def open_config_from_disk(self):
        if self.connector is None:
            return
        last_dir = tempfile.gettempdir()
        # try to get last used config
        last_project_settings = QSettings().value(self.LAST_USED_LAYER_AND_CONFIG, [None, None])[1]
        if last_project_settings:
            last_dir = last_project_settings
        ndff_settings_dir = QFileDialog.getExistingDirectory(
            self,
            f'Open Een Bestaande Configuratie Folder/Map',
            last_dir,
            QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks)
        #log.debug(f'Saving configuration into "{NdffConnector.NDFF_SETTINGS_DIR}" in "{ndff_settings_dir}"')
        if ndff_settings_dir is None or ndff_settings_dir == '':
            # user 'Cancelled' the dialog (in which case QFileDialog.getExistingDirectory returns None or '', so quit saving...
            return
        log.info(f'Opening an (existing) NDFF confiration from disk {ndff_settings_dir}')
        self.set_ndff_settings_dir(ndff_settings_dir, layer_or_config_changed=True)

    def save_new_config_to_disk(self):
        if self.connector is None:
            return
        last_dir = tempfile.gettempdir()
        # try to get last used config
        last_project_settings = QSettings().value(self.LAST_USED_LAYER_AND_CONFIG, [None, None])[1]
        if last_project_settings:
            last_dir = last_project_settings
        ndff_settings_dir = QFileDialog.getExistingDirectory(
            self,
            f'Open Een Folder/Map. Settings worden in een "{NdffConnector.NDFF_SETTINGS_DIR}" subfolder gezet)',
            last_dir,
            QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks)
        #log.debug(f'Saving configuration into "{NdffConnector.NDFF_SETTINGS_DIR}" in "{ndff_settings_dir}"')
        if ndff_settings_dir is None or ndff_settings_dir == '':
            # user 'Cancelled' the dialog (in which case QFileDialog.getExistingDirectory returns None or '', so quit saving...
            return
        self.set_ndff_settings_dir(ndff_settings_dir, layer_or_config_changed=False)
        # make it a Path to be able to check some things
        ndff_settings_dir = Path(ndff_settings_dir)
        if ndff_settings_dir.name != NdffConnector.NDFF_SETTINGS_DIR:
            ndff_settings_dir = ndff_settings_dir / NdffConnector.NDFF_SETTINGS_DIR
        if ndff_settings_dir.is_dir():
            reply = QMessageBox.question(self, self.PLUGIN_DIALOGS_TITLE, 'Deze directory bevat al een Settings/Configuratie. Deze overschrijven?')
            if reply == QMessageBox.Yes:
                log.debug(f'Overwriting existing dir: "{ndff_settings_dir}"')
            else:
                log.debug(f'NOT Overwriting existing dir: "{ndff_settings_dir}"')
                return
        self.save_config_to_disk(ndff_settings_dir)

    def save_config_to_disk(self, ndff_settings_dir=None):
        if self.connector is None:
            return
        if ndff_settings_dir in (None, True, False):  # TODO MESS: when clicked this is called with checked param
            ndff_settings_dir = self.get_ndff_settings_dir()
            ndff_settings_dir = Path(ndff_settings_dir)
        log.debug(f'Saving configuration into "{NdffConnector.NDFF_SETTINGS_DIR}" in "{ndff_settings_dir}"')
        if str(ndff_settings_dir) in (None, '', '.'):
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, f'Probleem bij opslaan in: "{ndff_settings_dir} (geen map geselecteerd)"\nKies een andere map via "Opslaan Als..."')
            return
        if ndff_settings_dir.name != NdffConnector.NDFF_SETTINGS_DIR:
            ndff_settings_dir = ndff_settings_dir / NdffConnector.NDFF_SETTINGS_DIR
        # IF this is a postgres OR a csv layer, QGIS can try to write a datasource.csv for it ...
        with_datasource = False
        if self.current_layer.providerType() == 'delimitedtext':
            qurl = QUrl(self.current_layer.dataProvider().uri().uri().strip())
            path = qurl.path()
            query = QUrlQuery(qurl.query().strip())
            delimiter = query.queryItemValue('delimiter')
            log.debug(f'Creating data_source.csv with datasource_type=csv, csv_delimiter="{delimiter}" and csv_file={path}')
            datasource_settings = {
                'datasource_type': 'csv',
                'csv_delimiter': delimiter,
                'csv_file': path,
            }
            # create a (not by QGIS client used) datasource, which can be written to configuration directory and used by cli
            self.connector.datasource_settings = datasource_settings
            with_datasource = True
        elif self.current_layer.providerType() == 'postgres':
            dsuri = self.current_layer.dataProvider().uri()
            host = dsuri.host()
            dbname = dsuri.database()
            user = dsuri.username()
            password = dsuri.password()
            port = dsuri.port()
            table = dsuri.quotedTablename()
            log.debug(f'Creating data_source.csv with datasource_type=postgres, postgres_host="{host}", postgres_port="{port}", postgres_dbname="{dbname}" postgres_user="{user}", postgres_password="{password}", postgres_table="{table}"')
            datasource_settings = {
                'datasource_type': 'postgres',
                'postgres_host': host,
                'postgres_dbname': dbname,
                'postgres_user': user,
                'postgres_password': password,
                'postgres_port': port,
                'postgres_timeout': '3',  # TODO
                'postgres_table': table,
                'postgres_write_log': 'no',  # TODO # no or yes
            }
            # create a (not by QGIS client used) datasource, which can be written to configuration directory and used by cli
            self.connector.datasource_settings = datasource_settings
            # NOTE!! we also have to 'fix' the field_mapping of the location, as the plugin is 'just' using the geometry
            geometry_column = dsuri.geometryColumn()
            self.connector.set_field_mapping('location', geometry_column)
            self.connector.set_field_mapping('location_x', '')
            self.connector.set_field_mapping('location_y', '')
            with_datasource = True
        else:
            log.warning(f'Not able to create a data_source.csv for a layer with providerType: "{self.current_layer.providerType()}"')
            # write the 'example datasource.csv to the dir
            # TODO
        if self.connector.save_to_directory(ndff_settings_dir, with_datasource):
            self.set_ndff_settings_dir(str(ndff_settings_dir))
            log.debug(f'Setting "{self.LAST_CONFIG_FOR_LAYER}" for this layer to "{self.get_ndff_settings_dir()}"')
            # also save the name of this 'config dir' into the layers custom properties (saved into the QGIS project)
            if self.current_layer:  # BUT only if there IS a layer
                self.current_layer.setCustomProperty(self.LAST_CONFIG_FOR_LAYER, str(ndff_settings_dir))
                QgsProject.instance().write()  # write the custom prop into the project
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, f'Succesvol opgeslagen: \n"{ndff_settings_dir}"')
            return
        QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, f'Probleem bij opslaan van: \n"{ndff_settings_dir}". \nKies een andere map...')

    def send_record_as_observation_to_ndff(self, overwrite=True):
        self.lbl_ndff_result.setText('')
        observation = self.connector.map_data_to_ndff_observation(self.record)
        self.check_fix_qgis_location(observation, self.record)

        # observation SHOULD be valid at this moment, but it is possible that user started with valid mappings and records
        # but there are data issues
        if not observation.is_valid()[0]:
            self.lbl_ndff_result.setText(f'Dit record met identity <a href="{observation.get("identity")}">{observation.get("identity")}</a> heeft een data issue: Repareer en valideer data eerst en probeer opnieuw')
            return -1

        try:
            result = self.connector.sent_observation_to_ndff(observation, overwrite=overwrite, test_modus=False)
        except ValueError as err:
            log.error(err)
            self.lbl_ndff_result.setText(f'Dit record met identity <a href="{observation.get("identity")}">{observation.get("identity")}</a> heeft een geometrie issue: {err}')
            return -1
        except Exception as e:
            self.lbl_ndff_result.setText(f'Er heeft zich een probleem voorgedaan: {e}')
            return -1

        #log.debug(f'Api-call: HTTP Method: {result["http_method"]} HTTP Status: {result["http_status"]} NDFF Uri: {result["ndff_uri"]} \nResult: {result}')
        #log.debug(f'HTTP Response: {result["http_response"]}')
        if str(result["http_status"]) == "400":
            # mmm an issue, find out what. Example response:
            # {"status": 400, "type": "https://acc-web02.ndff.nl/api/statuscodes/#400", "invalid-params": {"periodStart": ["Dit veld mag niet leeg zijn."]}, "title": "Validatie mislukt"}
            response = result['http_response']  # json.loads(result['http_response'])
            if "invalid-params" in response:
                self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} {response["title"]}  {response["invalid-params"]}')
            elif "detail" in response:
                self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} {response["title"]} {response["detail"]}')
            else:
                self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} {response["title"]}')
            return 400
        elif str(result["http_status"]) == "409" and overwrite:
            self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} deze identity <a href="">{observation.get("identity")}</a> bestaat al bij het NDFF...')
            return 409
        elif str(result["http_status"]) == "409":
            self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} deze identity <a href="">{observation.get("identity")}</a> bestaat al bij het NDFF...')
            return 409
        elif str(result["http_status"]) == "201":
            # all OK: created a NEW record
            self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} succesvol NIEUW aangemaakt <a href="{self.connector.get_api().add_domain_key_option(result["ndff_uri"])}">{self.connector.get_api().add_domain_key_option(result["ndff_uri"])}</a>...')
            return 201
        elif str(result["http_status"]) == "200":
            # all OK: UPDATED an existing record
            self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} UPDATE van <a href="{self.connector.get_api().add_domain_key_option(result["ndff_uri"])}">{self.connector.get_api().add_domain_key_option(result["ndff_uri"])}</a> gelukt... ')
            return 200
        elif str(result["http_status"]) == "500":
            self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} NDFF ERROR: er ging iets flink mis aan de API kant (overschrijven met incompatibele inhoud?). Neem evt contact op met de servicedesk')
            return 500
        elif str(result["http_status"]) == "404":
            # 404 means a Not Found...
            # BUT this can be trying to PUT or Search for an Observation URI which is NOT in your own domain
            # OR it can be that a certain (URI)-value within an observation is not OK
            # probably the result when searching for an already available record (from another domain?) which was not found in current user's domain
            # Below a wrong Code URI:
            # {
            #     "status": 404,
            #     "type": "https://acc-web02.ndff.nl/api/statuscodes/#404",
            #     "detail": "Code niet gevonden: http://ndff-ecogrid.nl/codes/domainvalues/survey/surveymethods/unkno",
            #     "title": "Niet gevonden"
            # }
            # THIS is what the api returns if you cannot update/PUT the observation because it is not in your domain:
            # {
            #   "status": 404,
            #   "type": "https://acc-web02.ndff.nl/api/statuscodes/#404",
            #   "instance": "/api/v2/domains/708/observations/162443019/",
            #   "detail": "No Observation matches the given query.",
            #   "title": "Niet gevonden"
            #  }
            response = result['http_response']  # json.loads(result['http_response'])
            if "invalid-params" in response:
                self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} {response["title"]}. {response["invalid-params"]}')
            elif "detail" in response:
                detail = response["detail"]
                if detail == "No Observation matches the given query.":
                    # Ok, user probably tries to PUT an Observation from another domain into it's own domain...
                    # Let's rephrase this a little
                    identity = observation.get("identity")
                    self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} deze identity <a href="{identity}f">{identity}</a> bestaat al ergens bij het NDFF, maar kan niet gevonden worden binnen uw "domain"...')
                else:
                    # probably some URI error for code:
                    self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} {response["title"]}. {detail}')
            else:
                self.lbl_ndff_result.setText(f'NDFF Resultaat: {result["http_status"]} {response["title"]}.')
            return 404

    def send_one_record_to_ndff(self):
        result = self.send_record_as_observation_to_ndff(overwrite=False)
        observation = self.connector.map_data_to_ndff_observation(self.record)
        if result in (409,):
            message_box = QMessageBox(parent=self)
            message_box.setText(f'Deze waarneming\n "{observation.identity}"\nlijkt al aanwezig, wat te doen?')
            btn_overwrite = QPushButton('Deze Overschrijven')
            message_box.addButton(btn_overwrite, QMessageBox.ApplyRole)
            btn_cancel = QPushButton('Stoppen / Annuleren')
            message_box.addButton(btn_cancel, QMessageBox.RejectRole)
            message_box.exec()
            clicked = message_box.clickedButton()
            if clicked == btn_overwrite:
                result = self.send_record_as_observation_to_ndff(overwrite=True)
            elif clicked == btn_cancel:
                return
        # this is either the result of a first try, or of a second try (user clicked 'overwrite')
        if result in (200, 201):
            # add this one as DONE
            if self.uploaded is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
                self.uploaded = {}
            self.uploaded[observation.identity] = True
        elif result in (404, ):
            # this is probably a record from another domain, so searching for it to update, resulted in a 404/Not Found
            # pass as the result is already shown by send_record_as_observation_to_ndff
            pass

    def beep(self):
        # notification: beep!
        # https://stackoverflow.com/questions/6537481/python-making-a-beep-noise
        # https://gitlab.com/rduivenvoorde/ndff-connector-plugin/-/issues/22
        # TODO check if this does not give issues on Windows!!
        print('\a')
        pass

    def send_dataset_to_ndff(self):

        if self.busy_sending == self.SENDING_STATE_BUSY:
            log.warning('Already busy sending data... STOPPING')
            self.busy_sending = self.SENDING_STATE_STOPPING
            return
        else:
            self.busy_sending = self.SENDING_STATE_BUSY

        # We always reset the dataset iterator here !!
        # so we are sure to start with the 'first' record
        if len(self.qgis_layer_select.currentLayer().selectedFeatures()) > 0:  # selectedFeatures returns a List
            # important: FIRST do feature_count THEN the iterator
            # the other way around you will get QGIS-freezes... but I cannot reproduce it outside of the plugin :-(
            feature_count = len(self.qgis_layer_select.currentLayer().selectedFeatures())
            iterator = self.qgis_layer_select.currentLayer().getSelectedFeatures()  # getSelectedFeaures returns an Iterator
        else:
            feature_count = self.current_layer.featureCount()
            iterator = self.qgis_layer_select.currentLayer().getFeatures()

        self.connector.set_data_records(iterator)

        if self.uploaded is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
            self.uploaded = {}
        upload_success = True
        observation = True
        start_time = time.time()

        skip_all = False
        overwrite_all = False
        overwrite_once = False
        questions_asked = False

        self.skipped = {}
        self.failed = {}
        self.updated = {}

        def ask_questions(_observation: NdffObservation, at_start=False):
            message_box = QMessageBox(parent=self)
            btn_cancel = QPushButton('Annuleren')
            btn_overwrite = QPushButton('Alleen Deze Overschrijven')
            btn_overwrite_all = QPushButton('Al Aanwezige Waarnemingen Overschrijven/Updaten')
            btn_skip = QPushButton('Skip Alle Al Aanwezige Waarnemingen')

            if at_start:
                message_box.setText(f'Er zijn {feature_count} waarnemingen, wat te doen wanneer een waarneming al bestaat bij NDFF???')
            else:
                message_box.setText(f'Deze {_observation.identity} lijkt al aanwezig, wat te doen???')
                message_box.addButton(btn_overwrite, QMessageBox.ApplyRole)
            message_box.addButton(btn_overwrite_all, QMessageBox.ApplyRole)
            message_box.addButton(btn_skip, QMessageBox.ApplyRole)
            message_box.addButton(btn_cancel, QMessageBox.RejectRole)

            message_box.exec()
            clicked = message_box.clickedButton()

            _skip_all = False
            _overwrite_all = False
            _overwrite_once = False
            _cancel = False

            if clicked == btn_skip:
                log.debug('SKIP ALL AVAILABLE !!!!!!!!!!!!!!!!!')
                # only when this is during a session, NOT when it is asked before the set
                if not at_start:
                    # add this one as DONE
                    self.uploaded[_observation.identity] = True
                _skip_all = True
            elif clicked == btn_overwrite:
                # ONLY overwrite this one
                log.debug('OVERWRITE ONCE (ONLY THIS ONE) !!!!!!!!!!!!!!!!!')
                _overwrite_once = True
                # only when this is during a session, NOT when it is asked before the set
                if not at_start:
                    _observation = None  # this will redo current one
            elif clicked == btn_overwrite_all:
                log.debug('ALL .... OVERWRITE ALL !!!!!!!!!!!!!!!!!')
                _overwrite_all = True
                # only when this is during a session, NOT when it is asked before the set
                if not at_start:
                    _observation = None  # this will redo current one
            else:
                log.debug('NOTHING CHOSEN (CANCEL) !!!!!!!!!!!!!!!!!')
                _cancel = True
            return _observation, _skip_all, _overwrite_all, _overwrite_once, _cancel

        # we REALLY need a user config directory (to save ndff_log.csv in), so the user HAS TO save the current config
        if self.get_ndff_settings_dir().strip() in ('', None):
            QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE,
                                    f'Bij het opsturen van de hele dataset, moet een log worden weggeschreven in de huidige configuratie-directory.\n'
                                    f'Maar er lijkt nog geen configuratie opgeslagen te zijn.\n'
                                    f'Sla alstublieft de huidige configuratie op via de knop "Opslaan Als Nieuwe NDFF Mapping Configuratie (Save As)"\n'
                                    f'Huidige actie wordt onderbroken nu...')
            self.busy_sending = self.SENDING_STATE_STOPPED
            return

        extra_message = ''

        while upload_success and self.isVisible() and self.busy_sending is not self.SENDING_STATE_STOPPING:

            # are we finished?
            if len(self.uploaded.keys()) == feature_count:
                seconds = int(time.time() - start_time)
                self.beep()
                QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, f"Waarnemingen ({len(self.uploaded.keys())} van {feature_count}) zijn verstuurd naar de NDFF.\n\n"
                                                                         f"{len(self.updated)} updates, {len(self.failed)} fouten en {len(self.skipped)} overgeslagen, in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}\n\n"
                                                                         f"In de map met uw configuratie is een administratie van de geleverde waarnemingen opgenomen.\n\n"
                                                                         f"U kunt een nieuwe dataset laden of de plugin afsluiten.")
                #self.show_ndff_results()
                break
                # below is an alternative
                # self.busy_sending = self.SENDING_STATE_STOPPED
                # self.qgis_layer_select.setCurrentIndex(0)
                # self.layer_or_config_changed()
                # return

            # this if/else part is only to be able to start/init with either current record OR next one (which will check if it is the last of the dataset)
            if observation:  # 'earlier' observation OR True (meaning it is the first one)
                observation = self.next_record()
                # self.show_observation()  # this forces a repaint of current record in view and model; NOT is already part of self.next_record() !!!
            else:
                # grab CURRENT record (and show it again to force that all fields are updated/fixed)
                observation = self.show_observation()

            # now we (should) have either current or a new observation from our dataset
            if observation:
                self.btn_send_dataset_to_ndff.setText(self.STOP_SENDING_TXT)
                if feature_count > 100 and not questions_asked:
                    questions_asked = True
                    observation, skip_all, overwrite_all, overwrite_once, cancel = ask_questions(observation, True)
                    if cancel:
                        break
                #log.warning(f'{observation.identity}')
                #log.debug(f'{self.uploaded.keys()}\n')
                if observation.identity in self.uploaded.keys():
                    # ok we already did this record apparently during our current session...
                    log.debug(f'SKIPPING {observation.identity}, as we already uploaded it during this session...')
                    pass  # ignore and try next record
                else:
                    overwrite = False
                    if overwrite_once:
                        overwrite = True
                        overwrite_once = False  # resetting it...
                    elif overwrite_all:
                        overwrite = True

                    # first attempt (first time overwrite will be False, so we can ask user)
                    result = self.send_record_as_observation_to_ndff(overwrite=overwrite)

                    if result in (200, 201):
                        if overwrite:
                            self.updated[observation.identity] = 'UPDATED'
                        self.uploaded[observation.identity] = True
                        log.debug(f'Sent record {observation.identity} - {len(self.uploaded.keys())} / {feature_count} to NDFF')
                        seconds = int(time.time() - start_time)
                        self.lbl_ndff_result.setText(self.lbl_ndff_result.text() + f'  -- {len(self.uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
                    elif result in (409, ):
                        # OK, this one was already there... ?
                        cancel = False
                        if not skip_all:
                            # let's ask the user:
                            observation, skip_all, overwrite_all, overwrite_once, cancel = ask_questions(observation, False)
                        if skip_all:
                            # add this one as DONE
                            self.uploaded[observation.identity] = True
                            self.skipped[observation.identity] = 'SKIPPED'
                            log.debug(f'{result}: skipping {observation.identity}, al {len(self.uploaded.keys())} record(s) van {feature_count} verwerkt')
                            seconds = int(time.time() - start_time)
                            self.lbl_ndff_result.setText(f'{observation.identity} overgeslagen, al {len(self.uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
                            QApplication.processEvents()
                            continue
                        if cancel:
                            break
                    else:
                        # 400 or ??
                        log.debug(f'Upload to NDFF resulted in: {result}, aborting')
                        #return  # not even show the amount of succesfully sent records!
                        self.failed[observation.identity] = 'FAILED'
                        self.uploaded[observation.identity] = False
                        extra_message = f'{result}: Opsturen van gegevens is afgebroken, probeer 1 enkele waarneming te versturen. '
                        # MMM should we do this?
                        break
            else:
                # ERR or user said we do NOT want to rewind the iterator
                upload_success = False
            QApplication.processEvents()

        self.beep()

        seconds = int(time.time()-start_time)
        self.lbl_ndff_result.setText(f'{extra_message}Er zijn {len(self.uploaded.keys())} record(s) verwerkt ({len(self.updated)} updates, {len(self.failed)} fouten, {len(self.skipped)} geskipped) in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
        log.debug(f'Succesfully send {len(self.uploaded.keys())}/{feature_count} records to NDFF in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
        log.warning(f'Updates: {len(self.updated.keys())}: {self.updated}')
        log.warning(f'Skipped {len(self.skipped.keys())}: {self.skipped}')
        log.warning(f'Failed {len(self.failed.keys())}: {self.failed}')
        self.show_ndff_results()
        self.busy_sending = self.SENDING_STATE_STOPPED
        self.btn_send_dataset_to_ndff.setText(self.SEND_DATASET_TXT)

    # 5843 records in 273s 0:04:33
    # def validate_dataset0(self):
    #     if self.validated is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
    #         self.validated = {}
    #     self.layer_changed()
    #     valid = True
    #     observation = None
    #     start_time = time.time()
    #     i = 0
    #     while valid and self.isVisible():
    #         if observation:
    #             observation = self.next_record()
    #         else:
    #             # grab CURRENT record (and show it again to force that all fields are updated/fixed)
    #             observation = self.show_observation()
    #         if observation:
    #             valid, reason = observation.is_valid()
    #             if valid:
    #                 if observation.identity not in self.validated.keys():
    #                     i += 1
    #                     self.validated[observation.identity] = True
    #                     self.show_observation()  # calling show_observation here to be sure the count in 'validated/of' line is OK
    #                 log.debug(f'{i} - {len(self.validated.keys())} record(s) OK ({observation.identity}) of {self.current_layer.featureCount()} records')
    #                 self.lbl_ndff_result.setText(f'{i} record(s) gechecked...')
    #                 if len(self.validated.keys()) == self.current_layer.featureCount():
    #                     QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, 'Alle waarnemingen zijn gevalideerd en kunnen verstuurd worden naar de NDFF.\n\n'
    #                                                                              'Deze configuratie kunt u in de toekomst opnieuw gebruiken door deze op te slaan (als een nieuwe configuratie).\n')
    #                     valid = False  # break if we (apparently) have validated all available features...
    #             if False:  # extra dev check: some validity rules did not return False, but threw exception in to_ndff_observation_json() TODO make configurable?
    #                 # extra json check
    #                 try:
    #                     observation.to_ndff_observation_json(observation_identity='http://test.nl/1', observation_dataset='http://test.nl/1')
    #                 except Exception as e:
    #                     log.error(f'Exception in to_ndff_observation_json() of observation {observation.identity} e={e}\n{observation}')
    #         else:
    #             valid = False
    #         QApplication.processEvents()
    #     seconds = int(time.time()-start_time)
    #     self.lbl_ndff_result.setText(f'Er zijn {i} record(s) gevalideerd in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')

    # 5843 records in 122s
    # 139s
    # 131s
    def validate_dataset(self):
        if self.validated is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
            self.validated = {}
        self.layer_changed()
        valid = True
        observation = None
        feature_count = self.current_layer.featureCount()
        start_time = time.time()
        i = 0
        self.layer_fields_and_values.setEnabled(False)

        while valid and self.isVisible():
            if observation:
                # normal flow...
                if i % 100 == 0:
                    observation = self.next_record(show_observation=True)
                    self.lbl_ndff_result.setText(f'{i} / {feature_count} record(s) gechecked...')
                    QApplication.processEvents()
                else:
                    observation = self.next_record(show_observation=False)
            else:
                # ONLY first time: grab CURRENT record (and show it again to force that all fields are updated/fixed)
                observation = self.show_observation()
            if observation:
                valid, reason = observation.is_valid()
                if valid:
                    if observation.identity not in self.validated.keys():
                        i += 1
                        self.validated[observation.identity] = True
                    log.debug(f'{i} - {len(self.validated.keys())} record(s) OK ({observation.identity}) of {feature_count} records')
                    if len(self.validated.keys()) == feature_count:
                        self.show_observation()  # just to make the counters be OK
                        self.beep()
                        QMessageBox.information(self, self.PLUGIN_DIALOGS_TITLE, 'Klaar? Alle records lijken valide...\n\nSchrijf huidige instellingen en mappings weg (in de ndff_settings map) door de Huidige Configuratie en Mappings (weer) op te slaan.\n')
                        #valid = False  # break if we (apparently) have validated all available features...
                        break

                # if False:  # extra dev check: some validity rules did not return False, but threw exception in to_ndff_observation_json() TODO make configurable?
                #     # extra json check
                #     try:
                #         observation.to_ndff_observation_json(observation_identity='http://test.nl/1', observation_dataset='http://test.nl/1')
                #     except Exception as e:
                #         log.error(f'Exception in to_ndff_observation_json() of observation {observation.identity} e={e}\n{observation}')
                #         log.error(f'Exception in to_ndff_observation_json() of observation {observation.identity} e={e}\n{observation}')
            else:
                valid = False

        if not valid:
            self.beep()

        self.show_observation()  # calling show_observation here to be sure the 'validation' method is updated
        seconds = int(time.time()-start_time)
        self.lbl_ndff_result.setText(f'Er zijn {i} record(s) gevalideerd in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
        self.layer_fields_and_values.setEnabled(True)

    def open_in_browser(self, link: str) -> object:
        QDesktopServices.openUrl(QUrl(link))

#     def send_dataset_to_ndff_2(self, size=-1, overwrite=True, threads=5, timeout=120):
#         max_records_to_sent = 1000000000  # sorry
#         if size > 0:
#             max_records_to_sent = size
#         count = 0
#         try:
#             start_time = time.time()
#             if self.uploaded is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
#                 self.uploaded = {}
#
#             # We always reset the dataset iterator here !!
#             # so we are sure to start with the 'first' record
#             if len(self.qgis_layer_select.currentLayer().selectedFeatures()) > 0:  # selectedFeatures returns a List
#                 # important: FIRST do feature_count THEN the iterator
#                 # the other way around you will get QGIS-freezes... but I cannot reproduce it outside of the plugin :-(
#                 feature_count = len(self.qgis_layer_select.currentLayer().selectedFeatures())
#                 iterator = self.qgis_layer_select.currentLayer().getSelectedFeatures()  # getSelectedFeaures returns an Iterator
#             else:
#                 feature_count = self.current_layer.featureCount()
#                 iterator = self.qgis_layer_select.currentLayer().getFeatures()
#             self.connector.set_data_records(iterator)
#
#             nc = self.connector
#
#             def task(feature):  # the records retrieved from within QGIS are actually features !!!
#
#                 if count >= max_records_to_sent:
#                     # sorry no easier way to stop...
#                     raise Exception(f'Gepland gestopt na ongeveer(!) {max_records_to_sent} records')
#                 elif count % 10 == 0:
#                     # throw signal?
#                     self.ndff_feedback_signal.emit(f'{count} reoords verwerkt')
#                     QApplication.processEvents()
#
#                 #observation = nc.map_data_to_ndff_observation(record.attributeMap())  # NOT working from within QGIS as this does not take the QDatetime etc from Qt/QGIS into account
#                 # doing about the same without touching ui or self:
#                 record = feature.attributeMap()
#                 # hack to be able to move around a simple dict instead of a QgsFeature:
#                 # we move the geometry IF available in the field 'geometry'
#                 if feature.hasGeometry():
#                     record['geometry'] = feature.geometry()
#                 # FIX special treatment of QDate en QDatetime and QTime!!
#                 for field in record:
#                     if type(record[field]) == QDateTime:
#                         record[field] = record[field].toString('yyyy-MM-ddThh:mm:ss')
#                     elif type(record[field]) == QDate:
#                         record[field] = record[field].toString('yyyy-MM-dd')
#                     elif type(record[field]) == QTime:
#                         record[field] = record[field].toString('hh:mm:ss')
#                 observation = nc.map_data_to_ndff_observation(record)
#                 self.check_fix_qgis_location(observation, record, show_feedback=False)
#
#                 if observation:
#                     valid, reasons = observation.is_valid()
#                     if valid:
#                         results = nc.sent_observation_to_ndff(observation, overwrite=overwrite)
#                         if str(results["http_status"]) == "200":
#                             log.info(f'NDFF Resultaat: UPDATE van {observation.identity} / {results["ndff_uri"]} gelukt (~{count}) ')
#                             self.uploaded[observation.identity] = True
#                         elif str(results["http_status"]) == "201":
#                             log.info(f'NDFF Resultaat: succesvol NIEUW {observation.identity} / {results["ndff_uri"]} aangemaakt (~{count}) ')
#                             self.uploaded[observation.identity] = True
#                         elif str(results["http_status"]) == "429":
#                             # normal 'X-Rate-Limit-Limit' are being h
#                             log.info(f'Hit X-Rate-Limit-Limit (429)... {observation.identity} / {results["ndff_uri"]}')
#                             results = nc.sent_observation_to_ndff(observation, overwrite=overwrite)
#                             if str(results["http_status"]) == "200":
#                                 log.info(f'NDFF Resultaat 2: UPDATE van  {observation.identity} / {results["ndff_uri"]} gelukt (~{count}) ')
#                                 self.uploaded[observation.identity] = True
#                             elif str(results["http_status"]) == "201":
#                                 log.info(f'NDFF Resultaat 2: succesvol NIEUW  {observation.identity} / {results["ndff_uri"]} aangemaakt (~{count}) ')
#                                 self.uploaded[observation.identity] = True
#                             else:
#                                 log.error(f'SECOND TRY 2, ALSO ERR... {observation.identity} {results}')
#                                 api = nc.get_api()
#                                 search_uri = api.add_domain_key_option(f'{api.api_url}/{api.domain}/observations/')
#                                 # OK, we tried twice both failing let's stop:
#                                 raise Exception(f'API limitatie: 2x geprobeerd te uploaden: {observation.identity} {search_uri}?identity={observation.identity}, waarschijnlijk NIET aangemaakt/aangepast:\n{results}')
#                         else:
#                             # not sure what, but apparently no 200, 201 or 429 let's stop
#                             raise Exception(f'Onduidelijke foutmelding ({results["http_status"]} {results})\n\nUpload verder afgebroken...')
#                     else:
#                         raise Exception(f'Waarneming is niet geldig: {reasons}\n\nZorg eerst dat de hele set valide is')
#                 else:
#                     log.info('OBSERVATION = FALSE... END OF DATASET????')
#                     # sorry no easier way to stop...
#                     raise Exception(f'Er lijkt geen waarneming (meer) te zijn: {observation}')
#             # END OF TASK
#
#             with ThreadPoolExecutor(threads, thread_name_prefix='NDFF_CLI') as executor:
#                 # note the timeout !!
#                 for task_result in executor.map(task, nc.data_records, timeout=timeout):
#                     count += 1  # indication(!), as addition is after (parallelized) action
#
#             seconds = int(time.time()-start_time)
#             self.lbl_ndff_result.setText(self.lbl_ndff_result.text() + f'  -- {len(self.uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#             QApplication.processEvents()
#
#         except FileNotFoundError as fnfe:
#             log.error(fnfe)
#             log.error(f'Huidige werk-directory is "{os.getcwd()}"')
#         except StopIteration:
#             log.info(f'Einde van deze dataset bereikt, {count} records verwerkt...')
#         except TimeoutError:
#             log.error(f'TimeOut Error in een van de ( {threads} concurrent ) tasks, Timeout waarde: {timeout}')
#         except Exception as e:
#             if len(str(e)) > 3:
#                 log.error(f'FOUT in fast_sent: {e}')
#             else:
#                 log.error(f'FOUT in fast_sent, error type = {type(e)}: {e}')
#
#     # https://superfastpython.com/threadpoolexecutor-in-python/#Step_1_Create_the_Thread_Pool
#     def send_dataset_to_ndff_1(self, size=-1, overwrite=True, threads=10, timeout=None):
#         max_to_sent = 100000000000  # sorry
#         if size > 0:
#             max_to_sent = size
#         count = 0
#         try:
#
#             start_time = time.time()
#             if self.uploaded is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
#                 self.uploaded = {}
#
#             # We always reset the dataset iterator here !!
#             # so we are sure to start with the 'first' record
#             if len(self.qgis_layer_select.currentLayer().selectedFeatures()) > 0:  # selectedFeatures returns a List
#                 # important: FIRST do feature_count THEN the iterator
#                 # the other way around you will get QGIS-freezes... but I cannot reproduce it outside of the plugin :-(
#                 feature_count = len(self.qgis_layer_select.currentLayer().selectedFeatures())
#                 iterator = self.qgis_layer_select.currentLayer().getSelectedFeatures()  # getSelectedFeaures returns an Iterator
#             else:
#                 feature_count = self.current_layer.featureCount()
#                 iterator = self.qgis_layer_select.currentLayer().getFeatures()
#
#             self.connector.set_data_records(iterator)
#
#             # def task(record):
#             #     observation = nc.map_data_to_ndff_observation(record)
#             #     valid, reasons = observation.is_valid()
#             #     if valid:
#             #         result = nc.sent_observation_to_ndff(observation, overwrite=overwrite)
#             #         if str(result["http_status"]) == "200":
#             #             log.info(f'NDFF Resultaat: UPDATE van {result["ndff_uri"]} gelukt')
#             #         elif str(result["http_status"]) == "201":
#             #             log.info(f'NDFF Resultaat: succesvol NIEUW aangemaakt {result["ndff_uri"]}')
#             #         else:
#             #             log.error(f'ERR... {result} {observation}')
#             #     else:
#             #         raise Exception(f'Waarneming is niet geldig: {reasons}\n\nZorg eerst dat de hele set valide is')
#
#             # with ThreadPoolExecutor(threads, thread_name_prefix='QGIS_NDFF_CONNECTOR') as executor:
#             #     # note the timeout !!
#             #     for result in executor.map(task, nc.data_records, timeout=timeout):
#             #         # retrieve the result / DO it
#             #         result
#
#             upload_success = True
#             with ThreadPoolExecutor(max_workers=threads) as executor:
#                 uploaded = {}
#                 while count < max_to_sent and upload_success:
#
#                     # DANGER: NOT self.connector.next_record() !! already maps to NdffObservation (by calling show_observation())
#                     # NOTE: by setting show_observation=False, we do NOT update the fields_and_mapping_widget
#                     observation = self.next_record(show_observation=True)
#                     if observation:
#                         count += 1
#                         #observation = self.connector.map_data_to_ndff_observation(record)
#                         valid, reasons = observation.is_valid()
#                         seconds = int(time.time()-start_time)
#                         if valid:
#                             result = self.connector.sent_observation_to_ndff(observation, overwrite=overwrite)
#                             if str(result["http_status"]) == "200":
#                                 log.info(f'NDFF Resultaat: UPDATE van {result["ndff_uri"]} gelukt')
#                                 uploaded[observation.identity] = True
#                                 #self.lbl_ndff_result.setText(f' {result["http_status"]} -- {len(uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#                             elif str(result["http_status"]) == "201":
#                                 log.info(f'NDFF Resultaat: succesvol NIEUW aangemaakt {result["ndff_uri"]}')
#                                 uploaded[observation.identity] = True
#                                 #self.lbl_ndff_result.setText(f' {result["http_status"]} -- {len(uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#                             else:
#                                 log.error(result)
#                                 #self.lbl_ndff_result.setText(f' {result["http_status"]} -- {result}')
#                         else:
#                             raise Exception(reasons)
#                             #self.lbl_ndff_result.setText(f' ERRR {result["http_status"]}')
#                     else:
#                         # ERR or user said we do NOT want to rewind the iterator
#                         upload_success = False
#
#                     self.lbl_ndff_result.setText(self.lbl_ndff_result.text() + f'  -- {len(uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#                     if len(uploaded.keys()) % 10:
#                         self.lbl_ndff_result.setText(f' {result["http_status"]} -- {len(uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#                         QApplication.processEvents()
#
#         except FileNotFoundError as fnfe:
#             log.error(fnfe)
#             log.error(f'Huidige werk-directory is "{os.getcwd()}"')
#         except StopIteration as s:
#             log.info(f'Einde van deze dataset bereikt, {count} records verwerkt...')
#         except TimeoutError as t:
#             log.error(f'TimeOut Error in een van de ( {threads} concurrent ) tasks, Timeout waarde:')
#         except Exception as e:
#             log.error(f'Err in fast-log type: {type(e)}')
#             log.error(e)
#
#
#
#     def send_dataset_to_ndff_3(self, size=50, overwrite=True, threads=5, timeout=120):
#         log.info('START send_dataset_to_ndff_3 ')
#         # We always reset the dataset iterator here !!
#         # so we are sure to start with the 'first' record
#         if len(self.qgis_layer_select.currentLayer().selectedFeatures()) > 0:  # selectedFeatures returns a List
#             # important: FIRST do feature_count THEN the iterator
#             # the other way around you will get QGIS-freezes... but I cannot reproduce it outside of the plugin :-(
#             feature_count = len(self.qgis_layer_select.currentLayer().selectedFeatures())
#             iterator = self.qgis_layer_select.currentLayer().getSelectedFeatures()  # getSelectedFeaures returns an Iterator
#         else:
#             feature_count = self.current_layer.featureCount()
#             iterator = self.qgis_layer_select.currentLayer().getFeatures()
#         self.connector.set_data_records(iterator)
#
#         # create one task (which will setup x threads)
#         upload_task = NdffParallelUploader('NdffParallelUploader Taak', size, overwrite, threads, timeout, self.connector)
#         upload_task.ndff_feedback_signal.connect(self.kaas)
#         QgsApplication.taskManager().addTask(upload_task)
#         log.info('START TASK? send_dataset_to_ndff_3 ')
#
#     def kaas(self, message):
#         #self.lbl_ndff_result.setText(self.lbl_ndff_result.text() + f'  -- {len(self.uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#         self.lbl_ndff_result.setText(message)
#
#
# class NdffParallelUploader(QgsTask):
#
#     MESSAGE_CATEGORY = 'NdffParallelUploader'
#
#     ndff_feedback_signal = pyqtSignal(str)
#
#     def __init__(self, description, size=-1, overwrite=True, threads=5, timeout=120, ndff_connector=None):
#         super().__init__(description, QgsTask.CanCancel)
#         self.size = size
#         self.overwrite = overwrite
#         self.threads = threads
#         self.timeout = timeout
#         self.connector = ndff_connector
#         if self.connector is None:
#             self.cancel()
#         # ??
#         self.uploaded = {}
#
#         # move exception from run to finished...
#         self.exception = None
#
#     def run(self):
#         """Here you implement your heavy lifting.
#         Should periodically test for isCanceled() to gracefully
#         abort.
#         This method MUST return True or False.
#         Raising exceptions will crash QGIS, so we handle them
#         internally and raise them in self.finished
#         """
#         log.info(f'Started task "{self.description()}"')
#         # MIJN SPULLEN:
#         max_records_to_sent = 1000000000  # sorry
#         if self.size > 0:
#             max_records_to_sent = self.size
#         count = self.threads
#         try:
#             start_time = time.time()
#             if self.uploaded is None:  # will be set to None/invalidated upon change of layer, field mapping etc etc
#                 self.uploaded = {}
#
#             def task(feature):  # the records retrieved from within QGIS are actually features !!!
#
#                 if count >= max_records_to_sent:
#                     # sorry no easier way to stop...
#                     self.exception = Exception(f'Gepland gestopt na ongeveer(!) {max_records_to_sent} records')
#                     self.ndff_feedback_signal.emit(f'Klaro count: {cound}  max_records_to_sent: {max_records_to_sent}')
#                     return True
#                 elif count < max_records_to_sent - 10:
#                     self.ndff_feedback_signal.emit(f'Bijna klaar: {count}/{max_records_to_sent} records verwerkt (signal)')
#                 elif count % 10 == 0:
#                     # throw signal?
#                     self.ndff_feedback_signal.emit(f'{count} records verwerkt (signal)')
#                     # QApplication.processEvents()  # not in this Thread
#
#                 #observation = nc.map_data_to_ndff_observation(record.attributeMap())  # NOT working from within QGIS as this does not take the QDatetime etc from Qt/QGIS into account
#                 # doing about the same without touching ui or self:
#                 record = feature.attributeMap()
#                 # hack to be able to move around a simple dict instead of a QgsFeature:
#                 # we move the geometry IF available in the field 'geometry'
#                 if feature.hasGeometry():
#                     record['geometry'] = feature.geometry()
#                 # FIX special treatment of QDate en QDatetime and QTime!!
#                 for field in record:
#                     if type(record[field]) == QDateTime:
#                         record[field] = record[field].toString('yyyy-MM-ddThh:mm:ss')
#                     elif type(record[field]) == QDate:
#                         record[field] = record[field].toString('yyyy-MM-dd')
#                     elif type(record[field]) == QTime:
#                         record[field] = record[field].toString('hh:mm:ss')
#                 observation = self.connector.map_data_to_ndff_observation(record)
#
#                 self.check_fix_qgis_location(observation, record, show_feedback=False)
#
#                 if observation:
#                     valid, reasons = observation.is_valid()
#                     if valid:
#
#                         results = self.connector.sent_observation_to_ndff(observation, overwrite=self.overwrite)
#                         if str(results["http_status"]) == "200":
#                             log.info(f'NDFF Resultaat: UPDATE van {observation.identity} / {results["ndff_uri"]} gelukt (~{count}) ')
#                             self.uploaded[observation.identity] = True
#                         elif str(results["http_status"]) == "201":
#                             log.info(f'NDFF Resultaat: succesvol NIEUW {observation.identity} / {results["ndff_uri"]} aangemaakt (~{count}) ')
#                             self.uploaded[observation.identity] = True
#                         elif str(results["http_status"]) == "429":
#                             # normal 'X-Rate-Limit-Limit' are being h
#                             log.info(f'Hit X-Rate-Limit-Limit (429)... {observation.identity} / {results["ndff_uri"]}')
#                             results = self.connector.sent_observation_to_ndff(observation, overwrite=self.overwrite)
#                             if str(results["http_status"]) == "200":
#                                 log.info(f'NDFF Resultaat 2: UPDATE van  {observation.identity} / {results["ndff_uri"]} gelukt (~{count}) ')
#                                 self.uploaded[observation.identity] = True
#                             elif str(results["http_status"]) == "201":
#                                 log.info(f'NDFF Resultaat 2: succesvol NIEUW  {observation.identity} / {results["ndff_uri"]} aangemaakt (~{count}) ')
#                                 self.uploaded[observation.identity] = True
#                             else:
#                                 log.error(f'SECOND TRY 2, ALSO ERR... {observation.identity} {results}')
#                                 api = self.connector.get_api()
#                                 search_uri = api.add_domain_key_option(f'{api.api_url}/{api.domain}/observations/')
#                                 # OK, we tried twice both failing let's stop:
#                                 #raise Exception(f'API limitatie: 2x geprobeerd te uploaden: {observation.identity} {search_uri}?identity={observation.identity}, waarschijnlijk NIET aangemaakt/aangepast:\n{results}')
#                                 self.exception = Exception(f'API limitatie: 2x geprobeerd te uploaden: {observation.identity} {search_uri}?identity={observation.identity}, waarschijnlijk NIET aangemaakt/aangepast:\n{results}')
#                         else:
#                             # not sure what, but apparently no 200, 201 or 429 let's stop
#                             #raise Exception(f'Onduidelijke foutmelding ({results["http_status"]} {results})\n\nUpload verder afgebroken...')
#                             self.exception = Exception(f'Onduidelijke foutmelding ({results["http_status"]} {results})\n\nUpload verder afgebroken...')
#                     else:
#                         #raise Exception(f'Waarneming is niet geldig: {reasons}\n\nZorg eerst dat de hele set valide is')
#                         self.exception = Exception(f'Waarneming is niet geldig: {reasons}\n\nZorg eerst dat de hele set valide is')
#                 else:
#                     log.info('OBSERVATION = FALSE... END OF DATASET????')
#                     # sorry no easier way to stop...
#                     #raise Exception(f'Er lijkt geen waarneming (meer) te zijn: {observation}')
#                     self.exception = Exception(f'Er lijkt geen waarneming (meer) te zijn: {observation}')
#                     self.ndff_feedback_signal.emit(f'Er lijkt geen waarneming (meer) te zijn: {observation}')
#                     return True
#             # END OF TASK
#
#             with ThreadPoolExecutor(self.threads, thread_name_prefix='NDFF_CLI') as executor:
#                 # note the timeout !!
#                 for task_result in executor.map(task, self.connector.data_records, timeout=self.timeout):
#                     count += 1  # indication(!), as addition is after (parallelized) action
#
#             seconds = int(time.time()-start_time)
#             #self.lbl_ndff_result.setText(self.lbl_ndff_result.text() + f'  -- {len(self.uploaded.keys())}/{feature_count} record(s) verwerkt in {seconds} seconden = {str(datetime.timedelta(seconds=seconds))}')
#             #QApplication.processEvents()
#
#         # TODO? DIT NAAR DE FINISHED ?
#         except FileNotFoundError as fnfe:
#             log.error(fnfe)
#             log.error(f'Huidige werk-directory is "{os.getcwd()}"')
#             return False
#         except StopIteration:
#             log.info(f'Einde van deze dataset bereikt, {count} records verwerkt...')
#             return False
#         except TimeoutError:
#             log.error(f'TimeOut Error in een van de ( {self.threads} concurrent ) tasks, Timeout waarde: {self.timeout}')
#             return False
#         except Exception as e:
#             if len(str(e)) > 3:
#                 log.error(f'FOUT in fast_sent: {e}')
#             else:
#                 log.error(f'FOUT in fast_sent, error type = {type(e)}: {e}')
#             return False
#
#         # check isCanceled() to handle cancellation
#         if self.isCanceled():
#             return False
#
#         return True
#
#     def finished(self, result: bool):
#         """
#         This function is automatically called when the task has
#         completed (successfully or not).
#         You implement finished() to do whatever follow-up stuff
#         should happen after the task is complete.
#         finished is always called from the main thread, so it's safe
#         to do GUI operations and raise Python exceptions here.
#         result is the return value from self.run.
#         """
#         if result:
#             QgsMessageLog.logMessage(
#                 'RandomTask "{name}" completed\n'\
#                 'RandomTotal: {total} (with {iterations} '\
#                 'iterations)'.format(
#                   name=self.description(),
#                   total=self.total,
#                   iterations=self.iterations),
#                 self.MESSAGE_CATEGORY, Qgis.Success)
#         else:
#             if self.exception is None:
#                 QgsMessageLog.logMessage(
#                     'RandomTask "{name}" not successful but without '\
#                     'exception (probably the task was manually '\
#                     'canceled by the user)'.format(
#                         name=self.description()),
#                     self.MESSAGE_CATEGORY, Qgis.Warning)
#             else:
#                 QgsMessageLog.logMessage(
#                     'RandomTask "{name}" Exception: {exception}'.format(
#                         name=self.description(),
#                         exception=self.exception),
#                     self.MESSAGE_CATEGORY, Qgis.Critical)
#                 raise self.exception
#
#     def cancel(self):
#         QgsMessageLog.logMessage(
#             'RandomTask "{name}" was canceled'.format(
#                 name=self.description()),
#             self.MESSAGE_CATEGORY, Qgis.Info)
#
#         super().cancel()
#
#     def check_fix_qgis_location(self, observation: NdffObservation, record: dict = None, show_feedback: bool = True):
#         # the location of the observation should now either be set via field mapping,
#         # OR all location fields are UNSET, and we can try to create a valid geometry
#         # here based on the Geometry of current Feature (IF available)
#         #log.debug(f'In show_observation, location now is: {observation.location}')
#         # SHOW FEEDBACK IN DIALOG
#         if observation.location and isinstance(observation.location, dict):
#             # mappings created a valid location: show it
#             if self.connector.get_field_mapping('location'):
#                 feedback = f"{self.connector.get_field_mapping('location')}  -->  {observation.get('location')}"
#             else:
#                 feedback = f"{self.connector.get_field_mapping('location_x')}, {self.connector.get_field_mapping('location_y')}  -->  {observation.get('location')}"
#         elif 'geometry' in record and isinstance(record['geometry'], QgsGeometry):
#             # We moved the geometry into the connector via WKT... BUT shapely wkt creation does not handle 3D geoms...
#             # TODO: find out if it is better to always use WKB !
#             if record['geometry'].get().is3D():
#                 # after asWkb the geometry is a QByteArray, let's cast that to a more pythonic bytes/bytearray for later use in connector/ndff lib
#                 # bytearray is mutable, bytes are not
#                 log.info('Geometry contains Z-value: currently DROPPING it!')
#                 record['geometry'].get().dropZValue()
#                 temp_loc_object = {'location': bytes(record['geometry'].asWkb())}
#             else:
#                 temp_loc_object = {'location': record['geometry'].asWkt()}
#             # valid QgsGeometry in the record, let's make the connector build a location here:
#             buffer = self.connector.get_field_default('location_buffer')
#             # check if it was mapped (probably not, users do not put that in data...)
#             if self.connector.get_field_mapping('location_buffer'):
#                 buffer = record[self.connector.get_field_mapping('location_buffer')]
#             if buffer:
#                 temp_loc_object['location_buffer'] = buffer
#             # SHOW FEEDBACK IN DIALOG
#             if self.connector.create_geom(observation, temp_loc_object) and show_feedback:
#                 # NOW show it
#                 if len(observation.get('location')) < 120:
#                     feedback = f"QGIS  -->  {observation.get('location')}"
#                 else:
#                     feedback = f"QGIS  -->  {observation.get('location')[0:120]} ..."
#             else:
#                 feedback = f"QGIS  -->  FOUT: {observation.get('location')}"
#         else:
#             # no location
#             feedback = ' ?? '
#
#         if show_feedback:
#             self.example_location.setText(feedback)
#         log.debug('Finished checking geom')

