# -*- coding: utf-8 -*-
# Copyright 2018, CS GROUP - France, https://www.csgroup.eu/
#
# This file is part of EODAG project
#     https://www.github.com/CS-SI/EODAG
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime as dt
import hashlib
import os
import re
import shutil
import threading
import time
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

import pytest
from stac_validator import stac_validator

from eodag.api.product.metadata_mapping import ONLINE_STATUS
from tests import TEST_RESOURCES_PATH
from tests.context import (
    GENERIC_COLLECTION,
    AuthenticationError,
    EODataAccessGateway,
    SearchResult,
    sanitize,
    uri_to_path,
)

GEODES_SEARCH_ARGS = [
    "geodes",
    "S2_MSI_L1C",
    "2024-08-08",
    "2024-08-16",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
AWSEOS_SEARCH_ARGS = [
    "aws_eos",
    "S2_MSI_L1C",
    "2020-01-01",
    "2020-01-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
EARTH_SEARCH_SEARCH_ARGS = [
    "earth_search",
    "S2_MSI_L1C",
    "2020-01-01",
    "2020-01-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
EARTH_SEARCH_GCS_SEARCH_ARGS = [
    "earth_search_gcs",
    "S2_MSI_L1C",
    "2020-01-01",
    "2020-01-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
USGS_SATAPI_AWS_SEARCH_ARGS = [
    "usgs_satapi_aws",
    "LANDSAT_C2L1",
    "2020-01-01",
    "2020-01-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
CREODIAS_SEARCH_ARGS = [
    "creodias",
    "S2_MSI_L1C",
    "2019-03-01",
    "2019-03-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
COP_DATASPACE_SEARCH_ARGS = [
    "cop_dataspace",
    "S2_MSI_L1C",
    "2019-03-01",
    "2019-03-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
COP_DATASPACE_S3_SEARCH_ARGS = [
    "cop_dataspace_s3",
    "S2_MSI_L1C",
    "2019-03-01",
    "2019-03-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
PLANETARY_COMPUTER_SEARCH_ARGS = [
    "planetary_computer",
    "S2_MSI_L2A",
    "2019-03-01",
    "2019-03-15",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
HYDROWBEB_NEXT_SEARCH_ARGS = [
    "hydroweb_next",
    "SWOT_PRIOR_RIVER_DATABASE",
    "2022-04-01",
    "2022-04-10",
    [0.2563590566012408, 43.19555008715042, 2.379835675499976, 43.907759172380565],
]
# As of 2021-01-14 the products previously required in 2020-08 were offline.
# Trying here to retrieve the most recent products which are more likely to be online.
today = dt.date.today()
week_span = dt.timedelta(days=7)
day_span = dt.timedelta(days=1)
USGS_RECENT_SEARCH_ARGS = [
    "usgs",
    "LANDSAT_C2L1",
    (today - 6 * week_span).isoformat(),
    (today - 5 * week_span).isoformat(),
    [50, 50, 50.3, 50.3],
]
USGS_OLD_SEARCH_ARGS = [
    "usgs",
    "LANDSAT_C2L1",
    "2017-03-01",
    "2017-03-10",
    [50, 50, 50.3, 50.3],
]
ECMWF_SEARCH_ARGS = [
    "ecmwf",
    "TIGGE_CF_SFC",
    "2017-03-01",
    "2017-03-02",
    # no need of an additional post-processing area extraction
    [-180, -90, 180, 90],
]
ECMWF_SEARCH_KWARGS = {
    # request for only 1 parameter instead of all available
    "param": "tcc",
}
COP_ADS_SEARCH_ARGS = [
    "cop_ads",
    "CAMS_EAC4",
    "2021-01-01",
    "2021-01-05",
    # no need of an additional post-processing area extraction
    [-180, -90, 180, 90],
]
COP_ADS_SEARCH_KWARGS = {
    # request for grib file instead of netcdf
    "format": "grib",
}
COP_CDS_SEARCH_ARGS = [
    "cop_cds",
    "ERA5_SL",
    "2021-01-01",
    "2021-01-02",
    # no need of an additional post-processing area extraction
    [-180, -90, 180, 90],
]
COP_CDS_SEARCH_KWARGS = {
    # request for grib file instead of netcdf
    "format": "grib",
}
COP_EWDS_SEARCH_ARGS = [
    "cop_ewds",
    "EFAS_SEASONAL",
    "2021-01-01",
    "2021-01-05",
    [-180, -90, 180, 90],
]
SARA_SEARCH_ARGS = [
    "sara",
    "S2_MSI_L1C",
    "2020-03-01",
    "2020-03-15",
    [150, -33, 151, -32],
]
METEOBLUE_SEARCH_ARGS = [
    "meteoblue",
    "NEMSGLOBAL_TCDC",
    (today + day_span).isoformat(),
    (today + 2 * day_span).isoformat(),
    [0.2, 43.2, 0.5, 43.5],
]
WEKEO_MAIN_SEARCH_ARGS = [
    "wekeo_main",
    "S5P_L1B_IR_ALL",
    "2018-04-30",
    "2018-11-18",
    [-180, -90, 180, 90],
]
WEKEO_ECMWF_SEARCH_ARGS = [
    "wekeo_ecmwf",
    "ERA5_SL_MONTHLY",
    "2024-01-01",
    "2024-01-01",
    [-180, -90, 180, 90],
]
EUMETSAT_DS_SEARCH_ARGS = [
    "eumetsat_ds",
    "S3_OLCI_L2WFR",
    "2021-03-11",
    "2021-03-11",
    [-69.3363, -75.9038, -39.3465, -68.2361],
]

FEDEO_CEDA_SEARCH_ARGS = [
    "fedeo_ceda",
    "MERIS_ALAMO_L2_V2.2",
    None,
    None,
    None,
]

EOCAT_SEARCH_ARGS = [
    "eocat",
    "GOCE_Thermosphere_Data",
    "2010-03-17",
    "2010-03-18",
    None,
]

COP_GHSL_SEARCH_ARGS = [
    "cop_ghsl",
    "GHS_BUILT_S",
    "2020-01-01",
    "2021-01-01",
    [-69.3363, -75.9038, -39.3465, -68.2361],
]
COP_GHSL_SEARCH_KWARGS = {
    "proj:code": "4326",
    "tile_size": "3ss",
    "classification": "TOTAL",
}


@pytest.mark.enable_socket
class EndToEndBase(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.stac = stac_validator.StacValidate()

    def execute_search(
        self,
        provider,
        collection,
        start,
        end,
        geom,
        offline=False,
        page=None,
        limit=None,
        check_product=True,
        search_kwargs_dict={},
    ):
        """Search products on provider:

        - First set the preferred provider as the one given in parameter
        - Then do the search
        - Then ensure that at least the first result originates from the provider
        - Return one product to be downloaded
        """
        search_criteria = {
            "collection": collection,
            "start": start,
            "end": end,
            "geom": geom,
            "raise_errors": True,
            **search_kwargs_dict,
        }
        if limit:
            search_criteria["limit"] = limit
        if page:
            search_criteria["page"] = page
        results = self.eodag.search(provider=provider, **search_criteria)
        if offline:
            results = [
                prod
                for prod in results
                if prod.properties.get("order:status", "") != ONLINE_STATUS
            ]
        if check_product:
            self.assertGreater(len(results), 0)
            one_product = results[0]
            self.assertEqual(one_product.provider, provider)

            # validate STAC serialization
            self.stac.validate_item_collection_dict(results.as_dict())
            for msg in self.stac.message:
                self.assertTrue(
                    msg["valid_stac"],
                    "%s %s" % (msg.get("error_message", msg), msg.get("failed_schema")),
                )
            self.stac.validate_dict(one_product.as_dict())
            for msg in self.stac.message:
                self.assertTrue(
                    msg["valid_stac"],
                    "%s %s" % (msg.get("error_message", msg), msg.get("failed_schema")),
                )
            return one_product
        else:
            return results

    def execute_search_all(
        self,
        provider,
        collection,
        start,
        end,
        geom,
        limit=None,
        check_products=True,
    ):
        """Search all the products on provider:

        - First set the preferred provider as the one given in parameter
        - Then do the search_all
        - Then ensure that at least the first result originates from the provider
        - Return all the products
        """
        search_criteria = {
            "start": start,
            "end": end,
            "geom": geom,
        }
        self.eodag.set_preferred_provider(provider)
        results = self.eodag.search_all(
            collection=collection, limit=limit, **search_criteria
        )
        if check_products:
            self.assertGreater(len(results), 0)
            one_product = results[0]
            self.assertEqual(one_product.provider, provider)
        return results


# @unittest.skip("skip auto run")
class TestEODagEndToEnd(EndToEndBase):
    """Make real case tests. This assume the existence of a user conf file in resources folder named user_conf.yml"""  # noqa

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        # use tests/resources/user_conf.yml if exists else default file ~/.config/eodag/eodag.yml
        tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
        if os.path.isfile(tests_user_conf):
            cls.eodag = EODataAccessGateway(
                user_conf_file_path=os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
            )
        else:
            cls.eodag = EODataAccessGateway()

        # temp download directory
        cls.tmp_download_dir = TemporaryDirectory()
        cls.tmp_download_path = cls.tmp_download_dir.name

    def setUp(self):
        self.downloaded_file_path = ""

    @classmethod
    def tearDownClass(cls):
        cls.tmp_download_dir.cleanup()

    def execute_download(
        self, product, expected_filename=None, wait_sec=5, timeout_sec=120
    ):
        """Download the product in a child thread, avoiding to perform the entire
        download, then do some checks and delete the downloaded result from the
        filesystem.
        """

        start_time = time.time()

        download_thread = threading.Thread(
            target=self.eodag.download,
            args=(product, None, None, wait_sec / 60, timeout_sec / 60),
            kwargs={"output_dir": self.tmp_download_path, "extract": False},
            daemon=True,
        )
        download_thread.start()

        max_wait_time = timeout_sec
        while (
            sum(
                f.stat().st_size
                for f in Path(self.tmp_download_path).glob("**/[!quicklooks]*")
                if f.is_file()
            )
            <= 0
            and max_wait_time > 0
        ):
            time.sleep(2)
            max_wait_time -= 2

        # Wait for the thread to complete or timeout
        download_thread.join(timeout=wait_sec)

        stop_time = time.time()
        print(stop_time - start_time)

        if expected_filename is None:
            expected_filename = product.properties["title"]

        self.assertIn(expected_filename, os.listdir(self.tmp_download_path))
        self.downloaded_file_path = os.path.join(
            self.tmp_download_path, expected_filename
        )
        # check whether expected_filename refers to a file or a dir
        if os.path.isdir(self.downloaded_file_path):
            product_directory = Path(self.downloaded_file_path)
            downloaded_size = sum(
                f.stat().st_size for f in product_directory.glob("**/*") if f.is_file()
            )
        else:
            downloaded_size = os.stat(self.downloaded_file_path).st_size
        # The partially downloaded file should be greater or equal to 5 KB
        self.assertGreaterEqual(downloaded_size, 5 * 2**10)

    def test_end_to_end_search_download_usgs_recent(self):
        product = self.execute_search(*USGS_RECENT_SEARCH_ARGS)
        expected_filename = "{}.tar.gz".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_usgs_old(self):
        product = self.execute_search(*USGS_OLD_SEARCH_ARGS)
        expected_filename = "{}.tar.gz".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_geodes(self):
        product = self.execute_search(*GEODES_SEARCH_ARGS)
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_creodias(self):
        product = self.execute_search(*CREODIAS_SEARCH_ARGS)
        self.eodag._providers.configs["creodias"].auth.credentials[
            "totp"
        ] = "PLEASE_CHANGE_ME"
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_cop_dataspace(self):
        product = self.execute_search(*COP_DATASPACE_SEARCH_ARGS)
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_cop_dataspace_s3(self):
        product = self.execute_search(*COP_DATASPACE_S3_SEARCH_ARGS)
        expected_filename = product.properties["title"]
        self.execute_download(product, expected_filename, wait_sec=15)

    def test_end_to_end_search_download_planetary_computer(self):
        product = self.execute_search(*PLANETARY_COMPUTER_SEARCH_ARGS)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename, wait_sec=20)

    def test_end_to_end_search_download_hydroweb_next(self):
        product = self.execute_search(*HYDROWBEB_NEXT_SEARCH_ARGS)
        expected_filename = "{}".format(
            sanitize(product.properties["title"])
            + "-"
            + sanitize(product.properties["id"])
        )
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_eocat(self):
        self.eodag.discover_collections(provider="eocat")
        products = self.execute_search(*EOCAT_SEARCH_ARGS, check_product=False)
        expected_filename = "{}.TGZ".format(products[0].properties["title"])
        self.execute_download(products[0], expected_filename)

    def test_end_to_end_search_download_fedeo_ceda(self):
        self.eodag.discover_collections(provider="fedeo_ceda")
        products = self.execute_search(*FEDEO_CEDA_SEARCH_ARGS, check_product=False)
        expected_filename = "{}".format(products[2].properties["title"])
        self.execute_download(products[2], expected_filename)

    def test_end_to_end_search_download_creodias_noresult(self):
        """Requesting a page on creodias with no results must return an empty SearchResult"""
        # As of 2021-03-19 this search at page 1 returns 31 products, so at page 2 there
        # are no products available and creodias returns a response without products (`hits`).
        product = self.execute_search(
            *CREODIAS_SEARCH_ARGS, page=2, limit=50, check_product=False
        )
        self.assertEqual(len(product), 0)

    # @unittest.skip("expired aws_eos api key")
    def test_end_to_end_search_download_aws_eos(self):
        product = self.execute_search(*AWSEOS_SEARCH_ARGS)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_earth_search(self):
        product = self.execute_search(*EARTH_SEARCH_SEARCH_ARGS)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename, wait_sec=15)

    def test_end_to_end_search_download_earth_search_gcs(self):
        product = self.execute_search(*EARTH_SEARCH_GCS_SEARCH_ARGS)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename, wait_sec=20)

    def test_end_to_end_search_download_usgs_satapi_aws(self):
        product = self.execute_search(*USGS_SATAPI_AWS_SEARCH_ARGS)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename, wait_sec=15)

    def test_end_to_end_search_download_ecmwf(self):
        product = self.execute_search(
            *ECMWF_SEARCH_ARGS, search_kwargs_dict=ECMWF_SEARCH_KWARGS
        )
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_cop_ads(self):
        product = self.execute_search(
            *COP_ADS_SEARCH_ARGS, search_kwargs_dict=COP_ADS_SEARCH_KWARGS
        )
        expected_filename = "{}.grib".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_cop_cds(self):
        product = self.execute_search(
            *COP_CDS_SEARCH_ARGS, search_kwargs_dict=COP_CDS_SEARCH_KWARGS
        )
        expected_filename = "{}.grib".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_cop_ewds(self):
        product = self.execute_search(*COP_EWDS_SEARCH_ARGS)
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_sara(self):
        product = self.execute_search(*SARA_SEARCH_ARGS)
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_meteoblue(self):
        product = self.execute_search(*METEOBLUE_SEARCH_ARGS)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_wekeo_main(self):
        product = self.execute_search(*WEKEO_MAIN_SEARCH_ARGS, page=2, limit=4)
        expected_filename = "{}".format(product.properties["title"])
        self.execute_download(product, expected_filename, timeout_sec=40)

    def test_end_to_end_search_download_wekeo_ecmwf(self):
        product = self.execute_search(*WEKEO_ECMWF_SEARCH_ARGS)
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename, timeout_sec=40)

    def test_end_to_end_search_download_eumetsat_ds(self):
        product = self.execute_search(*EUMETSAT_DS_SEARCH_ARGS)
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    def test_end_to_end_search_download_cop_ghsl(self):
        product = self.execute_search(
            *COP_GHSL_SEARCH_ARGS, search_kwargs_dict=COP_GHSL_SEARCH_KWARGS
        )
        expected_filename = "{}.zip".format(product.properties["title"])
        self.execute_download(product, expected_filename)

    # @unittest.skip("service unavailable for the moment")
    def test_get_quicklook_cop_dataspace(self):
        product = self.execute_search(
            "cop_dataspace",
            "S2_MSI_L1C",
            "2019-03-01",
            "2019-03-15",
            [50, 50, 50.3, 50.3],
        )
        quicklook_file_path = product.get_quicklook(filename="cop_dataspace_quicklook")
        # TearDown will remove quicklook_file_path on end
        self.downloaded_file_path = quicklook_file_path

        self.assertNotEqual(quicklook_file_path, "")
        self.assertEqual(
            os.path.basename(quicklook_file_path), "cop_dataspace_quicklook"
        )
        self.assertEqual(
            os.path.dirname(quicklook_file_path),
            os.path.join(product.downloader.config.output_dir, "quicklooks"),
        )
        self.assertGreaterEqual(os.stat(quicklook_file_path).st_size, 2**5)

    def test__search_by_id_creodias(self):
        # A single test with creodias to check that _search_by_id returns
        # correctly the exact product looked for.
        uid = "S2A_MSIL1C_20200810T030551_N0209_R075_T53WPU_20200810T050611"
        provider = "creodias"

        products = self.eodag._search_by_id(
            uid=uid, provider=provider, collection="S2_MSI_L1C"
        )
        product = products[0]

        self.assertEqual(product.properties["id"], uid)
        self.assertIsNotNone(product.collection)

    def test_search_by_tile(self):
        """Search by tileIdentifier should find results and correctly map found metadata"""
        # providers supporting search-by-tile
        supported_providers_collections = [
            ("cop_dataspace", "S2_MSI_L1C"),
            ("planetary_computer", "S2_MSI_L2A"),
            ("earth_search", "S2_MSI_L1C"),
        ]

        tile_id = "31TCJ"

        for provider, collection in supported_providers_collections:
            products = self.eodag.search(
                collection=collection,
                start="2021-06-01",
                end="2021-06-30",
                tileIdentifier=tile_id,
                provider=provider,
            )
            self.assertGreater(len(products), 0, msg=f"no result found for {provider}")
            self.assertEqual(
                products[0].properties["tileIdentifier"],
                tile_id,
                msg=f"tileIdentifier not mapped for {provider}",
            )

    def test_end_to_end_discover_collections_usgs_satapi_aws(self):
        """discover_collections() must return an external collections configuration for usgs_satapi_aws"""
        provider = "usgs_satapi_aws"
        ext_collections_conf = self.eodag.discover_collections(provider=provider)
        self.assertEqual(
            "landsat-c2l1",
            ext_collections_conf[provider]["providers_config"]["landsat-c2l1"][
                "_collection"
            ],
        )
        self.assertEqual(
            "Landsat Collection 2 Level-1 Product",
            ext_collections_conf[provider]["collections_config"]["landsat-c2l1"][
                "title"
            ],
        )
        self.assertEqual(
            "1972-07-25T00:00:00.000Z",
            ext_collections_conf[provider]["collections_config"]["landsat-c2l1"][
                "extent"
            ]["temporal"]["interval"][0][0],
        )
        # check that all pre-configured collections are listed by provider
        provider_collections = [
            v["_collection"]
            for k, v in self.eodag.providers.configs[provider].products.items()
            if k != GENERIC_COLLECTION
        ]
        for provider_collection in provider_collections:
            self.assertIn(
                provider_collection,
                ext_collections_conf[provider]["providers_config"],
            )

    def test_end_to_end_discover_collections_earth_search(self):
        """discover_collections() must return an external collections configuration for earth_search"""
        provider = "earth_search"
        ext_collections_conf = self.eodag.discover_collections(provider=provider)
        self.assertEqual(
            "sentinel-2-l1c",
            ext_collections_conf[provider]["providers_config"]["sentinel-2-l1c"][
                "_collection"
            ],
        )
        self.assertEqual(
            "Sentinel-2 Level-1C",
            ext_collections_conf[provider]["collections_config"]["sentinel-2-l1c"][
                "title"
            ],
        )
        self.assertEqual(
            "proprietary",
            ext_collections_conf[provider]["collections_config"]["sentinel-2-l1c"][
                "license"
            ],
        )
        # check that all pre-configured collections are listed by provider
        provider_collections = [
            v["_collection"]
            for k, v in self.eodag.providers.configs[provider].products.items()
            if k != GENERIC_COLLECTION
        ]
        for provider_collection in provider_collections:
            self.assertIn(
                provider_collection,
                ext_collections_conf[provider]["providers_config"],
            )

    def test_end_to_end_discover_collections_earth_search_gcs(self):
        """discover_collections() must return None for earth_search_gcs"""
        provider = "earth_search_gcs"
        ext_collections_conf = self.eodag.discover_collections(provider=provider)
        self.assertDictEqual(ext_collections_conf, {})

    def test_end_to_end_discover_collections_eocat(self):
        """discover_collections() must return an external collections configuration for eocat"""
        provider = "eocat"
        ext_collections_conf = self.eodag.discover_collections(provider=provider)
        self.assertIsNotNone(ext_collections_conf[provider])

    def test_end_to_end_discover_collections_fedeo_ceda(self):
        """discover_collections() must return an external collections configuration for fedeo ceda"""
        provider = "fedeo_ceda"
        ext_collections_conf = self.eodag.discover_collections(provider=provider)
        self.assertEqual(
            ["MERIS"],
            ext_collections_conf[provider]["collections_config"]["MERIS_ALAMO_L2_V2.2"][
                "instruments"
            ],
        )
        self.assertEqual(
            "other",
            ext_collections_conf[provider]["collections_config"]["MERIS_ALAMO_L2_V2.2"][
                "license"
            ],
        )


# @unittest.skip("skip auto run")
class TestEODagEndToEndComplete(EndToEndBase):
    """Make real and complete test cases that search for products, download them and
    extract them. There should be just a tiny number of these tests which can be quite
    long to run.

    There must be a user conf file in the test resources folder named user_conf.yml
    """

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        # use tests/resources/user_conf.yml if exists else default file ~/.config/eodag/eodag.yml
        tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
        if not os.path.isfile(tests_user_conf):
            tests_user_conf = None
        cls.eodag = EODataAccessGateway(user_conf_file_path=tests_user_conf)

        # temp download directory
        cls.tmp_download_dir = TemporaryDirectory()
        cls.tmp_download_path = cls.tmp_download_dir.name

    @classmethod
    def tearDownClass(cls):
        cls.tmp_download_dir.cleanup()

    def test_end_to_end_complete_cop_dataspace(self):
        """Complete end-to-end test with cop_dataspace for download and download_all"""

        self.eodag._providers.configs[
            "cop_dataspace"
        ].download.output_dir = self.tmp_download_path

        # Search for products that are online and as small as possible (smallest area first)
        today = dt.date.today()
        month_span = dt.timedelta(weeks=4)
        search_results = self.eodag.search(
            collection="S2_MSI_L1C",
            start=(today - month_span).isoformat(),
            end=today.isoformat(),
            geom={"lonmin": 1, "latmin": 42, "lonmax": 5, "latmax": 46},
            limit=100,
            provider="cop_dataspace",
        )
        prods_sorted_by_size = SearchResult(
            sorted(search_results, key=lambda p: p.geometry.area)
        )
        prods_online = prods_sorted_by_size.filter_online()
        if len(prods_online) < 2:
            unittest.skip(
                "Not enough online products found, update the search criteria."
            )

        # Retrieve one product to work with
        product = prods_online[0]

        prev_remote_location = product.remote_location
        prev_location = product.location
        # The expected product's archive filename is based on the product's title
        expected_product_name = f"{product.properties['title']}.zip"

        # Download the product, but DON'T extract it
        archive_file_path = self.eodag.download(product, extract=False)

        # The archive must have been downloaded
        self.assertTrue(os.path.isfile(archive_file_path))
        # Its name must be the "{product_title}.zip"
        self.assertIn(
            expected_product_name, os.listdir(product.downloader.config.output_dir)
        )
        # Its size should be >= 5 KB
        archive_size = os.stat(archive_file_path).st_size
        self.assertGreaterEqual(archive_size, 5 * 2**10)
        # The product remote_location should be the same
        self.assertEqual(prev_remote_location, product.remote_location)
        # However its location should have been update
        self.assertNotEqual(prev_location, product.location)
        # The location must follow the file URI scheme
        self.assertTrue(product.location.startswith("file://"))
        # That points to the downloaded archive
        self.assertEqual(uri_to_path(product.location), archive_file_path)
        # A .downloaded folder must have been created
        record_dir = os.path.join(self.tmp_download_path, ".downloaded")
        self.assertTrue(os.path.isdir(record_dir))
        # It must contain a file per product downloade, whose name is
        # the MD5 hash of the product's ``collection`` and ``properties['id']``
        expected_hash = product.collection + "-" + product.properties["id"]
        expected_hash = hashlib.md5(expected_hash.encode("utf-8")).hexdigest()
        record_file = os.path.join(record_dir, expected_hash)
        self.assertTrue(os.path.isfile(record_file))
        # Its content must be the product's remote location
        record_content = Path(record_file).read_text()
        self.assertEqual(record_content, product.remote_location)

        # The downloaded product should not be downloaded again if the download
        # method is executed again
        previous_archive_file_path = archive_file_path
        previous_location = product.location
        start_time = time.time()
        archive_file_path = self.eodag.download(product, extract=False)
        end_time = time.time()
        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
        # The paths should be the same as before
        self.assertEqual(archive_file_path, previous_archive_file_path)
        self.assertEqual(product.location, previous_location)

        # If we emulate that the product has just been found, it should not
        # be downloaded again since the record file is still present.
        product.location = product.remote_location
        # Pretty much the same checks as with the previous step
        previous_archive_file_path = archive_file_path
        start_time = time.time()
        archive_file_path = self.eodag.download(product, extract=False)
        end_time = time.time()
        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
        # The returned path should be the same as before
        self.assertEqual(archive_file_path, previous_archive_file_path)
        self.assertEqual(uri_to_path(product.location), archive_file_path)

        # Remove the archive
        os.remove(archive_file_path)

        # Now, the archive is removed but its associated record file
        # still exists. Downloading the product again should really
        # download it, if its location points to the remote location.
        # The product should be automatically extracted.
        product.location = product.remote_location
        product_dir_path = self.eodag.download(
            product, extract=True, delete_archive=False
        )

        # Its size should be >= 5 KB
        downloaded_size = sum(
            f.stat().st_size for f in Path(product_dir_path).glob("**/*") if f.is_file()
        )
        self.assertGreaterEqual(downloaded_size, 5 * 2**10)
        # The product remote_location should be the same
        self.assertEqual(prev_remote_location, product.remote_location)
        # However its location should have been update
        self.assertNotEqual(prev_location, product.location)
        # The location must follow the file URI scheme
        self.assertTrue(product.location.startswith("file://"))
        # The path must point to a directory
        self.assertTrue(os.path.isdir(product_dir_path))

        # The downloaded & extracted product should not be downloaded again if
        # the download method is executed again
        previous_product_dir_path = product_dir_path
        start_time = time.time()
        product_dir_path = self.eodag.download(product)
        end_time = time.time()
        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
        # The paths should be the same as before
        self.assertEqual(product_dir_path, previous_product_dir_path)

        # Remove the archive and extracted product and reset the product's location
        os.remove(archive_file_path)
        shutil.rmtree(Path(product_dir_path).parent)
        product.location = product.remote_location

        # Now let's check download_all
        products = prods_sorted_by_size[:2]
        # Pass a copy because download_all empties the list
        archive_paths = self.eodag.download_all(products[:], extract=False)

        # The returned paths must point to the downloaded archives
        # Each product's location must be a URI path to the archive
        self.assertEqual(len(archive_paths), len(products))
        for archive_path in archive_paths:
            self.assertTrue(os.path.isfile(archive_path))
        for product in products:
            self.assertIn(uri_to_path(product.location), archive_paths)

        # Downloading the product again should not download them, since
        # they are all already there.
        prev_archive_paths = archive_paths
        start_time = time.time()
        archive_paths = self.eodag.download_all(products[:], extract=False)
        end_time = time.time()
        self.assertLess(end_time - start_time, 2)  # Should be really fast (< 2s)
        self.assertEqual(sorted(archive_paths), sorted(prev_archive_paths))


# @unittest.skip("skip auto run")
class TestEODagEndToEndWrongCredentials(EndToEndBase):
    """Make real case tests with wrong credentials. This assumes the existence of a
    wrong_credentials_cong.yml file in resources folder named user_conf.yml"""

    @classmethod
    def setUpClass(cls):
        super().setUpClass()

        tests_wrong_conf = os.path.join(
            TEST_RESOURCES_PATH, "wrong_credentials_conf.yml"
        )
        cls.eodag = EODataAccessGateway(user_conf_file_path=tests_wrong_conf)
        # backup os.environ as it will be modified by tests
        cls.eodag_env_pattern = re.compile(r"EODAG_\w+")
        cls.eodag_env_backup = {
            k: v for k, v in os.environ.items() if cls.eodag_env_pattern.match(k)
        }

    @classmethod
    def tearDownClass(cls):
        super(TestEODagEndToEndWrongCredentials, cls).tearDownClass()
        # restore os.environ
        for k, v in os.environ.items():
            if cls.eodag_env_pattern.match(k):
                os.environ.pop(k)
        os.environ.update(cls.eodag_env_backup)

    def test_end_to_end_wrong_apikey_search_aws_eos(self):
        self.eodag.set_preferred_provider(AWSEOS_SEARCH_ARGS[0])
        with self.assertRaises(AuthenticationError):
            self.eodag.search(
                raise_errors=True,
                **dict(
                    zip(["collection", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:])
                ),
            )

    # @unittest.skip("expired aws_eos api key")
    def test_end_to_end_good_apikey_wrong_credentials_aws_eos(self):
        # Setup
        # We retrieve correct credentials from the user_conf.yml file
        tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
        if not os.path.isfile(tests_user_conf):
            self.skipTest("user_conf.yml file with credentials not found.")
        # But we set the access key id and the secret to wrong values
        try:
            os.environ[
                "EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_ACCESS_KEY_ID"
            ] = "badaccessid"
            os.environ[
                "EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_SECRET_ACCESS_KEY"
            ] = "badsecret"
            os.environ["EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_PROFILE"] = "badsecret"

            eodag = EODataAccessGateway(
                user_conf_file_path=os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
            )
            eodag.set_preferred_provider(AWSEOS_SEARCH_ARGS[0])
            results = eodag.search(
                raise_errors=True,
                **dict(
                    zip(["collection", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:])
                ),
            )
            self.assertGreater(len(results), 0)
            one_product = results[0]
            self.assertEqual(one_product.provider, AWSEOS_SEARCH_ARGS[0])
            with self.assertRaises(AuthenticationError):
                self.eodag.download(one_product)
        # Teardown
        finally:
            os.environ.pop("EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_ACCESS_KEY_ID")
            os.environ.pop("EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_SECRET_ACCESS_KEY")
            os.environ.pop("EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_PROFILE")

    def test_end_to_end_wrong_credentials_creodias(self):
        product = self.execute_search(*CREODIAS_SEARCH_ARGS)
        with self.assertRaises(AuthenticationError):
            self.eodag.download(product)

    def test_end_to_end_wrong_credentials_cop_dataspace(self):
        product = self.execute_search(*COP_DATASPACE_SEARCH_ARGS)
        with self.assertRaises(AuthenticationError):
            self.eodag.download(product)

    def test_end_to_end_wrong_credentials_search_usgs(self):
        # It should already fail while searching for the products.
        self.eodag.set_preferred_provider(USGS_RECENT_SEARCH_ARGS[0])
        with self.assertRaises(AuthenticationError):
            self.eodag.search(
                raise_errors=True,
                **dict(
                    zip(
                        ["collection", "start", "end", "geom"],
                        USGS_RECENT_SEARCH_ARGS[1:],
                    )
                ),
            )

    def test_end_to_end_wrong_credentials_search_meteoblue(self):
        # It should already fail while searching for the products.
        self.eodag.set_preferred_provider(METEOBLUE_SEARCH_ARGS[0])
        with self.assertRaises(AuthenticationError):
            self.eodag.search(
                raise_errors=True,
                **dict(
                    zip(
                        ["collection", "start", "end", "geom"],
                        METEOBLUE_SEARCH_ARGS[1:],
                    )
                ),
            )

    def test_end_to_end_wrong_credentials_search_hydroweb_next(self):
        # It should already fail while searching for the products.
        self.eodag.set_preferred_provider(HYDROWBEB_NEXT_SEARCH_ARGS[0])
        with self.assertRaises(AuthenticationError):
            self.eodag.search(
                raise_errors=True,
                **dict(
                    zip(
                        ["collection", "start", "end", "geom"],
                        HYDROWBEB_NEXT_SEARCH_ARGS[1:],
                    )
                ),
            )

    def test_end_to_end_wrong_credentials_search_wekeo_main(self):
        # It should already fail while searching for the products.
        self.eodag.set_preferred_provider(WEKEO_MAIN_SEARCH_ARGS[0])
        with self.assertRaises(AuthenticationError):
            results, _ = self.eodag.search(
                raise_errors=True,
                **dict(
                    zip(
                        ["collection", "start", "end", "geom"],
                        WEKEO_MAIN_SEARCH_ARGS[1:],
                    )
                ),
            )

    def test_end_to_end_wrong_credentials_search_wekeo_ecmwf(self):
        # It should already fail while searching for the products.
        self.eodag.set_preferred_provider(WEKEO_ECMWF_SEARCH_ARGS[0])
        with self.assertRaises(AuthenticationError):
            self.eodag.search(
                raise_errors=True,
                **dict(
                    zip(
                        ["collection", "start", "end", "geom"],
                        WEKEO_ECMWF_SEARCH_ARGS[1:],
                    )
                ),
            )

    def test_end_to_end_wrong_credentials_search_eumetsat_ds(self):
        product = self.execute_search(*EUMETSAT_DS_SEARCH_ARGS)
        with self.assertRaises(AuthenticationError):
            self.eodag.download(product)
