From 3da39f7d0c2d1401006ca48fab6bb1a4e2a90bf8 Mon Sep 17 00:00:00 2001 From: Dmitry Sorokin <40151847+DimedS@users.noreply.github.com> Date: Mon, 9 Sep 2024 20:54:35 +0100 Subject: [PATCH] fix(datasets): Cleanup of dependency (#822) * Update dependencies --------- Signed-off-by: Dmitry Sorokin <129520297+DmitrySorokinQB@users.noreply.github.com> Co-authored-by: Dmitry Sorokin <129520297+DmitrySorokinQB@users.noreply.github.com> --- kedro-datasets/pyproject.toml | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/kedro-datasets/pyproject.toml b/kedro-datasets/pyproject.toml index e1180f2f2..895caebfd 100644 --- a/kedro-datasets/pyproject.toml +++ b/kedro-datasets/pyproject.toml @@ -33,8 +33,9 @@ api = ["kedro-datasets[api-apidataset]"] biosequence-biosequencedataset = ["biopython~=1.73"] biosequence = ["kedro-datasets[biosequence-biosequencedataset]"] +dask-csvdataset = ["dask[dataframe]>=2021.10"] dask-parquetdataset = ["dask[complete]>=2021.10", "triad>=0.6.7, <1.0"] -dask = ["kedro-datasets[dask-parquetdataset]"] +dask = ["kedro-datasets[dask-parquetdataset, dask-csvdataset]"] databricks-managedtabledataset = ["kedro-datasets[spark-base,pandas-base,delta-base,hdfs-base,s3fs-base]"] databricks = ["kedro-datasets[databricks-managedtabledataset]"] @@ -92,7 +93,7 @@ pandas-featherdataset = ["kedro-datasets[pandas-base]"] pandas-gbqtabledataset = ["kedro-datasets[pandas-base]", "pandas-gbq>=0.12.0"] pandas-gbqquerydataset = ["kedro-datasets[pandas-base]", "pandas-gbq>=0.12.0"] pandas-genericdataset = ["kedro-datasets[pandas-base]"] -pandas-hdfdataset = ["kedro-datasets[pandas-base]", "tables~=3.6"] +pandas-hdfdataset = ["kedro-datasets[pandas-base]", "tables>=3.6"] pandas-jsondataset = ["kedro-datasets[pandas-base]"] pandas-parquetdataset = ["kedro-datasets[pandas-base]", "pyarrow>=6.0"] pandas-sqltabledataset = ["kedro-datasets[pandas-base]", "SQLAlchemy>=1.4, <3.0"] @@ -127,9 +128,12 @@ plotly = ["kedro-datasets[plotly-htmldataset,plotly-jsondataset,plotly-plotlydat polars-csvdataset = ["kedro-datasets[polars-base]"] polars-eagerpolarsdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "xlsx2csv>=0.8.0", "deltalake >= 0.6.2"] -polars-genericdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "xlsx2csv>=0.8.0", "deltalake >= 0.6.2"] polars-lazypolarsdataset = ["kedro-datasets[polars-base]", "pyarrow>=4.0", "deltalake >= 0.6.2"] -polars = ["kedro-datasets[polars-genericdataset]"] +polars = [ + """kedro-datasets[polars-csvdataset,\ + polars-eagerpolarsdataset,\ + polars-lazypolarsdataset]""" +] redis-pickledataset = ["redis~=4.1"] redis = ["kedro-datasets[redis-pickledataset]"] @@ -140,8 +144,15 @@ snowflake = ["kedro-datasets[snowflake-snowparktabledataset]"] spark-deltatabledataset = ["kedro-datasets[spark-base,hdfs-base,s3fs-base,delta-base]"] spark-sparkdataset = ["kedro-datasets[spark-base,hdfs-base,s3fs-base]"] spark-sparkhivedataset = ["kedro-datasets[spark-base,hdfs-base,s3fs-base]"] -spark-sparkjdbcdataset = ["kedro-datasets[spark-base,hdfs-base,s3fs-base]"] -spark = ["kedro-datasets[spark-deltatabledataset]"] +spark-sparkjdbcdataset = ["kedro-datasets[spark-base]"] +spark-sparkstreamingdataset = ["kedro-datasets[spark-base,hdfs-base,s3fs-base]"] +spark = [ + """kedro-datasets[spark-deltatabledataset,\ + spark-sparkdataset,\ + spark-sparkhivedataset,\ + spark-sparkjdbcdataset,\ + spark-sparkstreamingdataset]""" +] svmlight-svmlightdataset = ["scikit-learn>=1.0.2", "scipy~=1.7.3"] svmlight = ["kedro-datasets[svmlight-svmlightdataset]"] @@ -209,7 +220,7 @@ test = [ "ibis-framework[duckdb,examples]", "import-linter[toml]==1.2.6", "ipython>=7.31.1, <8.0", - "Jinja2<3.1.0", + "Jinja2<3.2.0", "joblib>=0.14", "jupyterlab>=3.0", "jupyter~=1.0", @@ -248,8 +259,7 @@ test = [ "scipy>=1.7.3", "packaging", "SQLAlchemy>=1.2", - "tables>=3.8.0; platform_system == 'Windows'", # Import issues with python 3.8 with pytables pinning to 3.8.0 fixes this https://github.com/PyTables/PyTables/issues/933#issuecomment-1555917593 - "tables~=3.6; platform_system != 'Windows'", + "tables>=3.6", "tensorflow-macos~=2.0; platform_system == 'Darwin' and platform_machine == 'arm64'", "tensorflow~=2.0; platform_system != 'Darwin' or platform_machine != 'arm64'", "triad>=0.6.7, <1.0",