pyproject.toml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
[tool.poetry]
name = "pyiceberg"
version = "0.10.0"
readme = "README.md"
homepage = "https://py.iceberg.apache.org/"
repository = "https://github.com/apache/iceberg-python"
description = "Apache Iceberg is an open table format for huge analytic datasets"
authors = ["Apache Software Foundation <dev@iceberg.apache.org>"]
license = "Apache License 2.0"
classifiers = [
  "License :: OSI Approved :: Apache Software License",
  "Operating System :: OS Independent",
  "Programming Language :: Python :: 3.9",
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
]
packages = [
  { include = "pyiceberg" },
  { from = "vendor", include = "fb303" },
  { from = "vendor", include = "hive_metastore" },
  { include = "tests", format = "sdist" },
  { include = "Makefile", format = "sdist" },
  { include = "NOTICE", format = [
    "sdist",
    "wheel",
  ] },
]
include = [
  { path = "dev", format = "sdist" },
  { path = "pyiceberg/**/*.so", format = "wheel" },
  { path = "pyiceberg/**/*.pyd", format = "wheel" },
]

[tool.poetry.dependencies]
python = "^3.9.2, !=3.9.7"
mmh3 = ">=4.0.0,<6.0.0"
requests = ">=2.20.0,<3.0.0"
click = ">=7.1.1,<9.0.0"
rich = ">=10.11.0,<15.0.0"
strictyaml = ">=1.7.0,<2.0.0" # CVE-2020-14343 was fixed in 5.4.
pydantic = ">=2.0,<3.0,!=2.4.0,!=2.4.1" # 2.4.0, 2.4.1 has a critical bug
sortedcontainers = "2.4.0"
fsspec = ">=2023.1.0"
pyparsing = ">=3.1.0,<4.0.0"
zstandard = ">=0.13.0,<1.0.0"
tenacity = ">=8.2.3,<10.0.0"
pyroaring = ">=1.0.0,<2.0.0"
pyarrow = { version = ">=17.0.0,<20.0.0", optional = true }
pandas = { version = ">=1.0.0,<3.0.0", optional = true }
duckdb = { version = ">=0.5.0,<2.0.0", optional = true }
ray = [
  { version = "==2.10.0", python = "<3.9", optional = true },
  { version = ">=2.10.0,<3.0.0", python = ">=3.9", optional = true },
]
python-snappy = { version = ">=0.6.0,<1.0.0", optional = true }
thrift = { version = ">=0.13.0,<1.0.0", optional = true }
mypy-boto3-glue = { version = ">=1.28.18", optional = true }
boto3 = { version = ">=1.24.59", optional = true }
s3fs = { version = ">=2023.1.0", optional = true }
adlfs = { version = ">=2023.1.0", optional = true }
gcsfs = { version = ">=2023.1.0", optional = true }
psycopg2-binary = { version = ">=2.9.6", optional = true }
sqlalchemy = { version = "^2.0.18", optional = true }
getdaft = { version = ">=0.2.12", optional = true }
cachetools = "^5.5.0"
pyiceberg-core = { version = "^0.4.0", optional = true }
polars = { version = "^1.21.0", optional = true }
thrift-sasl = { version = ">=0.4.3", optional = true }
kerberos = {version = "^1.3.1", optional = true}

[tool.poetry.group.dev.dependencies]
pytest = "7.4.4"
pytest-checkdocs = "2.13.0"
pytest-lazy-fixture = "0.6.3"
pre-commit = "4.2.0"
fastavro = "1.10.0"
coverage = { version = "^7.4.2", extras = ["toml"] }
requests-mock = "1.12.1"
moto = { version = "^5.0.2", extras = ["server"] }
typing-extensions = "4.13.2"
pytest-mock = "3.14.0"
pyspark = "3.5.5"
cython = "3.0.12"
deptry = ">=0.14,<0.24"
datafusion = ">=44,<47"
docutils = "!=0.21.post1"   # https://github.com/python-poetry/poetry/issues/9248#issuecomment-2026240520

[tool.poetry.group.docs.dependencies]
# for mkdocs
mkdocs = "1.6.1"
griffe = "1.7.3"
jinja2 = "3.1.6"
mkdocstrings = "0.29.1"
mkdocstrings-python = "1.16.10"
mkdocs-literate-nav = "0.6.2"
mkdocs-autorefs = "1.4.1"
mkdocs-gen-files = "0.5.0"
mkdocs-material = "9.6.12"
mkdocs-material-extensions = "1.3.1"
mkdocs-section-index = "0.3.10"

[[tool.mypy.overrides]]
module = "pytest_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyiceberg_core.*"
ignore_missing_imports = true

[tool.poetry.scripts]
pyiceberg = "pyiceberg.cli.console:run"

[build-system]
requires = ["poetry-core>=1.0.0", "wheel", "Cython>=3.0.0", "setuptools"]
build-backend = "poetry.core.masonry.api"

[tool.poetry.build]
generate-setup-file = false
script = "build-module.py"

[tool.poetry.extras]
pyarrow = ["pyarrow"]
pandas = ["pandas", "pyarrow"]
duckdb = ["duckdb", "pyarrow"]
ray = ["ray", "pyarrow", "pandas"]
daft = ["getdaft"]
polars = ["polars"]
snappy = ["python-snappy"]
hive = ["thrift"]
hive-kerberos = ["thrift", "thrift_sasl", "kerberos"]
s3fs = ["s3fs"]
glue = ["boto3", "mypy-boto3-glue"]
adlfs = ["adlfs"]
dynamodb = ["boto3"]
zstandard = ["zstandard"]
sql-postgres = ["sqlalchemy", "psycopg2-binary"]
sql-sqlite = ["sqlalchemy"]
gcsfs = ["gcsfs"]
rest-sigv4 = ["boto3"]
pyiceberg-core = ["pyiceberg-core"]

[tool.pytest.ini_options]
markers = [
  "unmarked: marks a test as a unittest",
  "s3: marks a test as requiring access to s3 compliant storage (use with --aws-access-key-id, --aws-secret-access-key, and --endpoint args)",
  "adls: marks a test as requiring access to adls compliant storage (use with --adls.account-name, --adls.account-key, and --adls.endpoint args)",
  "integration: marks integration tests against Apache Spark",
  "gcs: marks a test as requiring access to gcs compliant storage (use with --gs.token, --gs.project, and --gs.endpoint)",
  "benchmark: collection of tests to validate read/write performance before and after a change"
]

# Turns a warning into an error
filterwarnings = [
  "error",
  "ignore:A plugin raised an exception during an old-style hookwrapper teardown.",
  "ignore:unclosed <socket.socket",
  # Remove this in a future release of PySpark.
  "ignore:distutils Version classes are deprecated. Use packaging.version instead.",
  # Remove this in a future release of PySpark. https://github.com/apache/iceberg-python/issues/1349
  "ignore:datetime.datetime.utcfromtimestamp\\(\\) is deprecated and scheduled for removal in a future version.",
  # Remove this once https://github.com/boto/boto3/issues/3889 is fixed.
  "ignore:datetime.datetime.utcnow\\(\\) is deprecated and scheduled for removal in a future version.",
  # Latest PySpark version (v3.5.3) throws this error, remove in a future release of PySpark (possibly v4.0.0).
  "ignore:is_datetime64tz_dtype is deprecated and will be removed in a future version.",
]

[tool.black]
line-length = 130
target-version = ['py38']

[tool.pycln]
all = true

[tool.mypy]
mypy_path = "python"
no_implicit_optional = true
namespace_packages = false
warn_redundant_casts = true
warn_unreachable = true
warn_unused_ignores = true
disallow_any_generics = true
disallow_untyped_defs = true
implicit_reexport = false

[[tool.mypy.overrides]]
module = "pyarrow.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pandas.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "snappy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "zstandard.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pydantic_core.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pytest.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fastavro.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mmh3.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "hive_metastore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "thrift.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "requests_mock.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "click.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "rich.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "fsspec.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "s3fs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "azure.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "adlfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "gcsfs.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "packaging.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tests.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "boto3"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "botocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "mypy_boto3_glue.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "moto"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiobotocore.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "aiohttp.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "duckdb.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "ray.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "daft.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyparsing.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyspark.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "strictyaml.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sortedcontainers.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "sqlalchemy.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "Cython.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "setuptools.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "tenacity.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "polars.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "datafusion.*"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = "pyroaring.*"
ignore_missing_imports = true

[tool.coverage.run]
source = ['pyiceberg/']