pyproject.toml

# Code style
[tool.ruff]
line-length = 119

[tool.ruff.lint]
# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
# Never enforce `E501` (line length violations).
ignore = ["E501"]
select = ["C", "E", "F", "I", "W", "CPY"]
fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
preview = true

[tool.ruff.lint.isort]
lines-after-imports = 2
known-first-party = ["lighteval"]

[tool.ruff.format]
quote-style = "double" # Like Black, use double quotes for strings.
indent-style = "space" # Like Black, indent with spaces, rather than tabs.
skip-magic-trailing-comma = false # Like Black, respect magic trailing commas.
line-ending = "auto" # Like Black, automatically detect the appropriate line ending.

# Building package
[build-system]
requires = ["setuptools>=61.0"]
build-backend = "setuptools.build_meta"

[tool.setuptools.packages.find]
where = ["src"]

[project]
name = "lighteval"
version = "0.10.1.dev0"
authors = [
  { name="Nathan Habib", email="nathan.habib@huggingface.com" },
  { name="Clémentine Fourrier", email="clementine@huggingface.com" },
  { name="Thomas Wolf", email="thom@huggingface.com" },
]
maintainers = [
  { name="Nathan Habib", email="nathan.habib@huggingface.com" },
  { name="Clémentine Fourrier", email="clementine@huggingface.com" },
]
description = "A lightweight and configurable evaluation package"
readme = "README.md"
requires-python = ">=3.10"
license = {text = "MIT License"}
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Programming Language :: Python :: 3.10",
    "License :: OSI Approved :: MIT License",
    "Operating System :: OS Independent",
]
keywords = ["evaluation", "nlp", "llm"]
dependencies = [
    # Base dependencies
    "transformers>=4.51.0",
    "accelerate",
    "huggingface_hub[hf_xet]>=0.30.2",
    "torch>=2.0,<3.0",
    "GitPython>=3.1.41", # for logging
    "datasets>=3.5.0",
    "pydantic",
    "numpy<2",  # pinned to avoid incompatibilities
    # Prettiness
    "typer",
    "termcolor==2.3.0",
    "pytablewriter",
    "rich",
    "colorlog",
    # Extension of metrics
    "aenum==3.1.15",
    # Base metrics
    "nltk==3.9.1",
    "scikit-learn",
    "sacrebleu",
    "rouge_score==0.1.2",
    "sentencepiece>=0.1.99",
    "protobuf",
    "pycountry",
    "fsspec>=2023.12.2",
    "httpx == 0.27.2",
    "latex2sympy2_extended==1.0.6",
]

[project.optional-dependencies]
litellm = ["litellm", "diskcache"]
tgi = ["text-generation>=0.6.0"]
optimum = ["optimum==1.12.0"]
quantization = ["bitsandbytes>=0.41.0", "auto-gptq>=0.4.2"]
adapters = ["peft==0.3.0"]
nanotron = [
  "nanotron",
  "tensorboardX"
]
tensorboardX = ["tensorboardX"]
vllm = ["vllm>=0.8.4", "ray", "more_itertools"]
quality = ["ruff>=v0.11.0","pre-commit"]
tests = ["pytest>=7.4.0","deepdiff"]
dev = ["lighteval[accelerate,quality,tests,multilingual,math,extended_tasks,vllm]"]
docs = ["hf-doc-builder", "watchdog"]
extended_tasks = [
  "langdetect", # ifeval
  "openai==1.55.2", # llm as a judge using openai models
  "tiktoken"
]
s3 = ["s3fs"]
multilingual = [
    "stanza",
    "spacy[ja,ko,th]",
    "jieba", # for chinese tokenizer
    "pyvi", # for vietnamese tokenizer
]
math = ["latex2sympy2_extended==1.0.6"]
wandb = ["wandb"]

[project.urls]
Homepage = "https://github.com/huggingface/lighteval"
Issues = "https://github.com/huggingface/lighteval/issues"
# Documentation = ""
# Changelog = "https://github.com/huggingface/lighteval/blob/master/CHANGELOG.md"

[project.scripts]
lighteval = "lighteval.__main__:app"