[build-system]
requires = ["setuptools>=68"]
build-backend = "setuptools.build_meta"

[project]
name = "datainventor"
version = "0.2.2"
description = "Curate a pretraining corpus on your own machines: clean, calibrate, create, compose."
readme = "README.md"
requires-python = ">=3.10"
license = { text = "Apache-2.0" }
dependencies = ["pyyaml>=6"]

[project.optional-dependencies]
full = ["datasets>=2.18", "pyarrow>=14", "numpy>=1.24"]

[project.scripts]
datainventor = "datainventor.cli:main"

[tool.setuptools]
packages = ["datainventor"]

[tool.setuptools.package-data]
datainventor = ["data/*.json.gz"]