data_tooling/pyproject.toml at master · lvgithub3/data_tooling · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
[tool]
[tool.poetry]
name = "data-tooling"
version = "0.1.0"
description = "Tools for managing datasets for governance and training."
authors = ["BigScience <bigscience-contact@googlegroups.com>"]

[tool.poetry.dependencies]
python = "^3.7.10"

datasets = "^1.12.1"
transformers = "^4.12.3"
nltk = "^3.6.5"
scikit-learn = "^1.0.1"
fsspec = "^2021.11.0"
kenlm = {url = "https://github.com/kpu/kenlm/archive/master.zip", optional = true}
typer = "^0.4.0"
regex = "^2021.11.10"
simhash-py = "^0.4.0"
PyYAML = "^6.0"
tqdm = "^4.62.3"

[tool.poetry.dev-dependencies]
pdbpp = "^0.10.2"
isort = "^5.6.4"
flake8 = "^3.8.4"
black = "^21.7b0"
pytest = "^6.2.4"
jupyterlab = "^3.0.16"

[tool.poetry.extras]
kenlm = ["kenlm"]

[tool.isort]
profile = 'black'
treat_comments_as_code = "# %%"