movefasta/nixpkgs
1
0
Fork 0
mirror of https://github.com/NixOS/nixpkgs.git synced 2025-06-10 19:55:41 +03:00
Code Issues Projects Releases Packages Wiki Activity

nltk-data: make searchable, add all downloadables (#409952)

Browse source
This commit is contained in:
Yt 2025-05-24 09:24:00 -04:00 • committed by GitHub
commit 95b919fd5d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 220 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nixos/modules/services/web-apps/mealie.nix
View file

@ -76,7 +76,7 @@ in
API_PORT = toString cfg.port;
BASE_URL = "http://localhost:${toString cfg.port}";
DATA_DIR = "/var/lib/mealie";
NLTK_DATA = pkgs.nltk-data.averaged_perceptron_tagger_eng;
NLTK_DATA = pkgs.nltk-data.averaged-perceptron-tagger-eng;
} // (builtins.mapAttrs (_: val: toString val) cfg.settings);
serviceConfig = {

2
pkgs/by-name/me/mealie/package.nix
View file

@ -109,7 +109,7 @@ pythonpkgs.buildPythonApplication rec {
# Needed for tests
preCheck = ''
export NLTK_DATA=${nltk-data.averaged_perceptron_tagger_eng}
export NLTK_DATA=${nltk-data.averaged-perceptron-tagger-eng}
'';
disabledTestPaths = [

4
pkgs/by-name/pa/paperless-ngx/package.nix
View file

@ -305,8 +305,8 @@ python.pkgs.buildPythonApplication rec {
tesseract5
;
nltkData = with nltk-data; [
punkt_tab
snowball_data
punkt-tab
snowball-data
stopwords
];
tests = { inherit (nixosTests) paperless; };

2
pkgs/by-name/un/unstructured-api/package.nix
View file

@ -152,7 +152,7 @@ let
paths = [
nltk-data.punkt
nltk-data.averaged_perceptron_tagger
nltk-data.averaged-perceptron-tagger
];
};
in

2
pkgs/development/python-modules/aider-chat/default.nix
View file

@ -125,7 +125,7 @@ let
aider-nltk-data = symlinkJoin {
name = "aider-nltk-data";
paths = [
nltk-data.punkt_tab
nltk-data.punkt-tab
nltk-data.stopwords
];
};

2
pkgs/development/python-modules/ingredient-parser-nlp/default.nix
View file

@ -44,7 +44,7 @@ buildPythonPackage rec {
# Needed for tests
preCheck = ''
export NLTK_DATA=${nltk-data.averaged_perceptron_tagger_eng}
export NLTK_DATA=${nltk-data.averaged-perceptron-tagger-eng}
'';
meta = {

2
pkgs/development/python-modules/type-infer/default.nix
View file

@ -24,7 +24,7 @@ let
name = "nltk-test-data";
paths = [
nltk-data.punkt
nltk-data.punkt_tab
nltk-data.punkt-tab
nltk-data.stopwords
];
};

247
pkgs/tools/text/nltk-data/default.nix
View file

@ -10,12 +10,16 @@ let
version = "0-unstable-2024-07-29";
nativeBuildInputs = [ unzip ];
dontBuild = true;
dontFixup = true;
meta = with lib; {
description = "NLTK Data";
homepage = "https://github.com/nltk/nltk_data";
license = licenses.asl20;
platforms = platforms.all;
maintainers = with maintainers; [ happysalada ];
maintainers = with maintainers; [
bengsparks
happysalada
];
};
};
makeNltkDataPackage =
@ -50,41 +54,212 @@ let
'';
}
);
makeChunker =
pname:
makeNltkDataPackage {
inherit pname;
location = "chunkers";
hash = "sha256-kemjqaCM9hlKAdMw8oVJnp62EAC9rMQ50dKg7wlAwEc=";
};
makeCorpus =
pname:
makeNltkDataPackage {
inherit pname;
location = "corpora";
hash = "sha256-8lMjW5YI8h6dHJ/83HVY2OYGDyKPpgkUAKPISiAKqqk=";
};
makeGrammar =
pname:
makeNltkDataPackage {
inherit pname;
location = "grammars";
hash = "sha256-pyLEcX3Azv8j1kCGvVYonuiNgVJxtWt7veU0S/yNbIM=";
};
makeHelp =
pname:
makeNltkDataPackage {
inherit pname;
location = "help";
hash = "sha256-97mYLNES5WujLF5gD8Ul4cJ6LqSzz+jDzclUsdBeHNE=";
};
makeMisc =
pname:
makeNltkDataPackage {
inherit pname;
location = "misc";
hash = "sha256-XtizfEsc8TYWqvvC/eSFdha2ClC5/ZiJM8nue0vXLb4=";
};
makeModel =
pname:
makeNltkDataPackage {
inherit pname;
location = "models";
hash = "sha256-iq3weEgCci6rgLW2j28F2eRLprJtInGXKe/awJPSVG4=";
};
makeTagger =
pname:
makeNltkDataPackage {
inherit pname;
location = "taggers";
hash = "sha256-tl3Cn2okhBkUtTXvAmFRx72Brez6iTGRdmFTwFmpk3M=";
};
makeTokenizer =
pname:
makeNltkDataPackage {
inherit pname;
location = "tokenizers";
hash = "sha256-OzMkruoYbFKqzuimOXIpE5lhHz8tmSqOFoLT+fjdTVg=";
};
makeStemmer =
pname:
makeNltkDataPackage {
inherit pname;
location = "stemmers";
hash = "sha256-mNefwOPVJGz9kXV3LV4DuV7FJpNir/Nwg4ujd0CogEk=";
};
in
lib.makeScope newScope (self: {
punkt = makeNltkDataPackage {
pname = "punkt";
location = "tokenizers";
hash = "sha256-OzMkruoYbFKqzuimOXIpE5lhHz8tmSqOFoLT+fjdTVg=";
};
punkt_tab = makeNltkDataPackage {
pname = "punkt_tab";
location = "tokenizers";
hash = "sha256-OzMkruoYbFKqzuimOXIpE5lhHz8tmSqOFoLT+fjdTVg=";
};
averaged_perceptron_tagger = makeNltkDataPackage {
pname = "averaged_perceptron_tagger";
location = "taggers";
hash = "sha256-tl3Cn2okhBkUtTXvAmFRx72Brez6iTGRdmFTwFmpk3M=";
};
averaged_perceptron_tagger_eng = makeNltkDataPackage {
pname = "averaged_perceptron_tagger_eng";
location = "taggers";
hash = "sha256-tl3Cn2okhBkUtTXvAmFRx72Brez6iTGRdmFTwFmpk3M=";
};
snowball_data = makeNltkDataPackage {
pname = "snowball_data";
location = "stemmers";
hash = "sha256-mNefwOPVJGz9kXV3LV4DuV7FJpNir/Nwg4ujd0CogEk=";
};
stopwords = makeNltkDataPackage {
pname = "stopwords";
location = "corpora";
hash = "sha256-8lMjW5YI8h6dHJ/83HVY2OYGDyKPpgkUAKPISiAKqqk=";
};
wordnet = makeNltkDataPackage {
pname = "wordnet";
location = "corpora";
hash = "sha256-8lMjW5YI8h6dHJ/83HVY2OYGDyKPpgkUAKPISiAKqqk=";
};
## Chunkers
maxent-ne-chunker = makeChunker "maxent_ne_chunker";
maxent-ne-chunker-tab = makeChunker "maxent_ne_chunker_tab";
## Corpora
abc = makeCorpus "abc";
alpino = makeCorpus "alpino";
bcp47 = makeCorpus "bcp47";
biocreative-ppi = makeCorpus "biocreative_ppi";
brown = makeCorpus "brown";
brown-tei = makeCorpus "brown_tei";
cess-cat = makeCorpus "cess_cat";
cess-esp = makeCorpus "cess_esp";
chat80 = makeCorpus "chat80";
city-database = makeCorpus "city_database";
cmudict = makeCorpus "cmudict";
comparative-sentences = makeCorpus "comparative_sentences";
comtrans = makeCorpus "comtrans";
conll2000 = makeCorpus "conll2000";
conll2002 = makeCorpus "conll2002";
conll2007 = makeCorpus "conll2007";
crubadan = makeCorpus "crubadan";
dependency-treebank = makeCorpus "dependency_treebank";
dolch = makeCorpus "dolch";
europarl-raw = makeCorpus "europarl_raw";
extended-omw = makeCorpus "extended_omw";
floresta = makeCorpus "floresta";
framenet-v15 = makeCorpus "framenet_v15";
framenet-v17 = makeCorpus "framenet_v17";
gazetteers = makeCorpus "gazetteers";
genesis = makeCorpus "genesis";
gutenberg = makeCorpus "gutenberg";
ieer = makeCorpus "ieer";
inaugural = makeCorpus "inaugural";
indian = makeCorpus "indian";
jeita = makeCorpus "jeita";
kimmo = makeCorpus "kimmo";
knbc = makeCorpus "knbc";
lin-thesaurus = makeCorpus "lin_thesaurus";
mac-morpho = makeCorpus "mac_morpho";
machado = makeCorpus "machado";
masc-tagged = makeCorpus "masc_tagged";
movie-reviews = makeCorpus "movie_reviews";
mte-teip5 = makeCorpus "mte_teip5";
names = makeCorpus "names";
nombank-1-0 = makeCorpus "nombank.1.0";
nonbreaking-prefixes = makeCorpus "nonbreaking_prefixes";
nps-chat = makeCorpus "nps_chat";
omw = makeCorpus "omw";
omw-1-4 = makeCorpus "omw-1.4";
opinion-lexicon = makeCorpus "opinion_lexicon";
panlex-swadesh = makeCorpus "panlex_swadesh";
paradigms = makeCorpus "paradigms";
pe08 = makeCorpus "pe08";
pil = makeCorpus "pil";
pl196x = makeCorpus "pl196x";
ppattach = makeCorpus "ppattach";
problem-reports = makeCorpus "problem_reports";
product-reviews-1 = makeCorpus "product_reviews_1";
product-reviews-2 = makeCorpus "product_reviews_2";
propbank = makeCorpus "propbank";
pros-cons = makeCorpus "pros_cons";
ptb = makeCorpus "ptb";
qc = makeCorpus "qc";
reuters = makeCorpus "reuters";
rte = makeCorpus "rte";
semcor = makeCorpus "semcor";
senseval = makeCorpus "senseval";
sentence-polarity = makeCorpus "sentence_polarity";
sentiwordnet = makeCorpus "sentiwordnet";
shakespeare = makeCorpus "shakespeare";
sinica-treebank = makeCorpus "sinica_treebank";
smultron = makeCorpus "smultron";
state-union = makeCorpus "state_union";
stopwords = makeCorpus "stopwords";
subjectivity = makeCorpus "subjectivity";
swadesh = makeCorpus "swadesh";
switchboard = makeCorpus "switchboard";
timit = makeCorpus "timit";
toolbox = makeCorpus "toolbox";
treebank = makeCorpus "treebank";
twitter-samples = makeCorpus "twitter_samples";
udhr = makeCorpus "udhr";
udhr2 = makeCorpus "udhr2";
unicode-samples = makeCorpus "unicode_samples";
universal-treebanks-v20 = makeCorpus "universal_treebanks_v20";
verbnet = makeCorpus "verbnet";
verbnet3 = makeCorpus "verbnet3";
webtext = makeCorpus "webtext";
wordnet = makeCorpus "wordnet";
wordnet-ic = makeCorpus "wordnet_ic";
wordnet2021 = makeCorpus "wordnet2021";
wordnet2022 = makeCorpus "wordnet2022";
wordnet31 = makeCorpus "wordnet31";
words = makeCorpus "words";
ycoe = makeCorpus "ycoe";
## Grammars
basque-grammars = makeGrammar "basque_grammars";
book-grammars = makeGrammar "book_grammars";
large-grammars = makeGrammar "large_grammars";
sample-grammars = makeGrammar "sample_grammars";
spanish-grammars = makeGrammar "spanish_grammars";
## Help
tagsets-json = makeHelp "tagsets_json";
## Misc
mwa-ppdb = makeMisc "mwa_ppdb";
perluniprops = makeMisc "perluniprops";
## Models
bllip-wsj-no-aux = makeModel "bllip_wsj_no_aux";
moses-sample = makeModel "moses_sample";
wmt15-eval = makeModel "wmt15_eval";
word2vec-sample = makeModel "word2vec_sample";
## Taggers
averaged-perceptron-tagger = makeTagger "averaged_perceptron_tagger";
averaged-perceptron-tagger-eng = makeTagger "averaged_perceptron_tagger_eng";
averaged-perceptron-tagger-ru = makeTagger "averaged_perceptron_tagger_ru";
averaged-perceptron-tagger-rus = makeTagger "averaged_perceptron_tagger_rus";
maxent-treebank-pos-tagger = makeTagger "maxent_treebank_pos_tagger";
maxent-treebank-pos-tagger-tab = makeTagger "maxent_treebank_pos_tagger_tab";
universal-tagset = makeTagger "universal_tagset";
## Tokenizers
punkt = makeTokenizer "punkt";
punkt-tab = makeTokenizer "punkt_tab";
## Stemmers
porter-test = makeStemmer "porter_test";
rslp = makeStemmer "rslp";
snowball-data = makeStemmer "snowball_data";
})

2
pkgs/top-level/all-packages.nix
View file

@ -2344,7 +2344,7 @@ with pkgs;
mpd-sima = python3Packages.callPackage ../tools/audio/mpd-sima { };
nltk-data = callPackage ../tools/text/nltk-data { };
nltk-data = lib.recurseIntoAttrs (callPackage ../tools/text/nltk-data { });
seabios-coreboot = seabios.override { ___build-type = "coreboot"; };
seabios-csm = seabios.override { ___build-type = "csm"; };