Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 29 additions & 1 deletion docs/samples/python/gliner.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,35 @@ results = analyzer_engine.analyze(
print(results)
```

## Scoping Labels per Recognizer

By default, requested Presidio entities not covered by `entity_mapping` are also
sent to GLiNER as labels. When using multiple GLiNER recognizers with different
mappings or thresholds, set `include_requested_entities_as_labels` to `false` so
each recognizer sends only its configured labels to the model. In a unified
analyzer configuration, set the option on each recognizer that should be
restricted:

```yaml
recognizer_registry:
recognizers:
- name: OrganizationGLiNER
class_name: GLiNERRecognizer
type: predefined
entity_mapping:
organization: ORGANIZATION
threshold: 0.8
include_requested_entities_as_labels: false

- name: AddressGLiNER
class_name: GLiNERRecognizer
type: predefined
entity_mapping:
address: ADDRESS
threshold: 0.65
include_requested_entities_as_labels: false
```

## ONNX Runtime Support

GLiNERRecognizer supports using ONNX Runtime as a backend, which provides better CPU compatibility and can prevent crashes on older CPUs without AVX2 instruction set support (e.g., Intel Sandy Bridge).
Expand All @@ -101,4 +130,3 @@ gliner_recognizer = GLiNERRecognizer(
- Can provide better performance on certain CPU architectures

**Note:** Make sure `onnxruntime` is installed when using this feature. It's included in the `gliner` extra dependencies.

Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,12 @@ class GLiNERRecognizerConfig(PredefinedRecognizerConfig):
None, description="Use multi-label classification"
)
threshold: Optional[float] = Field(None, description="Confidence threshold")
include_requested_entities_as_labels: Optional[bool] = Field(
None,
description=(
"Add requested entities not covered by entity_mapping as GLiNER labels"
),
)
map_location: Optional[str] = Field(None, description="Device (cpu/gpu/etc.)")
load_onnx_model: Optional[bool] = Field(None, description="Load ONNX model")
onnx_model_file: Optional[str] = Field(None, description="ONNX model file name")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def __init__(
text_chunker: Optional[BaseTextChunker] = None,
load_onnx_model: bool = False,
onnx_model_file: str = "model.onnx",
include_requested_entities_as_labels: bool = True,
**model_kwargs,
):
"""GLiNER model based entity recognizer.
Expand Down Expand Up @@ -73,6 +74,10 @@ def __init__(
Only used when load_onnx_model is True. This is passed directly to
GLiNER.from_pretrained(). GLiNER looks for this file in the model
directory (downloaded or cached model path). Default is "model.onnx".
:param include_requested_entities_as_labels: Whether requested Presidio
entities not covered by entity_mapping should be added as ad-hoc GLiNER
labels. Defaults to True to preserve existing behavior. If False, only
configured GLiNER labels are sent to the model.
:param model_kwargs: Additional keyword arguments to pass to
GLiNER.from_pretrained(). This allows passing future parameters
to the GLiNER model without explicit support in this recognizer.
Expand Down Expand Up @@ -114,6 +119,7 @@ def __init__(
self.flat_ner = flat_ner
self.multi_label = multi_label
self.threshold = threshold
self.include_requested_entities_as_labels = include_requested_entities_as_labels
self.load_onnx_model = load_onnx_model
self.onnx_model_file = onnx_model_file
self.model_kwargs = model_kwargs
Expand Down Expand Up @@ -167,7 +173,6 @@ def analyze(
:param nlp_artifacts: N/A for this recognizer
"""

# combine the input labels as this model allows for ad-hoc labels
labels = self.__create_input_labels(entities)

# Process text with automatic chunking
Expand Down Expand Up @@ -216,9 +221,12 @@ def predict_func(text: str) -> List[RecognizerResult]:

return predictions

def __create_input_labels(self, entities):
"""Append the entities requested by the user to the list of labels if it's not there.""" # noqa: E501
def __create_input_labels(self, entities: List[str]) -> List[str]:
"""Build model labels from the mapping and, when enabled, the request."""
labels = list(self.gliner_labels)
if not self.include_requested_entities_as_labels:
return labels

for entity in entities:
if (
entity not in self.model_to_presidio_entity_mapping.values()
Expand Down
133 changes: 129 additions & 4 deletions presidio-analyzer/tests/test_gliner_recognizer.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
import sys

import pytest
from unittest.mock import MagicMock, patch

from presidio_analyzer.predefined_recognizers import GLiNERRecognizer
import pytest
from presidio_analyzer.chunkers import CharacterBasedTextChunker
from presidio_analyzer.input_validation import ConfigurationValidator
from presidio_analyzer.predefined_recognizers import GLiNERRecognizer
from presidio_analyzer.recognizer_registry.recognizers_loader_utils import (
RecognizerListLoader,
)


@pytest.fixture
Expand Down Expand Up @@ -261,6 +264,129 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label):
)


def test_requested_entities_are_added_as_ad_hoc_labels_by_default():
"""Requested entities not covered by the mapping remain labels by default."""
text = "Acme HQ is at 1 Main Street."
entities = ["ORGANIZATION", "PRODUCT", "ADDRESS"]
address_model = MagicMock()
address_model.predict_entities.return_value = [
{"label": "ORGANIZATION", "start": 0, "end": 4, "score": 0.70},
{"label": "address", "start": 14, "end": 27, "score": 0.90},
]

with patch(GLINER_MOCK_PATH) as mock_gliner_class:
mock_gliner_class.from_pretrained.return_value = address_model
address_recognizer = GLiNERRecognizer(
name="AddressGLiNER",
entity_mapping={"address": "ADDRESS"},
threshold=0.65,
map_location="cpu",
)

address_results = address_recognizer.analyze(text, entities)
address_call = address_model.predict_entities.call_args.kwargs

assert address_call["labels"] == ["address", "ORGANIZATION", "PRODUCT"]
assert address_call["threshold"] == 0.65
assert {result.entity_type for result in address_results} == {
"ADDRESS",
"ORGANIZATION",
}


def test_yaml_can_scope_multiple_recognizers_to_configured_labels():
"""YAML can keep each GLiNERRecognizer scoped to its configured labels."""
text = "Acme HQ is at 1 Main Street."
entities = ["ORGANIZATION", "PRODUCT", "ADDRESS"]
organization_model = MagicMock()
organization_model.predict_entities.return_value = []
address_model = MagicMock()
address_model.predict_entities.return_value = [
{"label": "address", "start": 14, "end": 27, "score": 0.90}
]
raw_config = {
"supported_languages": ["en"],
"global_regex_flags": 26,
"recognizers": [
{
"name": "OrganizationGLiNER",
"class_name": "GLiNERRecognizer",
"type": "predefined",
"supported_language": "en",
"entity_mapping": {"organization": "ORGANIZATION"},
"threshold": 0.80,
"include_requested_entities_as_labels": False,
"map_location": "cpu",
},
{
"name": "AddressGLiNER",
"class_name": "GLiNERRecognizer",
"type": "predefined",
"supported_language": "en",
"entity_mapping": {"address": "ADDRESS"},
"threshold": 0.65,
"include_requested_entities_as_labels": False,
"map_location": "cpu",
}
],
}
validated_config = (
ConfigurationValidator.validate_recognizer_registry_configuration(raw_config)
)

with patch(GLINER_MOCK_PATH) as mock_gliner_class:
mock_gliner_class.from_pretrained.side_effect = [
organization_model,
address_model,
]
recognizers = list(RecognizerListLoader.get(**validated_config))

recognizers_by_name = {recognizer.name: recognizer for recognizer in recognizers}
organization_results = recognizers_by_name["OrganizationGLiNER"].analyze(
text, entities
)
address_results = recognizers_by_name["AddressGLiNER"].analyze(text, entities)
organization_call = organization_model.predict_entities.call_args.kwargs
address_call = address_model.predict_entities.call_args.kwargs

assert organization_call["labels"] == ["organization"]
assert organization_call["threshold"] == 0.80
assert address_call["labels"] == ["address"]
assert address_call["threshold"] == 0.65
assert organization_results == []
assert [result.entity_type for result in address_results] == ["ADDRESS"]


def test_include_requested_entities_option_preserves_positional_arguments():
"""Existing positional constructor arguments keep their meaning."""
text_chunker = MagicMock()
with patch(GLINER_MOCK_PATH) as mock_gliner_class:
mock_gliner_class.from_pretrained.return_value = MagicMock()
recognizer = GLiNERRecognizer(
["PERSON"],
"PositionalGLiNER",
"en",
"1.0.0",
None,
None,
"custom/gliner-model",
True,
False,
0.42,
"cpu",
text_chunker,
True,
"custom.onnx",
)

assert recognizer.threshold == 0.42
assert recognizer.map_location == "cpu"
assert recognizer.text_chunker is text_chunker
assert recognizer.load_onnx_model is True
assert recognizer.onnx_model_file == "custom.onnx"
assert recognizer.include_requested_entities_as_labels is True


@pytest.mark.parametrize(
"load_onnx_model,onnx_model_file,expected_onnx_model,expected_file",
[
Expand Down Expand Up @@ -322,4 +448,3 @@ def test_when_model_kwargs_then_passes_to_from_pretrained():
assert call_kwargs["custom_param1"] == "value1"
assert call_kwargs["custom_param2"] == 42


4 changes: 4 additions & 0 deletions presidio-analyzer/tests/test_yaml_recognizer_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ def test_configuration_validator_uses_recognizer_specific_dump_rules():
assert gliner_recognizer["model_name"] == "custom/gliner-model"
assert "threshold" not in gliner_recognizer
assert "flat_ner" not in gliner_recognizer
assert "include_requested_entities_as_labels" not in gliner_recognizer
assert "entity_mapping" not in gliner_recognizer
assert predefined_recognizer["name"] == "CreditCardRecognizer"
assert predefined_recognizer["supported_language"] is None
Expand Down Expand Up @@ -757,6 +758,7 @@ def test_gliner_recognizer_config_model_name():
"threshold": 0.5,
"flat_ner": False,
"multi_label": True,
"include_requested_entities_as_labels": False,
}
]
}
Expand All @@ -769,6 +771,7 @@ def test_gliner_recognizer_config_model_name():
assert recognizer.threshold == 0.5
assert recognizer.flat_ner is False
assert recognizer.multi_label is True
assert recognizer.include_requested_entities_as_labels is False


def test_gliner_recognizer_config_model_dump_excludes_none():
Expand All @@ -788,6 +791,7 @@ def test_gliner_recognizer_config_model_dump_excludes_none():
# Fields not provided should be excluded, not set to None
assert "flat_ner" not in dumped
assert "threshold" not in dumped
assert "include_requested_entities_as_labels" not in dumped
assert "entity_mapping" not in dumped


Expand Down
Loading