diff --git a/docs/samples/python/gliner.md b/docs/samples/python/gliner.md index 13dfe11bc7..0e325a0f80 100644 --- a/docs/samples/python/gliner.md +++ b/docs/samples/python/gliner.md @@ -75,6 +75,35 @@ results = analyzer_engine.analyze( print(results) ``` +## Scoping Labels per Recognizer + +By default, requested Presidio entities not covered by `entity_mapping` are also +sent to GLiNER as labels. When using multiple GLiNER recognizers with different +mappings or thresholds, set `include_requested_entities_as_labels` to `false` so +each recognizer sends only its configured labels to the model. In a unified +analyzer configuration, set the option on each recognizer that should be +restricted: + +```yaml +recognizer_registry: + recognizers: + - name: OrganizationGLiNER + class_name: GLiNERRecognizer + type: predefined + entity_mapping: + organization: ORGANIZATION + threshold: 0.8 + include_requested_entities_as_labels: false + + - name: AddressGLiNER + class_name: GLiNERRecognizer + type: predefined + entity_mapping: + address: ADDRESS + threshold: 0.65 + include_requested_entities_as_labels: false +``` + ## ONNX Runtime Support GLiNERRecognizer supports using ONNX Runtime as a backend, which provides better CPU compatibility and can prevent crashes on older CPUs without AVX2 instruction set support (e.g., Intel Sandy Bridge). @@ -101,4 +130,3 @@ gliner_recognizer = GLiNERRecognizer( - Can provide better performance on certain CPU architectures **Note:** Make sure `onnxruntime` is installed when using this feature. It's included in the `gliner` extra dependencies. - diff --git a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py index 3b58101d92..b2f484afdd 100644 --- a/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py +++ b/presidio-analyzer/presidio_analyzer/input_validation/yaml_recognizer_models.py @@ -197,6 +197,12 @@ class GLiNERRecognizerConfig(PredefinedRecognizerConfig): None, description="Use multi-label classification" ) threshold: Optional[float] = Field(None, description="Confidence threshold") + include_requested_entities_as_labels: Optional[bool] = Field( + None, + description=( + "Add requested entities not covered by entity_mapping as GLiNER labels" + ), + ) map_location: Optional[str] = Field(None, description="Device (cpu/gpu/etc.)") load_onnx_model: Optional[bool] = Field(None, description="Load ONNX model") onnx_model_file: Optional[str] = Field(None, description="ONNX model file name") diff --git a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py index 4e12f7e634..6c66d092c1 100644 --- a/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py +++ b/presidio-analyzer/presidio_analyzer/predefined_recognizers/ner/gliner_recognizer.py @@ -42,6 +42,7 @@ def __init__( text_chunker: Optional[BaseTextChunker] = None, load_onnx_model: bool = False, onnx_model_file: str = "model.onnx", + include_requested_entities_as_labels: bool = True, **model_kwargs, ): """GLiNER model based entity recognizer. @@ -73,6 +74,10 @@ def __init__( Only used when load_onnx_model is True. This is passed directly to GLiNER.from_pretrained(). GLiNER looks for this file in the model directory (downloaded or cached model path). Default is "model.onnx". + :param include_requested_entities_as_labels: Whether requested Presidio + entities not covered by entity_mapping should be added as ad-hoc GLiNER + labels. Defaults to True to preserve existing behavior. If False, only + configured GLiNER labels are sent to the model. :param model_kwargs: Additional keyword arguments to pass to GLiNER.from_pretrained(). This allows passing future parameters to the GLiNER model without explicit support in this recognizer. @@ -114,6 +119,7 @@ def __init__( self.flat_ner = flat_ner self.multi_label = multi_label self.threshold = threshold + self.include_requested_entities_as_labels = include_requested_entities_as_labels self.load_onnx_model = load_onnx_model self.onnx_model_file = onnx_model_file self.model_kwargs = model_kwargs @@ -167,7 +173,6 @@ def analyze( :param nlp_artifacts: N/A for this recognizer """ - # combine the input labels as this model allows for ad-hoc labels labels = self.__create_input_labels(entities) # Process text with automatic chunking @@ -216,9 +221,12 @@ def predict_func(text: str) -> List[RecognizerResult]: return predictions - def __create_input_labels(self, entities): - """Append the entities requested by the user to the list of labels if it's not there.""" # noqa: E501 + def __create_input_labels(self, entities: List[str]) -> List[str]: + """Build model labels from the mapping and, when enabled, the request.""" labels = list(self.gliner_labels) + if not self.include_requested_entities_as_labels: + return labels + for entity in entities: if ( entity not in self.model_to_presidio_entity_mapping.values() diff --git a/presidio-analyzer/tests/test_gliner_recognizer.py b/presidio-analyzer/tests/test_gliner_recognizer.py index 528634441d..5089f6559b 100644 --- a/presidio-analyzer/tests/test_gliner_recognizer.py +++ b/presidio-analyzer/tests/test_gliner_recognizer.py @@ -1,10 +1,13 @@ import sys - -import pytest from unittest.mock import MagicMock, patch -from presidio_analyzer.predefined_recognizers import GLiNERRecognizer +import pytest from presidio_analyzer.chunkers import CharacterBasedTextChunker +from presidio_analyzer.input_validation import ConfigurationValidator +from presidio_analyzer.predefined_recognizers import GLiNERRecognizer +from presidio_analyzer.recognizer_registry.recognizers_loader_utils import ( + RecognizerListLoader, +) @pytest.fixture @@ -261,6 +264,129 @@ def mock_predict_entities(text, labels, flat_ner, threshold, multi_label): ) +def test_requested_entities_are_added_as_ad_hoc_labels_by_default(): + """Requested entities not covered by the mapping remain labels by default.""" + text = "Acme HQ is at 1 Main Street." + entities = ["ORGANIZATION", "PRODUCT", "ADDRESS"] + address_model = MagicMock() + address_model.predict_entities.return_value = [ + {"label": "ORGANIZATION", "start": 0, "end": 4, "score": 0.70}, + {"label": "address", "start": 14, "end": 27, "score": 0.90}, + ] + + with patch(GLINER_MOCK_PATH) as mock_gliner_class: + mock_gliner_class.from_pretrained.return_value = address_model + address_recognizer = GLiNERRecognizer( + name="AddressGLiNER", + entity_mapping={"address": "ADDRESS"}, + threshold=0.65, + map_location="cpu", + ) + + address_results = address_recognizer.analyze(text, entities) + address_call = address_model.predict_entities.call_args.kwargs + + assert address_call["labels"] == ["address", "ORGANIZATION", "PRODUCT"] + assert address_call["threshold"] == 0.65 + assert {result.entity_type for result in address_results} == { + "ADDRESS", + "ORGANIZATION", + } + + +def test_yaml_can_scope_multiple_recognizers_to_configured_labels(): + """YAML can keep each GLiNERRecognizer scoped to its configured labels.""" + text = "Acme HQ is at 1 Main Street." + entities = ["ORGANIZATION", "PRODUCT", "ADDRESS"] + organization_model = MagicMock() + organization_model.predict_entities.return_value = [] + address_model = MagicMock() + address_model.predict_entities.return_value = [ + {"label": "address", "start": 14, "end": 27, "score": 0.90} + ] + raw_config = { + "supported_languages": ["en"], + "global_regex_flags": 26, + "recognizers": [ + { + "name": "OrganizationGLiNER", + "class_name": "GLiNERRecognizer", + "type": "predefined", + "supported_language": "en", + "entity_mapping": {"organization": "ORGANIZATION"}, + "threshold": 0.80, + "include_requested_entities_as_labels": False, + "map_location": "cpu", + }, + { + "name": "AddressGLiNER", + "class_name": "GLiNERRecognizer", + "type": "predefined", + "supported_language": "en", + "entity_mapping": {"address": "ADDRESS"}, + "threshold": 0.65, + "include_requested_entities_as_labels": False, + "map_location": "cpu", + } + ], + } + validated_config = ( + ConfigurationValidator.validate_recognizer_registry_configuration(raw_config) + ) + + with patch(GLINER_MOCK_PATH) as mock_gliner_class: + mock_gliner_class.from_pretrained.side_effect = [ + organization_model, + address_model, + ] + recognizers = list(RecognizerListLoader.get(**validated_config)) + + recognizers_by_name = {recognizer.name: recognizer for recognizer in recognizers} + organization_results = recognizers_by_name["OrganizationGLiNER"].analyze( + text, entities + ) + address_results = recognizers_by_name["AddressGLiNER"].analyze(text, entities) + organization_call = organization_model.predict_entities.call_args.kwargs + address_call = address_model.predict_entities.call_args.kwargs + + assert organization_call["labels"] == ["organization"] + assert organization_call["threshold"] == 0.80 + assert address_call["labels"] == ["address"] + assert address_call["threshold"] == 0.65 + assert organization_results == [] + assert [result.entity_type for result in address_results] == ["ADDRESS"] + + +def test_include_requested_entities_option_preserves_positional_arguments(): + """Existing positional constructor arguments keep their meaning.""" + text_chunker = MagicMock() + with patch(GLINER_MOCK_PATH) as mock_gliner_class: + mock_gliner_class.from_pretrained.return_value = MagicMock() + recognizer = GLiNERRecognizer( + ["PERSON"], + "PositionalGLiNER", + "en", + "1.0.0", + None, + None, + "custom/gliner-model", + True, + False, + 0.42, + "cpu", + text_chunker, + True, + "custom.onnx", + ) + + assert recognizer.threshold == 0.42 + assert recognizer.map_location == "cpu" + assert recognizer.text_chunker is text_chunker + assert recognizer.load_onnx_model is True + assert recognizer.onnx_model_file == "custom.onnx" + assert recognizer.include_requested_entities_as_labels is True + + @pytest.mark.parametrize( "load_onnx_model,onnx_model_file,expected_onnx_model,expected_file", [ @@ -322,4 +448,3 @@ def test_when_model_kwargs_then_passes_to_from_pretrained(): assert call_kwargs["custom_param1"] == "value1" assert call_kwargs["custom_param2"] == 42 - diff --git a/presidio-analyzer/tests/test_yaml_recognizer_models.py b/presidio-analyzer/tests/test_yaml_recognizer_models.py index 72415031d8..2b40a20b35 100644 --- a/presidio-analyzer/tests/test_yaml_recognizer_models.py +++ b/presidio-analyzer/tests/test_yaml_recognizer_models.py @@ -301,6 +301,7 @@ def test_configuration_validator_uses_recognizer_specific_dump_rules(): assert gliner_recognizer["model_name"] == "custom/gliner-model" assert "threshold" not in gliner_recognizer assert "flat_ner" not in gliner_recognizer + assert "include_requested_entities_as_labels" not in gliner_recognizer assert "entity_mapping" not in gliner_recognizer assert predefined_recognizer["name"] == "CreditCardRecognizer" assert predefined_recognizer["supported_language"] is None @@ -757,6 +758,7 @@ def test_gliner_recognizer_config_model_name(): "threshold": 0.5, "flat_ner": False, "multi_label": True, + "include_requested_entities_as_labels": False, } ] } @@ -769,6 +771,7 @@ def test_gliner_recognizer_config_model_name(): assert recognizer.threshold == 0.5 assert recognizer.flat_ner is False assert recognizer.multi_label is True + assert recognizer.include_requested_entities_as_labels is False def test_gliner_recognizer_config_model_dump_excludes_none(): @@ -788,6 +791,7 @@ def test_gliner_recognizer_config_model_dump_excludes_none(): # Fields not provided should be excluded, not set to None assert "flat_ner" not in dumped assert "threshold" not in dumped + assert "include_requested_entities_as_labels" not in dumped assert "entity_mapping" not in dumped