fix(engine, java): reduce heap retention by releasing AST and detection state eagerly

engine/src/main/java/com/ibm/engine/detection/DetectionStore.java

@@ -259,6 +259,15 @@ public void analyse(@Nonnull final T tree) {
         this.statusReporting.emitFinding();
     }
 
+    public void release() {
+        for (DetectionStore<R, T, S, P> child : getChildren()) {
+            child.release();
+        }
+        detectionValues.clear();
+        children.clear();
+        actionValue = null;
+    }
+
     @SuppressWarnings("java:S3776")
     public void onReceivingNewDetection(@Nonnull IDetection<T> detection) {
         if (detection instanceof MethodDetection<T> methodDetection) {
@@ -293,34 +302,32 @@ public void onReceivingNewDetection(@Nonnull IDetection<T> detection) {
 
         } else if (detection instanceof ValueDetection<?, T> valueDetection) {
             final DetectableParameter<T> detectableParameter = valueDetection.detectableParameter();
+            Optional<IValue<T>> emittedValue =
+                    valueDetection.toValue(valueDetection.detectableParameter().getiValueFactory());
 
             final Optional<Integer> positionMove = detectableParameter.getShouldBeMovedUnder();
             // Check if the parameter should be moved under
             if (positionMove.isPresent()) {
                 final int id = positionMove.get();
                 // Get the iValue to be detected and store it in a variable
-                valueDetection
-                        .toValue(valueDetection.detectableParameter().getiValueFactory())
-                        .ifPresent(
-                                iValue -> {
-                                    // Create a detection store with the given parameters
-                                    DetectionStore<R, T, S, P> detectionStore =
-                                            new DetectionStore<>(
-                                                    level + 1,
-                                                    detectionRule,
-                                                    scanContext,
-                                                    handler,
-                                                    statusReporting);
-                                    // Compute the detection values for the given id
-                                    addValue(detectionStore, id, iValue);
-                                    // Attach the detection store to the given id
-                                    this.attach(id, detectionStore);
-                                });
+                emittedValue.ifPresent(
+                        iValue -> {
+                            // Create a detection store with the given parameters
+                            DetectionStore<R, T, S, P> detectionStore =
+                                    new DetectionStore<>(
+                                            level + 1,
+                                            detectionRule,
+                                            scanContext,
+                                            handler,
+                                            statusReporting);
+                            // Compute the detection values for the given id
+                            addValue(detectionStore, id, iValue);
+                            // Attach the detection store to the given id
+                            this.attach(id, detectionStore);
+                        });
             } else {
-                valueDetection
-                        .toValue(valueDetection.detectableParameter().getiValueFactory())
-                        .ifPresent(
-                                iValue -> addValue(this, detectableParameter.getIndex(), iValue));
+                emittedValue.ifPresent(
+                        iValue -> addValue(this, detectableParameter.getIndex(), iValue));
             }
 
             // follow method parameter related detection rules
@@ -392,6 +399,7 @@ public void onDetectedDependingParameter(
     }
 
     public void onNewHookRegistration(@Nonnull IHook<R, T, S, P> hook) {
+        statusReporting.onDeferredHookRegistration();
         handler.subscribeToHookDetectionObservable(hook, this);
     }
 

engine/src/main/java/com/ibm/engine/executive/DetectionExecutive.java

@@ -30,15 +30,18 @@
 import java.util.Collection;
 import java.util.List;
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public class DetectionExecutive<R, T, S, P>
         implements IStatusReporting<R, T, S, P>, IDomainEvent<Finding<R, T, S, P>> {
     @Nonnull private final List<IObserver<Finding<R, T, S, P>>> listeners = new ArrayList<>();
 
     @Nonnull private final DetectionStore<R, T, S, P> rootDetectionStore;
-    @Nonnull private final T tree;
+    @Nullable private T tree;
     private int expectedRuleVisits;
     private int visitedRules = 0;
+    private boolean deferredHookRegistered = false;
+    private boolean released = false;
 
     public DetectionExecutive(
             @Nonnull final T tree,
@@ -52,7 +55,11 @@ public DetectionExecutive(
     }
 
     public void start() {
-        this.rootDetectionStore.analyse(tree);
+        try {
+            this.rootDetectionStore.analyse(tree);
+        } finally {
+            this.tree = null;
+        }
     }
 
     @Override
@@ -65,12 +72,18 @@ public void emitFinding(@Nonnull final DetectionStore<R, T, S, P> rootDetectionS
         if (this.expectedRuleVisits != this.visitedRules) {
             return;
         }
-        getRootStoresWithValue(rootDetectionStore)
-                .forEach(
-                        store -> {
-                            final Finding<R, T, S, P> finding = new Finding<>(store);
-                            this.notify(finding);
-                        });
+        try {
+            getRootStoresWithValue(rootDetectionStore)
+                    .forEach(
+                            store -> {
+                                final Finding<R, T, S, P> finding = new Finding<>(store);
+                                this.notify(finding);
+                            });
+        } finally {
+            if (!deferredHookRegistered) {
+                releaseResources();
+            }
+        }
     }
 
     @Override
@@ -83,6 +96,23 @@ public void addAdditionalExpectedRuleVisits(int number) {
         this.expectedRuleVisits += number;
     }
 
+    @Override
+    public void onDeferredHookRegistration() {
+        this.deferredHookRegistered = true;
+    }
+
+    public boolean hasDeferredHooks() {
+        return deferredHookRegistered;
+    }
+
+    public boolean isReleased() {
+        return released;
+    }
+
+    public void releaseDeferredResources() {
+        releaseResources();
+    }
+
     @Nonnull
     private List<DetectionStore<R, T, S, P>> getRootStoresWithValue(
             @Nonnull DetectionStore<R, T, S, P> detectionStore) {
@@ -110,4 +140,14 @@ public void unsubscribe(@Nonnull IObserver<Finding<R, T, S, P>> listener) {
     public void notify(@Nonnull Finding<R, T, S, P> finding) {
         this.listeners.forEach(listener -> listener.update(finding));
     }
+
+    private void releaseResources() {
+        if (released) {
+            return;
+        }
+        released = true;
+        rootDetectionStore.release();
+        listeners.clear();
+        tree = null;
+    }
 }

engine/src/main/java/com/ibm/engine/executive/IStatusReporting.java

@@ -30,4 +30,8 @@ public interface IStatusReporting<R, T, S, P> {
     void incrementVisitedRules();
 
     void addAdditionalExpectedRuleVisits(int number);
+
+    default void onDeferredHookRegistration() {
+        // Optional lifecycle callback for deferred hook-based findings.
+    }
 }

engine/src/test/java/com/ibm/engine/detection/DetectionStoreReleaseTest.java

@@ -0,0 +1,148 @@
+/*
+ * Sonar Cryptography Plugin
+ * Copyright (C) 2026 PQCA
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.ibm.engine.detection;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.mockito.Mockito.mock;
+
+import com.ibm.engine.executive.IStatusReporting;
+import com.ibm.engine.language.IScanContext;
+import com.ibm.engine.model.IAction;
+import com.ibm.engine.model.IValue;
+import com.ibm.engine.rule.IDetectionRule;
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class DetectionStoreReleaseTest {
+
+    private IDetectionRule<Object> detectionRule;
+    private IScanContext<Object, Object> scanContext;
+    private Handler<Object, Object, Object, Object> handler;
+    private IStatusReporting<Object, Object, Object, Object> statusReporting;
+    private IValue<Object> mockValue;
+    private IAction<Object> mockAction;
+
+    private DetectionStore<Object, Object, Object, Object> store;
+
+    @SuppressWarnings("unchecked")
+    @BeforeEach
+    void setUp() {
+        detectionRule = mock(IDetectionRule.class);
+        scanContext = mock(IScanContext.class);
+        handler = mock(Handler.class);
+        statusReporting = mock(IStatusReporting.class);
+        mockValue = mock(IValue.class);
+        mockAction = mock(IAction.class);
+        store = new DetectionStore<>(0, detectionRule, scanContext, handler, statusReporting);
+    }
+
+    @Test
+    void release_clearsDetectionValues() {
+        store.detectionValues.put(0, new ArrayList<>(List.of(mockValue)));
+        assertThat(store.getDetectionValues()).isNotEmpty();
+
+        store.release();
+
+        assertThat(store.getDetectionValues()).isEmpty();
+    }
+
+    @Test
+    void release_clearsChildren() {
+        DetectionStore<Object, Object, Object, Object> child =
+                new DetectionStore<>(1, detectionRule, scanContext, handler, statusReporting);
+        store.children.put(0, new ArrayList<>(List.of(child)));
+        assertThat(store.getChildren()).isNotEmpty();
+
+        store.release();
+
+        assertThat(store.getChildren()).isEmpty();
+    }
+
+    @Test
+    void release_clearsActionValue() {
+        store.actionValue = mockAction;
+        assertThat(store.getActionValue()).isPresent();
+
+        store.release();
+
+        assertThat(store.getActionValue()).isEmpty();
+    }
+
+    @Test
+    void release_onEmptyStore_doesNotThrow() {
+        assertThatCode(() -> store.release()).doesNotThrowAnyException();
+        assertThat(store.getDetectionValues()).isEmpty();
+        assertThat(store.getChildren()).isEmpty();
+    }
+
+    @Test
+    void release_isRecursive_clearsChildDetectionValues() {
+        DetectionStore<Object, Object, Object, Object> child =
+                new DetectionStore<>(1, detectionRule, scanContext, handler, statusReporting);
+        child.detectionValues.put(0, new ArrayList<>(List.of(mockValue)));
+        store.children.put(0, new ArrayList<>(List.of(child)));
+
+        store.release();
+
+        assertThat(store.getChildren()).isEmpty();
+        assertThat(child.getDetectionValues()).isEmpty();
+    }
+
+    @Test
+    void release_isRecursive_clearsDeepNestedChildren() {
+        DetectionStore<Object, Object, Object, Object> child =
+                new DetectionStore<>(1, detectionRule, scanContext, handler, statusReporting);
+        DetectionStore<Object, Object, Object, Object> grandchild =
+                new DetectionStore<>(2, detectionRule, scanContext, handler, statusReporting);
+
+        grandchild.detectionValues.put(0, new ArrayList<>(List.of(mockValue)));
+        child.children.put(0, new ArrayList<>(List.of(grandchild)));
+        store.children.put(0, new ArrayList<>(List.of(child)));
+
+        store.release();
+
+        assertThat(store.getChildren()).isEmpty();
+        assertThat(child.getChildren()).isEmpty();
+        assertThat(grandchild.getDetectionValues()).isEmpty();
+    }
+
+    @Test
+    void release_multipleValuesAndChildren_allCleared() {
+        store.detectionValues.put(0, new ArrayList<>(List.of(mockValue)));
+        store.detectionValues.put(1, new ArrayList<>(List.of(mockValue)));
+        store.actionValue = mockAction;
+
+        DetectionStore<Object, Object, Object, Object> child1 =
+                new DetectionStore<>(1, detectionRule, scanContext, handler, statusReporting);
+        DetectionStore<Object, Object, Object, Object> child2 =
+                new DetectionStore<>(1, detectionRule, scanContext, handler, statusReporting);
+        store.children.put(0, new ArrayList<>(List.of(child1)));
+        store.children.put(1, new ArrayList<>(List.of(child2)));
+
+        store.release();
+
+        assertThat(store.getDetectionValues()).isEmpty();
+        assertThat(store.getChildren()).isEmpty();
+        assertThat(store.getActionValue()).isEmpty();
+    }
+}