Scope rework patch

bbot/core/event/base.py

@@ -1188,8 +1188,6 @@ def _host(self):
 
 
 class DNS_NAME(DnsEvent):
-    _always_emit_tags = ["affiliate", "seed"]
-
     def sanitize_data(self, data):
         return validators.validate_host(data)
 
@@ -1210,6 +1208,9 @@ def _words(self):
 
 
 class OPEN_TCP_PORT(BaseEvent):
+    # we generally don't care about open ports on affiliates
+    _always_emit_tags = ["seed"]
+
     def sanitize_data(self, data):
         return validators.validate_open_port(data)
 

bbot/core/modules.py

@@ -328,7 +328,6 @@ def preload_module(self, module_file):
         ansible_tasks = []
         config = {}
         options_desc = {}
-        accept_seeds = None  # None means use default (True for passive, False otherwise)
         python_code = open(module_file).read()
         # take a hash of the code so we can keep track of when it changes
         module_hash = sha1(python_code).hexdigest()
@@ -365,16 +364,6 @@ def preload_module(self, module_file):
                         elif any(target.id == "meta" for target in class_attr.targets):
                             meta = ast.literal_eval(class_attr.value)
 
-                    # class attributes that are boolean values (like accept_seeds)
-                    if type(class_attr) == ast.Assign:
-                        # Check for accept_seeds = True/False
-                        if any(target.id == "accept_seeds" for target in class_attr.targets):
-                            # Handle both ast.Constant (Python 3.8+) and ast.NameConstant (older)
-                            if isinstance(class_attr.value, ast.Constant):
-                                accept_seeds = class_attr.value.value
-                            elif isinstance(class_attr.value, ast.NameConstant):
-                                accept_seeds = class_attr.value.value
-
                     # class attributes that are lists
                     if type(class_attr) == ast.Assign and type(class_attr.value) == ast.List:
                         # flags
@@ -441,7 +430,6 @@ def preload_module(self, module_file):
             "config": config,
             "options_desc": options_desc,
             "hash": module_hash,
-            "accept_seeds": accept_seeds,
             "deps": {
                 "modules": sorted(deps_modules),
                 "pip": deps_pip,

bbot/models/pydantic.py

@@ -148,8 +148,8 @@ def from_scan(cls, scan):
 class Target(BBOTBaseModel):
     name: str = "Default Target"
     strict_dns_scope: bool = False
-    seeds: List = []
     target: List = []
+    seeds: Optional[List] = None
     blacklist: List = []
     hash: Annotated[str, "indexed", "unique"]
     scope_hash: Annotated[str, "indexed"]

bbot/models/sql.py

@@ -125,8 +125,8 @@ class Scan(BBOTBaseModel, table=True):
 class Target(BBOTBaseModel, table=True):
     name: str = "Default Target"
     strict_dns_scope: bool = False
-    seeds: List = Field(default=[], sa_type=JSON)
     target: List = Field(default=[], sa_type=JSON)
+    seeds: Optional[List] = Field(default=None, sa_type=JSON)
     blacklist: List = Field(default=[], sa_type=JSON)
     hash: str = Field(sa_column=Column("hash", String(length=255), unique=True, primary_key=True, index=True))
     scope_hash: str = Field(sa_column=Column("scope_hash", String(length=255), index=True))

bbot/modules/base.py

@@ -101,7 +101,6 @@ class BaseModule:
     per_domain_only = False
     scope_distance_modifier = 0
     target_only = False
-    _accept_seeds = None  # None means "use default based on flags"
     in_scope_only = False
     accept_url_special = False
     _module_threads = 1
@@ -744,18 +743,10 @@ def accept_seeds(self):
         """
         Returns whether the module accepts seed events.
         Defaults to True for passive modules, False otherwise.
-        Can be explicitly overridden by setting _accept_seeds.
         """
-        if self._accept_seeds is not None:
-            return self._accept_seeds
         # Default to True for passive modules, False otherwise
         return "passive" in self.flags
 
-    @accept_seeds.setter
-    def accept_seeds(self, value):
-        """Allow explicit setting of accept_seeds to override the default."""
-        self._accept_seeds = value
-
     @property
     def max_scope_distance(self):
         if self.in_scope_only or self.target_only:
@@ -806,9 +797,8 @@ def _event_precheck(self, event):
             return True, "it is a seed event and module accepts seeds"
         if not event_type_watched:
             return False, "its type is not in watched_events"
-        if self.target_only:
-            if "target" not in event.tags:
-                return False, "it did not meet target_only filter criteria"
+        if self.target_only and "target" not in event.tags:
+            return False, "it did not meet target_only filter criteria"
 
         # limit js URLs to modules that opt in to receive them
         if (not self.accept_url_special) and event.type.startswith("URL"):

bbot/modules/internal/dnsresolve.py

@@ -289,6 +289,12 @@ async def resolve_event(self, event, types):
     def get_dns_parent(self, event):
         """
         Get the first parent DNS_NAME / IP_ADDRESS of an event. If one isn't found, create it.
+
+        Returns a 4-tuple of:
+        - the parent event
+        - whether the parent is in target
+        - whether the parent is blacklisted
+        - whether the parent is a new event, i.e. it is newly created or is the current event
         """
         for parent in event.get_parents(include_self=True):
             if parent.host == event.host and parent.type in ("IP_ADDRESS", "DNS_NAME", "DNS_NAME_UNRESOLVED"):

bbot/modules/internal/excavate.py

@@ -1163,8 +1163,8 @@ async def handle_event(self, event, **kwargs):
                 await self.emit_custom_parameters(event, "http_cookies", "COOKIE", "Custom Cookie")
                 await self.emit_custom_parameters(event, "http_headers", "HEADER", "Custom Header")
 
-                # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the TARGET, create a WEB
-                if self.url_querystring_remove is False and str(event.parent.parent.module) == "TARGET":
+                # if parameter extraction is enabled, and querystring removal is disabled, and the event is directly from the SEED, create a WEB
+                if self.url_querystring_remove is False and str(event.parent.parent.module) == "SEED":
                     self.debug(f"Processing target URL [{urlunparse(event.parsed_url)}] for GET parameters")
                     for (
                         method,

bbot/modules/output/base.py

@@ -38,12 +38,12 @@ def _event_precheck(self, event):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
+        if event.always_emit:
+            return True, "event is always emitted"
+
         # omit certain event types
         if event._omit:
-            if "seed" in event.tags:
-                reason = "it's a seed"
-                self.debug(f"Allowing omitted event: {event} because {reason}")
-            elif event.type in self.get_watched_events():
+            if event.type in self.get_watched_events():
                 reason = "its type is explicitly in watched_events"
                 self.debug(f"Allowing omitted event: {event} because {reason}")
             else:

bbot/modules/output/mongo.py

@@ -1,4 +1,4 @@
-from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo import AsyncMongoClient
 
 from bbot.models.pydantic import Event, Scan, Target
 from bbot.modules.output.base import BaseOutputModule
@@ -29,13 +29,13 @@ class Mongo(BaseOutputModule):
         "password": "The password to use to connect to the database",
         "collection_prefix": "Prefix the name of each collection with this string",
     }
-    deps_pip = ["motor~=3.6.0"]
+    deps_pip = ["pymongo~=4.15"]
 
     async def setup(self):
         self.uri = self.config.get("uri", "mongodb://localhost:27017")
         self.username = self.config.get("username", "")
         self.password = self.config.get("password", "")
-        self.db_client = AsyncIOMotorClient(self.uri, username=self.username, password=self.password)
+        self.db_client = AsyncMongoClient(self.uri, username=self.username, password=self.password)
 
         # Ping the server to confirm a successful connection
         try:

bbot/scanner/manager.py

@@ -48,7 +48,7 @@ async def init_events(self, event_seeds=None):
             event_seeds = sorted(event_seeds, key=lambda e: (host_size_key(str(e.host)), e.data))
             # queue root scan event
             await self.queue_event(root_event, {})
-            target_module = self.scan._make_dummy_module(name="TARGET", _type="TARGET")
+            target_module = self.scan._make_dummy_module(name="SEED", _type="SEED")
             # queue each seed in turn
             for event_seed in event_seeds:
                 event = self.scan.make_event(

bbot/scanner/preset/args.py

@@ -105,11 +105,11 @@ def parsed(self):
     def preset_from_args(self):
         # the order here is important
         # first we make the preset
-        # -t/--targets becomes target_list (defines target, what in_target() checks)
+        # -t/--targets becomes target (defines target, what in_target() checks)
         # -s/--seeds becomes seeds (drives passive modules), defaults to targets if not specified
         seeds = self.parsed.seeds if self.parsed.seeds is not None else self.parsed.targets
         args_preset = self.preset.__class__(
-            target_list=self.parsed.targets if self.parsed.targets else None,
+            *(self.parsed.targets or []),
             seeds=seeds if seeds else None,
             blacklist=self.parsed.blacklist,
             name="args_preset",

bbot/scanner/preset/preset.py

@@ -116,7 +116,6 @@ class Preset(metaclass=BasePreset):
     def __init__(
         self,
         *target,
-        target_list=None,
         seeds=None,
         blacklist=None,
         modules=None,
@@ -146,9 +145,6 @@ def __init__(
             *target (str): Target(s) to scan. These ALWAYS become the target (what `in_target()` checks).
                 Types supported: hostnames, IPs, CIDRs, emails, open ports.
                 Note: Positional arguments always mean target, never seeds.
-            target_list (list, optional): Explicitly define what's in the target (what `in_target()` checks).
-                If specified, this takes precedence over positional *target arguments.
-                Note: This defines the target, NOT scope. Use `in_scope()` to check target AND blacklist.
             seeds (list, optional): Explicitly define seeds (initial events for passive modules).
                 If not specified, seeds will be backfilled from target when target is defined.
             blacklist (list, optional): Blacklisted target(s). Takes ultimate precedence. Defaults to empty.
@@ -269,27 +265,15 @@ def __init__(
 
         # target / seeds / blacklist
         # these are temporary receptacles until they all get .baked() together
-        # Positional arguments ALWAYS become target (never seeds).
-        # Seeds must be explicitly provided via the `seeds` parameter.
-        if target_list is not None:
-            # Explicit target_list takes precedence over positional args
-            self._target_list = set(target_list)
-        elif target:
-            # Positional args become target
-            self._target_list = set(target)
-        else:
-            # No positional args and no explicit target_list
-            self._target_list = None
-
-        # Handle seeds - must be explicitly provided
-        if seeds is not None:
-            self._seeds = set(seeds) if seeds else set()
-        else:
-            # Seeds not explicitly provided - will be backfilled in BBOTTarget if target is set
-            self._seeds = set()
-
+        self._target_list = set(target or [])
         self._blacklist = set(blacklist if blacklist else [])
+        # seeds are special. Instead of initializing them as an empty set, we use "None"
+        # to signify they haven't been explicitly set.
+        # after all the merging is done, if seeds are still untouched by the user
+        # (i.e. they are still None), we'll know it's okay to copy them from the targets.
+        self._seeds = set(seeds) if seeds else None
 
+        # _target doesn't get set until .bake()
         self._target = None
 
         # we don't fill self.modules yet (that happens in .bake())
@@ -382,12 +366,12 @@ def merge(self, other):
         self.flags.update(other.flags)
 
         # target / scope
-        self._seeds.update(other._seeds)
-        if other._target_list is not None:
-            if self._target_list is None:
-                self._target_list = set(other._target_list)
+        self._target_list.update(other._target_list)
+        if other._seeds is not None:
+            if self._seeds is None:
+                self._seeds = set(other._seeds)
             else:
-                self._target_list.update(other._target_list)
+                self._seeds.update(other._seeds)
         self._blacklist.update(other._blacklist)
 
         # module dirs
@@ -502,8 +486,8 @@ def bake(self, scan=None):
         from bbot.scanner.target import BBOTTarget
 
         baked_preset._target = BBOTTarget(
-            seeds=list(self._seeds),
-            target=list(self._target_list) if self._target_list else None,
+            seeds=list(self._seeds) if self._seeds else None,
+            target=list(self._target_list),
             blacklist=self._blacklist,
             strict_dns_scope=self.strict_scope,
         )
@@ -674,18 +658,13 @@ def from_dict(cls, preset_dict, name=None, _exclude=None, _log=False):
         Examples:
             >>> preset = Preset.from_dict({"target": ["evilcorp.com"], "modules": ["portscan"]})
         """
-        # Handle seeds and target_list from dict
-        # - "target" key represents target_list (what in_target() checks)
-        # - "targets" is legacy and also treated as target_list
-        # - "seeds" key represents explicit seeds (never positional args)
-        #
-        # If BOTH "target" and "targets" are present, treat this as a user typo
-        # and merge them into a single target_list (order-preserving, deduped).
+        # Handle seeds and targets from dict
+        # for user-friendliness, we allow both "target" and "targets" to be used. we merge them into a single list.
         target_vals = (preset_dict.get("target") or []) + (preset_dict.get("targets") or [])
-        target_list = list(dict.fromkeys(target_vals)) or None
+        targets = list(dict.fromkeys(target_vals))
         seeds = preset_dict.get("seeds")
         new_preset = cls(
-            target_list=target_list,
+            *targets,
             seeds=seeds,
             blacklist=preset_dict.get("blacklist"),
             modules=preset_dict.get("modules"),
@@ -815,15 +794,15 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False)
 
         # scope
         if include_target:
-            seeds = sorted(self.target.seeds.inputs)
-            target_list = []
-            if self.target.target is not None:
-                target_list = sorted(self.target.target.inputs)
+            target = sorted(self.target.target.inputs)
+            seeds = []
+            if self.target.seeds is not None:
+                seeds = sorted(self.target.seeds.inputs)
             blacklist = sorted(self.target.blacklist.inputs)
-            if seeds:
+            if target:
+                preset_dict["target"] = target
+            if seeds and seeds != target:
                 preset_dict["seeds"] = seeds
-            if target_list and target_list != seeds:
-                preset_dict["target"] = target_list
             if blacklist:
                 preset_dict["blacklist"] = blacklist
 

bbot/scanner/scanner.py

@@ -1012,8 +1012,8 @@ def root_event(self):
               "tags": [
                 "distance-0"
               ],
-              "module": "TARGET",
-              "module_sequence": "TARGET"
+              "module": "SEED",
+              "module_sequence": "SEED"
             }
             ```
         """
@@ -1040,7 +1040,7 @@ def make_root_event(self, context):
         root_event.scope_distance = 0
         root_event.parent = root_event
         root_event._dummy = False
-        root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET")
+        root_event.module = self._make_dummy_module(name="SEED", _type="SEED")
         return root_event
 
     @property

bbot/scanner/stats.py

@@ -72,7 +72,7 @@ def table(self):
         header = ["Module", "Produced", "Consumed"]
         table = []
         for mname, mstat in self.module_stats.items():
-            if mname == "TARGET" or mstat.module._stats_exclude:
+            if mname == "SEED" or mstat.module._stats_exclude:
                 continue
             table_row = []
             table_row.append(mname)