diff --git a/.github/legit-bots-sources.json b/.github/legit-bots-sources.json new file mode 100644 index 000000000..1c1324ed5 --- /dev/null +++ b/.github/legit-bots-sources.json @@ -0,0 +1,130 @@ +[ + { + "name": "anthropic", + "user_agent": "claude-searchbot|claude-user", + "comment": "Anthropic Claude-SearchBot / Claude-User - https://claude.com/crawling/bots.json (auto-synced). ClaudeBot (training crawler) shares these IPs but is deliberately excluded from the user_agent match.", + "urls": ["https://claude.com/crawling/bots.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "duckduckbot", + "user_agent": "duckduckbot", + "comment": "DuckDuckBot - https://duckduckgo.com/duckduckbot.json (auto-synced)", + "urls": ["https://duckduckgo.com/duckduckbot.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "duckassistbot", + "user_agent": "duckassistbot", + "comment": "DuckAssistBot (DuckDuckGo AI assistant) - https://duckduckgo.com/duckassistbot.json (auto-synced). DuckDuckGo serves one combined list for DuckDuckBot and DuckAssistBot; kept separate so search crawling can be allowed without AI assistant fetches.", + "urls": ["https://duckduckgo.com/duckassistbot.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "gptbot", + "user_agent": "gptbot", + "comment": "GPTBot (OpenAI) - https://openai.com/gptbot.json (auto-synced)", + "urls": ["https://openai.com/gptbot.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "openai-searchbot", + "user_agent": "oai-searchbot", + "comment": "OpenAI SearchBot - https://openai.com/searchbot.json (auto-synced)", + "urls": ["https://openai.com/searchbot.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "openai-chatgpt-user", + "user_agent": "chatgpt-user", + "comment": "OpenAI ChatGPT-User - https://openai.com/chatgpt-user.json (auto-synced)", + "urls": ["https://openai.com/chatgpt-user.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "openai-adsbot", + "user_agent": "oai-adsbot", + "comment": "OpenAI OAI-AdsBot - https://openai.com/adsbot.json (auto-synced)", + "urls": ["https://openai.com/adsbot.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "perplexitybot", + "user_agent": "perplexitybot", + "comment": "PerplexityBot - https://www.perplexity.ai/perplexitybot.json (auto-synced)", + "urls": ["https://www.perplexity.ai/perplexitybot.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "perplexity-user", + "user_agent": "perplexity-user", + "comment": "Perplexity-User (user-triggered fetches) - https://www.perplexity.ai/perplexity-user.json (auto-synced)", + "urls": ["https://www.perplexity.ai/perplexity-user.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "mistralai-index", + "user_agent": "mistralai-index", + "comment": "MistralAI-Index - https://mistral.ai/mistralai-index-ips.json (auto-synced)", + "urls": ["https://mistral.ai/mistralai-index-ips.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "mistralai-user", + "user_agent": "mistralai-user", + "comment": "MistralAI-User (user-triggered fetches) - https://mistral.ai/mistralai-user-ips.json (auto-synced)", + "urls": ["https://mistral.ai/mistralai-user-ips.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "ahrefs", + "user_agent": "ahrefsbot|ahrefssiteaudit", + "comment": "AhrefsBot / AhrefsSiteAudit - https://api.ahrefs.com/v3/public/crawler-ip-ranges (auto-synced). rDNS ahrefs.net does not forward-confirm, so this is IP based.", + "urls": ["https://api.ahrefs.com/v3/public/crawler-ip-ranges"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]" + }, + { + "name": "flipboard", + "user_agent": "flipboardproxy|flipboardbot|flipboardrss", + "comment": "Flipboard proxy / link preview - https://cdn.flipboard.com/flipboard_ip.txt (auto-synced). rDNS proxy.flipboard.com does not forward-confirm, so this is IP based.", + "urls": ["https://cdn.flipboard.com/flipboard_ip.txt"] + }, + { + "name": "telegram", + "user_agent": "telegrambot", + "comment": "Telegram - https://core.telegram.org/resources/cidr.txt (auto-synced)", + "urls": ["https://core.telegram.org/resources/cidr.txt"] + }, + { + "name": "uptimerobot", + "user_agent": "uptimerobot", + "comment": "UptimeRobot monitors - https://cdn.uptimerobot.com/api/IPv4andIPv6.txt (auto-synced)", + "urls": ["https://cdn.uptimerobot.com/api/IPv4andIPv6.txt"] + }, + { + "name": "datadog", + "user_agent": "datadog", + "comment": "Datadog Synthetics monitors - https://ip-ranges.datadoghq.com/ (.synthetics; auto-synced)", + "urls": ["https://ip-ranges.datadoghq.com/"], + "jq_filter": "[.synthetics.prefixes_ipv4[], .synthetics.prefixes_ipv6[]] | .[]" + }, + { + "name": "pagerduty", + "user_agent": "pagerduty-webhook", + "comment": "PagerDuty outbound webhooks (UA PagerDuty-Webhook/Vx.0) - US + EU service regions, https://developer.pagerduty.com/docs/webhook-ips (auto-synced)", + "urls": [ + "https://developer.pagerduty.com/ip-safelists/webhooks-us-service-region-json", + "https://developer.pagerduty.com/ip-safelists/webhooks-eu-service-region-json" + ], + "jq_filter": ".[]" + }, + { + "name": "lumar", + "user_agent": "lumar|deepcrawl", + "comment": "Lumar (ex-Deepcrawl) - see https://www.lumar.io/ for the current IP list", + "urls": ["https://www.lumar.io/wp-content/uploads/2026/02/lumar_ip_list_feb_2026.json"], + "jq_filter": "[.prefixes[] | .ipv4Prefix // .ipv6Prefix] | .[]", + "disabled": true, + "note": "Lumar publishes its IP list under a date-stamped wp-content URL that rotates (…/2026/02/lumar_ip_list_feb_2026.json), so it cannot be polled on a fixed URL. Refresh whitelists/benign_bots/legit_bots/lumar.json by hand and bump the URL here when Lumar publishes a new one." + } +] diff --git a/.github/scripts/sync-legit-bots.py b/.github/scripts/sync-legit-bots.py new file mode 100755 index 000000000..8ba2eaa12 --- /dev/null +++ b/.github/scripts/sync-legit-bots.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Refresh the IP-based entries in whitelists/benign_bots/legit_bots/ from their +upstream vendor sources. + +Driven by .github/legit-bots-sources.json. Each entry names the target bot, the +user_agent regex and file comment to keep, and one or more URLs to pull ranges +from. JSON sources are reduced with `jq_filter`; sources without one are read as +plain text, one address or CIDR per line. + +Entries verified by reverse DNS (googlebot, seznam, linkedin, ...) have nothing +to poll and are deliberately absent from the config - this script never touches +them. + +Writes GitHub Actions outputs (changed, names, files) when GITHUB_OUTPUT is set. +Exits non-zero only if every source failed, so one broken vendor endpoint cannot +block the others from updating. +""" +import ipaddress +import json +import os +import subprocess +import sys + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +CONFIG = os.environ.get("LEGIT_BOTS_CONFIG") or os.path.join( + ROOT, ".github", "legit-bots-sources.json") +TARGET_DIR = os.environ.get("LEGIT_BOTS_TARGET_DIR") or os.path.join( + ROOT, "whitelists", "benign_bots", "legit_bots") + +# A vendor serving a truncated or half-broken list would otherwise silently gut a +# whitelist. Below this ratio the change is still applied but called out loudly in +# the PR body for a human to confirm - a real shrink does happen (Perplexity went +# from 254 stale entries to the 8 they actually publish), so this must not hard-fail. +SHRINK_RATIO = 0.5 +SHRINK_FLOOR = 10 + +UA = "Mozilla/5.0 (compatible; crowdsec-sec-lists-sync)" + + +def fetch(url): + r = subprocess.run( + ["curl", "-fsSL", "--max-time", "60", "--retry", "3", "--retry-delay", "5", + "-A", UA, url], + capture_output=True, text=True, + ) + if r.returncode != 0: + raise RuntimeError(f"fetch failed ({r.returncode}): {r.stderr.strip()[:200]}") + return r.stdout + + +def apply_jq(payload, expr): + r = subprocess.run(["jq", "-er", expr], input=payload, + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(f"jq filter failed: {r.stderr.strip()[:200]}") + return r.stdout + + +def parse_ranges(text): + """Normalise a newline-separated list of addresses/CIDRs to canonical CIDRs.""" + out = [] + for raw in text.splitlines(): + line = raw.strip().strip('"').strip(",").strip('"') + if not line or line.startswith("#"): + continue + try: + if "/" in line: + net = ipaddress.ip_network(line, strict=True) + else: + addr = ipaddress.ip_address(line) + net = ipaddress.ip_network(f"{addr}/{addr.max_prefixlen}") + except ValueError as ex: + raise RuntimeError(f"invalid address {line!r}: {ex}") + if not net.is_global: + raise RuntimeError(f"refusing non-public range {line!r}") + out.append(net) + # deterministic order so a vendor reshuffling its list produces no diff + uniq = sorted(set(out), key=lambda n: (n.version, n.network_address, n.prefixlen)) + return [str(n) for n in uniq] + + +def render(entry, ranges): + body = json.dumps( + {"name": entry["name"], "user_agent": entry["user_agent"], "ranges": ranges}, + separators=(",", ":"), ensure_ascii=False, + ) + return f"# {entry['comment']}\n{body}\n" + + +def current_range_count(path): + try: + with open(path) as fh: + line = [l for l in fh.read().splitlines() if not l.startswith("#")][0] + return len(json.loads(line).get("ranges", [])) + except (OSError, IndexError, ValueError): + return 0 + + +def main(): + with open(CONFIG) as fh: + config = json.load(fh) + + changed, warnings, errors, skipped = [], [], [], [] + + for entry in config: + name = entry["name"] + if entry.get("disabled"): + skipped.append(name) + print(f"-- {name}: disabled ({entry.get('note', 'no reason given')})") + continue + + path = os.path.join(TARGET_DIR, f"{name}.json") + try: + ranges = [] + for url in entry["urls"]: + payload = fetch(url) + if "jq_filter" in entry: + payload = apply_jq(payload, entry["jq_filter"]) + ranges.extend(parse_ranges(payload)) + ranges = parse_ranges("\n".join(ranges)) # merge + re-sort across URLs + except RuntimeError as ex: + errors.append(f"{name}: {ex}") + print(f"!! {name}: {ex}", file=sys.stderr) + continue + + if not ranges: + errors.append(f"{name}: source returned no ranges, keeping existing file") + print(f"!! {name}: empty result, refusing to write", file=sys.stderr) + continue + + before = current_range_count(path) + if before >= SHRINK_FLOOR and len(ranges) < before * SHRINK_RATIO: + warnings.append( + f"`{name}` shrank sharply: {before} -> {len(ranges)} ranges. " + f"Confirm the vendor really retired these before merging." + ) + + new = render(entry, ranges) + old = open(path).read() if os.path.exists(path) else None + if old == new: + print(f" {name}: unchanged ({len(ranges)} ranges)") + continue + + with open(path, "w") as fh: + fh.write(new) + changed.append((name, before, len(ranges))) + print(f"++ {name}: {before} -> {len(ranges)} ranges") + + if errors and not changed and len(errors) == len([e for e in config if not e.get("disabled")]): + print("\nevery source failed", file=sys.stderr) + return 1 + + lines = [] + if changed: + lines.append("Automated refresh of `whitelists/benign_bots/legit_bots/` " + "from upstream vendor sources.\n") + lines.append("| bot | ranges before | ranges after |") + lines.append("| --- | ---: | ---: |") + for name, before, after in changed: + lines.append(f"| `{name}` | {before} | {after} |") + if warnings: + lines.append("\n**Review carefully:**\n") + lines.extend(f"- {w}" for w in warnings) + if errors: + lines.append("\n**Sources that failed this run** (their files were left " + "untouched):\n") + lines.extend(f"- {e}" for e in errors) + if skipped: + lines.append(f"\nNot polled (no stable URL): {', '.join(f'`{s}`' for s in skipped)}.") + body = "\n".join(lines) + + out = os.environ.get("GITHUB_OUTPUT") + # only materialise the PR body under CI, and outside the checkout, so running + # this locally never leaves a stray file in the working tree + body_file = os.path.join(os.environ.get("RUNNER_TEMP") or ROOT, "pr-body.md") + if body and out: + with open(body_file, "w") as fh: + fh.write(body + "\n") + elif body: + print("\n--- PR body ---\n" + body) + + if out: + with open(out, "a") as fh: + fh.write(f"changed={'true' if changed else 'false'}\n") + fh.write(f"names={' '.join(n for n, _, _ in changed)}\n") + fh.write(f"files={' '.join(f'whitelists/benign_bots/legit_bots/{n}.json' for n, _, _ in changed)}\n") + fh.write(f"body_file={body_file}\n") + + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary and body: + with open(summary, "a") as fh: + fh.write(body + "\n") + + print(f"\n{len(changed)} changed, {len(warnings)} warning(s), " + f"{len(errors)} error(s), {len(skipped)} skipped") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/sync-legit-bots.yml b/.github/workflows/sync-legit-bots.yml new file mode 100644 index 000000000..26e9631f4 --- /dev/null +++ b/.github/workflows/sync-legit-bots.yml @@ -0,0 +1,72 @@ +name: Sync Legit Bots + +on: + schedule: + - cron: "30 2 * * *" # every day, offset from the other sync jobs + workflow_dispatch: {} + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-legit-bots + cancel-in-progress: false + +env: + BRANCH_PREFIX: sync-legit-bots- + +jobs: + sync-legit-bots: + runs-on: ubuntu-latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout master + uses: actions/checkout@v4 + with: + ref: master + + - name: Refresh legit_bots entries from upstream sources + id: sync + run: python3 .github/scripts/sync-legit-bots.py + + - name: Open PR and close superseded ones + if: steps.sync.outputs.changed == 'true' + run: | + set -euo pipefail + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + BRANCH="${BRANCH_PREFIX}${{ github.run_id }}" + git checkout -b "$BRANCH" + + for f in ${{ steps.sync.outputs.files }}; do + git add "$f" + done + git commit -m "Sync legit bots:${{ steps.sync.outputs.names }}" + git push --set-upstream origin "$BRANCH" + + # Collect the still-open PRs from previous runs *before* opening the new + # one, so the new PR can never appear in its own supersede list. + SUPERSEDED=$(gh pr list --state open --limit 100 \ + --json number,headRefName \ + --jq ".[] | select(.headRefName | startswith(\"${BRANCH_PREFIX}\")) | .number") + + PR_URL=$(gh pr create \ + --base master \ + --head "$BRANCH" \ + --title "Sync legit bots:${{ steps.sync.outputs.names }}" \ + --body-file "${{ steps.sync.outputs.body_file }}") + echo "Opened $PR_URL" + + # Only now that the replacement exists do we retire the older ones, so a + # failure above leaves the previous PR open rather than dropping updates. + for n in $SUPERSEDED; do + echo "Closing superseded PR #$n" + gh pr close "$n" \ + --comment "Superseded by $PR_URL, which contains the current upstream state." \ + --delete-branch + done diff --git a/whitelists/benign_bots/legit_bots/ahrefs.json b/whitelists/benign_bots/legit_bots/ahrefs.json new file mode 100644 index 000000000..8e54973a5 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/ahrefs.json @@ -0,0 +1,2 @@ +# AhrefsBot / AhrefsSiteAudit - https://api.ahrefs.com/v3/public/crawler-ip-ranges (auto-synced). rDNS ahrefs.net does not forward-confirm, so this is IP based. +{"name":"ahrefs","user_agent":"ahrefsbot|ahrefssiteaudit","ranges":["5.39.1.224/27","5.39.109.160/27","15.235.27.0/24","15.235.96.0/24","15.235.98.0/24","37.59.204.128/27","51.68.247.192/27","51.75.236.128/27","51.89.129.0/24","51.161.37.0/24","51.161.65.0/24","51.195.183.0/24","51.195.215.0/24","51.195.244.0/24","51.222.95.0/24","51.222.168.0/24","51.222.253.0/26","54.36.148.0/23","54.37.118.64/27","54.38.147.0/24","54.39.0.0/24","54.39.6.0/24","54.39.89.0/24","54.39.136.0/24","54.39.203.0/24","54.39.210.0/24","92.222.104.192/27","92.222.108.96/27","94.23.188.192/27","142.44.220.0/24","142.44.225.0/24","142.44.228.0/24","142.44.233.0/24","148.113.128.0/24","148.113.130.0/24","167.114.139.0/24","168.100.149.0/24","176.31.139.0/27","198.244.168.0/24","198.244.183.0/24","198.244.186.193/32","198.244.186.194/31","198.244.186.196/30","198.244.186.200/31","198.244.186.202/32","198.244.226.0/24","198.244.240.0/24","198.244.242.0/24","202.8.40.0/22","202.94.84.110/31","202.94.84.112/31"]} diff --git a/whitelists/benign_bots/legit_bots/anthropic.json b/whitelists/benign_bots/legit_bots/anthropic.json new file mode 100644 index 000000000..8ee104986 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/anthropic.json @@ -0,0 +1,2 @@ +# Anthropic Claude-SearchBot / Claude-User - https://claude.com/crawling/bots.json (auto-synced). ClaudeBot (training crawler) shares these IPs but is deliberately excluded from the user_agent match. +{"name":"anthropic","user_agent":"claude-searchbot|claude-user","ranges":["34.11.34.31/32","34.85.172.162/32","34.150.241.79/32","34.162.191.81/32","34.162.230.222/32","34.162.244.71/32","34.182.140.95/32","34.182.161.143/32","34.182.218.27/32","34.182.220.85/32","34.182.222.37/32","34.182.225.167/32","34.182.226.151/32","34.182.226.221/32","34.186.108.163/32","35.221.29.174/32","35.245.89.239/32","35.245.175.129/32","136.107.176.208/32","216.73.216.0/22"]} diff --git a/whitelists/benign_bots/legit_bots/commoncrawl.json b/whitelists/benign_bots/legit_bots/commoncrawl.json new file mode 100644 index 000000000..290296a53 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/commoncrawl.json @@ -0,0 +1,2 @@ +# CCBot (Common Crawl) - forward-confirmed rDNS crawl.commoncrawl.org (https://index.commoncrawl.org/ccbot.json also publishes IP ranges) +{"name":"commoncrawl","user_agent":"ccbot","rdns":["(^|\\.)crawl\\.commoncrawl\\.org$"]} diff --git a/whitelists/benign_bots/legit_bots/datadog.json b/whitelists/benign_bots/legit_bots/datadog.json index bbd187e2b..25d4df733 100644 --- a/whitelists/benign_bots/legit_bots/datadog.json +++ b/whitelists/benign_bots/legit_bots/datadog.json @@ -1,2 +1,2 @@ -# Datadog Synthetics monitors - https://ip-ranges.datadoghq.com/ (.synthetics; version 62, modified 2026-02-24; refresh periodically) +# Datadog Synthetics monitors - https://ip-ranges.datadoghq.com/ (.synthetics; auto-synced) {"name":"datadog","user_agent":"datadog","ranges":["3.1.36.99/32","3.1.219.207/32","3.18.172.189/32","3.18.188.104/32","3.18.197.0/32","3.35.66.96/32","3.36.177.119/32","3.40.2.64/29","3.40.134.72/29","3.41.0.88/29","3.41.176.48/29","3.42.136.80/29","3.44.192.120/29","3.45.0.40/29","3.92.150.182/32","3.96.7.126/32","3.120.223.25/32","3.121.24.234/32","5.60.130.112/29","13.48.150.244/32","13.48.239.118/32","13.48.254.37/32","13.54.169.48/32","13.114.211.96/32","13.115.46.213/32","13.126.169.175/32","13.208.126.217/32","13.208.133.55/32","13.208.142.17/32","13.208.255.200/32","13.209.118.42/32","13.209.230.111/32","13.234.54.8/32","13.236.246.161/32","13.238.14.57/32","13.244.85.86/32","13.244.188.203/32","13.245.194.43/32","13.245.200.254/32","13.246.172.210/32","13.247.164.9/32","15.152.238.192/32","15.161.86.71/32","15.165.240.116/32","15.168.188.85/32","15.184.139.182/32","15.185.189.82/32","15.188.202.64/32","15.188.240.172/32","15.188.243.248/32","16.24.38.13/32","16.24.60.114/32","16.162.136.62/32","16.163.153.45/32","18.96.16.112/29","18.96.130.96/29","18.96.227.72/29","18.97.147.8/29","18.98.67.176/29","18.98.96.240/29","18.98.131.24/29","18.98.196.216/29","18.99.4.112/29","18.99.34.208/29","18.99.99.232/29","18.99.147.8/29","18.99.209.96/29","18.99.224.56/29","18.102.80.189/32","18.130.113.168/32","18.139.52.173/32","18.163.21.55/32","18.163.59.106/32","18.166.19.255/32","18.195.155.52/32","18.200.120.237/32","18.229.28.50/32","18.229.36.120/32","20.62.248.141/32","20.83.144.189/32","34.48.76.208/29","34.94.234.88/29","34.145.82.128/29","34.146.154.144/29","34.159.50.128/29","34.174.98.16/29","34.208.32.189/32","35.152.76.8/32","35.154.93.182/32","35.176.195.46/32","35.177.43.250/32","40.76.107.170/32","43.198.123.228/32","43.203.72.233/32","43.218.5.202/32","52.9.13.199/32","52.9.139.134/32","52.35.61.232/32","52.55.56.26/32","52.60.189.53/32","52.67.95.251/32","52.89.221.151/32","52.186.167.216/29","52.192.175.207/32","54.177.155.33/32","63.34.100.178/32","63.35.33.198/32","99.79.87.237/32","107.21.25.247/32","108.137.133.223/32","108.137.188.57/32","157.241.36.106/32","157.241.93.102/32"]} diff --git a/whitelists/benign_bots/legit_bots/duckassistbot.json b/whitelists/benign_bots/legit_bots/duckassistbot.json new file mode 100644 index 000000000..041e491d6 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/duckassistbot.json @@ -0,0 +1,2 @@ +# DuckAssistBot (DuckDuckGo AI assistant) - https://duckduckgo.com/duckassistbot.json (auto-synced). DuckDuckGo serves one combined list for DuckDuckBot and DuckAssistBot; kept separate so search crawling can be allowed without AI assistant fetches. +{"name":"duckassistbot","user_agent":"duckassistbot","ranges":["4.144.182.50/32","4.149.142.7/32","4.149.159.71/32","4.149.159.102/32","4.149.216.220/32","4.150.142.218/32","4.154.193.167/32","4.156.30.123/32","4.156.136.79/32","4.156.154.107/32","4.157.77.76/32","4.172.49.103/32","4.177.16.196/32","4.182.10.198/32","4.182.131.108/32","4.195.133.120/32","4.201.125.59/32","4.201.141.71/32","4.201.197.203/32","4.201.206.133/32","4.201.220.8/32","4.207.220.92/32","4.209.224.56/32","4.213.46.14/32","4.226.40.135/32","4.228.76.163/32","4.237.155.54/32","4.237.244.80/32","4.242.119.13/32","4.246.10.252/32","4.248.234.26/32","4.249.216.104/32","4.249.223.51/32","4.249.238.11/32","4.254.123.56/32","4.255.35.121/32","4.255.39.80/32","13.86.35.212/32","13.89.106.77/32","13.90.178.139/32","20.3.1.178/32","20.3.60.212/32","20.8.252.26/32","20.9.114.242/32","20.12.141.99/32","20.12.249.123/32","20.12.252.21/32","20.13.44.19/32","20.15.142.178/32","20.23.88.231/32","20.29.149.44/32","20.29.154.56/32","20.40.133.240/32","20.40.147.172/32","20.43.150.85/32","20.43.150.93/32","20.43.152.28/32","20.43.172.120/32","20.44.222.1/32","20.49.129.236/32","20.49.136.28/32","20.50.48.159/32","20.50.48.192/32","20.50.49.0/32","20.50.49.25/32","20.50.49.40/32","20.50.49.55/32","20.50.49.237/32","20.50.50.46/32","20.50.50.118/32","20.50.50.121/32","20.50.50.123/32","20.50.50.130/32","20.50.50.134/32","20.50.50.145/32","20.50.50.146/32","20.50.50.163/32","20.50.168.91/32","20.53.78.106/32","20.53.78.123/32","20.53.78.138/32","20.53.78.144/32","20.53.78.236/32","20.53.91.2/32","20.53.92.211/32","20.53.134.160/32","20.54.224.39/32","20.56.197.58/32","20.56.197.63/32","20.59.0.65/32","20.61.34.40/32","20.61.142.192/32","20.62.224.44/32","20.64.185.45/32","20.69.74.250/32","20.69.129.205/32","20.69.131.45/32","20.69.138.149/32","20.69.138.189/32","20.69.144.137/32","20.69.146.171/32","20.69.147.21/32","20.69.171.204/32","20.71.12.143/32","20.71.69.210/32","20.72.197.110/32","20.72.203.250/32","20.72.218.29/32","20.72.221.33/32","20.72.221.45/32","20.72.225.13/32","20.72.242.93/32","20.72.242.209/32","20.73.132.240/32","20.73.202.147/32","20.75.144.152/32","20.75.254.61/32","20.77.146.108/32","20.79.226.26/32","20.79.238.198/32","20.79.239.66/32","20.80.114.203/32","20.80.120.41/32","20.80.129.80/32","20.80.135.7/32","20.80.156.88/32","20.81.115.13/32","20.82.24.207/32","20.84.138.61/32","20.84.219.45/32","20.84.251.223/32","20.85.138.75/32","20.85.167.56/32","20.85.187.146/32","20.88.168.65/32","20.93.28.24/32","20.93.49.96/32","20.99.154.113/32","20.99.255.235/32","20.100.136.36/32","20.100.140.155/32","20.101.17.173/32","20.102.34.205/32","20.109.129.162/32","20.109.130.190/32","20.109.241.18/32","20.112.58.44/32","20.113.3.121/32","20.113.14.159/32","20.115.181.187/32","20.118.11.251/32","20.119.127.146/32","20.121.75.32/32","20.166.171.150/32","20.170.75.54/32","20.175.232.228/32","20.185.79.15/32","20.185.79.47/32","20.190.21.172/32","20.191.44.16/32","20.191.44.22/32","20.191.44.119/32","20.191.44.234/32","20.191.45.212/32","20.193.12.126/32","20.193.24.10/32","20.193.24.251/32","20.193.25.197/32","20.193.27.215/32","20.193.45.113/32","20.195.58.189/32","20.195.108.47/32","20.197.209.11/32","20.197.209.27/32","20.198.132.126/32","20.198.152.208/32","20.201.15.208/32","20.204.228.200/32","20.204.240.172/32","20.204.241.148/32","20.204.242.19/32","20.204.242.101/32","20.204.243.55/32","20.204.246.81/32","20.204.246.254/32","20.207.72.11/32","20.207.72.21/32","20.207.72.110/32","20.207.72.113/32","20.207.97.190/32","20.207.99.197/32","20.207.107.181/32","20.207.117.66/32","20.212.90.107/32","20.212.139.34/32","20.216.200.223/32","20.219.43.246/32","20.219.45.67/32","20.219.45.190/32","20.221.10.115/32","20.221.84.185/32","20.226.133.105/32","20.231.254.179/32","20.232.51.46/32","20.232.249.95/32","20.236.40.93/32","20.236.195.196/32","20.236.198.192/32","20.236.227.113/32","20.236.233.23/32","20.236.247.180/32","20.237.1.51/32","20.241.135.19/32","20.241.147.151/32","20.241.153.203/32","20.241.164.5/32","20.241.241.100/32","20.242.133.53/32","20.242.135.165/32","20.242.211.139/32","20.242.215.53/32","20.242.237.240/32","20.246.132.168/32","20.246.225.34/32","20.247.235.11/32","20.250.51.113/32","20.252.9.6/32","20.252.16.193/32","20.252.41.18/32","20.252.63.162/32","20.252.88.10/32","20.253.59.76/32","20.253.96.199/32","20.253.114.51/32","40.64.67.220/32","40.64.78.45/32","40.64.88.145/32","40.64.105.247/32","40.64.106.11/32","40.64.106.152/32","40.65.78.86/32","40.76.162.17/32","40.76.162.191/32","40.76.162.208/32","40.76.162.247/32","40.76.163.7/32","40.76.163.23/32","40.76.173.151/32","40.81.250.205/32","40.82.218.203/32","40.88.21.235/32","40.88.195.62/32","40.89.243.175/32","40.114.182.45/32","40.114.182.153/32","40.114.182.172/32","40.114.183.88/32","40.114.183.196/32","40.114.183.251/32","40.119.232.50/32","40.119.232.146/32","40.119.232.215/32","40.119.232.218/32","40.119.232.251/32","40.121.121.54/32","40.127.154.196/32","48.192.16.86/32","48.192.74.178/32","48.200.33.227/32","48.202.155.200/32","48.202.171.124/32","48.204.32.87/32","48.206.18.152/32","48.217.23.236/32","48.217.129.210/32","48.217.212.89/32","48.223.209.110/32","51.8.31.28/32","51.8.71.117/32","51.8.253.152/32","51.104.144.31/32","51.104.146.225/32","51.104.146.235/32","51.104.160.167/32","51.104.160.177/32","51.104.161.32/32","51.104.162.149/32","51.104.163.250/32","51.104.164.109/32","51.104.164.147/32","51.104.164.189/32","51.104.164.215/32","51.104.166.111/32","51.104.167.19/32","51.104.167.52/32","51.104.167.54/32","51.104.167.61/32","51.104.167.71/32","51.104.167.87/32","51.104.167.88/32","51.104.167.95/32","51.104.167.96/32","51.104.167.104/32","51.104.167.110/32","51.104.180.26/32","51.104.180.47/32","51.104.180.53/32","51.107.40.209/32","51.116.131.221/32","51.120.48.122/32","51.138.90.161/32","51.138.90.206/32","51.138.90.233/32","51.143.114.45/32","51.145.185.16/32","52.142.24.149/32","52.142.26.175/32","52.143.95.162/32","52.143.95.204/32","52.143.241.111/32","52.143.242.6/32","52.143.243.117/32","52.143.244.81/32","52.143.247.235/32","52.146.57.51/32","52.146.58.236/32","52.146.59.12/32","52.146.59.154/32","52.146.59.156/32","52.146.63.80/32","52.148.161.87/32","52.148.165.38/32","52.149.25.43/32","52.149.28.18/32","52.149.28.83/32","52.149.30.45/32","52.149.56.151/32","52.149.58.27/32","52.149.58.69/32","52.149.58.139/32","52.149.58.173/32","52.149.60.38/32","52.149.61.51/32","52.154.60.82/32","52.154.169.50/32","52.154.169.200/32","52.154.170.26/32","52.154.170.28/32","52.154.170.88/32","52.154.170.96/32","52.154.170.113/32","52.154.170.117/32","52.154.170.122/32","52.154.170.209/32","52.154.170.229/32","52.154.170.243/32","52.154.171.0/32","52.154.171.44/32","52.154.171.70/32","52.154.171.87/32","52.154.171.150/32","52.154.171.196/32","52.154.171.205/32","52.154.171.235/32","52.154.171.250/32","52.154.172.2/32","52.154.204.93/32","52.185.93.208/32","52.186.37.211/32","52.188.89.106/32","52.189.70.103/32","52.190.37.160/32","52.191.82.242/32","52.191.222.158/32","52.224.16.221/32","52.224.16.229/32","52.224.19.152/32","52.224.20.174/32","52.224.20.181/32","52.224.20.186/32","52.224.20.190/32","52.224.20.193/32","52.224.20.203/32","52.224.20.204/32","52.224.20.223/32","52.224.20.227/32","52.224.20.249/32","52.224.21.4/32","52.224.21.19/32","52.224.21.20/32","52.224.21.23/32","52.224.21.27/32","52.224.21.49/32","52.224.21.51/32","52.224.21.53/32","52.224.21.55/32","52.224.21.61/32","52.224.147.21/32","52.224.191.245/32","52.242.224.168/32","52.250.46.221/32","57.151.44.140/32","57.152.17.237/32","57.152.62.107/32","57.152.72.128/32","57.155.18.44/32","57.165.205.172/32","57.167.88.38/32","64.236.15.33/32","64.236.93.100/32","64.236.118.43/32","68.219.152.220/32","72.144.95.185/32","74.163.169.106/32","74.163.208.206/32","74.179.200.126/32","74.179.232.116/32","74.242.194.48/32","98.70.219.37/32","104.43.54.127/32","104.43.55.116/32","104.43.55.117/32","104.43.55.166/32","104.43.55.167/32","108.141.83.74/32","128.203.82.154/32","128.203.132.152/32","128.203.181.79/32","128.203.214.241/32","128.203.236.110/32","130.107.228.224/32","131.145.39.118/32","131.189.183.241/32","132.164.209.198/32","132.196.106.146/32","132.220.1.71/32","134.33.207.189/32","135.233.63.237/32","135.234.221.112/32","135.235.183.95/32","172.168.12.93/32","172.168.43.43/32","172.168.51.216/32","172.168.53.53/32","172.168.81.217/32","172.168.94.160/32","172.168.115.250/32","172.168.133.76/32","172.168.137.102/32","172.168.152.88/32","172.168.173.230/32","172.168.190.186/32","172.168.227.120/32","172.168.254.119/32","172.169.7.33/32","172.169.17.165/32","172.169.28.184/32","172.169.36.43/32","172.169.53.208/32","172.169.60.134/32","172.169.93.148/32","172.169.164.109/32","172.169.177.131/32","172.169.181.237/32","172.171.37.221/32","172.179.105.87/32","172.179.128.118/32","172.189.40.18/32","172.193.184.42/32","172.193.197.146/32","172.193.245.229/32","172.194.141.49/32","172.199.55.212/32","172.210.111.68/32","191.233.3.197/32","191.233.3.202/32","191.234.216.4/32","191.234.216.178/32","191.235.201.214/32","191.235.202.38/32","191.235.202.48/32"]} diff --git a/whitelists/benign_bots/legit_bots/duckduckbot.json b/whitelists/benign_bots/legit_bots/duckduckbot.json index 7e0f8fc87..43bac8dcd 100644 --- a/whitelists/benign_bots/legit_bots/duckduckbot.json +++ b/whitelists/benign_bots/legit_bots/duckduckbot.json @@ -1,2 +1,2 @@ -# DuckDuckBot - https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/ (from search_engine_crawlers/ip_seo_bots.txt) -{"name":"duckduckbot","user_agent":"duckduckbot","ranges":["23.21.227.69/32","40.88.21.235/32","50.16.241.113/32","50.16.241.114/32","50.16.241.117/32","50.16.247.234/32","52.204.97.54/32","52.5.190.19/32","54.197.234.188/32","54.208.100.253/32","54.208.102.37/32","107.21.1.8/32"]} +# DuckDuckBot - https://duckduckgo.com/duckduckbot.json (auto-synced) +{"name":"duckduckbot","user_agent":"duckduckbot","ranges":["4.144.182.50/32","4.149.142.7/32","4.149.159.71/32","4.149.159.102/32","4.149.216.220/32","4.150.142.218/32","4.154.193.167/32","4.156.30.123/32","4.156.136.79/32","4.156.154.107/32","4.157.77.76/32","4.172.49.103/32","4.177.16.196/32","4.182.10.198/32","4.182.131.108/32","4.195.133.120/32","4.201.125.59/32","4.201.141.71/32","4.201.197.203/32","4.201.206.133/32","4.201.220.8/32","4.207.220.92/32","4.209.224.56/32","4.213.46.14/32","4.226.40.135/32","4.228.76.163/32","4.237.155.54/32","4.237.244.80/32","4.242.119.13/32","4.246.10.252/32","4.248.234.26/32","4.249.216.104/32","4.249.223.51/32","4.249.238.11/32","4.254.123.56/32","4.255.35.121/32","4.255.39.80/32","13.86.35.212/32","13.89.106.77/32","13.90.178.139/32","20.3.1.178/32","20.3.60.212/32","20.8.252.26/32","20.9.114.242/32","20.12.141.99/32","20.12.249.123/32","20.12.252.21/32","20.13.44.19/32","20.15.142.178/32","20.23.88.231/32","20.29.149.44/32","20.29.154.56/32","20.40.133.240/32","20.40.147.172/32","20.43.150.85/32","20.43.150.93/32","20.43.152.28/32","20.43.172.120/32","20.44.222.1/32","20.49.129.236/32","20.49.136.28/32","20.50.48.159/32","20.50.48.192/32","20.50.49.0/32","20.50.49.25/32","20.50.49.40/32","20.50.49.55/32","20.50.49.237/32","20.50.50.46/32","20.50.50.118/32","20.50.50.121/32","20.50.50.123/32","20.50.50.130/32","20.50.50.134/32","20.50.50.145/32","20.50.50.146/32","20.50.50.163/32","20.50.168.91/32","20.53.78.106/32","20.53.78.123/32","20.53.78.138/32","20.53.78.144/32","20.53.78.236/32","20.53.91.2/32","20.53.92.211/32","20.53.134.160/32","20.54.224.39/32","20.56.197.58/32","20.56.197.63/32","20.59.0.65/32","20.61.34.40/32","20.61.142.192/32","20.62.224.44/32","20.64.185.45/32","20.69.74.250/32","20.69.129.205/32","20.69.131.45/32","20.69.138.149/32","20.69.138.189/32","20.69.144.137/32","20.69.146.171/32","20.69.147.21/32","20.69.171.204/32","20.71.12.143/32","20.71.69.210/32","20.72.197.110/32","20.72.203.250/32","20.72.218.29/32","20.72.221.33/32","20.72.221.45/32","20.72.225.13/32","20.72.242.93/32","20.72.242.209/32","20.73.132.240/32","20.73.202.147/32","20.75.144.152/32","20.75.254.61/32","20.77.146.108/32","20.79.226.26/32","20.79.238.198/32","20.79.239.66/32","20.80.114.203/32","20.80.120.41/32","20.80.129.80/32","20.80.135.7/32","20.80.156.88/32","20.81.115.13/32","20.82.24.207/32","20.84.138.61/32","20.84.219.45/32","20.84.251.223/32","20.85.138.75/32","20.85.167.56/32","20.85.187.146/32","20.88.168.65/32","20.93.28.24/32","20.93.49.96/32","20.99.154.113/32","20.99.255.235/32","20.100.136.36/32","20.100.140.155/32","20.101.17.173/32","20.102.34.205/32","20.109.129.162/32","20.109.130.190/32","20.109.241.18/32","20.112.58.44/32","20.113.3.121/32","20.113.14.159/32","20.115.181.187/32","20.118.11.251/32","20.119.127.146/32","20.121.75.32/32","20.166.171.150/32","20.170.75.54/32","20.175.232.228/32","20.185.79.15/32","20.185.79.47/32","20.190.21.172/32","20.191.44.16/32","20.191.44.22/32","20.191.44.119/32","20.191.44.234/32","20.191.45.212/32","20.193.12.126/32","20.193.24.10/32","20.193.24.251/32","20.193.25.197/32","20.193.27.215/32","20.193.45.113/32","20.195.58.189/32","20.195.108.47/32","20.197.209.11/32","20.197.209.27/32","20.198.132.126/32","20.198.152.208/32","20.201.15.208/32","20.204.228.200/32","20.204.240.172/32","20.204.241.148/32","20.204.242.19/32","20.204.242.101/32","20.204.243.55/32","20.204.246.81/32","20.204.246.254/32","20.207.72.11/32","20.207.72.21/32","20.207.72.110/32","20.207.72.113/32","20.207.97.190/32","20.207.99.197/32","20.207.107.181/32","20.207.117.66/32","20.212.90.107/32","20.212.139.34/32","20.216.200.223/32","20.219.43.246/32","20.219.45.67/32","20.219.45.190/32","20.221.10.115/32","20.221.84.185/32","20.226.133.105/32","20.231.254.179/32","20.232.51.46/32","20.232.249.95/32","20.236.40.93/32","20.236.195.196/32","20.236.198.192/32","20.236.227.113/32","20.236.233.23/32","20.236.247.180/32","20.237.1.51/32","20.241.135.19/32","20.241.147.151/32","20.241.153.203/32","20.241.164.5/32","20.241.241.100/32","20.242.133.53/32","20.242.135.165/32","20.242.211.139/32","20.242.215.53/32","20.242.237.240/32","20.246.132.168/32","20.246.225.34/32","20.247.235.11/32","20.250.51.113/32","20.252.9.6/32","20.252.16.193/32","20.252.41.18/32","20.252.63.162/32","20.252.88.10/32","20.253.59.76/32","20.253.96.199/32","20.253.114.51/32","40.64.67.220/32","40.64.78.45/32","40.64.88.145/32","40.64.105.247/32","40.64.106.11/32","40.64.106.152/32","40.65.78.86/32","40.76.162.17/32","40.76.162.191/32","40.76.162.208/32","40.76.162.247/32","40.76.163.7/32","40.76.163.23/32","40.76.173.151/32","40.81.250.205/32","40.82.218.203/32","40.88.21.235/32","40.88.195.62/32","40.89.243.175/32","40.114.182.45/32","40.114.182.153/32","40.114.182.172/32","40.114.183.88/32","40.114.183.196/32","40.114.183.251/32","40.119.232.50/32","40.119.232.146/32","40.119.232.215/32","40.119.232.218/32","40.119.232.251/32","40.121.121.54/32","40.127.154.196/32","48.192.16.86/32","48.192.74.178/32","48.200.33.227/32","48.202.155.200/32","48.202.171.124/32","48.204.32.87/32","48.206.18.152/32","48.217.23.236/32","48.217.129.210/32","48.217.212.89/32","48.223.209.110/32","51.8.31.28/32","51.8.71.117/32","51.8.253.152/32","51.104.144.31/32","51.104.146.225/32","51.104.146.235/32","51.104.160.167/32","51.104.160.177/32","51.104.161.32/32","51.104.162.149/32","51.104.163.250/32","51.104.164.109/32","51.104.164.147/32","51.104.164.189/32","51.104.164.215/32","51.104.166.111/32","51.104.167.19/32","51.104.167.52/32","51.104.167.54/32","51.104.167.61/32","51.104.167.71/32","51.104.167.87/32","51.104.167.88/32","51.104.167.95/32","51.104.167.96/32","51.104.167.104/32","51.104.167.110/32","51.104.180.26/32","51.104.180.47/32","51.104.180.53/32","51.107.40.209/32","51.116.131.221/32","51.120.48.122/32","51.138.90.161/32","51.138.90.206/32","51.138.90.233/32","51.143.114.45/32","51.145.185.16/32","52.142.24.149/32","52.142.26.175/32","52.143.95.162/32","52.143.95.204/32","52.143.241.111/32","52.143.242.6/32","52.143.243.117/32","52.143.244.81/32","52.143.247.235/32","52.146.57.51/32","52.146.58.236/32","52.146.59.12/32","52.146.59.154/32","52.146.59.156/32","52.146.63.80/32","52.148.161.87/32","52.148.165.38/32","52.149.25.43/32","52.149.28.18/32","52.149.28.83/32","52.149.30.45/32","52.149.56.151/32","52.149.58.27/32","52.149.58.69/32","52.149.58.139/32","52.149.58.173/32","52.149.60.38/32","52.149.61.51/32","52.154.60.82/32","52.154.169.50/32","52.154.169.200/32","52.154.170.26/32","52.154.170.28/32","52.154.170.88/32","52.154.170.96/32","52.154.170.113/32","52.154.170.117/32","52.154.170.122/32","52.154.170.209/32","52.154.170.229/32","52.154.170.243/32","52.154.171.0/32","52.154.171.44/32","52.154.171.70/32","52.154.171.87/32","52.154.171.150/32","52.154.171.196/32","52.154.171.205/32","52.154.171.235/32","52.154.171.250/32","52.154.172.2/32","52.154.204.93/32","52.185.93.208/32","52.186.37.211/32","52.188.89.106/32","52.189.70.103/32","52.190.37.160/32","52.191.82.242/32","52.191.222.158/32","52.224.16.221/32","52.224.16.229/32","52.224.19.152/32","52.224.20.174/32","52.224.20.181/32","52.224.20.186/32","52.224.20.190/32","52.224.20.193/32","52.224.20.203/32","52.224.20.204/32","52.224.20.223/32","52.224.20.227/32","52.224.20.249/32","52.224.21.4/32","52.224.21.19/32","52.224.21.20/32","52.224.21.23/32","52.224.21.27/32","52.224.21.49/32","52.224.21.51/32","52.224.21.53/32","52.224.21.55/32","52.224.21.61/32","52.224.147.21/32","52.224.191.245/32","52.242.224.168/32","52.250.46.221/32","57.151.44.140/32","57.152.17.237/32","57.152.62.107/32","57.152.72.128/32","57.155.18.44/32","57.165.205.172/32","57.167.88.38/32","64.236.15.33/32","64.236.93.100/32","64.236.118.43/32","68.219.152.220/32","72.144.95.185/32","74.163.169.106/32","74.163.208.206/32","74.179.200.126/32","74.179.232.116/32","74.242.194.48/32","98.70.219.37/32","104.43.54.127/32","104.43.55.116/32","104.43.55.117/32","104.43.55.166/32","104.43.55.167/32","108.141.83.74/32","128.203.82.154/32","128.203.132.152/32","128.203.181.79/32","128.203.214.241/32","128.203.236.110/32","130.107.228.224/32","131.145.39.118/32","131.189.183.241/32","132.164.209.198/32","132.196.106.146/32","132.220.1.71/32","134.33.207.189/32","135.233.63.237/32","135.234.221.112/32","135.235.183.95/32","172.168.12.93/32","172.168.43.43/32","172.168.51.216/32","172.168.53.53/32","172.168.81.217/32","172.168.94.160/32","172.168.115.250/32","172.168.133.76/32","172.168.137.102/32","172.168.152.88/32","172.168.173.230/32","172.168.190.186/32","172.168.227.120/32","172.168.254.119/32","172.169.7.33/32","172.169.17.165/32","172.169.28.184/32","172.169.36.43/32","172.169.53.208/32","172.169.60.134/32","172.169.93.148/32","172.169.164.109/32","172.169.177.131/32","172.169.181.237/32","172.171.37.221/32","172.179.105.87/32","172.179.128.118/32","172.189.40.18/32","172.193.184.42/32","172.193.197.146/32","172.193.245.229/32","172.194.141.49/32","172.199.55.212/32","172.210.111.68/32","191.233.3.197/32","191.233.3.202/32","191.234.216.4/32","191.234.216.178/32","191.235.201.214/32","191.235.202.38/32","191.235.202.48/32"]} diff --git a/whitelists/benign_bots/legit_bots/flipboard.json b/whitelists/benign_bots/legit_bots/flipboard.json new file mode 100644 index 000000000..f61180ee0 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/flipboard.json @@ -0,0 +1,2 @@ +# Flipboard proxy / link preview - https://cdn.flipboard.com/flipboard_ip.txt (auto-synced). rDNS proxy.flipboard.com does not forward-confirm, so this is IP based. +{"name":"flipboard","user_agent":"flipboardproxy|flipboardbot|flipboardrss","ranges":["3.82.72.4/32","3.87.166.241/32","3.88.123.119/32","3.89.188.234/32","3.210.153.17/32","3.211.213.154/32","3.212.152.239/32","3.213.81.156/32","3.217.67.126/32","3.217.88.249/32","3.219.250.90/32","3.221.15.39/32","3.222.161.177/32","3.223.194.35/32","3.225.61.40/32","3.227.81.86/32","3.227.115.211/32","3.229.45.93/32","3.229.154.173/32","3.229.207.56/32","3.233.45.217/32","13.223.151.250/32","18.205.102.232/32","18.206.23.5/32","18.210.151.148/32","18.211.238.213/32","18.214.90.210/32","18.215.24.55/32","18.234.6.71/32","23.23.178.169/32","23.23.178.171/32","34.193.144.40/32","34.195.200.175/32","34.196.252.233/32","34.199.68.255/32","34.200.164.25/32","34.202.57.157/32","34.204.246.33/32","34.206.70.112/32","34.206.132.111/32","34.225.109.8/32","34.226.158.201/32","34.233.228.195/32","34.234.196.58/32","34.235.44.187/32","34.235.92.62/32","34.239.194.123/32","34.239.204.86/32","35.153.195.135/32","35.169.12.119/32","35.170.49.60/32","35.170.117.39/32","35.173.15.235/32","44.193.230.34/32","44.194.168.123/32","44.197.40.151/32","44.197.61.161/32","44.198.88.55/32","44.198.101.184/32","44.198.200.118/32","44.199.75.207/32","44.205.70.55/32","44.205.132.32/32","44.212.89.250/32","44.214.24.190/32","44.216.112.73/32","44.217.169.208/32","44.218.23.81/32","44.218.35.1/32","44.218.52.98/32","44.221.217.207/32","50.19.17.18/32","52.1.12.216/32","52.1.35.231/32","52.1.208.2/32","52.3.54.4/32","52.3.188.44/32","52.4.100.114/32","52.5.173.251/32","52.7.251.54/32","52.21.63.80/32","52.21.80.49/32","52.23.132.202/32","52.23.141.29/32","52.72.61.119/32","52.72.76.81/32","52.86.249.24/32","52.87.67.247/32","52.200.111.40/32","52.200.251.70/32","52.201.43.8/32","52.206.233.135/32","52.207.51.144/32","54.85.75.76/32","54.85.241.50/32","54.86.175.34/32","54.92.133.139/32","54.144.13.129/32","54.144.104.111/32","54.145.102.149/32","54.147.79.168/32","54.157.8.225/32","54.157.101.234/32","54.158.62.223/32","54.165.1.171/32","54.205.240.79/32","54.221.224.40/32","54.224.200.22/32","54.225.95.39/32","54.227.231.100/32","54.234.74.14/32","54.236.186.40/32","54.243.130.36/32","54.243.243.38/32","54.243.253.33/32","98.82.66.104/32","98.82.104.6/32","98.82.206.182/32","98.83.72.162/32","98.83.73.112/32","98.83.89.0/32","98.83.92.111/32","98.83.194.235/32","98.83.228.90/32","98.84.145.161/32","98.86.24.102/32","98.86.47.53/32","100.24.176.99/32","100.24.185.125/32","100.27.143.229/32","100.27.154.89/32","100.27.184.209/32","100.29.31.150/32","100.55.135.214/32","107.20.222.68/32","107.22.185.233/32"]} diff --git a/whitelists/benign_bots/legit_bots/gptbot.json b/whitelists/benign_bots/legit_bots/gptbot.json index 8546b3305..fead2e9cb 100644 --- a/whitelists/benign_bots/legit_bots/gptbot.json +++ b/whitelists/benign_bots/legit_bots/gptbot.json @@ -1,2 +1,2 @@ -# GPTBot (OpenAI) - https://openai.com/gptbot.json (creationTime 2025-10-30; refresh periodically) +# GPTBot (OpenAI) - https://openai.com/gptbot.json (auto-synced) {"name":"gptbot","user_agent":"gptbot","ranges":["4.227.36.0/25","20.125.66.80/28","20.171.206.0/24","20.171.207.0/24","52.230.152.0/24","74.7.175.128/25","74.7.227.0/25","74.7.227.128/25","74.7.228.0/25","74.7.230.0/25","74.7.241.0/25","74.7.241.128/25","74.7.242.0/25","74.7.243.128/25","74.7.244.0/25","132.196.86.0/24","172.182.202.0/25","172.182.204.0/24","172.182.207.0/25","172.182.214.0/24","172.182.215.0/24"]} diff --git a/whitelists/benign_bots/legit_bots/internet-archive.json b/whitelists/benign_bots/legit_bots/internet-archive.json new file mode 100644 index 000000000..9de94e26f --- /dev/null +++ b/whitelists/benign_bots/legit_bots/internet-archive.json @@ -0,0 +1,2 @@ +# Internet Archive (Wayback Machine) - forward-confirmed rDNS archive.org +{"name":"internet-archive","user_agent":"ia_archiver|archive\\.org_bot","rdns":["(^|\\.)archive\\.org$"]} diff --git a/whitelists/benign_bots/legit_bots/linkedin.json b/whitelists/benign_bots/legit_bots/linkedin.json new file mode 100644 index 000000000..590f55da4 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/linkedin.json @@ -0,0 +1,2 @@ +# LinkedInBot link-preview - forward-confirmed rDNS fwd.linkedin.com +{"name":"linkedin","user_agent":"linkedinbot","rdns":["(^|\\.)fwd\\.linkedin\\.com$"]} diff --git a/whitelists/benign_bots/legit_bots/lumar.json b/whitelists/benign_bots/legit_bots/lumar.json new file mode 100644 index 000000000..161ea5745 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/lumar.json @@ -0,0 +1,2 @@ +# Lumar (ex-Deepcrawl) - https://www.lumar.io/wp-content/uploads/2026/02/lumar_ip_list_feb_2026.json (creationTime 2026-02-06T16:47:39.230Z; URL is date-stamped and rotates, check https://www.lumar.io/ for the current list; refresh periodically) +{"name":"lumar","user_agent":"lumar|deepcrawl","ranges":["45.84.231.187/32","45.114.241.26/32","45.154.120.1/32","45.154.120.246/32","45.154.121.6/32","45.154.192.120/32","45.159.8.185/32","52.5.118.182/32","52.86.188.211/32","58.97.134.135/32","62.241.63.0/32","78.156.189.19/32","78.156.190.58/32","83.229.119.118/32","89.104.122.84/32","89.184.192.251/32","89.223.23.247/32","92.43.85.17/32","92.43.85.20/32","92.43.86.96/32","92.118.208.185/32","92.255.0.44/32","92.255.52.54/32","92.255.82.52/32","94.139.231.67/32","94.176.132.129/32","94.176.133.250/32","95.175.69.231/32","95.175.71.242/32","103.30.12.14/32","103.241.52.5/32","104.218.194.48/32","109.198.49.168/32","110.239.208.213/32","110.239.211.210/32","110.239.215.76/32","110.239.216.183/32","110.239.219.46/32","110.239.221.50/32","119.13.209.196/32","119.13.234.31/32","119.13.235.120/32","119.13.237.234/32","121.91.87.145/32","161.123.94.195/32","161.123.172.42/32","162.43.226.212/32","176.53.223.125/32","188.190.107.50/32","188.241.57.89/32","193.42.57.127/32","193.42.59.80/32","193.56.25.78/32","193.56.25.182/32","194.110.50.161/32","194.156.114.188/32","195.66.216.177/32","204.217.149.77/32","206.204.2.98/32","206.204.7.194/32","206.204.13.77/32","206.204.14.137/32","206.232.41.10/32","208.66.95.116/32","209.20.175.57/32","217.9.19.167/32"]} diff --git a/whitelists/benign_bots/legit_bots/mistralai-index.json b/whitelists/benign_bots/legit_bots/mistralai-index.json new file mode 100644 index 000000000..748f07529 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/mistralai-index.json @@ -0,0 +1,2 @@ +# MistralAI-Index - https://mistral.ai/mistralai-index-ips.json (auto-synced) +{"name":"mistralai-index","user_agent":"mistralai-index","ranges":["20.240.192.194/32","135.225.57.92/32"]} diff --git a/whitelists/benign_bots/legit_bots/mistralai-user.json b/whitelists/benign_bots/legit_bots/mistralai-user.json new file mode 100644 index 000000000..1d1e45a77 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/mistralai-user.json @@ -0,0 +1,2 @@ +# MistralAI-User (user-triggered fetches) - https://mistral.ai/mistralai-user-ips.json (auto-synced) +{"name":"mistralai-user","user_agent":"mistralai-user","ranges":["20.240.160.1/32","20.240.160.161/32","20.240.194.83/32","51.12.243.114/32"]} diff --git a/whitelists/benign_bots/legit_bots/naver.json b/whitelists/benign_bots/legit_bots/naver.json new file mode 100644 index 000000000..f436f1dcd --- /dev/null +++ b/whitelists/benign_bots/legit_bots/naver.json @@ -0,0 +1,2 @@ +# Naver Yeti - forward-confirmed rDNS web.naver.com (https://searchadvisor.naver.com/doc/naverbot.json also publishes IP ranges) +{"name":"naver","user_agent":"yeti","rdns":["(^|\\.)web\\.naver\\.com$"]} diff --git a/whitelists/benign_bots/legit_bots/openai-adsbot.json b/whitelists/benign_bots/legit_bots/openai-adsbot.json new file mode 100644 index 000000000..b11352fa4 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/openai-adsbot.json @@ -0,0 +1,2 @@ +# OpenAI OAI-AdsBot - https://openai.com/adsbot.json (auto-synced) +{"name":"openai-adsbot","user_agent":"oai-adsbot","ranges":["20.168.90.128/25","130.131.229.128/25"]} diff --git a/whitelists/benign_bots/legit_bots/openai-chatgpt-user.json b/whitelists/benign_bots/legit_bots/openai-chatgpt-user.json index 53e6ea6b6..8b391d667 100644 --- a/whitelists/benign_bots/legit_bots/openai-chatgpt-user.json +++ b/whitelists/benign_bots/legit_bots/openai-chatgpt-user.json @@ -1,2 +1,2 @@ -# OpenAI ChatGPT-User - https://openai.com/chatgpt-user.json (creationTime 2025-02-07; refresh periodically) -{"name":"openai-chatgpt-user","user_agent":"chatgpt-user","ranges":["107.20.236.150/32","3.224.62.45/32","18.210.92.235/32","3.222.232.239/32","3.211.124.183/32","3.231.139.107/32","18.97.1.228/30","18.97.9.96/29"]} +# OpenAI ChatGPT-User - https://openai.com/chatgpt-user.json (auto-synced) +{"name":"openai-chatgpt-user","user_agent":"chatgpt-user","ranges":["4.151.71.176/28","4.151.119.48/28","4.151.241.240/28","4.189.118.208/28","4.189.119.48/28","4.196.118.112/28","4.197.19.176/28","4.197.22.112/28","4.197.64.0/28","4.197.64.16/28","4.197.64.48/28","4.197.64.64/28","4.197.115.112/28","4.198.72.16/28","4.198.96.112/28","4.201.232.64/28","4.201.232.80/28","4.203.96.80/28","4.205.128.176/28","4.226.200.16/28","4.226.226.32/28","9.129.0.0/17","9.160.34.144/28","9.160.128.16/28","9.160.128.32/28","9.160.164.128/28","9.234.96.192/28","9.234.97.96/28","9.234.97.128/28","9.235.40.32/28","13.65.138.96/28","13.65.138.112/28","13.67.72.16/28","13.70.107.160/28","13.71.2.208/28","13.76.32.208/28","13.76.115.224/28","13.76.115.240/28","13.76.116.80/28","13.83.167.128/28","13.83.237.176/28","20.17.108.96/28","20.27.94.128/28","20.45.178.144/28","20.52.125.160/28","20.55.129.0/28","20.55.229.144/28","20.57.199.192/28","20.63.221.64/28","20.79.59.112/28","20.83.243.176/28","20.97.189.96/28","20.102.212.144/28","20.113.225.112/28","20.125.112.224/28","20.125.144.144/28","20.161.75.208/28","20.168.7.192/28","20.168.7.240/28","20.169.72.112/28","20.169.73.32/28","20.169.73.64/28","20.169.73.176/28","20.169.78.48/28","20.169.78.64/28","20.169.78.80/28","20.169.78.96/28","20.169.78.112/28","20.169.78.128/28","20.169.78.144/28","20.169.78.160/28","20.169.78.176/28","20.169.78.192/28","20.169.78.208/28","20.169.86.224/28","20.169.86.240/28","20.169.87.112/28","20.170.184.16/28","20.170.184.32/28","20.170.184.48/28","20.170.184.64/28","20.172.29.32/28","20.193.50.32/28","20.193.233.240/28","20.194.0.208/28","20.194.1.0/28","20.198.67.96/28","20.199.211.160/28","20.199.242.0/28","20.204.24.240/28","20.210.154.128/28","20.210.174.208/28","20.210.211.192/28","20.215.187.208/28","20.215.188.192/28","20.215.214.16/28","20.215.219.128/28","20.215.219.160/28","20.215.219.208/28","20.215.220.64/28","20.215.220.80/28","20.215.220.96/28","20.215.220.112/28","20.215.220.128/28","20.215.220.144/28","20.215.220.160/28","20.215.220.176/28","20.215.220.192/28","20.215.220.208/28","20.218.30.240/28","20.226.32.80/28","20.227.140.32/28","20.228.106.176/28","20.235.75.208/28","20.235.87.224/28","20.249.63.208/28","20.250.6.128/28","20.250.136.64/28","20.250.136.80/28","23.98.142.176/28","23.98.179.16/28","23.98.186.64/28","23.98.186.96/28","23.98.186.176/28","23.98.186.192/28","23.102.140.144/28","23.102.141.32/28","40.67.183.160/28","40.67.183.176/28","40.75.14.224/28","40.78.161.48/28","40.81.134.128/28","40.81.134.144/28","40.81.234.144/28","40.84.181.32/28","40.84.221.208/28","40.84.221.224/28","40.116.73.208/28","40.122.235.112/28","48.193.44.32/28","48.221.40.176/28","48.221.184.80/28","48.221.184.96/28","48.221.184.112/28","51.8.155.48/28","51.8.155.64/28","51.8.155.80/28","51.56.40.80/28","51.57.0.96/28","51.59.24.64/28","51.59.24.80/28","51.59.40.80/28","51.59.40.96/28","51.59.48.80/28","51.59.48.96/28","51.107.70.192/28","51.116.2.80/28","51.116.221.96/28","52.148.129.32/28","52.153.130.64/28","52.154.22.48/28","52.156.77.144/28","52.159.227.32/28","52.159.249.96/28","52.161.49.96/28","52.161.49.224/28","52.165.212.16/28","52.165.212.32/28","52.165.212.48/28","52.172.129.160/28","52.172.251.112/28","52.173.219.96/28","52.173.219.112/28","52.173.221.16/28","52.173.221.176/28","52.173.221.208/28","52.173.234.16/28","52.173.234.80/28","52.173.235.80/28","52.183.217.240/28","52.187.246.128/28","52.190.137.16/28","52.190.137.144/28","52.190.139.48/28","52.190.142.64/28","52.190.190.16/28","52.225.75.208/28","52.230.163.32/28","52.230.164.176/28","52.231.30.48/28","52.231.34.176/28","52.231.39.144/28","52.231.39.192/28","52.231.49.48/28","52.231.50.64/28","52.236.94.144/28","52.241.146.208/28","52.242.132.224/28","52.242.132.240/28","52.242.245.208/28","52.252.113.240/28","52.255.109.80/28","52.255.109.96/28","52.255.109.112/28","52.255.109.128/28","52.255.109.144/28","52.255.111.0/28","52.255.111.32/28","52.255.111.48/28","52.255.111.80/28","52.255.111.112/28","57.154.174.112/28","57.154.175.0/28","57.154.187.32/28","68.154.28.96/28","68.218.30.112/28","68.220.57.64/28","68.221.67.192/28","68.221.67.224/28","68.221.67.240/28","70.153.76.16/28","70.153.189.192/28","70.153.190.16/28","70.156.144.64/28","70.156.152.80/28","74.7.35.48/28","74.7.35.112/28","74.7.36.64/28","74.7.36.80/28","74.7.36.96/28","74.161.200.96/28","74.226.253.160/28","74.249.86.176/28","85.211.241.128/28","104.210.139.192/28","104.210.139.224/28","128.85.198.32/28","132.196.82.48/28","135.119.134.128/28","135.119.134.192/28","135.220.73.208/28","135.220.73.240/28","135.237.131.208/28","135.237.133.48/28","137.135.191.176/28","138.91.30.48/28","138.91.46.96/28","145.132.136.96/28","145.133.0.176/28","168.63.252.240/28","172.162.248.64/28","172.170.1.80/28","172.170.8.208/28","172.170.225.0/28","172.170.241.80/28","172.171.4.176/28","172.175.152.224/28","172.178.140.144/28","172.178.141.112/28","172.178.141.128/28","172.183.143.224/28","172.183.222.128/28","172.192.97.32/28","172.197.170.80/28","172.197.203.16/28","172.202.102.112/28","172.204.27.16/28","172.204.96.32/28","172.204.96.48/28","172.208.128.32/28","172.208.128.48/28","172.212.159.64/28","172.212.172.160/28","172.213.21.16/28","172.215.215.32/28","172.215.218.96/28","191.233.1.112/28","191.233.1.128/28","191.233.2.0/28","191.233.194.32/28","191.233.196.112/28","191.233.199.160/28","191.235.66.16/28","191.235.99.80/28","191.237.249.64/28","191.239.245.16/28"]} diff --git a/whitelists/benign_bots/legit_bots/openai-searchbot.json b/whitelists/benign_bots/legit_bots/openai-searchbot.json index 0b335a4eb..97bc3fcee 100644 --- a/whitelists/benign_bots/legit_bots/openai-searchbot.json +++ b/whitelists/benign_bots/legit_bots/openai-searchbot.json @@ -1,2 +1,2 @@ -# OpenAI SearchBot - https://openai.com/searchbot.json (creationTime 2026-01-02; refresh periodically) -{"name":"openai-searchbot","user_agent":"oai-searchbot","ranges":["104.210.140.128/28","135.234.64.0/24","172.182.193.224/28","172.182.193.80/28","172.182.194.144/28","172.182.194.32/28","172.182.195.48/28","172.182.209.208/28","172.182.211.192/28","172.182.213.192/28","172.182.224.0/28","172.203.190.128/28","20.14.99.96/28","20.168.18.32/28","20.169.6.224/28","20.169.7.48/28","20.169.77.0/25","20.171.123.64/28","20.171.53.224/28","20.25.151.224/28","20.42.10.176/28","4.227.36.0/25","40.67.175.0/25","40.90.214.16/28","51.8.102.0/24","74.7.175.128/25","74.7.228.0/25","74.7.228.128/25","74.7.229.0/25","74.7.229.128/25","74.7.230.0/25","74.7.241.128/25","74.7.242.128/25","74.7.243.0/25","74.7.244.0/25"]} +# OpenAI SearchBot - https://openai.com/searchbot.json (auto-synced) +{"name":"openai-searchbot","user_agent":"oai-searchbot","ranges":["4.227.36.0/25","20.14.99.96/28","20.25.151.224/28","20.42.10.176/28","20.168.18.32/28","20.169.6.224/28","20.169.7.48/28","20.169.77.0/25","20.171.53.224/28","20.171.123.64/28","40.67.175.0/25","40.90.214.16/28","51.8.102.0/24","74.7.175.128/25","74.7.228.0/25","74.7.228.128/25","74.7.229.0/25","74.7.229.128/25","74.7.230.0/25","74.7.241.128/25","74.7.242.128/25","74.7.243.0/25","74.7.244.0/25","104.210.140.128/28","135.234.64.0/24","172.182.193.80/28","172.182.193.224/28","172.182.194.32/28","172.182.194.144/28","172.182.195.48/28","172.182.209.208/28","172.182.211.192/28","172.182.213.192/28","172.182.224.0/28","172.203.190.128/28"]} diff --git a/whitelists/benign_bots/legit_bots/pagerduty.json b/whitelists/benign_bots/legit_bots/pagerduty.json index abc9a4142..b40bcdb34 100644 --- a/whitelists/benign_bots/legit_bots/pagerduty.json +++ b/whitelists/benign_bots/legit_bots/pagerduty.json @@ -1,2 +1,2 @@ -# PagerDuty outbound webhooks (UA PagerDuty-Webhook/Vx.0) - US https://app.pagerduty.com/webhook_ips + EU https://app.eu.pagerduty.com/webhook_ips (fetched 2026-07-01; refresh periodically) -{"name":"pagerduty","user_agent":"pagerduty-webhook","ranges":["34.210.57.30/32","34.210.242.134/32","52.34.208.156/32","44.242.69.192/32","52.89.71.166/32","54.213.187.133/32","35.86.21.47/32","52.88.94.18/32","44.238.89.29/32","54.241.68.46/32","54.176.72.216/32","54.177.81.67/32","13.56.49.27/32","18.192.91.93/32","18.158.120.237/32","18.194.177.30/32","18.158.199.216/32","3.126.25.87/32","18.197.187.16/32","54.76.3.62/32","54.170.2.90/32","52.213.188.110/32","54.195.179.238/32","34.250.91.200/32","54.76.225.71/32"]} +# PagerDuty outbound webhooks (UA PagerDuty-Webhook/Vx.0) - US + EU service regions, https://developer.pagerduty.com/docs/webhook-ips (auto-synced) +{"name":"pagerduty","user_agent":"pagerduty-webhook","ranges":["3.126.25.87/32","13.56.49.27/32","18.158.120.237/32","18.158.199.216/32","18.192.91.93/32","18.194.177.30/32","18.197.187.16/32","34.210.57.30/32","34.210.242.134/32","34.250.91.200/32","35.86.21.47/32","44.238.89.29/32","44.242.69.192/32","52.34.208.156/32","52.88.94.18/32","52.89.71.166/32","52.213.188.110/32","54.76.3.62/32","54.76.225.71/32","54.170.2.90/32","54.176.72.216/32","54.177.81.67/32","54.195.179.238/32","54.213.187.133/32","54.241.68.46/32"]} diff --git a/whitelists/benign_bots/legit_bots/perplexity-user.json b/whitelists/benign_bots/legit_bots/perplexity-user.json new file mode 100644 index 000000000..eac606e64 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/perplexity-user.json @@ -0,0 +1,2 @@ +# Perplexity-User (user-triggered fetches) - https://www.perplexity.ai/perplexity-user.json (auto-synced) +{"name":"perplexity-user","user_agent":"perplexity-user","ranges":["18.97.21.0/30","18.97.43.80/29","34.193.163.52/32","44.208.221.197/32"]} diff --git a/whitelists/benign_bots/legit_bots/perplexitybot.json b/whitelists/benign_bots/legit_bots/perplexitybot.json index 5fa994c43..2c360073e 100644 --- a/whitelists/benign_bots/legit_bots/perplexitybot.json +++ b/whitelists/benign_bots/legit_bots/perplexitybot.json @@ -1,2 +1,2 @@ -# PerplexityBot - https://www.perplexity.ai/perplexitybot.json (creationTime 2026-06-24; refresh periodically) -{"name":"perplexitybot","user_agent":"perplexitybot","ranges":["104.210.139.192/28","104.210.139.224/28","13.65.138.112/28","13.65.138.96/28","13.67.72.16/28","13.70.107.160/28","13.71.2.208/28","13.76.115.224/28","13.76.115.240/28","13.76.116.80/28","13.76.32.208/28","13.83.167.128/28","13.83.237.176/28","132.196.82.48/28","135.119.134.128/28","135.119.134.192/28","135.220.73.208/28","135.220.73.240/28","135.237.131.208/28","135.237.133.48/28","137.135.191.176/28","138.91.30.48/28","138.91.46.96/28","168.63.252.240/28","172.170.8.208/28","172.171.4.176/28","172.178.140.144/28","172.178.141.112/28","172.178.141.128/28","172.183.143.224/28","172.183.222.128/28","172.196.40.208/28","172.202.102.112/28","172.204.16.64/28","172.204.27.16/28","172.212.159.64/28","172.212.172.160/28","172.213.11.144/28","172.213.21.16/28","172.215.215.32/28","172.215.218.96/28","191.233.1.112/28","191.233.1.128/28","191.233.1.224/28","191.233.194.32/28","191.233.196.112/28","191.233.199.160/28","191.233.2.0/28","191.234.167.128/28","191.235.66.16/28","191.235.98.144/28","191.235.99.80/28","191.237.249.64/28","191.239.245.16/28","20.0.53.96/28","20.102.212.144/28","20.113.225.112/28","20.125.112.224/28","20.125.144.144/28","20.161.75.208/28","20.168.7.192/28","20.168.7.240/28","20.169.72.112/28","20.169.72.96/28","20.169.73.176/28","20.169.73.32/28","20.169.73.64/28","20.169.78.112/28","20.169.78.128/28","20.169.78.144/28","20.169.78.160/28","20.169.78.176/28","20.169.78.192/28","20.169.78.208/28","20.169.78.48/28","20.169.78.64/28","20.169.78.80/28","20.169.78.96/28","20.169.86.224/28","20.169.86.240/28","20.169.87.112/28","20.17.108.96/28","20.170.184.16/28","20.172.29.32/28","20.193.233.240/28","20.193.50.32/28","20.194.0.208/28","20.194.1.0/28","20.194.157.176/28","20.198.67.96/28","20.199.211.160/28","20.203.245.32/28","20.204.24.240/28","20.206.107.192/28","20.210.154.128/28","20.210.174.208/28","20.210.211.192/28","20.215.187.208/28","20.215.188.192/28","20.215.214.16/28","20.215.219.128/28","20.215.219.160/28","20.215.219.208/28","20.215.220.112/28","20.215.220.128/28","20.215.220.144/28","20.215.220.160/28","20.215.220.176/28","20.215.220.192/28","20.215.220.208/28","20.215.220.64/28","20.215.220.80/28","20.215.220.96/28","20.226.32.80/28","20.227.140.32/28","20.228.106.176/28","20.235.75.208/28","20.235.87.224/28","20.249.63.208/28","20.27.94.128/28","20.42.250.32/28","20.45.178.144/28","20.52.125.160/28","20.55.129.0/28","20.55.229.144/28","20.57.199.192/28","20.63.221.64/28","20.79.59.112/28","20.97.189.96/28","23.102.140.144/28","23.102.141.32/28","23.97.109.224/28","23.98.142.176/28","23.98.179.16/28","23.98.186.176/28","23.98.186.192/28","23.98.186.64/28","23.98.186.96/28","4.151.119.48/28","4.151.241.240/28","4.151.71.176/28","4.189.118.208/28","4.189.119.48/28","4.196.118.112/28","4.196.198.80/28","4.197.115.112/28","4.197.19.176/28","4.197.22.112/28","4.197.64.0/28","4.197.64.16/28","4.197.64.48/28","4.197.64.64/28","4.198.72.16/28","4.205.128.176/28","4.226.226.32/28","40.116.73.208/28","40.122.235.112/28","40.67.183.160/28","40.67.183.176/28","40.75.14.224/28","40.81.134.128/28","40.81.134.144/28","40.81.234.144/28","40.84.181.32/28","40.84.221.208/28","40.84.221.224/28","48.193.44.32/28","51.107.70.192/28","51.116.2.64/28","51.116.2.80/28","51.8.155.48/28","51.8.155.64/28","51.8.155.80/28","52.148.129.32/28","52.153.130.48/28","52.153.130.64/28","52.154.22.48/28","52.156.77.144/28","52.159.227.32/28","52.159.249.96/28","52.161.49.224/28","52.161.49.96/28","52.165.212.16/28","52.165.212.32/28","52.165.212.48/28","52.172.129.160/28","52.172.251.112/28","52.173.123.0/28","52.173.219.112/28","52.173.219.96/28","52.173.221.16/28","52.173.221.176/28","52.173.221.208/28","52.173.234.16/28","52.173.234.80/28","52.173.235.80/28","52.176.139.176/28","52.187.246.128/28","52.190.137.144/28","52.190.137.16/28","52.190.139.48/28","52.190.142.64/28","52.190.190.16/28","52.225.75.208/28","52.230.163.32/28","52.230.164.176/28","52.231.30.48/28","52.231.34.176/28","52.231.39.144/28","52.231.39.192/28","52.231.49.48/28","52.231.50.64/28","52.236.94.144/28","52.241.146.208/28","52.242.132.224/28","52.242.132.240/28","52.242.245.208/28","52.252.113.240/28","52.255.109.112/28","52.255.109.128/28","52.255.109.144/28","52.255.109.80/28","52.255.109.96/28","52.255.111.0/28","52.255.111.112/28","52.255.111.16/28","52.255.111.32/28","52.255.111.48/28","52.255.111.80/28","57.151.131.224/28","57.154.174.112/28","57.154.175.0/28","57.154.187.32/28","68.154.28.96/28","68.218.30.112/28","68.220.57.64/28","68.221.67.160/28","68.221.67.192/28","68.221.67.224/28","68.221.67.240/28","68.221.75.16/28","70.153.76.16/28","74.226.253.160/28","74.249.86.176/28","74.7.35.112/28","74.7.35.48/28","74.7.36.64/28","74.7.36.80/28","74.7.36.96/28","85.211.241.128/28","9.129.0.0/17","9.160.163.224/28","9.160.164.128/28","9.234.96.192/28"]} +# PerplexityBot - https://www.perplexity.ai/perplexitybot.json (auto-synced) +{"name":"perplexitybot","user_agent":"perplexitybot","ranges":["3.211.124.183/32","3.222.232.239/32","3.224.62.45/32","3.231.139.107/32","18.97.1.228/30","18.97.9.96/29","18.210.92.235/32","107.20.236.150/32"]} diff --git a/whitelists/benign_bots/legit_bots/semrush.json b/whitelists/benign_bots/legit_bots/semrush.json new file mode 100644 index 000000000..6c9a0d184 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/semrush.json @@ -0,0 +1,2 @@ +# SemrushBot and friends - forward-confirmed rDNS bot.semrush.com; Semrush publishes no IP list (https://www.semrush.com/bot/) +{"name":"semrush","user_agent":"semrushbot|siteauditbot|splitsignalbot|rytebot","rdns":["(^|\\.)bot\\.semrush\\.com$"]} diff --git a/whitelists/benign_bots/legit_bots/seznam.json b/whitelists/benign_bots/legit_bots/seznam.json new file mode 100644 index 000000000..ddf2afa10 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/seznam.json @@ -0,0 +1,2 @@ +# SeznamBot - forward-confirmed rDNS seznam.cz (fulltextrobot-*.seznam.cz); https://search.seznam.cz/ipranges/seznambot.json also publishes IP ranges +{"name":"seznam","user_agent":"seznambot","rdns":["(^|\\.)seznam\\.cz$"]} diff --git a/whitelists/benign_bots/legit_bots/telegram.json b/whitelists/benign_bots/legit_bots/telegram.json index c6075449a..0f4e0275e 100644 --- a/whitelists/benign_bots/legit_bots/telegram.json +++ b/whitelists/benign_bots/legit_bots/telegram.json @@ -1,2 +1,2 @@ -# Telegram - https://core.telegram.org/resources/cidr.txt (fetched 2026-07-01; refresh periodically) -{"name":"telegram","user_agent":"telegrambot","ranges":["91.108.56.0/22","91.108.4.0/22","91.108.8.0/22","91.108.16.0/22","91.108.12.0/22","149.154.160.0/20","91.105.192.0/23","91.108.20.0/22","185.76.151.0/24","2001:b28:f23d::/48","2001:b28:f23f::/48","2001:67c:4e8::/48","2001:b28:f23c::/48","2a0a:f280::/32"]} +# Telegram - https://core.telegram.org/resources/cidr.txt (auto-synced) +{"name":"telegram","user_agent":"telegrambot","ranges":["91.105.192.0/23","91.108.4.0/22","91.108.8.0/22","91.108.12.0/22","91.108.16.0/22","91.108.20.0/22","91.108.56.0/22","149.154.160.0/20","185.76.151.0/24","2001:67c:4e8::/48","2001:b28:f23c::/48","2001:b28:f23d::/48","2001:b28:f23f::/48","2a0a:f280::/32"]} diff --git a/whitelists/benign_bots/legit_bots/twitterbot.json b/whitelists/benign_bots/legit_bots/twitterbot.json index a28cf6605..6f0696798 100644 --- a/whitelists/benign_bots/legit_bots/twitterbot.json +++ b/whitelists/benign_bots/legit_bots/twitterbot.json @@ -1,2 +1,2 @@ # Twitterbot (X) - AS13414, https://developer.twitter.com/en/docs/twitter-for-websites/cards/guides/troubleshooting-cards (fetched 2026-07-01; hand-maintained, refresh periodically) -{"name":"twitterbot","user_agent":"twitterbot","ranges":["199.16.156.0/22","199.59.148.0/22","192.133.76.0/22"]} +{"name":"twitterbot","user_agent":"twitterbot","ranges":["192.133.76.0/22","199.16.156.0/22","199.59.148.0/22"]} diff --git a/whitelists/benign_bots/legit_bots/uptimerobot.json b/whitelists/benign_bots/legit_bots/uptimerobot.json index e6b3800b9..4ef00ea1d 100644 --- a/whitelists/benign_bots/legit_bots/uptimerobot.json +++ b/whitelists/benign_bots/legit_bots/uptimerobot.json @@ -1,2 +1,2 @@ -# UptimeRobot monitors - https://cdn.uptimerobot.com/api/IPv4andIPv6.txt (fetched 2026-07-01; probe IPs change, refresh periodically) -{"name":"uptimerobot","user_agent":"uptimerobot","ranges":["3.12.251.153/32","3.20.63.178/32","3.77.67.4/32","3.79.134.69/32","3.105.133.239/32","3.105.190.221/32","3.133.226.214/32","3.149.57.90/32","3.212.128.62/32","5.161.61.238/32","5.161.73.160/32","5.161.75.7/32","5.161.113.195/32","5.161.117.52/32","5.161.177.47/32","5.161.194.92/32","5.161.215.244/32","5.223.43.32/32","5.223.53.147/32","5.223.57.22/32","18.116.205.62/32","18.180.208.214/32","18.192.166.72/32","18.193.252.127/32","24.144.78.39/32","24.144.78.185/32","34.198.201.66/32","45.55.123.175/32","45.55.127.146/32","49.13.24.81/32","49.13.130.29/32","49.13.134.145/32","49.13.164.148/32","49.13.167.123/32","52.15.147.27/32","52.22.236.30/32","52.28.162.93/32","52.59.43.236/32","52.87.72.16/32","54.64.67.106/32","54.79.28.129/32","54.87.112.51/32","54.167.223.174/32","54.249.170.27/32","63.178.84.147/32","64.225.81.248/32","64.225.82.147/32","78.46.190.63/32","78.46.215.1/32","78.47.98.55/32","78.47.173.76/32","88.99.80.227/32","91.99.101.207/32","128.140.41.193/32","128.140.106.114/32","129.212.132.140/32","134.199.240.137/32","138.197.53.117/32","138.197.53.138/32","138.197.54.143/32","138.197.54.247/32","138.197.63.92/32","139.59.50.44/32","142.132.180.39/32","143.198.249.237/32","143.198.250.89/32","143.244.196.21/32","143.244.196.211/32","143.244.221.177/32","144.126.251.21/32","146.190.9.187/32","152.42.149.135/32","157.90.155.240/32","157.90.156.63/32","159.69.158.189/32","159.223.243.219/32","161.35.247.201/32","167.99.18.52/32","167.235.143.113/32","168.119.53.160/32","168.119.96.239/32","168.119.123.75/32","170.64.250.64/32","170.64.250.132/32","170.64.250.235/32","178.156.181.172/32","178.156.184.20/32","178.156.185.127/32","178.156.185.231/32","178.156.187.238/32","178.156.189.113/32","178.156.189.249/32","188.166.201.79/32","206.189.241.133/32","209.38.49.1/32","209.38.49.206/32","209.38.49.226/32","209.38.51.43/32","209.38.53.7/32","209.38.124.252/32","216.144.248.27/32","216.144.248.28/32","216.144.248.29/32","2a01:4f8:1c1a:3d53::1/128","2a01:4f8:1c1b:4ef4::1/128","2a01:4f8:1c1b:5b5a::1/128","2a01:4f8:1c1b:7ecc::1/128","2a01:4f8:1c1c:11aa::1/128","2a01:4f8:1c1c:5353::1/128","2a01:4f8:1c1c:7240::1/128","2a01:4f8:1c1c:a98a::1/128","2a01:4f8:c0c:83fa::1/128","2a01:4f8:c2c:9fc6::1/128","2a01:4f8:c2c:beae::1/128","2a01:4f8:c012:c60e::1/128","2a01:4f8:c013:3b0f::1/128","2a01:4f8:c013:3c52::1/128","2a01:4f8:c013:3c53::1/128","2a01:4f8:c013:3c54::1/128","2a01:4f8:c013:3c55::1/128","2a01:4f8:c013:3c56::1/128","2a01:4f8:c013:34c0::1/128","2a01:4f8:c013:c18::1/128","2a01:4f8:c17:42e4::1/128","2a01:4ff:2f0:3b3a::1/128","2a01:4ff:2f0:27de::1/128","2a01:4ff:2f0:193c::1/128","2a01:4ff:f0:3e03::1/128","2a01:4ff:f0:5f80::1/128","2a01:4ff:f0:7fad::1/128","2a01:4ff:f0:9c5f::1/128","2a01:4ff:f0:2219::1/128","2a01:4ff:f0:b2f2::1/128","2a01:4ff:f0:b6f1::1/128","2a01:4ff:f0:bfd::1/128","2a01:4ff:f0:d3cd::1/128","2a01:4ff:f0:d283::1/128","2a01:4ff:f0:e9cf::1/128","2a01:4ff:f0:e516::1/128","2a01:4ff:f0:eccb::1/128","2a01:4ff:f0:efd1::1/128","2a01:4ff:f0:fdc7::1/128","2a03:b0c0:2:f0::bd91:f001/128","2a03:b0c0:2:f0::bd92:1/128","2a03:b0c0:2:f0::bd92:1001/128","2a03:b0c0:2:f0::bd92:2001/128","2a03:b0c0:2:f0::bd92:4001/128","2a03:b0c0:2:f0::bd92:5001/128","2a03:b0c0:2:f0::bd92:6001/128","2a03:b0c0:2:f0::bd92:7001/128","2a03:b0c0:2:f0::bd92:8001/128","2a03:b0c0:2:f0::bd92:9001/128","2a03:b0c0:2:f0::bd92:a001/128","2a03:b0c0:2:f0::bd92:b001/128","2a03:b0c0:2:f0::bd92:c001/128","2a03:b0c0:2:f0::bd92:e001/128","2a03:b0c0:2:f0::bd92:f001/128","2a05:d014:1815:3400:6d:9235:c1c0:96ad/128","2a05:d014:1815:3400:90b4:4ef9:5631:b170/128","2a05:d014:1815:3400:654f:bd37:724c:212b/128","2a05:d014:1815:3400:9779:d8e9:100a:9642/128","2a05:d014:1815:3400:af29:e95e:64ff:df81/128","2a05:d014:1815:3400:c7d6:f7f3:6cc1:30d1/128","2a05:d014:1815:3400:d784:e5dd:8e0:67cb/128","2406:da1c:9c8:dc02:7ae1:f2ea:ab91:2fde/128","2406:da1c:9c8:dc02:7db9:f38b:7b9f:402e/128","2406:da1c:9c8:dc02:82b2:f0fd:ee96:579/128","2406:da14:94d:8601:9d0d:7754:bedf:e4f5/128","2406:da14:94d:8601:b325:ff58:2bba:7934/128","2406:da14:94d:8601:db4b:c5ac:2cbe:9a79/128","2600:1f16:775:3a00:3f24:5bb0:95d7:5a6b/128","2600:1f16:775:3a00:8c2c:2ba6:778f:5be5/128","2600:1f16:775:3a00:37bf:6026:e54a:f03a/128","2600:1f16:775:3a00:91ac:3120:ff38:92b5/128","2600:1f16:775:3a00:ac3:c5eb:7081:942e/128","2600:1f16:775:3a00:dbbe:36b0:3c45:da32/128","2600:1f18:179:f900:4b7d:d1cc:2d10:211/128","2600:1f18:179:f900:5c68:91b6:5d75:5d7/128","2600:1f18:179:f900:71:af9a:ade7:d772/128","2600:1f18:179:f900:2406:9399:4ae6:c5d3/128","2600:1f18:179:f900:4696:7729:7bb3:f52f/128","2600:1f18:179:f900:e8dd:eed1:a6c:183b/128","2604:a880:800:14:0:1:68ba:d000/128","2604:a880:800:14:0:1:68ba:e000/128","2604:a880:800:14:0:1:68bb:0/128","2604:a880:800:14:0:1:68bb:1000/128","2604:a880:800:14:0:1:68bb:3000/128","2604:a880:800:14:0:1:68bb:4000/128","2604:a880:800:14:0:1:68bb:5000/128","2604:a880:800:14:0:1:68bb:6000/128","2604:a880:800:14:0:1:68bb:7000/128","2604:a880:800:14:0:1:68bb:a000/128","2604:a880:800:14:0:1:68bb:b000/128","2604:a880:800:14:0:1:68bb:c000/128","2604:a880:800:14:0:1:68bb:d000/128","2604:a880:800:14:0:1:68bb:e000/128","2604:a880:800:14:0:1:68bb:f000/128","2607:ff68:107::33/128","2607:ff68:107::57/128","2607:ff68:107::59/128"]} +# UptimeRobot monitors - https://cdn.uptimerobot.com/api/IPv4andIPv6.txt (auto-synced) +{"name":"uptimerobot","user_agent":"uptimerobot","ranges":["3.12.251.153/32","3.20.63.178/32","3.77.67.4/32","3.79.134.69/32","3.105.133.239/32","3.105.190.221/32","3.133.226.214/32","3.149.57.90/32","3.212.128.62/32","5.161.61.238/32","5.161.73.160/32","5.161.75.7/32","5.161.113.195/32","5.161.117.52/32","5.161.177.47/32","5.161.194.92/32","5.161.215.244/32","5.223.43.32/32","5.223.53.147/32","5.223.57.22/32","18.116.205.62/32","18.180.208.214/32","18.192.166.72/32","18.193.252.127/32","24.144.78.39/32","24.144.78.185/32","34.198.201.66/32","45.55.123.175/32","45.55.127.146/32","49.13.24.81/32","49.13.130.29/32","49.13.134.145/32","49.13.164.148/32","49.13.167.123/32","52.15.147.27/32","52.22.236.30/32","52.28.162.93/32","52.59.43.236/32","52.87.72.16/32","54.64.67.106/32","54.79.28.129/32","54.87.112.51/32","54.167.223.174/32","54.249.170.27/32","63.178.84.147/32","64.225.81.248/32","64.225.82.147/32","78.46.190.63/32","78.46.215.1/32","78.47.98.55/32","78.47.173.76/32","88.99.80.227/32","91.99.101.207/32","128.140.41.193/32","128.140.106.114/32","129.212.132.140/32","134.199.240.137/32","138.197.53.117/32","138.197.53.138/32","138.197.54.143/32","138.197.54.247/32","138.197.63.92/32","139.59.50.44/32","142.132.180.39/32","143.198.249.237/32","143.198.250.89/32","143.244.196.21/32","143.244.196.211/32","143.244.221.177/32","144.126.251.21/32","146.190.9.187/32","152.42.149.135/32","157.90.155.240/32","157.90.156.63/32","159.69.158.189/32","159.223.243.219/32","161.35.247.201/32","167.99.18.52/32","167.235.143.113/32","168.119.53.160/32","168.119.96.239/32","168.119.123.75/32","170.64.250.64/32","170.64.250.132/32","170.64.250.235/32","178.156.181.172/32","178.156.184.20/32","178.156.185.127/32","178.156.185.231/32","178.156.187.238/32","178.156.189.113/32","178.156.189.249/32","188.166.201.79/32","206.189.241.133/32","209.38.49.1/32","209.38.49.206/32","209.38.49.226/32","209.38.51.43/32","209.38.53.7/32","209.38.124.252/32","216.144.248.27/32","216.144.248.28/32","216.144.248.29/32","2400:6180:10:200::56a0:b000/128","2400:6180:10:200::56a0:c000/128","2400:6180:10:200::56a0:e000/128","2400:6180:100:d0::94b6:4001/128","2400:6180:100:d0::94b6:5001/128","2400:6180:100:d0::94b6:7001/128","2406:da14:94d:8601:9d0d:7754:bedf:e4f5/128","2406:da14:94d:8601:b325:ff58:2bba:7934/128","2406:da14:94d:8601:db4b:c5ac:2cbe:9a79/128","2406:da1c:9c8:dc02:7ae1:f2ea:ab91:2fde/128","2406:da1c:9c8:dc02:7db9:f38b:7b9f:402e/128","2406:da1c:9c8:dc02:82b2:f0fd:ee96:579/128","2600:1f16:775:3a00:ac3:c5eb:7081:942e/128","2600:1f16:775:3a00:37bf:6026:e54a:f03a/128","2600:1f16:775:3a00:3f24:5bb0:95d7:5a6b/128","2600:1f16:775:3a00:8c2c:2ba6:778f:5be5/128","2600:1f16:775:3a00:91ac:3120:ff38:92b5/128","2600:1f16:775:3a00:dbbe:36b0:3c45:da32/128","2600:1f18:179:f900:71:af9a:ade7:d772/128","2600:1f18:179:f900:2406:9399:4ae6:c5d3/128","2600:1f18:179:f900:4696:7729:7bb3:f52f/128","2600:1f18:179:f900:4b7d:d1cc:2d10:211/128","2600:1f18:179:f900:5c68:91b6:5d75:5d7/128","2600:1f18:179:f900:e8dd:eed1:a6c:183b/128","2604:a880:800:14:0:1:68ba:d000/128","2604:a880:800:14:0:1:68ba:e000/128","2604:a880:800:14:0:1:68bb:0/128","2604:a880:800:14:0:1:68bb:1000/128","2604:a880:800:14:0:1:68bb:3000/128","2604:a880:800:14:0:1:68bb:4000/128","2604:a880:800:14:0:1:68bb:5000/128","2604:a880:800:14:0:1:68bb:6000/128","2604:a880:800:14:0:1:68bb:7000/128","2604:a880:800:14:0:1:68bb:a000/128","2604:a880:800:14:0:1:68bb:b000/128","2604:a880:800:14:0:1:68bb:c000/128","2604:a880:800:14:0:1:68bb:d000/128","2604:a880:800:14:0:1:68bb:e000/128","2604:a880:800:14:0:1:68bb:f000/128","2607:ff68:107::33/128","2607:ff68:107::57/128","2607:ff68:107::59/128","2a01:4f8:c0c:83fa::1/128","2a01:4f8:c17:42e4::1/128","2a01:4f8:c2c:9fc6::1/128","2a01:4f8:c2c:beae::1/128","2a01:4f8:1c1a:3d53::1/128","2a01:4f8:1c1b:4ef4::1/128","2a01:4f8:1c1b:5b5a::1/128","2a01:4f8:1c1b:7ecc::1/128","2a01:4f8:1c1c:11aa::1/128","2a01:4f8:1c1c:5353::1/128","2a01:4f8:1c1c:7240::1/128","2a01:4f8:1c1c:a98a::1/128","2a01:4f8:c012:c60e::1/128","2a01:4f8:c013:c18::1/128","2a01:4f8:c013:34c0::1/128","2a01:4f8:c013:3b0f::1/128","2a01:4f8:c013:3c52::1/128","2a01:4f8:c013:3c53::1/128","2a01:4f8:c013:3c54::1/128","2a01:4f8:c013:3c55::1/128","2a01:4f8:c013:3c56::1/128","2a01:4ff:f0:bfd::1/128","2a01:4ff:f0:2219::1/128","2a01:4ff:f0:3e03::1/128","2a01:4ff:f0:5f80::1/128","2a01:4ff:f0:7fad::1/128","2a01:4ff:f0:9c5f::1/128","2a01:4ff:f0:b2f2::1/128","2a01:4ff:f0:b6f1::1/128","2a01:4ff:f0:d283::1/128","2a01:4ff:f0:d3cd::1/128","2a01:4ff:f0:e516::1/128","2a01:4ff:f0:e9cf::1/128","2a01:4ff:f0:eccb::1/128","2a01:4ff:f0:efd1::1/128","2a01:4ff:f0:fdc7::1/128","2a01:4ff:2f0:193c::1/128","2a01:4ff:2f0:27de::1/128","2a01:4ff:2f0:3b3a::1/128","2a03:b0c0:2:f0::bd91:f001/128","2a03:b0c0:2:f0::bd92:1/128","2a03:b0c0:2:f0::bd92:1001/128","2a03:b0c0:2:f0::bd92:2001/128","2a03:b0c0:2:f0::bd92:4001/128","2a03:b0c0:2:f0::bd92:5001/128","2a03:b0c0:2:f0::bd92:6001/128","2a03:b0c0:2:f0::bd92:7001/128","2a03:b0c0:2:f0::bd92:8001/128","2a03:b0c0:2:f0::bd92:9001/128","2a03:b0c0:2:f0::bd92:a001/128","2a03:b0c0:2:f0::bd92:b001/128","2a03:b0c0:2:f0::bd92:c001/128","2a03:b0c0:2:f0::bd92:e001/128","2a03:b0c0:2:f0::bd92:f001/128","2a05:d014:1815:3400:6d:9235:c1c0:96ad/128","2a05:d014:1815:3400:654f:bd37:724c:212b/128","2a05:d014:1815:3400:90b4:4ef9:5631:b170/128","2a05:d014:1815:3400:9779:d8e9:100a:9642/128","2a05:d014:1815:3400:af29:e95e:64ff:df81/128","2a05:d014:1815:3400:c7d6:f7f3:6cc1:30d1/128","2a05:d014:1815:3400:d784:e5dd:8e0:67cb/128"]} diff --git a/whitelists/benign_bots/legit_bots/youbot.json b/whitelists/benign_bots/legit_bots/youbot.json new file mode 100644 index 000000000..5445a0e95 --- /dev/null +++ b/whitelists/benign_bots/legit_bots/youbot.json @@ -0,0 +1,2 @@ +# YouBot (You.com) - forward-confirmed rDNS search.you.com +{"name":"youbot","user_agent":"youbot","rdns":["(^|\\.)search\\.you\\.com$"]}