OSINT enrichments degradation

[leandrosardi]

Problem

Osint platforms start asking for captchas, logout automatically, or ban accounts.

Solution

• ✔️ reduce max perfmed enrichments per day from 2000 to 200
• ✔️ reduce max aborted enrichments per day from 2000 to 200
• implement automated fingerprints rotation (accounts shouldn't be logged in)

Helper Queries

Generic restart of all enrichments

update enrichment set status=0 where status=3;
update enrichment set status=2 where hit is not null;
update enrichment set status=0 where hit is null;
update "enrichment" set done_time=null where status=0 and done_time is not null;
update "enrichment" set error_description=null where status=0 and error_description is not null;

Analize generic hit rate in last days

select e.hit, count(distinct e.id) as n
--select count(distinct e.id)
--select r.id, r.name
from "rule" r
join "action" a on a.id=r.id_action
join "enrichment_type" t on t.id=a.id_enrichment_type
join rule_instance i on r.id=i.id_rule
join "enrichment" e on i.id=e.id_rule_instance
where r.active = true -- rule must be active
and t.name ilike '%osint%' -- enrichment is an osint
and e.status = 2 -- performed
--and e.hit = false -- hit failed
and e.done_time > current_timestamp - interval '12 hours'
group by e.hit
order by e.hit

Query to restart OSINT enrichments

update "enrichment" set hit=null, id_profile=null, status=0 where "id" in (
	select distinct e.id
	--select count(distinct e.id)
	--select r.id, r.name
	from "rule" r
	join "action" a on a.id=r.id_action
	join "enrichment_type" t on t.id=a.id_enrichment_type
	join rule_instance i on r.id=i.id_rule
	join "enrichment" e on i.id=e.id_rule_instance
	where r.active = true -- rule must be active
	and t.name ilike '%osint%' -- enrichment is an osint
	and e.status = 2 -- performed
	and e.hit = false -- hit failed
	and e.done_time > current_timestamp - interval '7 days'
);