| 1 | """Concrete OSINT enrichers. All passive, public-source lookups. Network calls are guarded |
| 2 | so a missing dependency or offline environment degrades to a clean error envelope rather than |
| 3 | raising. These enrich an investigation that has crossed into the digital domain; they are |
| 4 | deliberately secondary to the ALPR core.""" |
| 5 | from __future__ import annotations |
| 6 | |
| 7 | import socket |
| 8 | import ssl |
| 9 | from datetime import datetime, timezone |
| 10 | |
| 11 | from app.services.enrichment.base import ( |
| 12 | BaseEnricher, EnrichEntity, EnrichRelationship, EnrichResult, |
| 13 | ) |
| 14 | |
| 15 | try: |
| 16 | import httpx |
| 17 | except Exception: |
| 18 | httpx = None |
| 19 | |
| 20 | |
| 21 | def _http_get(url: str, timeout: float = 8.0, headers: dict | None = None): |
| 22 | if httpx is None: |
| 23 | return None, "httpx not installed" |
| 24 | try: |
| 25 | r = httpx.get(url, timeout=timeout, headers=headers or {}, follow_redirects=True) |
| 26 | return r, None |
| 27 | except Exception as e: |
| 28 | return None, str(e) |
| 29 | |
| 30 | |
| 31 | class DnsEnricher(BaseEnricher): |
| 32 | name, kind, requires_network = "dns", "domain", True |
| 33 | |
| 34 | def run(self, target: str) -> EnrichResult: |
| 35 | res = EnrichResult(enricher=self.name, target=target, confidence=0.6) |
| 36 | try: |
| 37 | infos = socket.getaddrinfo(target, None) |
| 38 | ips = sorted({i[4][0] for i in infos}) |
| 39 | except OSError as e: |
| 40 | return self._empty(target, f"dns resolution failed: {e}") |
| 41 | res.entities.append(EnrichEntity("domain", target)) |
| 42 | for ip in ips: |
| 43 | res.entities.append(EnrichEntity("ip", ip)) |
| 44 | res.relationships.append(EnrichRelationship("domain", target, "ip", ip, "resolves_to", 0.9)) |
| 45 | res.evidence.append({"type": "dns_a", "records": ips}) |
| 46 | return res |
| 47 | |
| 48 | |
| 49 | class RdapEnricher(BaseEnricher): |
| 50 | name, kind, requires_network = "rdap", "domain", True |
| 51 | |
| 52 | def run(self, target: str) -> EnrichResult: |
| 53 | r, err = _http_get(f"https://rdap.org/domain/{target}") |
| 54 | if err or r is None: |
| 55 | return self._empty(target, f"rdap lookup failed: {err}") |
| 56 | if r.status_code != 200: |
| 57 | return self._empty(target, f"rdap status {r.status_code}") |
| 58 | data = r.json() |
| 59 | attrs = { |
| 60 | "handle": data.get("handle"), |
| 61 | "status": data.get("status"), |
| 62 | "events": [e.get("eventAction") for e in data.get("events", [])], |
| 63 | } |
| 64 | res = EnrichResult(enricher=self.name, target=target, confidence=0.8, |
| 65 | raw_response=data, source_metadata={"endpoint": "rdap.org"}) |
| 66 | res.entities.append(EnrichEntity("domain", target, attrs)) |
| 67 | res.evidence.append({"type": "rdap", "handle": attrs["handle"], "status": attrs["status"]}) |
| 68 | return res |
| 69 | |
| 70 | |
| 71 | class CrtShEnricher(BaseEnricher): |
| 72 | name, kind, requires_network = "crtsh", "domain", True |
| 73 | |
| 74 | def run(self, target: str) -> EnrichResult: |
| 75 | r, err = _http_get(f"https://crt.sh/?q=%25.{target}&output=json") |
| 76 | if err or r is None: |
| 77 | return self._empty(target, f"crt.sh failed: {err}") |
| 78 | try: |
| 79 | rows = r.json() |
| 80 | except Exception: |
| 81 | return self._empty(target, "crt.sh returned non-json") |
| 82 | names = sorted({n.strip().lower() for row in rows |
| 83 | for n in str(row.get("name_value", "")).splitlines() if n.strip()}) |
| 84 | res = EnrichResult(enricher=self.name, target=target, confidence=0.75, |
| 85 | source_metadata={"endpoint": "crt.sh", "cert_count": len(rows)}) |
| 86 | res.entities.append(EnrichEntity("domain", target)) |
| 87 | for n in names[:200]: |
| 88 | res.entities.append(EnrichEntity("domain", n, {"discovered_via": "cert_transparency"})) |
| 89 | res.relationships.append(EnrichRelationship("domain", target, "domain", n, "shares_cert_san", 0.7)) |
| 90 | res.evidence.append({"type": "cert_transparency", "subdomains": names[:200]}) |
| 91 | return res |
| 92 | |
| 93 | |
| 94 | class IpAsnEnricher(BaseEnricher): |
| 95 | name, kind, requires_network = "ipasn", "ip", True |
| 96 | |
| 97 | def run(self, target: str) -> EnrichResult: |
| 98 | r, err = _http_get(f"https://ipapi.co/{target}/json/") |
| 99 | if err or r is None: |
| 100 | return self._empty(target, f"ip lookup failed: {err}") |
| 101 | if r.status_code != 200: |
| 102 | return self._empty(target, f"ip lookup status {r.status_code}") |
| 103 | data = r.json() |
| 104 | attrs = {k: data.get(k) for k in ("asn", "org", "city", "region", "country_name", "latitude", "longitude")} |
| 105 | res = EnrichResult(enricher=self.name, target=target, confidence=0.7, |
| 106 | raw_response=data, source_metadata={"endpoint": "ipapi.co"}) |
| 107 | res.entities.append(EnrichEntity("ip", target, attrs)) |
| 108 | if attrs.get("asn"): |
| 109 | res.entities.append(EnrichEntity("asn", str(attrs["asn"]), {"org": attrs.get("org")})) |
| 110 | res.relationships.append(EnrichRelationship("ip", target, "asn", str(attrs["asn"]), "announced_by", 0.85)) |
| 111 | res.evidence.append({"type": "ip_geo_asn", **attrs}) |
| 112 | return res |
| 113 | |
| 114 | |
| 115 | class SecurityTxtEnricher(BaseEnricher): |
| 116 | name, kind, requires_network = "securitytxt", "domain", True |
| 117 | |
| 118 | def run(self, target: str) -> EnrichResult: |
| 119 | for path in (f"https://{target}/.well-known/security.txt", f"https://{target}/security.txt"): |
| 120 | r, err = _http_get(path) |
| 121 | if r is not None and r.status_code == 200 and "text" in r.headers.get("content-type", ""): |
| 122 | res = EnrichResult(enricher=self.name, target=target, confidence=0.6, |
| 123 | source_metadata={"path": path}) |
| 124 | res.entities.append(EnrichEntity("domain", target, {"has_security_txt": True})) |
| 125 | res.evidence.append({"type": "security_txt", "path": path, "body": r.text[:2000]}) |
| 126 | return res |
| 127 | return self._empty(target, "no security.txt found") |
| 128 | |
| 129 | |
| 130 | class RobotsEnricher(BaseEnricher): |
| 131 | name, kind, requires_network = "robots", "domain", True |
| 132 | |
| 133 | def run(self, target: str) -> EnrichResult: |
| 134 | r, err = _http_get(f"https://{target}/robots.txt") |
| 135 | if r is None or r.status_code != 200: |
| 136 | return self._empty(target, f"robots.txt unavailable: {err or r.status_code}") |
| 137 | disallows = [ln.split(":", 1)[1].strip() for ln in r.text.splitlines() |
| 138 | if ln.lower().startswith("disallow:")] |
| 139 | res = EnrichResult(enricher=self.name, target=target, confidence=0.5, |
| 140 | source_metadata={"path": "/robots.txt"}) |
| 141 | res.entities.append(EnrichEntity("domain", target, {"robots_disallow_count": len(disallows)})) |
| 142 | res.evidence.append({"type": "robots", "disallow": disallows[:100]}) |
| 143 | return res |
| 144 | |
| 145 | |
| 146 | class GithubEnricher(BaseEnricher): |
| 147 | name, kind, requires_network = "github", "company", True |
| 148 | |
| 149 | def run(self, target: str) -> EnrichResult: |
| 150 | r, err = _http_get(f"https://api.github.com/users/{target}", |
| 151 | headers={"Accept": "application/vnd.github+json"}) |
| 152 | if r is None or r.status_code != 200: |
| 153 | return self._empty(target, f"github lookup failed: {err or r.status_code}") |
| 154 | data = r.json() |
| 155 | attrs = {k: data.get(k) for k in ("type", "name", "company", "blog", "location", |
| 156 | "public_repos", "followers", "created_at")} |
| 157 | res = EnrichResult(enricher=self.name, target=target, confidence=0.7, |
| 158 | raw_response=data, source_metadata={"endpoint": "api.github.com"}) |
| 159 | res.entities.append(EnrichEntity("company", target, attrs)) |
| 160 | if attrs.get("blog"): |
| 161 | res.relationships.append(EnrichRelationship("company", target, "domain", |
| 162 | str(attrs["blog"]), "owns_site", 0.6)) |
| 163 | res.evidence.append({"type": "github_profile", **attrs}) |
| 164 | return res |
| 165 | |
| 166 | |
| 167 | class TlsCertEnricher(BaseEnricher): |
| 168 | name, kind, requires_network = "tlscert", "domain", True |
| 169 | |
| 170 | def run(self, target: str) -> EnrichResult: |
| 171 | try: |
| 172 | ctx = ssl.create_default_context() |
| 173 | with socket.create_connection((target, 443), timeout=8) as sock: |
| 174 | with ctx.wrap_socket(sock, server_hostname=target) as ssock: |
| 175 | cert = ssock.getpeercert() |
| 176 | except Exception as e: |
| 177 | return self._empty(target, f"tls handshake failed: {e}") |
| 178 | subject = dict(x[0] for x in cert.get("subject", [])) |
| 179 | issuer = dict(x[0] for x in cert.get("issuer", [])) |
| 180 | res = EnrichResult(enricher=self.name, target=target, confidence=0.7, |
| 181 | source_metadata={"port": 443}) |
| 182 | res.entities.append(EnrichEntity("domain", target, { |
| 183 | "tls_subject_cn": subject.get("commonName"), |
| 184 | "tls_issuer": issuer.get("organizationName"), |
| 185 | "tls_not_after": cert.get("notAfter"), |
| 186 | })) |
| 187 | res.evidence.append({"type": "tls_cert", "subject": subject, "issuer": issuer, |
| 188 | "not_after": cert.get("notAfter")}) |
| 189 | return res |