● Zion Boggan
repos/Sightline/backend/app/services/enrichment/plugins.py
zionboggan.com ↗
189 lines · python
History for this file →
1
"""Concrete OSINT enrichers. All passive, public-source lookups. Network calls are guarded
2
so a missing dependency or offline environment degrades to a clean error envelope rather than
3
raising. These enrich an investigation that has crossed into the digital domain; they are
4
deliberately secondary to the ALPR core."""
5
from __future__ import annotations
6
 
7
import socket
8
import ssl
9
from datetime import datetime, timezone
10
 
11
from app.services.enrichment.base import (
12
    BaseEnricher, EnrichEntity, EnrichRelationship, EnrichResult,
13
)
14
 
15
try:
16
    import httpx
17
except Exception:
18
    httpx = None
19
 
20
 
21
def _http_get(url: str, timeout: float = 8.0, headers: dict | None = None):
22
    if httpx is None:
23
        return None, "httpx not installed"
24
    try:
25
        r = httpx.get(url, timeout=timeout, headers=headers or {}, follow_redirects=True)
26
        return r, None
27
    except Exception as e:
28
        return None, str(e)
29
 
30
 
31
class DnsEnricher(BaseEnricher):
32
    name, kind, requires_network = "dns", "domain", True
33
 
34
    def run(self, target: str) -> EnrichResult:
35
        res = EnrichResult(enricher=self.name, target=target, confidence=0.6)
36
        try:
37
            infos = socket.getaddrinfo(target, None)
38
            ips = sorted({i[4][0] for i in infos})
39
        except OSError as e:
40
            return self._empty(target, f"dns resolution failed: {e}")
41
        res.entities.append(EnrichEntity("domain", target))
42
        for ip in ips:
43
            res.entities.append(EnrichEntity("ip", ip))
44
            res.relationships.append(EnrichRelationship("domain", target, "ip", ip, "resolves_to", 0.9))
45
        res.evidence.append({"type": "dns_a", "records": ips})
46
        return res
47
 
48
 
49
class RdapEnricher(BaseEnricher):
50
    name, kind, requires_network = "rdap", "domain", True
51
 
52
    def run(self, target: str) -> EnrichResult:
53
        r, err = _http_get(f"https://rdap.org/domain/{target}")
54
        if err or r is None:
55
            return self._empty(target, f"rdap lookup failed: {err}")
56
        if r.status_code != 200:
57
            return self._empty(target, f"rdap status {r.status_code}")
58
        data = r.json()
59
        attrs = {
60
            "handle": data.get("handle"),
61
            "status": data.get("status"),
62
            "events": [e.get("eventAction") for e in data.get("events", [])],
63
        }
64
        res = EnrichResult(enricher=self.name, target=target, confidence=0.8,
65
                           raw_response=data, source_metadata={"endpoint": "rdap.org"})
66
        res.entities.append(EnrichEntity("domain", target, attrs))
67
        res.evidence.append({"type": "rdap", "handle": attrs["handle"], "status": attrs["status"]})
68
        return res
69
 
70
 
71
class CrtShEnricher(BaseEnricher):
72
    name, kind, requires_network = "crtsh", "domain", True
73
 
74
    def run(self, target: str) -> EnrichResult:
75
        r, err = _http_get(f"https://crt.sh/?q=%25.{target}&output=json")
76
        if err or r is None:
77
            return self._empty(target, f"crt.sh failed: {err}")
78
        try:
79
            rows = r.json()
80
        except Exception:
81
            return self._empty(target, "crt.sh returned non-json")
82
        names = sorted({n.strip().lower() for row in rows
83
                        for n in str(row.get("name_value", "")).splitlines() if n.strip()})
84
        res = EnrichResult(enricher=self.name, target=target, confidence=0.75,
85
                           source_metadata={"endpoint": "crt.sh", "cert_count": len(rows)})
86
        res.entities.append(EnrichEntity("domain", target))
87
        for n in names[:200]:
88
            res.entities.append(EnrichEntity("domain", n, {"discovered_via": "cert_transparency"}))
89
            res.relationships.append(EnrichRelationship("domain", target, "domain", n, "shares_cert_san", 0.7))
90
        res.evidence.append({"type": "cert_transparency", "subdomains": names[:200]})
91
        return res
92
 
93
 
94
class IpAsnEnricher(BaseEnricher):
95
    name, kind, requires_network = "ipasn", "ip", True
96
 
97
    def run(self, target: str) -> EnrichResult:
98
        r, err = _http_get(f"https://ipapi.co/{target}/json/")
99
        if err or r is None:
100
            return self._empty(target, f"ip lookup failed: {err}")
101
        if r.status_code != 200:
102
            return self._empty(target, f"ip lookup status {r.status_code}")
103
        data = r.json()
104
        attrs = {k: data.get(k) for k in ("asn", "org", "city", "region", "country_name", "latitude", "longitude")}
105
        res = EnrichResult(enricher=self.name, target=target, confidence=0.7,
106
                           raw_response=data, source_metadata={"endpoint": "ipapi.co"})
107
        res.entities.append(EnrichEntity("ip", target, attrs))
108
        if attrs.get("asn"):
109
            res.entities.append(EnrichEntity("asn", str(attrs["asn"]), {"org": attrs.get("org")}))
110
            res.relationships.append(EnrichRelationship("ip", target, "asn", str(attrs["asn"]), "announced_by", 0.85))
111
        res.evidence.append({"type": "ip_geo_asn", **attrs})
112
        return res
113
 
114
 
115
class SecurityTxtEnricher(BaseEnricher):
116
    name, kind, requires_network = "securitytxt", "domain", True
117
 
118
    def run(self, target: str) -> EnrichResult:
119
        for path in (f"https://{target}/.well-known/security.txt", f"https://{target}/security.txt"):
120
            r, err = _http_get(path)
121
            if r is not None and r.status_code == 200 and "text" in r.headers.get("content-type", ""):
122
                res = EnrichResult(enricher=self.name, target=target, confidence=0.6,
123
                                   source_metadata={"path": path})
124
                res.entities.append(EnrichEntity("domain", target, {"has_security_txt": True}))
125
                res.evidence.append({"type": "security_txt", "path": path, "body": r.text[:2000]})
126
                return res
127
        return self._empty(target, "no security.txt found")
128
 
129
 
130
class RobotsEnricher(BaseEnricher):
131
    name, kind, requires_network = "robots", "domain", True
132
 
133
    def run(self, target: str) -> EnrichResult:
134
        r, err = _http_get(f"https://{target}/robots.txt")
135
        if r is None or r.status_code != 200:
136
            return self._empty(target, f"robots.txt unavailable: {err or r.status_code}")
137
        disallows = [ln.split(":", 1)[1].strip() for ln in r.text.splitlines()
138
                     if ln.lower().startswith("disallow:")]
139
        res = EnrichResult(enricher=self.name, target=target, confidence=0.5,
140
                           source_metadata={"path": "/robots.txt"})
141
        res.entities.append(EnrichEntity("domain", target, {"robots_disallow_count": len(disallows)}))
142
        res.evidence.append({"type": "robots", "disallow": disallows[:100]})
143
        return res
144
 
145
 
146
class GithubEnricher(BaseEnricher):
147
    name, kind, requires_network = "github", "company", True
148
 
149
    def run(self, target: str) -> EnrichResult:
150
        r, err = _http_get(f"https://api.github.com/users/{target}",
151
                           headers={"Accept": "application/vnd.github+json"})
152
        if r is None or r.status_code != 200:
153
            return self._empty(target, f"github lookup failed: {err or r.status_code}")
154
        data = r.json()
155
        attrs = {k: data.get(k) for k in ("type", "name", "company", "blog", "location",
156
                                          "public_repos", "followers", "created_at")}
157
        res = EnrichResult(enricher=self.name, target=target, confidence=0.7,
158
                           raw_response=data, source_metadata={"endpoint": "api.github.com"})
159
        res.entities.append(EnrichEntity("company", target, attrs))
160
        if attrs.get("blog"):
161
            res.relationships.append(EnrichRelationship("company", target, "domain",
162
                                                        str(attrs["blog"]), "owns_site", 0.6))
163
        res.evidence.append({"type": "github_profile", **attrs})
164
        return res
165
 
166
 
167
class TlsCertEnricher(BaseEnricher):
168
    name, kind, requires_network = "tlscert", "domain", True
169
 
170
    def run(self, target: str) -> EnrichResult:
171
        try:
172
            ctx = ssl.create_default_context()
173
            with socket.create_connection((target, 443), timeout=8) as sock:
174
                with ctx.wrap_socket(sock, server_hostname=target) as ssock:
175
                    cert = ssock.getpeercert()
176
        except Exception as e:
177
            return self._empty(target, f"tls handshake failed: {e}")
178
        subject = dict(x[0] for x in cert.get("subject", []))
179
        issuer = dict(x[0] for x in cert.get("issuer", []))
180
        res = EnrichResult(enricher=self.name, target=target, confidence=0.7,
181
                           source_metadata={"port": 443})
182
        res.entities.append(EnrichEntity("domain", target, {
183
            "tls_subject_cn": subject.get("commonName"),
184
            "tls_issuer": issuer.get("organizationName"),
185
            "tls_not_after": cert.get("notAfter"),
186
        }))
187
        res.evidence.append({"type": "tls_cert", "subject": subject, "issuer": issuer,
188
                             "not_after": cert.get("notAfter")})
189
        return res