● Zion Boggan
repos/TreeTrace/src/redact.js
zionboggan.com ↗
332 lines · javascript
History for this file →
1
import { createInterface } from 'node:readline/promises';
2
import { sha256, shannonEntropy, truncate, c } from './util.js';
3
 
4
export const RULES = [
5
 
6
  { id: 'private-key-block', severity: 'high', re: /-----BEGIN [A-Z ]*PRIVATE KEY( BLOCK)?-----[\s\S]*?(-----END [A-Z ]*PRIVATE KEY( BLOCK)?-----|$)/g },
7
  { id: 'aws-access-key', severity: 'high', re: /\b(AKIA|ASIA)[0-9A-Z]{16}\b/g },
8
  { id: 'github-token', severity: 'high', re: /\b(ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/g },
9
  { id: 'github-fine-grained', severity: 'high', re: /\bgithub_pat_[A-Za-z0-9_]{22,}\b/g },
10
  { id: 'gitlab-token', severity: 'high', re: /\bglpat-[0-9a-zA-Z_-]{20,}\b/g },
11
  { id: 'anthropic-key', severity: 'high', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g },
12
  { id: 'openai-key', severity: 'high', re: /\bsk-(?!ant-)[A-Za-z0-9_-]{20,}\b/g },
13
  { id: 'slack-token', severity: 'high', re: /\bxox[baprs]-[0-9A-Za-z-]{10,}\b/g },
14
  { id: 'stripe-live-key', severity: 'high', re: /\b[sr]k_live_[0-9a-zA-Z]{10,}\b/g },
15
  { id: 'npm-token', severity: 'high', re: /\bnpm_[A-Za-z0-9]{36}\b/g },
16
  { id: 'tailscale-key', severity: 'high', re: /\btskey-[a-zA-Z0-9-]{10,}\b/g },
17
  { id: 'google-api-key', severity: 'high', re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
18
  { id: 'sendgrid-key', severity: 'high', re: /\bSG\.[A-Za-z0-9_-]{16,32}\.[A-Za-z0-9_-]{16,64}\b/g },
19
  { id: 'twilio-key', severity: 'high', re: /\bSK[0-9a-fA-F]{32}\b/g },
20
  { id: 'telegram-bot-token', severity: 'high', re: /\b\d{8,10}:AA[A-Za-z0-9_-]{32,33}\b/g },
21
  { id: 'discord-webhook', severity: 'high', re: /https:\/\/(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/[A-Za-z0-9_-]+/g },
22
  { id: 'jwt', severity: 'high', re: /\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}\b/g },
23
 
24
  { id: 'hex-token', severity: 'medium', re: /\b[0-9a-fA-F]{32,512}\b/g },
25
  { id: 'wireguard-key', severity: 'medium', re: /\b(PrivateKey|PresharedKey)\s*=\s*[A-Za-z0-9+/]{42,44}=?/g },
26
  { id: 'url-basic-auth', severity: 'medium', re: /\b[a-z][a-z0-9+.-]{0,30}:\/\/[^/\s:@'"`]{2,256}:[^/\s@'"`]{2,256}@[^\s'"`]{1,512}/gi },
27
  { id: 'bearer-header', severity: 'medium', re: /\bBearer\s+[A-Za-z0-9._+/=-]{20,}\b/gi },
28
  { id: 'secret-assignment', severity: 'medium', re: /["'`]?\b(password|passwd|pwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token|client[_-]?secret|secret[_-]?key|token|bearer)\b["'`]?\s*[:=]\s*(?!(?:["'`]?\s*)?(?:\$\{|\$\(|<|%|\*{3}|\.{3}|REDACTED|\[REDACTED|xxx+|placeholder|changeme|example|your[-_]|null\b|true\b|false\b))(?:"(?:[^"\\]|\\.){4,512}"|'(?:[^'\\]|\\.){4,512}'|`(?:[^`\\]|\\.){4,512}`|[^\s'"`,;){}]{6,512})/gi },
29
  { id: 'secret-assignment', severity: 'medium', re: /["'`]?\b(password|passwd|pwd|secret|api[_-]?key|access[_-]?token|auth[_-]?token|client[_-]?secret|secret[_-]?key|token|bearer)\b["'`]?\s*[:=]\s*(?!(?:["'`]?\s*)?(?:\$\{|\$\(|<|%|\*{3}|\.{3}|REDACTED|\[REDACTED|xxx+|placeholder|changeme|example|your[-_]|null\b|true\b|false\b))(?:"(?:[^"\\]|\\.)*?\\.(?:[^"\\]|\\.)*?"|'(?:[^'\\]|\\.)*?\\.(?:[^'\\]|\\.)*?'|`(?:[^`\\]|\\.)*?\\.(?:[^`\\]|\\.)*?`)/gi },
30
 
31
  { id: 'email', severity: 'soft', re: /\b[A-Za-z0-9._%+-]+@(?!(?:users\.noreply\.github\.com|example\.(?:com|org)))[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g },
32
  { id: 'ipv4', severity: 'soft', re: /\b(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\b(?!\.\d)/g },
33
  { id: 'home-dir-username', severity: 'soft', re: /(?:\/(?:home|Users)\/|C:\\Users\\)([A-Za-z][A-Za-z0-9._-]{2,30})\b/g },
34
];
35
 
36
const HEX_RE = /^[0-9a-fA-F]+$/;
37
const ENTROPY_CANDIDATE_RE = /\b[A-Za-z0-9+/_=-]{32,4096}\b/g;
38
const MAX_TOKEN_LEN = 4096;
39
const TOKEN_CHAR_RE = /[A-Za-z0-9+/_=-]/;
40
const VERSION_LIKE_RE = /^\d+[.\d-]*$/;
41
const JOIN_SEPARATOR_RE = /[\s\u200B-\u200D\uFEFF]/;
42
const JOINED_SCAN_RULE_IDS = new Set([
43
  'aws-access-key',
44
  'github-token',
45
  'github-fine-grained',
46
  'gitlab-token',
47
  'anthropic-key',
48
  'openai-key',
49
  'slack-token',
50
  'stripe-live-key',
51
  'npm-token',
52
  'tailscale-key',
53
  'google-api-key',
54
  'sendgrid-key',
55
  'twilio-key',
56
  'telegram-bot-token',
57
  'jwt',
58
]);
59
 
60
const LOOSE_RULES = RULES.filter((r) => JOINED_SCAN_RULE_IDS.has(r.id)).map((r) => ({
61
  id: r.id,
62
  severity: r.severity,
63
  re: new RegExp(
64
    r.re.source.replace(/^\\b/, '').replace(/\\b$/, '').replace(/\{(\d+),\}/g, '{$1,128}'),
65
    'g'
66
  ),
67
}));
68
 
69
function findOversizedRuns(text) {
70
  const runs = [];
71
  let start = -1;
72
  for (let i = 0; i <= text.length; i++) {
73
    const isTok = i < text.length && TOKEN_CHAR_RE.test(text[i]);
74
    if (isTok) {
75
      if (start === -1) start = i;
76
    } else if (start !== -1) {
77
      if (i - start > MAX_TOKEN_LEN) runs.push([start, i]);
78
      start = -1;
79
    }
80
  }
81
  return runs;
82
}
83
 
84
const GIT_SHA_LENGTHS = new Set([40, 64]);
85
 
86
export function isGitShaCandidate(match, text, index) {
87
  if (!match || !GIT_SHA_LENGTHS.has(match.length)) return false;
88
  if (!/^[0-9a-fA-F]+$/.test(match)) return false;
89
  const before = text.slice(Math.max(0, index - 48), index);
90
  if (/\b(?:commit|tree|parent|object|merge|ref|refs|origin|HEAD|tag|blob|cherry|rebase|bisect|stash)\b[\s:./-]*$/i.test(before)) {
91
    return true;
92
  }
93
  const atLineStart = index === 0 || text[index - 1] === '\n';
94
  return atLineStart && text[index + match.length] === ' ';
95
}
96
 
97
export function scanText(text) {
98
  const oversized = text.length > MAX_TOKEN_LEN ? findOversizedRuns(text) : [];
99
  let scanInput = text;
100
  if (oversized.length) {
101
    const chars = text.split('');
102
    for (const [s, e] of oversized) {
103
      for (let i = s; i < e; i++) chars[i] = '\n';
104
    }
105
    scanInput = chars.join('');
106
  }
107
 
108
  const findings = [];
109
  for (const [s, e] of oversized) {
110
    findings.push({
111
      ruleId: 'oversized-token',
112
      severity: 'medium',
113
      match: text.slice(s, e),
114
      index: s,
115
    });
116
  }
117
 
118
  for (const rule of RULES) {
119
    rule.re.lastIndex = 0;
120
    let m;
121
    while ((m = rule.re.exec(scanInput)) !== null) {
122
      const finding = {
123
        ruleId: rule.id,
124
        severity: rule.severity,
125
        match: m[0],
126
        index: m.index,
127
      };
128
      if (rule.id === 'hex-token') finding.gitShaCandidate = isGitShaCandidate(m[0], scanInput, m.index);
129
      findings.push(finding);
130
      if (m.index === rule.re.lastIndex) rule.re.lastIndex++;
131
    }
132
  }
133
 
134
  const seenSpans = findings.map((f) => [f.index, f.index + f.match.length]);
135
  ENTROPY_CANDIDATE_RE.lastIndex = 0;
136
  let m;
137
  while ((m = ENTROPY_CANDIDATE_RE.exec(scanInput)) !== null) {
138
    const tok = m[0];
139
    if (HEX_RE.test(tok) || VERSION_LIKE_RE.test(tok)) continue;
140
    const classes = (/[A-Z]/.test(tok) ? 1 : 0) + (/[a-z]/.test(tok) ? 1 : 0) + (/[0-9]/.test(tok) ? 1 : 0);
141
    if (classes < 2) continue;
142
    const entropy = shannonEntropy(tok);
143
    if (entropy < 4.4) {
144
      if (!looksLikeLowercaseDigitSecret(tok, entropy)) continue;
145
      const start = m.index;
146
      if (seenSpans.some(([s, e]) => start >= s && start < e)) continue;
147
      findings.push({ ruleId: 'lowercase-digit-token', severity: 'medium', match: tok, index: start });
148
      continue;
149
    }
150
    const start = m.index;
151
    if (seenSpans.some(([s, e]) => start >= s && start < e)) continue;
152
    findings.push({ ruleId: 'high-entropy-token', severity: 'medium', match: tok, index: start });
153
  }
154
 
155
  findings.push(...scanJoinedProviderTokens(scanInput, findings, text));
156
  return findings;
157
}
158
 
159
function looksLikeLowercaseDigitSecret(tok, entropy) {
160
  if (tok.length < 32) return false;
161
  if (!/^[a-z0-9_-]+$/.test(tok)) return false;
162
  if (!/[a-z]/.test(tok) || !/[0-9]/.test(tok)) return false;
163
  if (VERSION_LIKE_RE.test(tok)) return false;
164
  return entropy >= 3.0;
165
}
166
 
167
function scanJoinedProviderTokens(scanInput, existing, original = scanInput) {
168
  const chars = [];
169
  const indexMap = [];
170
  for (let i = 0; i < scanInput.length; i++) {
171
    if (JOIN_SEPARATOR_RE.test(scanInput[i])) continue;
172
    chars.push(scanInput[i]);
173
    indexMap.push(i);
174
  }
175
  if (chars.length === scanInput.length) return [];
176
 
177
  const joined = chars.join('');
178
  const existingSpans = existing.map((f) => [f.index, f.index + f.match.length]);
179
  const findings = [];
180
  for (const rule of LOOSE_RULES) {
181
    rule.re.lastIndex = 0;
182
    let m;
183
    while ((m = rule.re.exec(joined)) !== null) {
184
      if (m[0].length <= 256) {
185
        const start = indexMap[m.index];
186
        const end = indexMap[m.index + m[0].length - 1] + 1;
187
        const slice = original.slice(start, end);
188
        if (JOIN_SEPARATOR_RE.test(slice) && !existingSpans.some(([s, e]) => start >= s && start < e)) {
189
          findings.push({ ruleId: rule.id, severity: rule.severity, match: slice, index: start });
190
        }
191
      }
192
      if (m.index === rule.re.lastIndex) rule.re.lastIndex++;
193
    }
194
  }
195
  return findings;
196
}
197
 
198
export function maskFor(finding) {
199
  return `[REDACTED:${finding.ruleId}]`;
200
}
201
 
202
export async function resolveFindings(findings, priorDecisions, { interactive, autoRedact, keepGitShas = false } = {}) {
203
  const decisions = { ...priorDecisions };
204
  const unique = new Map();
205
  for (const f of findings) {
206
    const h = sha256(f.match);
207
    if (!unique.has(h)) unique.set(h, { finding: f, count: 0 });
208
    unique.get(h).count++;
209
  }
210
 
211
  let autoKeptGitShas = 0;
212
  if (keepGitShas) {
213
    const highHashes = new Set();
214
    for (const f of findings) if (f.severity === 'high') highHashes.add(sha256(f.match));
215
    for (const [h, { finding }] of unique) {
216
      if (finding.gitShaCandidate && !decisions[h] && !highHashes.has(h)) {
217
        decisions[h] = { action: 'keep', ruleId: 'git-commit-sha' };
218
        autoKeptGitShas++;
219
      }
220
    }
221
  }
222
 
223
  const autoMode = !interactive || autoRedact;
224
  let overriddenKeeps = 0;
225
  if (autoMode) {
226
    for (const [h, { finding }] of unique) {
227
      const prior = decisions[h];
228
      if (prior && prior.action === 'keep' && (finding.severity === 'high' || finding.severity === 'medium')) {
229
        if (keepGitShas && finding.gitShaCandidate) continue;
230
        delete decisions[h];
231
        overriddenKeeps++;
232
      }
233
    }
234
  }
235
 
236
  const unresolved = [...unique.entries()].filter(([h]) => !decisions[h]);
237
  if (!unresolved.length) return { decisions, asked: 0, overriddenKeeps, autoKeptGitShas };
238
 
239
  if (autoMode) {
240
    for (const [h, { finding }] of unresolved) {
241
      decisions[h] = { action: 'redact', replacement: maskFor(finding), ruleId: finding.ruleId };
242
    }
243
    return { decisions, asked: 0, autoRedacted: unresolved.length, overriddenKeeps, autoKeptGitShas };
244
  }
245
 
246
  const rl = createInterface({ input: process.stdin, output: process.stderr });
247
  process.stderr.write(
248
    `\n${c.bold(`${unresolved.length} potential secret${unresolved.length === 1 ? '' : 's'} found`)}. Nothing is exported until each is resolved.\n\n`
249
  );
250
  let i = 0;
251
  for (const [h, { finding, count }] of unresolved) {
252
    i++;
253
    const sev =
254
      finding.severity === 'high' ? c.red(finding.severity)
255
      : finding.severity === 'medium' ? c.yellow(finding.severity)
256
      : c.gray(finding.severity);
257
    process.stderr.write(
258
      `${c.dim(`[${i}/${unresolved.length}]`)} ${sev} ${c.bold(finding.ruleId)} ×${count}\n    ${c.cyan(truncate(finding.match, 72))}\n`
259
 
260
    );
261
    let answer;
262
    for (;;) {
263
      answer = (await rl.question(`    ${c.bold('[r]')}edact  ${c.bold('[k]')}eep  ${c.bold('[e]')}dit replacement › `))
264
        .trim()
265
        .toLowerCase();
266
      if (['r', 'k', 'e', 'redact', 'keep', 'edit', ''].includes(answer)) break;
267
    }
268
    if (answer === 'k' || answer === 'keep') {
269
      decisions[h] = { action: 'keep', ruleId: finding.ruleId };
270
    } else if (answer === 'e' || answer === 'edit') {
271
      const replacement = (await rl.question('    replacement text › ')).trim() || maskFor(finding);
272
      decisions[h] = { action: 'redact', replacement, ruleId: finding.ruleId };
273
    } else {
274
      decisions[h] = { action: 'redact', replacement: maskFor(finding), ruleId: finding.ruleId };
275
    }
276
  }
277
  rl.close();
278
  return { decisions, asked: unresolved.length, autoKeptGitShas };
279
}
280
 
281
export function applyDecisions(text, findings, decisions) {
282
  const toRedact = new Map();
283
  for (const f of findings) {
284
    const d = decisions[sha256(f.match)];
285
    if (d && d.action === 'redact') {
286
      toRedact.set(f.match, d.replacement || maskFor(f));
287
    }
288
  }
289
  let out = text;
290
 
291
  for (const [original, replacement] of [...toRedact.entries()].sort(
292
    (a, b) => b[0].length - a[0].length
293
  )) {
294
    out = out.split(original).join(replacement);
295
  }
296
  return out;
297
}
298
 
299
export function shadowScan(renderedText, decisions) {
300
  const leaks = [];
301
  for (const f of scanText(renderedText)) {
302
    if (f.severity === 'soft') continue;
303
    const d = decisions[sha256(f.match)];
304
    if (d && d.action === 'keep') continue;
305
    if (f.match.startsWith('[REDACTED:')) continue;
306
    leaks.push(f);
307
  }
308
  return leaks;
309
}
310
 
311
export function patchResiduals(text, decisions) {
312
  const leaks = shadowScan(text, decisions);
313
  if (!leaks.length) return text;
314
 
315
  for (const f of leaks) {
316
    const h = sha256(f.match);
317
    if (!decisions[h]) {
318
      decisions[h] = { action: 'redact', replacement: maskFor(f), ruleId: f.ruleId };
319
    }
320
  }
321
 
322
  let out = applyDecisions(text, leaks, decisions);
323
 
324
  const residual = shadowScan(out, decisions);
325
  if (residual.length) {
326
    throw new Error(
327
      `patchResiduals: ${residual.length} leak(s) remain after auto-redaction ` +
328
        `(${[...new Set(residual.map((l) => l.ruleId))].join(', ')}). Refusing to write.`
329
    );
330
  }
331
  return out;
332
}