examples/api-key-auth/.treetrace/failures.json

128 lines · json

{
  "schemaVersion": "0.3",
  "project": {
    "name": "api-key-auth",
    "generatedAt": "2026-06-19T06:50:12.466Z"
  },
  "summary": {
    "totalFailureSignals": 5,
    "topFailureTypes": [
      {
        "type": "security_or_privacy_risk",
        "count": 2
      },
      {
        "type": "dependency_or_environment_mismatch",
        "count": 1
      },
      {
        "type": "user_frustration",
        "count": 1
      },
      {
        "type": "user_rejected_action",
        "count": 1
      }
    ],
    "tierCounts": {
      "verified": 0,
      "high": 2,
      "confirmed": 2,
      "inferred": 1
    },
    "models": [
      "assistant-model"
    ],
    "thinkingBlocks": 0,
    "correctionChains": 2,
    "evalCandidates": 4,
    "lessons": 4
  },
  "failures": [
    {
      "id": "failure_001",
      "type": "user_rejected_action",
      "tier": "high",
      "confidence": 0.8,
      "model": "assistant-model",
      "firstSeenNodeId": "node_002",
      "correctedByNodeId": null,
      "summary": "The user explicitly told the agent to stop or not proceed near \"No, do not hardcode the secret in the source.\".",
      "evidence": "user_text_decline (text): \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
      "lesson": "Future agents should not retry a tool action the user just declined without first explaining why the action is still worth taking. Specifically: user_text_decline (text): \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
      "evalCandidate": true
    },
    {
      "id": "failure_002",
      "type": "security_or_privacy_risk",
      "tier": "inferred",
      "confidence": 0.62,
      "model": "assistant-model",
      "firstSeenNodeId": "node_001",
      "correctedByNodeId": "node_002",
      "summary": "A human security correction was raised near \"Add API key authentication to the /admin route in our Express app.\" with no matching action-level signal.",
      "evidence": "Human flagged a security concern about a prior action with no security label [signal: human security correction]: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
      "lesson": "Future agents should not weaken local-first privacy, redaction, or no-network guarantees without explicit approval. Specifically: Human flagged a security concern about a prior action with no security label [signal: human security correction]: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
      "evalCandidate": true
    },
    {
      "id": "failure_003",
      "type": "dependency_or_environment_mismatch",
      "tier": "confirmed",
      "confidence": 0.82,
      "model": "assistant-model",
      "firstSeenNodeId": "node_001",
      "correctedByNodeId": "node_002",
      "summary": "A possible dependency or environment mismatch occurred near \"Add API key authentication to the /admin route in our Express app.\"; corrected by \"No, do not hardcode the secret in the source.\".",
      "evidence": "User said: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
      "lesson": "Future agents should validate environment assumptions before choosing dependencies or runtime paths. Specifically: User said: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
      "evalCandidate": true
    },
    {
      "id": "failure_004",
      "type": "security_or_privacy_risk",
      "tier": "high",
      "confidence": 0.84,
      "model": "assistant-model",
      "firstSeenNodeId": "node_003",
      "correctedByNodeId": null,
      "summary": "An agent action touched auth, secrets, or access control near \"The auth tests are failing.\".",
      "evidence": "Agent action touched risky-command [signals: risky command]: \"git commit -am \"wip: api key auth\" --no-verify && git push --force\"",
      "lesson": "Future agents should not weaken local-first privacy, redaction, or no-network guarantees without explicit approval. Specifically: Agent action touched risky-command [signals: risky command]: \"git commit -am \"wip: api key auth\" --no-verify && git push --force\"",
      "evalCandidate": true
    },
    {
      "id": "failure_005",
      "type": "user_frustration",
      "tier": "confirmed",
      "confidence": 0.82,
      "model": "assistant-model",
      "firstSeenNodeId": "node_001",
      "correctedByNodeId": "node_004",
      "summary": "User frustration signaled that the prior path near \"Add API key authentication to the /admin route in our Express app.\" was not meeting expectations.",
      "evidence": "User said: \"Here is my test key [REDACTED:anthropic-key], confirm the admin route rejects a bad key.\"",
      "lesson": "Future agents should treat frustration as a signal to slow down, verify assumptions, and correct course. Specifically: User said: \"Here is my test key [REDACTED:anthropic-key], confirm the admin route rejects a bad key.\"",
      "evalCandidate": true
    }
  ],
  "correctionChains": [
    {
      "id": "chain_001",
      "failureNodeId": "node_001",
      "correctionNodeId": "node_002",
      "resolvedNodeId": "node_003",
      "failureType": "security_or_privacy_risk",
      "confidence": "low",
      "summary": "A human security correction was raised near \"Add API key authentication to the /admin route in our Express app.\" with no matching action-level signal."
    },
    {
      "id": "chain_002",
      "failureNodeId": "node_001",
      "correctionNodeId": "node_004",
      "resolvedNodeId": null,
      "failureType": "user_frustration",
      "confidence": "high",
      "summary": "User frustration signaled that the prior path near \"Add API key authentication to the /admin route in our Express app.\" was not meeting expectations."
    }
  ]
}

1	{
2	"schemaVersion": "0.3",
3	"project": {
4	"name": "api-key-auth",
5	"generatedAt": "2026-06-19T06:50:12.466Z"
6	},
7	"summary": {
8	"totalFailureSignals": 5,
9	"topFailureTypes": [
10	{
11	"type": "security_or_privacy_risk",
12	"count": 2
13	},
14	{
15	"type": "dependency_or_environment_mismatch",
16	"count": 1
17	},
18	{
19	"type": "user_frustration",
20	"count": 1
21	},
22	{
23	"type": "user_rejected_action",
24	"count": 1
25	}
26	],
27	"tierCounts": {
28	"verified": 0,
29	"high": 2,
30	"confirmed": 2,
31	"inferred": 1
32	},
33	"models": [
34	"assistant-model"
35	],
36	"thinkingBlocks": 0,
37	"correctionChains": 2,
38	"evalCandidates": 4,
39	"lessons": 4
40	},
41	"failures": [
42	{
43	"id": "failure_001",
44	"type": "user_rejected_action",
45	"tier": "high",
46	"confidence": 0.8,
47	"model": "assistant-model",
48	"firstSeenNodeId": "node_002",
49	"correctedByNodeId": null,
50	"summary": "The user explicitly told the agent to stop or not proceed near \"No, do not hardcode the secret in the source.\".",
51	"evidence": "user_text_decline (text): \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
52	"lesson": "Future agents should not retry a tool action the user just declined without first explaining why the action is still worth taking. Specifically: user_text_decline (text): \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
53	"evalCandidate": true
54	},
55	{
56	"id": "failure_002",
57	"type": "security_or_privacy_risk",
58	"tier": "inferred",
59	"confidence": 0.62,
60	"model": "assistant-model",
61	"firstSeenNodeId": "node_001",
62	"correctedByNodeId": "node_002",
63	"summary": "A human security correction was raised near \"Add API key authentication to the /admin route in our Express app.\" with no matching action-level signal.",
64	"evidence": "Human flagged a security concern about a prior action with no security label [signal: human security correction]: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
65	"lesson": "Future agents should not weaken local-first privacy, redaction, or no-network guarantees without explicit approval. Specifically: Human flagged a security concern about a prior action with no security label [signal: human security correction]: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
66	"evalCandidate": true
67	},
68	{
69	"id": "failure_003",
70	"type": "dependency_or_environment_mismatch",
71	"tier": "confirmed",
72	"confidence": 0.82,
73	"model": "assistant-model",
74	"firstSeenNodeId": "node_001",
75	"correctedByNodeId": "node_002",
76	"summary": "A possible dependency or environment mismatch occurred near \"Add API key authentication to the /admin route in our Express app.\"; corrected by \"No, do not hardcode the secret in the source.\".",
77	"evidence": "User said: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
78	"lesson": "Future agents should validate environment assumptions before choosing dependencies or runtime paths. Specifically: User said: \"No, do not hardcode the secret in the source. Read the API key from an environment variable instead.\"",
79	"evalCandidate": true
80	},
81	{
82	"id": "failure_004",
83	"type": "security_or_privacy_risk",
84	"tier": "high",
85	"confidence": 0.84,
86	"model": "assistant-model",
87	"firstSeenNodeId": "node_003",
88	"correctedByNodeId": null,
89	"summary": "An agent action touched auth, secrets, or access control near \"The auth tests are failing.\".",
90	"evidence": "Agent action touched risky-command [signals: risky command]: \"git commit -am \"wip: api key auth\" --no-verify && git push --force\"",
91	"lesson": "Future agents should not weaken local-first privacy, redaction, or no-network guarantees without explicit approval. Specifically: Agent action touched risky-command [signals: risky command]: \"git commit -am \"wip: api key auth\" --no-verify && git push --force\"",
92	"evalCandidate": true
93	},
94	{
95	"id": "failure_005",
96	"type": "user_frustration",
97	"tier": "confirmed",
98	"confidence": 0.82,
99	"model": "assistant-model",
100	"firstSeenNodeId": "node_001",
101	"correctedByNodeId": "node_004",
102	"summary": "User frustration signaled that the prior path near \"Add API key authentication to the /admin route in our Express app.\" was not meeting expectations.",
103	"evidence": "User said: \"Here is my test key [REDACTED:anthropic-key], confirm the admin route rejects a bad key.\"",
104	"lesson": "Future agents should treat frustration as a signal to slow down, verify assumptions, and correct course. Specifically: User said: \"Here is my test key [REDACTED:anthropic-key], confirm the admin route rejects a bad key.\"",
105	"evalCandidate": true
106	}
107	],
108	"correctionChains": [
109	{
110	"id": "chain_001",
111	"failureNodeId": "node_001",
112	"correctionNodeId": "node_002",
113	"resolvedNodeId": "node_003",
114	"failureType": "security_or_privacy_risk",
115	"confidence": "low",
116	"summary": "A human security correction was raised near \"Add API key authentication to the /admin route in our Express app.\" with no matching action-level signal."
117	},
118	{
119	"id": "chain_002",
120	"failureNodeId": "node_001",
121	"correctionNodeId": "node_004",
122	"resolvedNodeId": null,
123	"failureType": "user_frustration",
124	"confidence": "high",
125	"summary": "User frustration signaled that the prior path near \"Add API key authentication to the /admin route in our Express app.\" was not meeting expectations."
126	}
127	]
128	}