Skip to content

Commit b0add63

Browse files
RulaKhaledRulaKhaled
authored
ref(core): Set system message as separate attribute (#18978)
So far we put all the messages including the system prompt on gen_ai.request.messages. Instead, the system prompt should be pulled out and set as a separate attribute called gen_ai.system_instructions. For now we will search for the first system message and use that. Closes #18917
1 parent 9a2b6a4 commit b0add63

23 files changed

Lines changed: 655 additions & 52 deletions

File tree

‎.size-limit.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@ module.exports = [
287287
import: createImport('init'),
288288
ignore: [...builtinModules, ...nodePrefixedBuiltinModules],
289289
gzip: true,
290-
limit: '166 KB',
290+
limit: '167 KB',
291291
},
292292
{
293293
name: '@sentry/node - without tracing',

‎dev-packages/node-integration-tests/suites/tracing/anthropic/scenario-system-instructions.mjs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
import Anthropic from '@anthropic-ai/sdk';
2+
import * as Sentry from '@sentry/node';
3+
import express from 'express';
4+
5+
function startMockAnthropicServer() {
6+
const app = express();
7+
app.use(express.json());
8+
9+
app.post('/anthropic/v1/messages', (req, res) => {
10+
res.send({
11+
id: 'msg_system_test',
12+
type: 'message',
13+
model: req.body.model,
14+
role: 'assistant',
15+
content: [
16+
{
17+
type: 'text',
18+
text: 'Response',
19+
},
20+
],
21+
stop_reason: 'end_turn',
22+
stop_sequence: null,
23+
usage: {
24+
input_tokens: 10,
25+
output_tokens: 5,
26+
},
27+
});
28+
});
29+
30+
return new Promise(resolve => {
31+
const server = app.listen(0, () => {
32+
resolve(server);
33+
});
34+
});
35+
}
36+
37+
async function run() {
38+
const server = await startMockAnthropicServer();
39+
40+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
41+
const client = new Anthropic({
42+
apiKey: 'mock-api-key',
43+
baseURL: `http://localhost:${server.address().port}/anthropic`,
44+
});
45+
46+
await client.messages.create({
47+
model: 'claude-3-5-sonnet-20241022',
48+
max_tokens: 1024,
49+
system: 'You are a helpful assistant',
50+
messages: [{ role: 'user', content: 'Hello' }],
51+
});
52+
});
53+
54+
server.close();
55+
}
56+
57+
run();

‎dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import {
1717
GEN_AI_RESPONSE_TEXT_ATTRIBUTE,
1818
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
1919
GEN_AI_SYSTEM_ATTRIBUTE,
20+
GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE,
2021
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
2122
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
2223
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
@@ -749,4 +750,32 @@ describe('Anthropic integration', () => {
749750
.completed();
750751
});
751752
});
753+
754+
createEsmAndCjsTests(
755+
__dirname,
756+
'scenario-system-instructions.mjs',
757+
'instrument-with-pii.mjs',
758+
(createRunner, test) => {
759+
test('extracts system instructions from messages', async () => {
760+
await createRunner()
761+
.ignore('event')
762+
.expect({
763+
transaction: {
764+
transaction: 'main',
765+
spans: expect.arrayContaining([
766+
expect.objectContaining({
767+
data: expect.objectContaining({
768+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([
769+
{ type: 'text', content: 'You are a helpful assistant' },
770+
]),
771+
}),
772+
}),
773+
]),
774+
},
775+
})
776+
.start()
777+
.completed();
778+
});
779+
},
780+
);
752781
});

‎dev-packages/node-integration-tests/suites/tracing/google-genai/scenario-system-instructions.mjs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { instrumentGoogleGenAIClient } from '@sentry/core';
2+
import * as Sentry from '@sentry/node';
3+
4+
class MockGoogleGenAI {
5+
constructor(config) {
6+
this.apiKey = config.apiKey;
7+
this.models = {
8+
generateContent: async params => {
9+
await new Promise(resolve => setTimeout(resolve, 10));
10+
return {
11+
response: {
12+
text: () => 'Response',
13+
modelVersion: params.model,
14+
usageMetadata: {
15+
promptTokenCount: 10,
16+
candidatesTokenCount: 5,
17+
totalTokenCount: 15,
18+
},
19+
candidates: [
20+
{
21+
content: {
22+
parts: [{ text: 'Response' }],
23+
role: 'model',
24+
},
25+
finishReason: 'STOP',
26+
},
27+
],
28+
},
29+
};
30+
},
31+
};
32+
}
33+
}
34+
35+
async function run() {
36+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
37+
const mockClient = new MockGoogleGenAI({ apiKey: 'mock-api-key' });
38+
const client = instrumentGoogleGenAIClient(mockClient);
39+
40+
await client.models.generateContent({
41+
model: 'gemini-1.5-flash',
42+
config: {
43+
systemInstruction: 'You are a helpful assistant',
44+
},
45+
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
46+
});
47+
});
48+
}
49+
50+
run();

‎dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import {
1616
GEN_AI_RESPONSE_TEXT_ATTRIBUTE,
1717
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
1818
GEN_AI_SYSTEM_ATTRIBUTE,
19+
GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE,
1920
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
2021
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
2122
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
@@ -572,4 +573,32 @@ describe('Google GenAI integration', () => {
572573
});
573574
},
574575
);
576+
577+
createEsmAndCjsTests(
578+
__dirname,
579+
'scenario-system-instructions.mjs',
580+
'instrument-with-pii.mjs',
581+
(createRunner, test) => {
582+
test('extracts system instructions from messages', async () => {
583+
await createRunner()
584+
.ignore('event')
585+
.expect({
586+
transaction: {
587+
transaction: 'main',
588+
spans: expect.arrayContaining([
589+
expect.objectContaining({
590+
data: expect.objectContaining({
591+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([
592+
{ type: 'text', content: 'You are a helpful assistant' },
593+
]),
594+
}),
595+
}),
596+
]),
597+
},
598+
})
599+
.start()
600+
.completed();
601+
});
602+
},
603+
);
575604
});

‎dev-packages/node-integration-tests/suites/tracing/langchain/scenario-system-instructions.mjs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { ChatAnthropic } from '@langchain/anthropic';
2+
import * as Sentry from '@sentry/node';
3+
import express from 'express';
4+
5+
function startMockServer() {
6+
const app = express();
7+
app.use(express.json());
8+
9+
app.post('/v1/messages', (req, res) => {
10+
res.json({
11+
id: 'msg_test123',
12+
type: 'message',
13+
role: 'assistant',
14+
content: [
15+
{
16+
type: 'text',
17+
text: 'Response',
18+
},
19+
],
20+
model: req.body.model,
21+
stop_reason: 'end_turn',
22+
stop_sequence: null,
23+
usage: {
24+
input_tokens: 10,
25+
output_tokens: 5,
26+
},
27+
});
28+
});
29+
30+
return new Promise(resolve => {
31+
const server = app.listen(0, () => {
32+
resolve(server);
33+
});
34+
});
35+
}
36+
37+
async function run() {
38+
const server = await startMockServer();
39+
const baseUrl = `http://localhost:${server.address().port}`;
40+
41+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
42+
const model = new ChatAnthropic({
43+
model: 'claude-3-5-sonnet-20241022',
44+
apiKey: 'mock-api-key',
45+
clientOptions: {
46+
baseURL: baseUrl,
47+
},
48+
});
49+
50+
await model.invoke([
51+
{ role: 'system', content: 'You are a helpful assistant' },
52+
{ role: 'user', content: 'Hello' },
53+
]);
54+
});
55+
56+
await Sentry.flush(2000);
57+
58+
server.close();
59+
}
60+
61+
run();

‎dev-packages/node-integration-tests/suites/tracing/langchain/test.ts‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
GEN_AI_RESPONSE_TEXT_ATTRIBUTE,
1515
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
1616
GEN_AI_SYSTEM_ATTRIBUTE,
17+
GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE,
1718
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
1819
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
1920
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
@@ -242,7 +243,8 @@ describe('LangChain integration', () => {
242243
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
243244
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
244245
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022',
245-
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3,
246+
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2,
247+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.any(String),
246248
// Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs)
247249
[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/),
248250
}),
@@ -259,7 +261,8 @@ describe('LangChain integration', () => {
259261
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
260262
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
261263
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022',
262-
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3,
264+
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2,
265+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.any(String),
263266
// Small message should be kept intact
264267
[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([
265268
{ role: 'user', content: 'This is a small message that fits within the limit' },
@@ -345,4 +348,32 @@ describe('LangChain integration', () => {
345348
// This test fails on CJS because we use dynamic imports to simulate importing LangChain after the Anthropic client is created
346349
{ failsOnCjs: true },
347350
);
351+
352+
createEsmAndCjsTests(
353+
__dirname,
354+
'scenario-system-instructions.mjs',
355+
'instrument-with-pii.mjs',
356+
(createRunner, test) => {
357+
test('extracts system instructions from messages', async () => {
358+
await createRunner()
359+
.ignore('event')
360+
.expect({
361+
transaction: {
362+
transaction: 'main',
363+
spans: expect.arrayContaining([
364+
expect.objectContaining({
365+
data: expect.objectContaining({
366+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: JSON.stringify([
367+
{ type: 'text', content: 'You are a helpful assistant' },
368+
]),
369+
}),
370+
}),
371+
]),
372+
},
373+
})
374+
.start()
375+
.completed();
376+
});
377+
},
378+
);
348379
});

‎dev-packages/node-integration-tests/suites/tracing/langchain/v1/test.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
GEN_AI_RESPONSE_TEXT_ATTRIBUTE,
1515
GEN_AI_RESPONSE_TOOL_CALLS_ATTRIBUTE,
1616
GEN_AI_SYSTEM_ATTRIBUTE,
17+
GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE,
1718
GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE,
1819
GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE,
1920
GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE,
@@ -285,7 +286,8 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => {
285286
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
286287
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
287288
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022',
288-
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3,
289+
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2,
290+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.stringMatching(/^\[\{"type":"text","content":"A+"\}\]$/),
289291
// Messages should be present (truncation happened) and should be a JSON array of a single index (contains only Cs)
290292
[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.stringMatching(/^\[\{"role":"user","content":"C+"\}\]$/),
291293
}),
@@ -302,7 +304,9 @@ conditionalTest({ min: 20 })('LangChain integration (v1)', () => {
302304
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.langchain',
303305
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
304306
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-5-sonnet-20241022',
305-
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 3,
307+
[GEN_AI_INPUT_MESSAGES_ORIGINAL_LENGTH_ATTRIBUTE]: 2,
308+
[GEN_AI_SYSTEM_INSTRUCTIONS_ATTRIBUTE]: expect.stringMatching(/^\[\{"type":"text","content":"A+"\}\]$/),
309+
306310
// Small message should be kept intact
307311
[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: JSON.stringify([
308312
{ role: 'user', content: 'This is a small message that fits within the limit' },

‎dev-packages/node-integration-tests/suites/tracing/langgraph/scenario-system-instructions.mjs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import { END, MessagesAnnotation, START, StateGraph } from '@langchain/langgraph';
2+
import * as Sentry from '@sentry/node';
3+
4+
async function run() {
5+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
6+
const mockLlm = () => {
7+
return {
8+
messages: [
9+
{
10+
role: 'assistant',
11+
content: 'Response',
12+
response_metadata: {
13+
model_name: 'mock-model',
14+
finish_reason: 'stop',
15+
tokenUsage: {
16+
promptTokens: 10,
17+
completionTokens: 5,
18+
totalTokens: 15,
19+
},
20+
},
21+
},
22+
],
23+
};
24+
};
25+
26+
const graph = new StateGraph(MessagesAnnotation)
27+
.addNode('agent', mockLlm)
28+
.addEdge(START, 'agent')
29+
.addEdge('agent', END)
30+
.compile({ name: 'test-agent' });
31+
32+
await graph.invoke({
33+
messages: [
34+
{ role: 'system', content: 'You are a helpful assistant' },
35+
{ role: 'user', content: 'Hello' },
36+
],
37+
});
38+
});
39+
40+
await Sentry.flush(2000);
41+
}
42+
43+
run();

0 commit comments

Comments
 (0)