add normalized request data in http

Author: mydea

packages/core/src/integrations/requestdata.ts

@@ -1,5 +1,6 @@
 import type { IntegrationFn } from '@sentry/types';
 import type { AddRequestDataToEventOptions, TransactionNamingScheme } from '@sentry/utils';
+import { addNormalizedRequestDataToEvent } from '@sentry/utils';
 import { addRequestDataToEvent } from '@sentry/utils';
 import { defineIntegration } from '../integration';
 
@@ -73,15 +74,24 @@ const _requestDataIntegration = ((options: RequestDataIntegrationOptions = {}) =
       // that's happened, it will be easier to add this logic in without worrying about unexpected side effects.)
 
       const { sdkProcessingMetadata = {} } = event;
-      const req = sdkProcessingMetadata.request;
+      const { request, normalizedRequest } = sdkProcessingMetadata;
 
-      if (!req) {
-        return event;
+      const addRequestDataOptions = convertReqDataIntegrationOptsToAddReqDataOpts(_options);
+
+      // If this is set, it takes precedence over the plain request object
+      if (normalizedRequest) {
+        // Some other data is not available in standard HTTP requests, but can sometimes be augmented by e.g. Express or Next.js
+        const ipAddress = request ? request.ip || (request.socket && request.socket.remoteAddress) : undefined;
+        const user = request ? request.user : undefined;
+
+        return addNormalizedRequestDataToEvent(event, normalizedRequest, { ipAddress, user }, addRequestDataOptions);
       }
 
-      const addRequestDataOptions = convertReqDataIntegrationOptsToAddReqDataOpts(_options);
+      if (!request) {
+        return event;
+      }
 
-      return addRequestDataToEvent(event, req, addRequestDataOptions);
+      return addRequestDataToEvent(event, request, addRequestDataOptions);
     },
   };
 }) satisfies IntegrationFn;

packages/node/src/transports/http-module.ts

@@ -10,7 +10,8 @@ export type HTTPModuleRequestOptions = HTTPRequestOptions | HTTPSRequestOptions
 export interface HTTPModuleRequestIncomingMessage {
   headers: IncomingHttpHeaders;
   statusCode?: number;
-  on(event: 'data' | 'end', listener: () => void): void;
+  on(event: 'data' | 'end', listener: (chunk: Buffer) => void): void;
+  off(event: 'data' | 'end', listener: (chunk: Buffer) => void): void;
   setEncoding(encoding: string): void;
 }
 

packages/types/src/envelope.ts

@@ -101,7 +101,7 @@ export type ProfileItem = BaseEnvelopeItem<ProfileItemHeaders, Profile>;
 export type ProfileChunkItem = BaseEnvelopeItem<ProfileChunkItemHeaders, ProfileChunk>;
 export type SpanItem = BaseEnvelopeItem<SpanItemHeaders, Partial<SpanJSON>>;
 
-export type EventEnvelopeHeaders = { event_id: string; sent_at: string; trace?: DynamicSamplingContext };
+export type EventEnvelopeHeaders = { event_id: string; sent_at: string; trace?: Partial<DynamicSamplingContext> };
 type SessionEnvelopeHeaders = { sent_at: string };
 type CheckInEnvelopeHeaders = { trace?: DynamicSamplingContext };
 type ClientReportEnvelopeHeaders = BaseEnvelopeHeaders;

packages/types/src/event.ts

@@ -2,13 +2,15 @@ import type { Attachment } from './attachment';
 import type { Breadcrumb } from './breadcrumb';
 import type { Contexts } from './context';
 import type { DebugMeta } from './debugMeta';
+import type { DynamicSamplingContext } from './envelope';
 import type { Exception } from './exception';
 import type { Extras } from './extra';
 import type { Measurements } from './measurement';
 import type { Mechanism } from './mechanism';
 import type { Primitive } from './misc';
+import type { PolymorphicRequest } from './polymorphics';
 import type { Request } from './request';
-import type { CaptureContext } from './scope';
+import type { CaptureContext, Scope } from './scope';
 import type { SdkInfo } from './sdkinfo';
 import type { SeverityLevel } from './severity';
 import type { MetricSummary, SpanJSON } from './span';
@@ -51,7 +53,15 @@ export interface Event {
   measurements?: Measurements;
   debug_meta?: DebugMeta;
   // A place to stash data which is needed at some point in the SDK's event processing pipeline but which shouldn't get sent to Sentry
-  sdkProcessingMetadata?: { [key: string]: any };
+  // Note: This is considered internal and is subject to change in minors
+  sdkProcessingMetadata?: { [key: string]: unknown } & {
+    request?: PolymorphicRequest;
+    normalizedRequest?: Request;
+    dynamicSamplingContext?: Partial<DynamicSamplingContext>;
+    capturedSpanScope?: Scope;
+    capturedSpanIsolationScope?: Scope;
+    spanCountBeforeProcessing?: number;
+  };
   transaction_info?: {
     source: TransactionSource;
   };

packages/utils/src/requestdata.ts

@@ -1,7 +1,9 @@
+/* eslint-disable max-lines */
 import type {
   Event,
   ExtractedNodeRequestData,
   PolymorphicRequest,
+  Request,
   TransactionSource,
   WebFetchHeaders,
   WebFetchRequest,
@@ -12,6 +14,7 @@ import { DEBUG_BUILD } from './debug-build';
 import { isPlainObject, isString } from './is';
 import { logger } from './logger';
 import { normalize } from './normalize';
+import { truncate } from './string';
 import { stripUrlQueryAndFragment } from './url';
 import { getClientIPAddress, ipHeaderNames } from './vendor/getIpAddress';
 
@@ -228,14 +231,27 @@ export function extractRequestData(
         if (method === 'GET' || method === 'HEAD') {
           break;
         }
+        // NOTE: As of v8, request is (unless a user sets this manually) ALWAYS a http request
+        // Which does not have a body by default
+        // However, in our http instrumentation, we patch the request to capture the body and store it on the
+        // request as `.body` anyhow
+        // In v9, we may update requestData to only work with plain http requests
         // body data:
         //   express, koa, nextjs: req.body
         //
         //   when using node by itself, you have to read the incoming stream(see
         //   https://nodejs.dev/learn/get-http-request-body-data-using-nodejs); if a user is doing that, we can't know
         //   where they're going to store the final result, so they'll have to capture this data themselves
-        if (req.body !== undefined) {
-          requestData.data = isString(req.body) ? req.body : JSON.stringify(normalize(req.body));
+        const body = req.body;
+        if (body !== undefined) {
+          const stringBody: string = isString(body)
+            ? body
+            : isPlainObject(body)
+              ? JSON.stringify(normalize(body))
+              : truncate(`${body}`, 1024);
+          if (stringBody) {
+            requestData.data = stringBody;
+          }
         }
         break;
       }
@@ -250,6 +266,62 @@ export function extractRequestData(
   return requestData;
 }
 
+/**
+ * Add already normalized request data to an event.
+ */
+export function addNormalizedRequestDataToEvent(
+  event: Event,
+  req: Request,
+  // This is non-standard data that is not part of the regular HTTP request
+  additionalData: { ipAddress?: string; user?: Record<string, unknown> },
+  options: AddRequestDataToEventOptions,
+): Event {
+  const include = {
+    ...DEFAULT_INCLUDES,
+    ...(options && options.include),
+  };
+
+  if (include.request) {
+    const includeRequest = Array.isArray(include.request) ? [...include.request] : [...DEFAULT_REQUEST_INCLUDES];
+    if (include.ip) {
+      includeRequest.push('ip');
+    }
+
+    const extractedRequestData = extractNormalizedRequestData(req, { include: includeRequest });
+
+    event.request = {
+      ...event.request,
+      ...extractedRequestData,
+    };
+  }
+
+  if (include.user) {
+    const extractedUser =
+      additionalData.user && isPlainObject(additionalData.user)
+        ? extractUserData(additionalData.user, include.user)
+        : {};
+
+    if (Object.keys(extractedUser).length) {
+      event.user = {
+        ...event.user,
+        ...extractedUser,
+      };
+    }
+  }
+
+  if (include.ip) {
+    const ip = (req.headers && getClientIPAddress(req.headers)) || additionalData.ipAddress;
+    if (ip) {
+      event.user = {
+        ...event.user,
+        ip_address: ip,
+      };
+    }
+  }
+
+  return event;
+}
+
 /**
  * Add data from the given request to the given event
  *
@@ -374,3 +446,51 @@ export function winterCGRequestToRequestData(req: WebFetchRequest): PolymorphicR
     headers,
   };
 }
+
+function extractNormalizedRequestData(normalizedRequest: Request, { include }: { include: string[] }): Request {
+  const includeKeys = include ? (Array.isArray(include) ? include : DEFAULT_REQUEST_INCLUDES) : [];
+
+  const requestData: Request = {};
+
+  const { headers } = normalizedRequest;
+
+  if (includeKeys.includes('headers')) {
+    requestData.headers = headers;
+
+    // Remove the Cookie header in case cookie data should not be included in the event
+    if (!include.includes('cookies')) {
+      delete (headers as { cookie?: string }).cookie;
+    }
+
+    // Remove IP headers in case IP data should not be included in the event
+    if (!include.includes('ip')) {
+      ipHeaderNames.forEach(ipHeaderName => {
+        // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+        delete (headers as Record<string, unknown>)[ipHeaderName];
+      });
+    }
+  }
+
+  if (includeKeys.includes('method')) {
+    requestData.method = normalizedRequest.method;
+  }
+
+  if (includeKeys.includes('url')) {
+    requestData.url = normalizedRequest.url;
+  }
+
+  if (includeKeys.includes('cookies')) {
+    const cookies = normalizedRequest.cookies || (headers && headers.cookie ? parseCookie(headers.cookie) : undefined);
+    requestData.cookies = cookies;
+  }
+
+  if (includeKeys.includes('query_string')) {
+    requestData.query_string = normalizedRequest.query_string;
+  }
+
+  if (includeKeys.includes('data')) {
+    requestData.data = normalizedRequest.data;
+  }
+
+  return requestData;
+}

yarn.lock

@@ -12964,6 +12964,24 @@ [email protected], body-parser@^1.18.3, body-parser@^1.19.0:
     type-is "~1.6.18"
     unpipe "1.0.0"
 
+body-parser@^1.20.3:
+  version "1.20.3"
+  resolved "https://registry.yarnpkg.com/body-parser/-/body-parser-1.20.3.tgz#1953431221c6fb5cd63c4b36d53fab0928e548c6"
+  integrity sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==
+  dependencies:
+    bytes "3.1.2"
+    content-type "~1.0.5"
+    debug "2.6.9"
+    depd "2.0.0"
+    destroy "1.2.0"
+    http-errors "2.0.0"
+    iconv-lite "0.4.24"
+    on-finished "2.4.1"
+    qs "6.13.0"
+    raw-body "2.5.2"
+    type-is "~1.6.18"
+    unpipe "1.0.0"
+
 body@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/body/-/body-5.1.0.tgz#e4ba0ce410a46936323367609ecb4e6553125069"
@@ -28287,6 +28305,13 @@ qs@^6.4.0:
   dependencies:
     side-channel "^1.0.4"
 
+[email protected]:
+  version "6.13.0"
+  resolved "https://registry.yarnpkg.com/qs/-/qs-6.13.0.tgz#6ca3bd58439f7e245655798997787b0d88a51906"
+  integrity sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==
+  dependencies:
+    side-channel "^1.0.6"
+
 query-string@^4.2.2:
   version "4.3.4"
   resolved "https://registry.yarnpkg.com/query-string/-/query-string-4.3.4.tgz#bbb693b9ca915c232515b228b1a02b609043dbeb"