test(replay): Test compressed recording payloads (#7082)

Lms24 · web-flow · commit f4564cc9e322 · 2023-02-08T14:26:41.000+01:00
Add Playwright integration test utilities to test Replay's zlib compression. We now have a dedicated parser that attempts to decompress a payload, if it encounters a zlib-compressed replay recording payload.

Additionally, add a simple test to check that an initial full compressed snapshot has the correct content.
diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json
@@ -32,6 +32,7 @@
     "@playwright/test": "^1.29.2",
     "babel-loader": "^8.2.2",
     "html-webpack-plugin": "^5.5.0",
+    "pako": "^2.1.0",
     "playwright": "^1.29.2",
     "typescript": "^4.5.2",
     "webpack": "^5.52.0"
diff --git a/packages/integration-tests/suites/replay/compression/init.js b/packages/integration-tests/suites/replay/compression/init.js
@@ -0,0 +1,18 @@
+import * as Sentry from '@sentry/browser';
+import { Replay } from '@sentry/replay';
+
+window.Sentry = Sentry;
+window.Replay = new Replay({
+  flushMinDelay: 500,
+  flushMaxDelay: 500,
+  useCompression: true,
+});
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  sampleRate: 0,
+  replaysSessionSampleRate: 1.0,
+  replaysOnErrorSampleRate: 0.0,
+
+  integrations: [window.Replay],
+});
diff --git a/packages/integration-tests/suites/replay/compression/subject.js b/packages/integration-tests/suites/replay/compression/subject.js
@@ -0,0 +1,6 @@
+document.getElementById('go-background').addEventListener('click', () => {
+  Object.defineProperty(document, 'hidden', { value: true, writable: true });
+  const ev = document.createEvent('Event');
+  ev.initEvent('visibilitychange');
+  document.dispatchEvent(ev);
+});
diff --git a/packages/integration-tests/suites/replay/compression/template.html b/packages/integration-tests/suites/replay/compression/template.html
@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+  </head>
+  <body>
+    <button id="go-background">New Tab</button>
+  </body>
+</html>
diff --git a/packages/integration-tests/suites/replay/compression/test.ts b/packages/integration-tests/suites/replay/compression/test.ts
@@ -0,0 +1,38 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../utils/fixtures';
+import { getExpectedReplayEvent } from '../../../utils/replayEventTemplates';
+import { getFullRecordingSnapshots, getReplayEvent, waitForReplayRequest } from '../../../utils/replayHelpers';
+
+sentryTest('replay recording should be compressed by default', async ({ getLocalTestPath, page }) => {
+  // Replay bundles are es6 only
+  if (process.env.PW_BUNDLE && process.env.PW_BUNDLE.startsWith('bundle_es5')) {
+    sentryTest.skip();
+  }
+
+  const reqPromise0 = waitForReplayRequest(page, 0);
+
+  await page.route('https://dsn.ingest.sentry.io/**/*', route => {
+    return route.fulfill({
+      status: 200,
+      contentType: 'application/json',
+      body: JSON.stringify({ id: 'test-id' }),
+    });
+  });
+
+  const url = await getLocalTestPath({ testDir: __dirname });
+
+  await page.goto(url);
+  const replayEvent0 = getReplayEvent(await reqPromise0);
+  expect(replayEvent0).toEqual(getExpectedReplayEvent());
+
+  const snapshots = getFullRecordingSnapshots(await reqPromise0);
+  expect(snapshots.length).toEqual(1);
+
+  const stringifiedSnapshot = JSON.stringify(snapshots[0]);
+  expect(stringifiedSnapshot).toContain('"tagName":"body"');
+  expect(stringifiedSnapshot).toContain('"tagName":"html"');
+  expect(stringifiedSnapshot).toContain('"tagName":"button"');
+  expect(stringifiedSnapshot).toContain('"textContent":"*** ***"');
+  expect(stringifiedSnapshot).toContain('"id":"go-background"');
+});
diff --git a/packages/integration-tests/utils/replayHelpers.ts b/packages/integration-tests/utils/replayHelpers.ts
@@ -1,5 +1,6 @@
 import type { RecordingEvent, ReplayContainer } from '@sentry/replay/build/npm/types/types';
 import type { Breadcrumb, Event, ReplayEvent } from '@sentry/types';
+import pako from 'pako';
 import type { Page, Request } from 'playwright';
 
 import { envelopeRequestParser } from './helpers';
@@ -13,6 +14,17 @@ type PerformanceSpan = {
   data: Record<string, number>;
 };
 
+export type RecordingSnapshot = {
+  node: SnapshotNode;
+  initialOffset: number;
+};
+
+type SnapshotNode = {
+  type: number;
+  id: number;
+  childNodes: SnapshotNode[];
+};
+
 /**
  * Waits for a replay request to be sent by the page and returns it.
  *
@@ -85,7 +97,7 @@ export function getCustomRecordingEvents(replayRequest: Request): {
   breadcrumbs: Breadcrumb[];
   performanceSpans: PerformanceSpan[];
 } {
-  const recordingEvents = envelopeRequestParser(replayRequest, 5) as RecordingEvent[];
+  const recordingEvents = getDecompressedRecordingEvents(replayRequest);
 
   const breadcrumbs = getReplayBreadcrumbs(recordingEvents);
   const performanceSpans = getReplayPerformanceSpans(recordingEvents);
@@ -108,3 +120,60 @@ function getReplayPerformanceSpans(recordingEvents: RecordingEvent[]): Performan
     .filter(data => data.tag === 'performanceSpan')
     .map(data => data.payload) as PerformanceSpan[];
 }
+
+export function getFullRecordingSnapshots(replayRequest: Request): RecordingSnapshot[] {
+  const events = getDecompressedRecordingEvents(replayRequest) as RecordingEvent[];
+  return events.filter(event => event.type === 2).map(event => event.data as RecordingSnapshot);
+}
+
+function getDecompressedRecordingEvents(replayRequest: Request): RecordingEvent[] {
+  return replayEnvelopeRequestParser(replayRequest, 5) as RecordingEvent[];
+}
+
+/**
+ * Copy of the envelopeParser from ./helpers.ts, but with the ability
+ * to decompress zlib-compressed envelope payloads which we need to inspect for replay recordings.
+ * This parser can handle uncompressed as well as compressed replay recordings.
+ */
+const replayEnvelopeRequestParser = (request: Request | null, envelopeIndex = 2): Event => {
+  const envelope = replayEnvelopeParser(request);
+  return envelope[envelopeIndex] as Event;
+};
+
+const replayEnvelopeParser = (request: Request | null): unknown[] => {
+  // https://develop.sentry.dev/sdk/envelopes/
+  const envelopeBytes = request?.postDataBuffer() || '';
+
+  // first, we convert the bugger to string to split and go through the uncompressed lines
+  const envelopeString = envelopeBytes.toString();
+
+  const lines = envelopeString.split('\n').map(line => {
+    try {
+      return JSON.parse(line);
+    } catch (error) {
+      // If we fail to parse a line, we _might_ have found a compressed payload,
+      // so let's check if this is actually the case.
+      // This is quite hacky but we can't go through `line` because the prior operations
+      // seem to have altered its binary content. Hence, we take the raw envelope and
+      // look up the place where the zlib compression header(0x78 0x9c) starts
+      for (let i = 0; i < envelopeBytes.length; i++) {
+        if (envelopeBytes[i] === 0x78 && envelopeBytes[i + 1] === 0x9c) {
+          try {
+            // We found a zlib-compressed payload - let's decompress it
+            const payload = envelopeBytes.slice(i);
+            // now we return the decompressed payload as JSON
+            const decompressedPayload = pako.inflate(payload as unknown as Uint8Array, { to: 'string' });
+            return JSON.parse(decompressedPayload);
+          } catch {
+            // Let's log that something went wrong
+            return line;
+          }
+        }
+      }
+
+      return line;
+    }
+  });
+
+  return lines;
+};
diff --git a/packages/replay/src/worker/worker.js b/packages/replay/src/worker/worker.js
diff --git a/yarn.lock b/yarn.lock
@@ -18616,7 +18616,7 @@ pad@^3.2.0:
   dependencies:
     wcwidth "^1.0.1"
 
-pako@^2.0.4:
+pako@^2.0.4, pako@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/pako/-/pako-2.1.0.tgz#266cc37f98c7d883545d11335c00fbd4062c9a86"
   integrity sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==
