Skip to content

Use lazy encoding in utf-8 encoded string comparison #6706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 14 commits into from
Mar 3, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions firebase-firestore/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Unreleased
* [fixed] Use lazy encoding in UTF-8 encoded byte comparison for strings to solve performance issues. [#6706](//github.com/firebase/firebase-android-sdk/pull/6706)
* [changed] Updated `protolite-well-known-types` dependency to `18.0.1`. [#6716]


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1658,17 +1658,33 @@ public void sdkOrdersQueryByDocumentIdTheSameWayOnlineAndOffline() {
public void snapshotListenerSortsUnicodeStringsAsServer() {
Map<String, Map<String, Object>> testDocs =
map(
"a", map("value", "Łukasiewicz"),
"b", map("value", "Sierpiński"),
"c", map("value", "岩澤"),
"d", map("value", "🄟"),
"e", map("value", "P"),
"f", map("value", "︒"),
"g", map("value", "🐵"));
"a",
map("value", "Łukasiewicz"),
"b",
map("value", "Sierpiński"),
"c",
map("value", "岩澤"),
"d",
map("value", "🄟"),
"e",
map("value", "P"),
"f",
map("value", "︒"),
"g",
map("value", "🐵"),
"h",
map("value", "你好"),
"i",
map("value", "你顥"),
"j",
map("value", "😁"),
"k",
map("value", "😀"));

CollectionReference colRef = testCollectionWithDocs(testDocs);
Query orderedQuery = colRef.orderBy("value");
List<String> expectedDocIds = Arrays.asList("b", "a", "c", "f", "e", "d", "g");
List<String> expectedDocIds =
Arrays.asList("b", "a", "h", "i", "c", "f", "e", "d", "g", "k", "j");

QuerySnapshot getSnapshot = waitFor(orderedQuery.get());
List<String> getSnapshotDocIds =
Expand Down Expand Up @@ -1699,17 +1715,33 @@ public void snapshotListenerSortsUnicodeStringsAsServer() {
public void snapshotListenerSortsUnicodeStringsInArrayAsServer() {
Map<String, Map<String, Object>> testDocs =
map(
"a", map("value", Arrays.asList("Łukasiewicz")),
"b", map("value", Arrays.asList("Sierpiński")),
"c", map("value", Arrays.asList("岩澤")),
"d", map("value", Arrays.asList("🄟")),
"e", map("value", Arrays.asList("P")),
"f", map("value", Arrays.asList("︒")),
"g", map("value", Arrays.asList("🐵")));
"a",
map("value", Arrays.asList("Łukasiewicz")),
"b",
map("value", Arrays.asList("Sierpiński")),
"c",
map("value", Arrays.asList("岩澤")),
"d",
map("value", Arrays.asList("🄟")),
"e",
map("value", Arrays.asList("P")),
"f",
map("value", Arrays.asList("︒")),
"g",
map("value", Arrays.asList("🐵")),
"h",
map("value", Arrays.asList("你好")),
"i",
map("value", Arrays.asList("你顥")),
"j",
map("value", Arrays.asList("😁")),
"k",
map("value", Arrays.asList("😀")));

CollectionReference colRef = testCollectionWithDocs(testDocs);
Query orderedQuery = colRef.orderBy("value");
List<String> expectedDocIds = Arrays.asList("b", "a", "c", "f", "e", "d", "g");
List<String> expectedDocIds =
Arrays.asList("b", "a", "h", "i", "c", "f", "e", "d", "g", "k", "j");

QuerySnapshot getSnapshot = waitFor(orderedQuery.get());
List<String> getSnapshotDocIds =
Expand Down Expand Up @@ -1740,17 +1772,33 @@ public void snapshotListenerSortsUnicodeStringsInArrayAsServer() {
public void snapshotListenerSortsUnicodeStringsInMapAsServer() {
Map<String, Map<String, Object>> testDocs =
map(
"a", map("value", map("foo", "Łukasiewicz")),
"b", map("value", map("foo", "Sierpiński")),
"c", map("value", map("foo", "岩澤")),
"d", map("value", map("foo", "🄟")),
"e", map("value", map("foo", "P")),
"f", map("value", map("foo", "︒")),
"g", map("value", map("foo", "🐵")));
"a",
map("value", map("foo", "Łukasiewicz")),
"b",
map("value", map("foo", "Sierpiński")),
"c",
map("value", map("foo", "岩澤")),
"d",
map("value", map("foo", "🄟")),
"e",
map("value", map("foo", "P")),
"f",
map("value", map("foo", "︒")),
"g",
map("value", map("foo", "🐵")),
"h",
map("value", map("foo", "你好")),
"i",
map("value", map("foo", "你顥")),
"j",
map("value", map("foo", "😁")),
"k",
map("value", map("foo", "😀")));

CollectionReference colRef = testCollectionWithDocs(testDocs);
Query orderedQuery = colRef.orderBy("value");
List<String> expectedDocIds = Arrays.asList("b", "a", "c", "f", "e", "d", "g");
List<String> expectedDocIds =
Arrays.asList("b", "a", "h", "i", "c", "f", "e", "d", "g", "k", "j");

QuerySnapshot getSnapshot = waitFor(orderedQuery.get());
List<String> getSnapshotDocIds =
Expand Down Expand Up @@ -1781,17 +1829,33 @@ public void snapshotListenerSortsUnicodeStringsInMapAsServer() {
public void snapshotListenerSortsUnicodeStringsInMapKeyAsServer() {
Map<String, Map<String, Object>> testDocs =
map(
"a", map("value", map("Łukasiewicz", "foo")),
"b", map("value", map("Sierpiński", "foo")),
"c", map("value", map("岩澤", "foo")),
"d", map("value", map("🄟", "foo")),
"e", map("value", map("P", "foo")),
"f", map("value", map("︒", "foo")),
"g", map("value", map("🐵", "foo")));
"a",
map("value", map("Łukasiewicz", "foo")),
"b",
map("value", map("Sierpiński", "foo")),
"c",
map("value", map("岩澤", "foo")),
"d",
map("value", map("🄟", "foo")),
"e",
map("value", map("P", "foo")),
"f",
map("value", map("︒", "foo")),
"g",
map("value", map("🐵", "foo")),
"h",
map("value", map("你好", "foo")),
"i",
map("value", map("你顥", "foo")),
"j",
map("value", map("😁", "foo")),
"k",
map("value", map("😀", "foo")));

CollectionReference colRef = testCollectionWithDocs(testDocs);
Query orderedQuery = colRef.orderBy("value");
List<String> expectedDocIds = Arrays.asList("b", "a", "c", "f", "e", "d", "g");
List<String> expectedDocIds =
Arrays.asList("b", "a", "h", "i", "c", "f", "e", "d", "g", "k", "j");

QuerySnapshot getSnapshot = waitFor(orderedQuery.get());
List<String> getSnapshotDocIds =
Expand Down Expand Up @@ -1822,18 +1886,83 @@ public void snapshotListenerSortsUnicodeStringsInMapKeyAsServer() {
public void snapshotListenerSortsUnicodeStringsInDocumentKeyAsServer() {
Map<String, Map<String, Object>> testDocs =
map(
"Łukasiewicz", map("value", "foo"),
"Sierpiński", map("value", "foo"),
"岩澤", map("value", "foo"),
"🄟", map("value", "foo"),
"P", map("value", "foo"),
"︒", map("value", "foo"),
"🐵", map("value", "foo"));
"Łukasiewicz",
map("value", "foo"),
"Sierpiński",
map("value", "foo"),
"岩澤",
map("value", "foo"),
"🄟",
map("value", "foo"),
"P",
map("value", "foo"),
"︒",
map("value", "foo"),
"🐵",
map("value", "foo"),
"你好",
map("value", "foo"),
"你顥",
map("value", "foo"),
"😁",
map("value", "foo"),
"😀",
map("value", "foo"));

CollectionReference colRef = testCollectionWithDocs(testDocs);
Query orderedQuery = colRef.orderBy(FieldPath.documentId());
List<String> expectedDocIds =
Arrays.asList("Sierpiński", "Łukasiewicz", "岩澤", "︒", "P", "🄟", "🐵");
Arrays.asList(
"Sierpiński", "Łukasiewicz", "你好", "你顥", "岩澤", "︒", "P", "🄟", "🐵", "😀", "😁");

QuerySnapshot getSnapshot = waitFor(orderedQuery.get());
List<String> getSnapshotDocIds =
getSnapshot.getDocuments().stream().map(ds -> ds.getId()).collect(Collectors.toList());

EventAccumulator<QuerySnapshot> eventAccumulator = new EventAccumulator<QuerySnapshot>();
ListenerRegistration registration =
orderedQuery.addSnapshotListener(eventAccumulator.listener());

List<String> watchSnapshotDocIds = new ArrayList<>();
try {
QuerySnapshot watchSnapshot = eventAccumulator.await();
watchSnapshotDocIds =
watchSnapshot.getDocuments().stream()
.map(documentSnapshot -> documentSnapshot.getId())
.collect(Collectors.toList());
} finally {
registration.remove();
}

assertTrue(getSnapshotDocIds.equals(expectedDocIds));
assertTrue(watchSnapshotDocIds.equals(expectedDocIds));

checkOnlineAndOfflineResultsMatch(orderedQuery, expectedDocIds.toArray(new String[0]));
}

@Test
public void snapshotListenerSortsInvalidUnicodeStringsAsServer() {
// Note: Protocol Buffer converts any invalid surrogates to "?".
Map<String, Map<String, Object>> testDocs =
map(
"a",
map("value", "Z"),
"b",
map("value", "你好"),
"c",
map("value", "😀"),
"d",
map("value", "ab\uD800"), // Lone high surrogate
"e",
map("value", "ab\uDC00"), // Lone low surrogate
"f",
map("value", "ab\uD800\uD800"), // Unpaired high surrogate
"g",
map("value", "ab\uDC00\uDC00")); // Unpaired low surrogate

CollectionReference colRef = testCollectionWithDocs(testDocs);
Query orderedQuery = colRef.orderBy("value");
List<String> expectedDocIds = Arrays.asList("a", "d", "e", "f", "g", "b", "c");

QuerySnapshot getSnapshot = waitFor(orderedQuery.get());
List<String> getSnapshotDocIds =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,44 @@ public static int compareIntegers(int i1, int i2) {

/** Compare strings in UTF-8 encoded byte order */
public static int compareUtf8Strings(String left, String right) {
ByteString leftBytes = ByteString.copyFromUtf8(left);
ByteString rightBytes = ByteString.copyFromUtf8(right);
return compareByteStrings(leftBytes, rightBytes);
int i = 0;
while (i < left.length() && i < right.length()) {
int leftCodePoint = left.codePointAt(i);
int rightCodePoint = right.codePointAt(i);

if (leftCodePoint != rightCodePoint) {
if (leftCodePoint < 128 && rightCodePoint < 128) {
// ASCII comparison
return Integer.compare(leftCodePoint, rightCodePoint);
} else {
// substring and do UTF-8 encoded byte comparison
ByteString leftBytes = ByteString.copyFromUtf8(getUtf8SafeBytes(left, i));
ByteString rightBytes = ByteString.copyFromUtf8(getUtf8SafeBytes(right, i));
int comp = compareByteStrings(leftBytes, rightBytes);
if (comp != 0) {
return comp;
} else {
// EXTREMELY RARE CASE: Code points differ, but their UTF-8 byte representations are
// identical. This can happen with malformed input (invalid surrogate pairs), where
// Java's encoding leads to unexpected byte sequences. Meanwhile, any invalid surrogate
// inputs get converted to "?" by protocol buffer while round tripping, so we almost
// never receive invalid strings from backend.
// Fallback to code point comparison for graceful handling.
return Integer.compare(leftCodePoint, rightCodePoint);
}
}
}
// Increment by 2 for surrogate pairs, 1 otherwise.
i += Character.charCount(leftCodePoint);
}

// Compare lengths if all characters are equal
return Integer.compare(left.length(), right.length());
}

private static String getUtf8SafeBytes(String str, int index) {
int firstCodePoint = str.codePointAt(index);
return str.substring(index, index + Character.charCount(firstCodePoint));
}

/**
Expand Down
Loading
Loading