Skip to content

Commit 0665c24

Browse files
BenWhiteheadBenWhitehead
authored
fix: update gRPC object list implementation to include synthetic directories (#1824)
Rework ITObjectTest.testListBlobsCurrentDirectory to have a more descriptive name along with a more straight forward implementation and assertions. (Removed all the listing retry stuff, object list is strongly consistent in GCS.) * test: remove obsolete test which is validating field mask which is tested separately
1 parent a3b46cf commit 0665c24

File tree

3 files changed

+227
-211
lines changed

3 files changed

+227
-211
lines changed

‎google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcStorageImpl.java

Lines changed: 135 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
import com.google.storage.v2.ListBucketsRequest;
9393
import com.google.storage.v2.ListHmacKeysRequest;
9494
import com.google.storage.v2.ListObjectsRequest;
95+
import com.google.storage.v2.ListObjectsResponse;
9596
import com.google.storage.v2.LockBucketRetentionPolicyRequest;
9697
import com.google.storage.v2.Object;
9798
import com.google.storage.v2.ObjectAccessControl;
@@ -101,8 +102,6 @@
101102
import com.google.storage.v2.RewriteObjectRequest;
102103
import com.google.storage.v2.RewriteResponse;
103104
import com.google.storage.v2.StorageClient;
104-
import com.google.storage.v2.StorageClient.ListObjectsPage;
105-
import com.google.storage.v2.StorageClient.ListObjectsPagedResponse;
106105
import com.google.storage.v2.UpdateBucketRequest;
107106
import com.google.storage.v2.UpdateHmacKeyRequest;
108107
import com.google.storage.v2.UpdateObjectRequest;
@@ -127,6 +126,7 @@
127126
import java.nio.file.StandardOpenOption;
128127
import java.util.Arrays;
129128
import java.util.List;
129+
import java.util.Map;
130130
import java.util.Objects;
131131
import java.util.Optional;
132132
import java.util.Set;
@@ -461,22 +461,18 @@ public Page<Bucket> list(BucketListOption... options) {
461461

462462
@Override
463463
public Page<Blob> list(String bucket, BlobListOption... options) {
464-
UnaryCallable<ListObjectsRequest, ListObjectsPagedResponse> listObjectsCallable =
465-
storageClient.listObjectsPagedCallable();
466464
Opts<ObjectListOpt> opts = Opts.unwrap(options);
467465
GrpcCallContext grpcCallContext =
468466
opts.grpcMetadataMapper().apply(GrpcCallContext.createDefault());
469467
ListObjectsRequest.Builder builder =
470468
ListObjectsRequest.newBuilder().setParent(bucketNameCodec.encode(bucket));
471469
ListObjectsRequest req = opts.listObjectsRequest().apply(builder).build();
472470
try {
473-
ListObjectsPagedResponse call = listObjectsCallable.call(req, grpcCallContext);
474-
ListObjectsPage page = call.getPage();
475-
return new TransformingPageDecorator<>(
476-
page,
477-
syntaxDecoders.blob.andThen(opts.clearBlobFields()),
471+
return Retrying.run(
478472
getOptions(),
479-
retryAlgorithmManager.getFor(req));
473+
retryAlgorithmManager.getFor(req),
474+
() -> storageClient.listObjectsCallable().call(req, grpcCallContext),
475+
resp -> new ListObjectsWithSyntheticDirectoriesPage(grpcCallContext, req, resp));
480476
} catch (Exception e) {
481477
throw StorageException.coalesce(e);
482478
}
@@ -1440,6 +1436,95 @@ private final class SyntaxDecoders {
14401436
b -> codecs.bucketInfo().decode(b).asBucket(GrpcStorageImpl.this);
14411437
}
14421438

1439+
/**
1440+
* Today {@link com.google.cloud.storage.spi.v1.HttpStorageRpc#list(String, Map)} creates
1441+
* synthetic objects to represent {@code prefixes} ("directories") returned as part of a list
1442+
* objects response. Specifically, a StorageObject with an `isDirectory` attribute added.
1443+
*
1444+
* <p>This approach is not sound, and presents an otherwise ephemeral piece of metadata as an
1445+
* actual piece of data. (A {@code prefix} is not actually an object, and therefor can't be
1446+
* queried for other object metadata.)
1447+
*
1448+
* <p>In an effort to preserve compatibility with the current public API, this class attempts to
1449+
* encapsulate the process of producing these Synthetic Directory Objects and lifting them into
1450+
* the Page.
1451+
*
1452+
* <p>This behavior should NOT be carried forward to any possible new API for the storage client.
1453+
*/
1454+
private final class ListObjectsWithSyntheticDirectoriesPage implements Page<Blob> {
1455+
1456+
private final GrpcCallContext ctx;
1457+
private final ListObjectsRequest req;
1458+
private final ListObjectsResponse resp;
1459+
1460+
private ListObjectsWithSyntheticDirectoriesPage(
1461+
GrpcCallContext ctx, ListObjectsRequest req, ListObjectsResponse resp) {
1462+
this.ctx = ctx;
1463+
this.req = req;
1464+
this.resp = resp;
1465+
}
1466+
1467+
@Override
1468+
public boolean hasNextPage() {
1469+
return !resp.getNextPageToken().isEmpty();
1470+
}
1471+
1472+
@Override
1473+
public String getNextPageToken() {
1474+
return resp.getNextPageToken();
1475+
}
1476+
1477+
@Override
1478+
public Page<Blob> getNextPage() {
1479+
ListObjectsRequest nextPageReq =
1480+
req.toBuilder().setPageToken(resp.getNextPageToken()).build();
1481+
try {
1482+
ListObjectsResponse nextPageResp =
1483+
Retrying.run(
1484+
GrpcStorageImpl.this.getOptions(),
1485+
retryAlgorithmManager.getFor(nextPageReq),
1486+
() -> storageClient.listObjectsCallable().call(nextPageReq, ctx),
1487+
Decoder.identity());
1488+
return new ListObjectsWithSyntheticDirectoriesPage(ctx, nextPageReq, nextPageResp);
1489+
} catch (Exception e) {
1490+
throw StorageException.coalesce(e);
1491+
}
1492+
}
1493+
1494+
@Override
1495+
public Iterable<Blob> iterateAll() {
1496+
// drop to our interface type to help type inference below with the stream.
1497+
Page<Blob> curr = this;
1498+
Predicate<Page<Blob>> exhausted = p -> p != null && p.hasNextPage();
1499+
// Create a stream which will attempt to call getNextPage repeatedly until we meet our
1500+
// condition of exhaustion. By doing this we are able to rely on the retry logic in
1501+
// getNextPage
1502+
return () ->
1503+
streamIterate(curr, exhausted, Page::getNextPage)
1504+
.filter(Objects::nonNull)
1505+
.flatMap(p -> StreamSupport.stream(p.getValues().spliterator(), false))
1506+
.iterator();
1507+
}
1508+
1509+
@Override
1510+
public Iterable<Blob> getValues() {
1511+
return () -> {
1512+
String bucketName = bucketNameCodec.decode(req.getParent());
1513+
return Streams.concat(
1514+
resp.getObjectsList().stream().map(syntaxDecoders.blob::decode),
1515+
resp.getPrefixesList().stream()
1516+
.map(
1517+
prefix ->
1518+
BlobInfo.newBuilder(bucketName, prefix)
1519+
.setSize(0L)
1520+
.setIsDirectory(true)
1521+
.build())
1522+
.map(info -> info.asBlob(GrpcStorageImpl.this)))
1523+
.iterator();
1524+
};
1525+
}
1526+
}
1527+
14431528
static final class TransformingPageDecorator<
14441529
RequestT,
14451530
ResponseT,
@@ -1511,50 +1596,50 @@ public Iterable<ModelT> getValues() {
15111596
.map(translator::decode)
15121597
.iterator();
15131598
}
1599+
}
15141600

1515-
private static <T> Stream<T> streamIterate(
1516-
T seed, Predicate<? super T> shouldComputeNext, UnaryOperator<T> computeNext) {
1517-
requireNonNull(seed, "seed must be non null");
1518-
requireNonNull(shouldComputeNext, "shouldComputeNext must be non null");
1519-
requireNonNull(computeNext, "computeNext must be non null");
1520-
Spliterator<T> spliterator =
1521-
new AbstractSpliterator<T>(Long.MAX_VALUE, 0) {
1522-
T prev;
1523-
boolean started = false;
1524-
boolean done = false;
1525-
1526-
@Override
1527-
public boolean tryAdvance(Consumer<? super T> action) {
1528-
// if we haven't started, emit our seed and return
1529-
if (!started) {
1530-
started = true;
1531-
action.accept(seed);
1532-
prev = seed;
1601+
private static <T> Stream<T> streamIterate(
1602+
T seed, Predicate<? super T> shouldComputeNext, UnaryOperator<T> computeNext) {
1603+
requireNonNull(seed, "seed must be non null");
1604+
requireNonNull(shouldComputeNext, "shouldComputeNext must be non null");
1605+
requireNonNull(computeNext, "computeNext must be non null");
1606+
Spliterator<T> spliterator =
1607+
new AbstractSpliterator<T>(Long.MAX_VALUE, 0) {
1608+
T prev;
1609+
boolean started = false;
1610+
boolean done = false;
1611+
1612+
@Override
1613+
public boolean tryAdvance(Consumer<? super T> action) {
1614+
// if we haven't started, emit our seed and return
1615+
if (!started) {
1616+
started = true;
1617+
action.accept(seed);
1618+
prev = seed;
1619+
return true;
1620+
}
1621+
// if we've previously finished quickly return
1622+
if (done) {
1623+
return false;
1624+
}
1625+
// test whether we should try and compute the next value
1626+
if (shouldComputeNext.test(prev)) {
1627+
// compute the next value and figure out if we can use it
1628+
T next = computeNext.apply(prev);
1629+
if (next != null) {
1630+
action.accept(next);
1631+
prev = next;
15331632
return true;
15341633
}
1535-
// if we've previously finished quickly return
1536-
if (done) {
1537-
return false;
1538-
}
1539-
// test whether we should try and compute the next value
1540-
if (shouldComputeNext.test(prev)) {
1541-
// compute the next value and figure out if we can use it
1542-
T next = computeNext.apply(prev);
1543-
if (next != null) {
1544-
action.accept(next);
1545-
prev = next;
1546-
return true;
1547-
}
1548-
}
1549-
1550-
// fallthrough, if we haven't taken an action by now consider the stream done and
1551-
// return
1552-
done = true;
1553-
return false;
15541634
}
1555-
};
1556-
return StreamSupport.stream(spliterator, false);
1557-
}
1635+
1636+
// fallthrough, if we haven't taken an action by now consider the stream done and
1637+
// return
1638+
done = true;
1639+
return false;
1640+
}
1641+
};
1642+
return StreamSupport.stream(spliterator, false);
15581643
}
15591644

15601645
static <T> T throwHttpJsonOnly(String methodName) {

‎google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITKmsTest.java

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import static org.junit.Assert.assertNull;
2525
import static org.junit.Assert.assertTrue;
2626

27-
import com.google.api.gax.paging.Page;
2827
import com.google.cloud.WriteChannel;
2928
import com.google.cloud.storage.Blob;
3029
import com.google.cloud.storage.BlobId;
@@ -41,19 +40,14 @@
4140
import com.google.cloud.storage.it.runner.annotations.Backend;
4241
import com.google.cloud.storage.it.runner.annotations.CrossRun;
4342
import com.google.cloud.storage.it.runner.annotations.Inject;
44-
import com.google.cloud.storage.it.runner.annotations.ParallelFriendly;
4543
import com.google.cloud.storage.it.runner.registry.Generator;
4644
import com.google.cloud.storage.it.runner.registry.KmsFixture;
4745
import com.google.common.collect.ImmutableMap;
48-
import com.google.common.collect.ImmutableSet;
49-
import com.google.common.collect.Iterators;
5046
import com.google.common.io.BaseEncoding;
5147
import java.io.IOException;
5248
import java.nio.ByteBuffer;
5349
import java.security.Key;
54-
import java.util.Iterator;
5550
import java.util.Random;
56-
import java.util.Set;
5751
import javax.crypto.spec.SecretKeySpec;
5852
import org.junit.Rule;
5953
import org.junit.Test;
@@ -63,7 +57,6 @@
6357
@CrossRun(
6458
transports = {Transport.HTTP, Transport.GRPC},
6559
backends = Backend.PROD)
66-
@ParallelFriendly
6760
public class ITKmsTest {
6861

6962
private static final long seed = -7071346537822433445L;
@@ -186,46 +179,6 @@ public void testGetBlobKmsKeyNameField() {
186179
assertNull(remoteBlob.getContentType());
187180
}
188181

189-
@Test(timeout = 5000)
190-
public void testListBlobsKmsKeySelectedFields() throws InterruptedException {
191-
String[] blobNames = {
192-
"test-list-blobs-selected-field-kms-key-name-blob1",
193-
"test-list-blobs-selected-field-kms-key-name-blob2"
194-
};
195-
BlobInfo blob1 = BlobInfo.newBuilder(bucket, blobNames[0]).setContentType(CONTENT_TYPE).build();
196-
BlobInfo blob2 = BlobInfo.newBuilder(bucket, blobNames[1]).setContentType(CONTENT_TYPE).build();
197-
Blob remoteBlob1 =
198-
storage.create(blob1, Storage.BlobTargetOption.kmsKeyName(kms.getKey1().getName()));
199-
Blob remoteBlob2 =
200-
storage.create(blob2, Storage.BlobTargetOption.kmsKeyName(kms.getKey1().getName()));
201-
assertNotNull(remoteBlob1);
202-
assertNotNull(remoteBlob2);
203-
Page<Blob> page =
204-
storage.list(
205-
bucket.getName(),
206-
Storage.BlobListOption.prefix("test-list-blobs-selected-field-kms-key-name-blob"),
207-
Storage.BlobListOption.fields(BlobField.KMS_KEY_NAME));
208-
// Listing blobs is eventually consistent, we loop until the list is of the expected size. The
209-
// test fails if timeout is reached.
210-
while (Iterators.size(page.iterateAll().iterator()) != 2) {
211-
Thread.sleep(500);
212-
page =
213-
storage.list(
214-
bucket.getName(),
215-
Storage.BlobListOption.prefix("test-list-blobs-selected-field-kms-key-name-blob"),
216-
Storage.BlobListOption.fields(BlobField.KMS_KEY_NAME));
217-
}
218-
Set<String> blobSet = ImmutableSet.of(blobNames[0], blobNames[1]);
219-
Iterator<Blob> iterator = page.iterateAll().iterator();
220-
while (iterator.hasNext()) {
221-
Blob remoteBlob = iterator.next();
222-
assertEquals(bucket.getName(), remoteBlob.getBucket());
223-
assertTrue(blobSet.contains(remoteBlob.getName()));
224-
assertTrue(remoteBlob.getKmsKeyName().startsWith(kms.getKey1().getName()));
225-
assertNull(remoteBlob.getContentType());
226-
}
227-
}
228-
229182
@Test
230183
public void testRotateFromCustomerEncryptionToKmsKey() {
231184
String sourceBlobName = "test-copy-blob-encryption-key-source";

0 commit comments

Comments
 (0)