Skip to content

Commit 9b16ee4

Browse files
thomasvlthomasvl
committed
[ObjC] Parsing helper and tests around unknown fields.
Make a new internal api for collecting up the unknown group fields that will be used in a future change. Add testing for the new api and for unknown field parsing of groups in general. PiperOrigin-RevId: 654875605
1 parent fad7b78 commit 9b16ee4

5 files changed

+493
-1
lines changed

‎objectivec/GPBCodedInputStream.m

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -267,6 +267,60 @@ int32_t GPBCodedInputStreamReadTag(GPBCodedInputStreamState *state) {
267267
return result;
268268
}
269269

270+
static void SkipToEndGroupInternal(GPBCodedInputStreamState *state, uint32_t endGroupTag) {
271+
CheckRecursionLimit(state);
272+
++state->recursionDepth;
273+
while (YES) {
274+
uint32_t tag = GPBCodedInputStreamReadTag(state);
275+
if (tag == endGroupTag || tag == 0) {
276+
GPBCodedInputStreamCheckLastTagWas(state, endGroupTag); // Will fail for end of input.
277+
--state->recursionDepth;
278+
return;
279+
}
280+
switch (GPBWireFormatGetTagWireType(tag)) {
281+
case GPBWireFormatVarint:
282+
(void)ReadRawVarint64(state);
283+
break;
284+
case GPBWireFormatFixed64:
285+
SkipRawData(state, sizeof(uint64_t));
286+
break;
287+
case GPBWireFormatLengthDelimited: {
288+
uint64_t size = ReadRawVarint64(state);
289+
CheckFieldSize(size);
290+
size_t size2 = (size_t)size; // Cast safe on 32bit because of CheckFieldSize() above.
291+
SkipRawData(state, size2);
292+
break;
293+
}
294+
case GPBWireFormatStartGroup:
295+
SkipToEndGroupInternal(state, GPBWireFormatMakeTag(GPBWireFormatGetTagFieldNumber(tag),
296+
GPBWireFormatEndGroup));
297+
break;
298+
case GPBWireFormatEndGroup:
299+
GPBRaiseStreamError(GPBCodedInputStreamErrorInvalidTag, @"Unmatched end group");
300+
break;
301+
case GPBWireFormatFixed32:
302+
SkipRawData(state, sizeof(uint32_t));
303+
break;
304+
}
305+
}
306+
}
307+
308+
// This doesn't include the start group, but it collects all bytes until the end group including
309+
// the end group tag.
310+
NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state,
311+
int32_t fieldNumber) {
312+
// Better have just read the start of the group.
313+
GPBCodedInputStreamCheckLastTagWas(state,
314+
GPBWireFormatMakeTag(fieldNumber, GPBWireFormatStartGroup));
315+
const uint8_t *start = state->bytes + state->bufferPos;
316+
SkipToEndGroupInternal(state, GPBWireFormatMakeTag(fieldNumber, GPBWireFormatEndGroup));
317+
// This will be after the end group tag.
318+
const uint8_t *end = state->bytes + state->bufferPos;
319+
return [[NSData alloc] initWithBytesNoCopy:(void *)start
320+
length:(NSUInteger)(end - start)
321+
freeWhenDone:NO];
322+
}
323+
270324
size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit) {
271325
byteLimit += state->bufferPos;
272326
size_t oldLimit = state->currentLimit;

‎objectivec/GPBCodedInputStream_PackagePrivate.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ NSData *GPBCodedInputStreamReadRetainedBytes(GPBCodedInputStreamState *state)
7676
__attribute((ns_returns_retained));
7777
NSData *GPBCodedInputStreamReadRetainedBytesNoCopy(GPBCodedInputStreamState *state)
7878
__attribute((ns_returns_retained));
79+
NSData *GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(GPBCodedInputStreamState *state,
80+
int32_t fieldNumber)
81+
__attribute((ns_returns_retained));
7982

8083
size_t GPBCodedInputStreamPushLimit(GPBCodedInputStreamState *state, size_t byteLimit);
8184
void GPBCodedInputStreamPopLimit(GPBCodedInputStreamState *state, size_t oldLimit);

‎objectivec/Tests/GPBCodedInputStreamTests.m

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@
66
// https://developers.google.com/open-source/licenses/bsd
77

88
#import <Foundation/Foundation.h>
9-
#import "GPBTestUtilities.h"
109

1110
#import "GPBCodedInputStream.h"
11+
#import "GPBCodedInputStream_PackagePrivate.h"
1212
#import "GPBCodedOutputStream.h"
13+
#import "GPBTestUtilities.h"
1314
#import "GPBUnknownFieldSet_PackagePrivate.h"
1415
#import "GPBUtilities_PackagePrivate.h"
16+
#import "GPBWireFormat.h"
1517
#import "objectivec/Tests/Unittest.pbobjc.h"
1618

1719
@interface CodedInputStreamTests : GPBTestCase
@@ -444,4 +446,102 @@ - (void)testBOMWithinStrings {
444446
}
445447
}
446448

449+
- (void)assertReadByteToEndGroupFails:(NSData*)data {
450+
GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data];
451+
uint32_t tag = [input readTag];
452+
XCTAssertThrows(GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(
453+
&input->state_, GPBWireFormatGetTagFieldNumber(tag)));
454+
}
455+
456+
- (void)assertReadByteToEndGroup:(NSData*)data value:(NSData*)value {
457+
GPBCodedInputStream* input = [GPBCodedInputStream streamWithData:data];
458+
uint32_t tag = [input readTag];
459+
NSData* readValue = GPBCodedInputStreamReadRetainedBytesToEndGroupNoCopy(
460+
&input->state_, GPBWireFormatGetTagFieldNumber(tag));
461+
XCTAssertNotNil(readValue);
462+
XCTAssertEqualObjects(readValue, value);
463+
[readValue release];
464+
}
465+
466+
static NSData* DataForGroupsOfDepth(NSUInteger depth) {
467+
NSMutableData* data = [NSMutableData dataWithCapacity:0];
468+
469+
uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group
470+
for (NSUInteger i = 0; i < depth; ++i) {
471+
[data appendBytes:&byte length:1];
472+
}
473+
474+
byte = 8; // 8 = 0b1000, -> field 1/varint
475+
[data appendBytes:&byte length:1];
476+
byte = 1; // 1 -> varint value of 1
477+
[data appendBytes:&byte length:1];
478+
479+
byte = 36; // 36 = 0b100100 -> field 4/end group
480+
for (NSUInteger i = 0; i < depth; ++i) {
481+
[data appendBytes:&byte length:1];
482+
}
483+
return data;
484+
}
485+
486+
- (void)testBytesToEndGroup {
487+
// 35 = 0b100011 -> field 4/start group
488+
// 36 = 0b100100 -> field 4/end group
489+
// 43 = 0b101011 -> field 5/end group
490+
// 44 = 0b101100 -> field 5/end group
491+
// 8 = 0b1000, 1 -> field 1/varint, value of 1
492+
// 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678
493+
// 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64,
494+
// value of 0x123456789abcdef0LL
495+
// 50 = 0b110010, 0x0 -> field 6/length delimited, length 0
496+
// 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42
497+
// 0 -> field 0 which is invalid/varint
498+
// 15 = 0b1111 -> field 1, wire type 7 which is invalid
499+
500+
[self assertReadByteToEndGroup:bytes(35, 36) value:bytes(36)]; // empty group
501+
[self assertReadByteToEndGroup:bytes(35, 8, 1, 36) value:bytes(8, 1, 36)]; // varint
502+
[self assertReadByteToEndGroup:bytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36) // fixed32
503+
value:bytes(21, 0x78, 0x56, 0x34, 0x12, 36)];
504+
[self assertReadByteToEndGroup:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12,
505+
36) // fixed64
506+
value:bytes(25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12, 36)];
507+
[self assertReadByteToEndGroup:bytes(35, 50, 0, 36)
508+
value:bytes(50, 0, 36)]; // length delimited, length 0
509+
[self assertReadByteToEndGroup:bytes(35, 50, 1, 42, 36)
510+
value:bytes(50, 1, 42, 36)]; // length delimited, length 1, byte 42
511+
[self assertReadByteToEndGroup:bytes(35, 43, 44, 36) value:bytes(43, 44, 36)]; // Sub group
512+
[self assertReadByteToEndGroup:bytes(35, 8, 1, 43, 8, 1, 44,
513+
36) // varint and sub group with varint
514+
value:bytes(8, 1, 43, 8, 1, 44, 36)];
515+
516+
[self assertReadByteToEndGroupFails:bytes(35, 0, 36)]; // Invalid field number
517+
[self assertReadByteToEndGroupFails:bytes(35, 15, 36)]; // Invalid wire type
518+
[self assertReadByteToEndGroupFails:bytes(35, 21, 0x78, 0x56, 0x34)]; // truncated fixed32
519+
[self assertReadByteToEndGroupFails:bytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56,
520+
0x34)]; // truncated fixed64
521+
522+
// Mising end group
523+
[self assertReadByteToEndGroupFails:bytes(35)];
524+
[self assertReadByteToEndGroupFails:bytes(35, 8, 1)];
525+
[self assertReadByteToEndGroupFails:bytes(35, 43)];
526+
[self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1)];
527+
528+
// Wrong end group
529+
[self assertReadByteToEndGroupFails:bytes(35, 44)];
530+
[self assertReadByteToEndGroupFails:bytes(35, 8, 1, 44)];
531+
[self assertReadByteToEndGroupFails:bytes(35, 43, 36)];
532+
[self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 36)];
533+
[self assertReadByteToEndGroupFails:bytes(35, 43, 44, 44)];
534+
[self assertReadByteToEndGroupFails:bytes(35, 43, 8, 1, 44, 44)];
535+
536+
// This is the same limit as within GPBCodedInputStream.
537+
const NSUInteger kDefaultRecursionLimit = 100;
538+
// That depth parses.
539+
NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit);
540+
[self assertReadByteToEndGroup:testData
541+
value:[testData subdataWithRange:NSMakeRange(1, testData.length - 1)]];
542+
// One more level deep fails.
543+
testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1);
544+
[self assertReadByteToEndGroupFails:testData];
545+
}
546+
447547
@end

‎objectivec/Tests/GPBUnknownFieldSetTest.m

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
// https://developers.google.com/open-source/licenses/bsd
77

88
#import "GPBTestUtilities.h"
9+
#import "GPBUnknownFieldSet.h"
910

1011
#import "GPBUnknownFieldSet_PackagePrivate.h"
1112
#import "GPBUnknownField_PackagePrivate.h"
@@ -376,6 +377,180 @@ - (void)testLargeVarint {
376377
XCTAssertEqual(0x7FFFFFFFFFFFFFFFULL, [field2.varintList valueAtIndex:0]);
377378
}
378379

380+
static NSData* DataForGroupsOfDepth(NSUInteger depth) {
381+
NSMutableData* data = [NSMutableData dataWithCapacity:0];
382+
383+
uint32_t byte = 35; // 35 = 0b100011 -> field 4/start group
384+
for (NSUInteger i = 0; i < depth; ++i) {
385+
[data appendBytes:&byte length:1];
386+
}
387+
388+
byte = 8; // 8 = 0b1000, -> field 1/varint
389+
[data appendBytes:&byte length:1];
390+
byte = 1; // 1 -> varint value of 1
391+
[data appendBytes:&byte length:1];
392+
393+
byte = 36; // 36 = 0b100100 -> field 4/end group
394+
for (NSUInteger i = 0; i < depth; ++i) {
395+
[data appendBytes:&byte length:1];
396+
}
397+
return data;
398+
}
399+
400+
- (void)testParsingNestingGroupData {
401+
// 35 = 0b100011 -> field 4/start group
402+
// 36 = 0b100100 -> field 4/end group
403+
// 43 = 0b101011 -> field 5/end group
404+
// 44 = 0b101100 -> field 5/end group
405+
// 8 = 0b1000, 1 -> field 1/varint, value of 1
406+
// 21 = 0b10101, 0x78, 0x56, 0x34, 0x12 -> field 2/fixed32, value of 0x12345678
407+
// 25 = 0b11001, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12 -> field 3/fixed64,
408+
// value of 0x123456789abcdef0LL
409+
// 50 = 0b110010, 0x0 -> field 6/length delimited, length 0
410+
// 50 = 0b110010, 0x1, 42 -> field 6/length delimited, length 1, byte 42
411+
// 0 -> field 0 which is invalid/varint
412+
// 15 = 0b1111 -> field 1, wire type 7 which is invalid
413+
414+
TestEmptyMessage* m = [TestEmptyMessage parseFromData:DataFromBytes(35, 36)
415+
error:NULL]; // empty group
416+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
417+
GPBUnknownField* field = [m.unknownFields getField:4];
418+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
419+
GPBUnknownFieldSet* group = field.groupList[0];
420+
XCTAssertEqual(group.countOfFields, (NSUInteger)0);
421+
422+
m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 36) error:NULL]; // varint
423+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
424+
field = [m.unknownFields getField:4];
425+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
426+
group = field.groupList[0];
427+
field = [group getField:1];
428+
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
429+
XCTAssertEqual([field.varintList valueAtIndex:0], 1);
430+
431+
m = [TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34, 0x12, 36)
432+
error:NULL]; // fixed32
433+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
434+
field = [m.unknownFields getField:4];
435+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
436+
group = field.groupList[0];
437+
field = [group getField:2];
438+
XCTAssertEqual(field.fixed32List.count, (NSUInteger)1);
439+
XCTAssertEqual([field.fixed32List valueAtIndex:0], 0x12345678);
440+
441+
m = [TestEmptyMessage
442+
parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12,
443+
36)
444+
error:NULL]; // fixed64
445+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
446+
field = [m.unknownFields getField:4];
447+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
448+
group = field.groupList[0];
449+
field = [group getField:3];
450+
XCTAssertEqual(field.fixed64List.count, (NSUInteger)1);
451+
XCTAssertEqual([field.fixed64List valueAtIndex:0], 0x123456789abcdef0LL);
452+
453+
m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 0, 36)
454+
error:NULL]; // length delimited, length 0
455+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
456+
field = [m.unknownFields getField:4];
457+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
458+
group = field.groupList[0];
459+
field = [group getField:6];
460+
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1);
461+
XCTAssertEqualObjects(field.lengthDelimitedList[0], [NSData data]);
462+
463+
m = [TestEmptyMessage parseFromData:DataFromBytes(35, 50, 1, 42, 36)
464+
error:NULL]; // length delimited, length 1, byte 42
465+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
466+
field = [m.unknownFields getField:4];
467+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
468+
group = field.groupList[0];
469+
field = [group getField:6];
470+
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)1);
471+
XCTAssertEqualObjects(field.lengthDelimitedList[0], DataFromBytes(42));
472+
473+
m = [TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 36) error:NULL]; // Sub group
474+
field = [m.unknownFields getField:4];
475+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
476+
group = field.groupList[0];
477+
XCTAssertEqual(group.countOfFields, (NSUInteger)1);
478+
field = [group getField:5];
479+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
480+
group = field.groupList[0];
481+
XCTAssertEqual(group.countOfFields, (NSUInteger)0);
482+
483+
m = [TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 43, 8, 2, 44, 36)
484+
error:NULL]; // varint and sub group with varint
485+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
486+
field = [m.unknownFields getField:4];
487+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
488+
group = field.groupList[0];
489+
XCTAssertEqual(group.countOfFields, (NSUInteger)2);
490+
field = [group getField:1];
491+
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
492+
XCTAssertEqual([field.varintList valueAtIndex:0], 1);
493+
field = [group getField:5];
494+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
495+
group = field.groupList[0];
496+
field = [group getField:1];
497+
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
498+
XCTAssertEqual([field.varintList valueAtIndex:0], 2);
499+
500+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 0, 36)
501+
error:NULL]); // Invalid field number
502+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 15, 36)
503+
error:NULL]); // Invalid wire type
504+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 21, 0x78, 0x56, 0x34)
505+
error:NULL]); // truncated fixed32
506+
XCTAssertNil([TestEmptyMessage
507+
parseFromData:DataFromBytes(35, 25, 0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56,
508+
0x34)
509+
error:NULL]); // truncated fixed64
510+
511+
// Mising end group
512+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35) error:NULL]);
513+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1) error:NULL]);
514+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43) error:NULL]);
515+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1) error:NULL]);
516+
517+
// Wrong end group
518+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 44) error:NULL]);
519+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 8, 1, 44) error:NULL]);
520+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 36) error:NULL]);
521+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 36) error:NULL]);
522+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 44, 44) error:NULL]);
523+
XCTAssertNil([TestEmptyMessage parseFromData:DataFromBytes(35, 43, 8, 1, 44, 44) error:NULL]);
524+
525+
// This is the same limit as within GPBCodedInputStream.
526+
const NSUInteger kDefaultRecursionLimit = 100;
527+
// That depth parses.
528+
NSData* testData = DataForGroupsOfDepth(kDefaultRecursionLimit);
529+
m = [TestEmptyMessage parseFromData:testData error:NULL];
530+
XCTAssertEqual(m.unknownFields.countOfFields, (NSUInteger)1);
531+
field = [m.unknownFields getField:4];
532+
for (NSUInteger i = 0; i < kDefaultRecursionLimit; ++i) {
533+
XCTAssertEqual(field.varintList.count, (NSUInteger)0);
534+
XCTAssertEqual(field.fixed32List.count, (NSUInteger)0);
535+
XCTAssertEqual(field.fixed64List.count, (NSUInteger)0);
536+
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0);
537+
XCTAssertEqual(field.groupList.count, (NSUInteger)1);
538+
group = field.groupList[0];
539+
XCTAssertEqual(group.countOfFields, (NSUInteger)1);
540+
field = [group getField:(i < (kDefaultRecursionLimit - 1) ? 4 : 1)];
541+
}
542+
// field is of the inner most group
543+
XCTAssertEqual(field.varintList.count, (NSUInteger)1);
544+
XCTAssertEqual([field.varintList valueAtIndex:0], (NSUInteger)1);
545+
XCTAssertEqual(field.fixed32List.count, (NSUInteger)0);
546+
XCTAssertEqual(field.fixed64List.count, (NSUInteger)0);
547+
XCTAssertEqual(field.lengthDelimitedList.count, (NSUInteger)0);
548+
XCTAssertEqual(field.groupList.count, (NSUInteger)0);
549+
// One more level deep fails.
550+
testData = DataForGroupsOfDepth(kDefaultRecursionLimit + 1);
551+
XCTAssertNil([TestEmptyMessage parseFromData:testData error:NULL]);
552+
}
553+
379554
#pragma mark - Field tests
380555
// Some tests directly on fields since the dictionary in FieldSet can gate
381556
// testing some of these.

0 commit comments

Comments
 (0)