Skip to content

Commit 3689da3

Browse files
billyevansjeffwidman
authored andcommitted
Pre-compile pack/unpack function calls
I noticed that pack/unpack functions from https://github.com/dpkp/kafka-python/blob/master/kafka/protocol/types.py might be slightly improved. I made pre-compilation for them. It gives about 10% better performance compared to the current implementation. Consumption of 100msg: ``` 239884 0.187 0.000 0.287 0.000 types.py:18(_unpack) # new version 239884 0.192 0.000 0.323 0.000 types.py:17(_unpack) ``` I also made some profiling for producers/consumers. It gives about 1-1.5% time savings.
1 parent 4d13713 commit 3689da3

File tree

1 file changed

+29
-13
lines changed

1 file changed

+29
-13
lines changed

kafka/protocol/types.py

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,14 @@
11
from __future__ import absolute_import
22

3-
from struct import pack, unpack, error
3+
import struct
4+
from struct import error
45

56
from kafka.protocol.abstract import AbstractType
67

78

89
def _pack(f, value):
910
try:
10-
return pack(f, value)
11+
return f(value)
1112
except error as e:
1213
raise ValueError("Error encountered when attempting to convert value: "
1314
"{!r} to struct format: '{}', hit error: {}"
@@ -16,7 +17,7 @@ def _pack(f, value):
1617

1718
def _unpack(f, data):
1819
try:
19-
(value,) = unpack(f, data)
20+
(value,) = f(data)
2021
return value
2122
except error as e:
2223
raise ValueError("Error encountered when attempting to convert value: "
@@ -25,43 +26,55 @@ def _unpack(f, data):
2526

2627

2728
class Int8(AbstractType):
29+
_pack = struct.Struct('>b').pack
30+
_unpack = struct.Struct('>b').unpack
31+
2832
@classmethod
2933
def encode(cls, value):
30-
return _pack('>b', value)
34+
return _pack(cls._pack, value)
3135

3236
@classmethod
3337
def decode(cls, data):
34-
return _unpack('>b', data.read(1))
38+
return _unpack(cls._unpack, data.read(1))
3539

3640

3741
class Int16(AbstractType):
42+
_pack = struct.Struct('>h').pack
43+
_unpack = struct.Struct('>h').unpack
44+
3845
@classmethod
3946
def encode(cls, value):
40-
return _pack('>h', value)
47+
return _pack(cls._pack, value)
4148

4249
@classmethod
4350
def decode(cls, data):
44-
return _unpack('>h', data.read(2))
51+
return _unpack(cls._unpack, data.read(2))
4552

4653

4754
class Int32(AbstractType):
55+
_pack = struct.Struct('>i').pack
56+
_unpack = struct.Struct('>i').unpack
57+
4858
@classmethod
4959
def encode(cls, value):
50-
return _pack('>i', value)
60+
return _pack(cls._pack, value)
5161

5262
@classmethod
5363
def decode(cls, data):
54-
return _unpack('>i', data.read(4))
64+
return _unpack(cls._unpack, data.read(4))
5565

5666

5767
class Int64(AbstractType):
68+
_pack = struct.Struct('>q').pack
69+
_unpack = struct.Struct('>q').unpack
70+
5871
@classmethod
5972
def encode(cls, value):
60-
return _pack('>q', value)
73+
return _pack(cls._pack, value)
6174

6275
@classmethod
6376
def decode(cls, data):
64-
return _unpack('>q', data.read(8))
77+
return _unpack(cls._unpack, data.read(8))
6578

6679

6780
class String(AbstractType):
@@ -108,13 +121,16 @@ def repr(cls, value):
108121

109122

110123
class Boolean(AbstractType):
124+
_pack = struct.Struct('>?').pack
125+
_unpack = struct.Struct('>?').unpack
126+
111127
@classmethod
112128
def encode(cls, value):
113-
return _pack('>?', value)
129+
return _pack(cls._pack, value)
114130

115131
@classmethod
116132
def decode(cls, data):
117-
return _unpack('>?', data.read(1))
133+
return _unpack(cls._unpack, data.read(1))
118134

119135

120136
class Schema(AbstractType):

0 commit comments

Comments
 (0)