kafka.protocol package¶
Submodules¶
kafka.protocol.abstract module¶
kafka.protocol.admin module¶
-
class
kafka.protocol.admin.
ApiVersionRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 18¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
ApiVersionResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.admin.
ApiVersionResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 18¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.admin.
DescribeGroupsRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 15¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
DescribeGroupsResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.admin.
DescribeGroupsResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 15¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.admin.
ListGroupsRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 16¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
ListGroupsResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.admin.
ListGroupsResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 16¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.api module¶
-
class
kafka.protocol.api.
RequestHeader
(request, correlation_id=0, client_id='kafka-python')¶ Bases:
kafka.protocol.struct.Struct
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.commit module¶
-
class
kafka.protocol.commit.
GroupCoordinatorRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 10¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
GroupCoordinatorResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
GroupCoordinatorResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 10¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetCommitRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 8¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
OffsetCommitResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetCommitRequest_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 8¶
-
API_VERSION
= 1¶
-
RESPONSE_TYPE
¶ alias of
OffsetCommitResponse_v1
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetCommitRequest_v2
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 8¶
-
API_VERSION
= 2¶
-
DEFAULT_GENERATION_ID
= -1¶
-
DEFAULT_RETENTION_TIME
= -1¶
-
RESPONSE_TYPE
¶ alias of
OffsetCommitResponse_v2
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetCommitResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 8¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetCommitResponse_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 8¶
-
API_VERSION
= 1¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetCommitResponse_v2
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 8¶
-
API_VERSION
= 2¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetFetchRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 9¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
OffsetFetchResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetFetchRequest_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 9¶
-
API_VERSION
= 1¶
-
RESPONSE_TYPE
¶ alias of
OffsetFetchResponse_v1
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetFetchResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 9¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.commit.
OffsetFetchResponse_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 9¶
-
API_VERSION
= 1¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.fetch module¶
-
class
kafka.protocol.fetch.
FetchRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 1¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
FetchResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.fetch.
FetchRequest_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 1¶
-
API_VERSION
= 1¶
-
RESPONSE_TYPE
¶ alias of
FetchResponse_v1
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.fetch.
FetchRequest_v2
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 1¶
-
API_VERSION
= 2¶
-
RESPONSE_TYPE
¶ alias of
FetchResponse_v2
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.fetch.
FetchResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 1¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.fetch.
FetchResponse_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 1¶
-
API_VERSION
= 1¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.fetch.
FetchResponse_v2
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 1¶
-
API_VERSION
= 2¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.group module¶
-
class
kafka.protocol.group.
HeartbeatRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 12¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
HeartbeatResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
HeartbeatResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 12¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
JoinGroupRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 11¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
JoinGroupResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
UNKNOWN_MEMBER_ID
= ''¶
-
-
class
kafka.protocol.group.
JoinGroupResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 11¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
LeaveGroupRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 13¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
LeaveGroupResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
LeaveGroupResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 13¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
MemberAssignment
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
ProtocolMetadata
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
SyncGroupRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 14¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
SyncGroupResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.group.
SyncGroupResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 14¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.legacy module¶
-
class
kafka.protocol.legacy.
KafkaProtocol
¶ Bases:
object
Class to encapsulate all of the protocol encoding/decoding. This class does not have any state associated with it, it is purely for organization.
-
CONSUMER_METADATA_KEY
= 10¶
-
FETCH_KEY
= 1¶
-
METADATA_KEY
= 3¶
-
OFFSET_COMMIT_KEY
= 8¶
-
OFFSET_FETCH_KEY
= 9¶
-
OFFSET_KEY
= 2¶
-
PRODUCE_KEY
= 0¶
-
classmethod
decode_consumer_metadata_response
(data)¶ Decode bytes to a ConsumerMetadataResponse
Parameters: data – bytes to decode
-
classmethod
decode_fetch_response
(response)¶ Decode FetchResponse struct to FetchResponsePayloads
Parameters: response – FetchResponse
-
classmethod
decode_message_set
(messages)¶
-
classmethod
decode_metadata_response
(response)¶
-
classmethod
decode_offset_commit_response
(response)¶ Decode OffsetCommitResponse to an OffsetCommitResponsePayload
Parameters: response – OffsetCommitResponse
-
classmethod
decode_offset_fetch_response
(response)¶ Decode OffsetFetchResponse to OffsetFetchResponsePayloads
Parameters: response – OffsetFetchResponse
-
classmethod
decode_offset_response
(response)¶ Decode OffsetResponse into OffsetResponsePayloads
Parameters: response – OffsetResponse Returns: list of OffsetResponsePayloads
-
classmethod
decode_produce_response
(response)¶ Decode ProduceResponse to ProduceResponsePayload
Parameters: response – ProduceResponse Return: list of ProduceResponsePayload
-
classmethod
encode_consumer_metadata_request
(client_id, correlation_id, payloads)¶ Encode a ConsumerMetadataRequest
Parameters: - client_id – string
- correlation_id – int
- payloads – string (consumer group)
-
classmethod
encode_fetch_request
(payloads=(), max_wait_time=100, min_bytes=4096)¶ Encodes a FetchRequest struct
Parameters: - payloads – list of FetchRequestPayload
- max_wait_time (int, optional) – ms to block waiting for min_bytes data. Defaults to 100.
- min_bytes (int, optional) – minimum bytes required to return before max_wait_time. Defaults to 4096.
Return: FetchRequest
-
classmethod
encode_metadata_request
(topics=(), payloads=None)¶ Encode a MetadataRequest
Parameters: topics – list of strings
-
classmethod
encode_offset_commit_request
(group, payloads)¶ Encode an OffsetCommitRequest struct
Parameters: - group – string, the consumer group you are committing offsets for
- payloads – list of OffsetCommitRequestPayload
-
classmethod
encode_offset_fetch_request
(group, payloads, from_kafka=False)¶ Encode an OffsetFetchRequest struct. The request is encoded using version 0 if from_kafka is false, indicating a request for Zookeeper offsets. It is encoded using version 1 otherwise, indicating a request for Kafka offsets.
Parameters: - group – string, the consumer group you are fetching offsets for
- payloads – list of OffsetFetchRequestPayload
- from_kafka – bool, default False, set True for Kafka-committed offsets
-
classmethod
encode_offset_request
(payloads=())¶
-
classmethod
encode_produce_request
(payloads=(), acks=1, timeout=1000)¶ Encode a ProduceRequest struct
Parameters: - payloads – list of ProduceRequestPayload
- acks – How “acky” you want the request to be 1: written to disk by the leader 0: immediate response -1: waits for all replicas to be in sync
- timeout – Maximum time (in ms) the server will wait for replica acks. This is _not_ a socket timeout
Returns: ProduceRequest
-
-
kafka.protocol.legacy.
create_gzip_message
(payloads, key=None, compresslevel=None)¶ Construct a Gzipped Message containing multiple Messages
The given payloads will be encoded, compressed, and sent as a single atomic message to Kafka.
Parameters: - payloads – list(bytes), a list of payload to send be sent to Kafka
- key – bytes, a key used for partition routing (optional)
-
kafka.protocol.legacy.
create_message
(payload, key=None)¶ Construct a Message
Parameters: - payload – bytes, the payload to send to Kafka
- key – bytes, a key used for partition routing (optional)
-
kafka.protocol.legacy.
create_message_set
(messages, codec=0, key=None, compresslevel=None)¶ Create a message set using the given codec.
If codec is CODEC_NONE, return a list of raw Kafka messages. Otherwise, return a list containing a single codec-encoded message.
-
kafka.protocol.legacy.
create_snappy_message
(payloads, key=None)¶ Construct a Snappy Message containing multiple Messages
The given payloads will be encoded, compressed, and sent as a single atomic message to Kafka.
Parameters: - payloads – list(bytes), a list of payload to send be sent to Kafka
- key – bytes, a key used for partition routing (optional)
kafka.protocol.message module¶
-
class
kafka.protocol.message.
Message
(value, key=None, magic=0, attributes=0, crc=0, timestamp=None)¶ Bases:
kafka.protocol.struct.Struct
-
CODEC_GZIP
= 1¶
-
CODEC_LZ4
= 3¶
-
CODEC_MASK
= 7¶
-
CODEC_SNAPPY
= 2¶
-
HEADER_SIZE
= 22¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
SCHEMAS
= [<kafka.protocol.types.Schema object at 0x7f989cebf190>, <kafka.protocol.types.Schema object at 0x7f989cebf2d0>]¶
-
TIMESTAMP_TYPE_MASK
= 8¶
-
classmethod
decode
(data)¶
-
decompress
()¶
-
is_compressed
()¶
-
timestamp_type
¶ 0 for CreateTime; 1 for LogAppendTime; None if unsupported.
Value is determined by broker; produced messages should always set to 0 Requires Kafka >= 0.10 / message version >= 1
-
validate_crc
()¶
-
-
class
kafka.protocol.message.
MessageSet
¶ Bases:
kafka.protocol.abstract.AbstractType
-
HEADER_SIZE
= 12¶
-
ITEM
= <kafka.protocol.types.Schema object>¶
-
classmethod
decode
(data, bytes_to_read=None)¶ Compressed messages should pass in bytes_to_read (via message size) otherwise, we decode from data as Int32
-
classmethod
encode
(items, size=True, recalc_message_size=True)¶
-
classmethod
repr
(messages)¶
-
-
class
kafka.protocol.message.
PartialMessage
¶ Bases:
str
kafka.protocol.metadata module¶
-
class
kafka.protocol.metadata.
MetadataRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 3¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
MetadataResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.metadata.
MetadataResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 3¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.offset module¶
-
class
kafka.protocol.offset.
OffsetRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 2¶
-
API_VERSION
= 0¶
-
DEFAULTS
= {'replica_id': -1}¶
-
RESPONSE_TYPE
¶ alias of
OffsetResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.offset.
OffsetResetStrategy
¶ Bases:
object
-
EARLIEST
= -2¶
-
LATEST
= -1¶
-
NONE
= 0¶
-
-
class
kafka.protocol.offset.
OffsetResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 2¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.pickle module¶
kafka.protocol.produce module¶
-
class
kafka.protocol.produce.
ProduceRequest_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 0¶
-
API_VERSION
= 0¶
-
RESPONSE_TYPE
¶ alias of
ProduceResponse_v0
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.produce.
ProduceRequest_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 0¶
-
API_VERSION
= 1¶
-
RESPONSE_TYPE
¶ alias of
ProduceResponse_v1
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.produce.
ProduceRequest_v2
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 0¶
-
API_VERSION
= 2¶
-
RESPONSE_TYPE
¶ alias of
ProduceResponse_v2
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.produce.
ProduceResponse_v0
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 0¶
-
API_VERSION
= 0¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.produce.
ProduceResponse_v1
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 0¶
-
API_VERSION
= 1¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
-
class
kafka.protocol.produce.
ProduceResponse_v2
(*args, **kwargs)¶ Bases:
kafka.protocol.struct.Struct
-
API_KEY
= 0¶
-
API_VERSION
= 2¶
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
kafka.protocol.struct module¶
-
class
kafka.protocol.struct.
Struct
(*args, **kwargs)¶ Bases:
kafka.protocol.abstract.AbstractType
-
SCHEMA
= <kafka.protocol.types.Schema object>¶
-
classmethod
decode
(data)¶
-
classmethod
encode
(item)¶
-
kafka.protocol.types module¶
-
class
kafka.protocol.types.
Array
(*array_of)¶ Bases:
kafka.protocol.abstract.AbstractType
-
decode
(data)¶
-
encode
(items)¶
-
repr
(list_of_items)¶
-
-
class
kafka.protocol.types.
Bytes
¶ Bases:
kafka.protocol.abstract.AbstractType
-
classmethod
decode
(data)¶
-
classmethod
encode
(value)¶
-
classmethod
-
class
kafka.protocol.types.
Int16
¶ Bases:
kafka.protocol.abstract.AbstractType
-
classmethod
decode
(data)¶
-
classmethod
encode
(value)¶
-
classmethod
-
class
kafka.protocol.types.
Int32
¶ Bases:
kafka.protocol.abstract.AbstractType
-
classmethod
decode
(data)¶
-
classmethod
encode
(value)¶
-
classmethod
-
class
kafka.protocol.types.
Int64
¶ Bases:
kafka.protocol.abstract.AbstractType
-
classmethod
decode
(data)¶
-
classmethod
encode
(value)¶
-
classmethod
-
class
kafka.protocol.types.
Int8
¶ Bases:
kafka.protocol.abstract.AbstractType
-
classmethod
decode
(data)¶
-
classmethod
encode
(value)¶
-
classmethod
-
class
kafka.protocol.types.
Schema
(*fields)¶ Bases:
kafka.protocol.abstract.AbstractType
-
decode
(data)¶
-
encode
(item)¶
-
repr
(value)¶
-
-
class
kafka.protocol.types.
String
(encoding='utf-8')¶ Bases:
kafka.protocol.abstract.AbstractType
-
decode
(data)¶
-
encode
(value)¶
-