kafka.protocol package

Submodules

kafka.protocol.abstract module

class kafka.protocol.abstract.AbstractType[source]

Bases: object

decode(data)[source]
encode(value)[source]
classmethod repr(value)[source]

kafka.protocol.admin module

class kafka.protocol.admin.ApiVersionRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 18
API_VERSION = 0
RESPONSE_TYPE

alias of ApiVersionResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ApiVersionRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 18
API_VERSION = 1
RESPONSE_TYPE

alias of ApiVersionResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ApiVersionResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 18
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ApiVersionResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 18
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.CreateTopicsRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 19
API_VERSION = 0
RESPONSE_TYPE

alias of CreateTopicsResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.CreateTopicsRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 19
API_VERSION = 1
RESPONSE_TYPE

alias of CreateTopicsResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.CreateTopicsRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 19
API_VERSION = 2
RESPONSE_TYPE

alias of CreateTopicsResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.CreateTopicsResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 19
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.CreateTopicsResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 19
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.CreateTopicsResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 19
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DeleteTopicsRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 20
API_VERSION = 0
RESPONSE_TYPE

alias of DeleteTopicsResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DeleteTopicsRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 20
API_VERSION = 1
RESPONSE_TYPE

alias of DeleteTopicsResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DeleteTopicsResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 20
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DeleteTopicsResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 20
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DescribeGroupsRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 15
API_VERSION = 0
RESPONSE_TYPE

alias of DescribeGroupsResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DescribeGroupsRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 15
API_VERSION = 1
RESPONSE_TYPE

alias of DescribeGroupsResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DescribeGroupsResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 15
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.DescribeGroupsResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 15
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ListGroupsRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 16
API_VERSION = 0
RESPONSE_TYPE

alias of ListGroupsResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ListGroupsRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 16
API_VERSION = 1
RESPONSE_TYPE

alias of ListGroupsResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ListGroupsResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 16
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.ListGroupsResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 16
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.SaslHandShakeRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 17
API_VERSION = 0
RESPONSE_TYPE

alias of SaslHandShakeResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.admin.SaslHandShakeResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 17
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.api module

class kafka.protocol.api.Request(*args, **kwargs)[source]

Bases: kafka.protocol.struct.Struct

API_KEY

Integer identifier for api request

API_VERSION

Integer of api request version

RESPONSE_TYPE

The Response class associated with the api request

SCHEMA

An instance of Schema() representing the request structure

expect_response()[source]

Override this method if an api request does not always generate a response

class kafka.protocol.api.RequestHeader(request, correlation_id=0, client_id='kafka-python')[source]

Bases: kafka.protocol.struct.Struct

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.api.Response(*args, **kwargs)[source]

Bases: kafka.protocol.struct.Struct

API_KEY

Integer identifier for api request/response

API_VERSION

Integer of api request/response version

SCHEMA

An instance of Schema() representing the response structure

kafka.protocol.commit module

class kafka.protocol.commit.GroupCoordinatorRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 10
API_VERSION = 0
RESPONSE_TYPE

alias of GroupCoordinatorResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.GroupCoordinatorRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 10
API_VERSION = 1
RESPONSE_TYPE

alias of GroupCoordinatorResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.GroupCoordinatorResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 10
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.GroupCoordinatorResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 10
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 8
API_VERSION = 0
RESPONSE_TYPE

alias of OffsetCommitResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 8
API_VERSION = 1
RESPONSE_TYPE

alias of OffsetCommitResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 8
API_VERSION = 2
DEFAULT_GENERATION_ID = -1
DEFAULT_RETENTION_TIME = -1
RESPONSE_TYPE

alias of OffsetCommitResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitRequest_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 8
API_VERSION = 3
RESPONSE_TYPE

alias of OffsetCommitResponse_v3

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 8
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 8
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 8
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetCommitResponse_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 8
API_VERSION = 3
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 9
API_VERSION = 0
RESPONSE_TYPE

alias of OffsetFetchResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 9
API_VERSION = 1
RESPONSE_TYPE

alias of OffsetFetchResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 9
API_VERSION = 2
RESPONSE_TYPE

alias of OffsetFetchResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchRequest_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 9
API_VERSION = 3
RESPONSE_TYPE

alias of OffsetFetchResponse_v3

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 9
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 9
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 9
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.commit.OffsetFetchResponse_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 9
API_VERSION = 3
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.fetch module

class kafka.protocol.fetch.FetchRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 1
API_VERSION = 0
RESPONSE_TYPE

alias of FetchResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 1
API_VERSION = 1
RESPONSE_TYPE

alias of FetchResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 1
API_VERSION = 2
RESPONSE_TYPE

alias of FetchResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchRequest_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 1
API_VERSION = 3
RESPONSE_TYPE

alias of FetchResponse_v3

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchRequest_v4(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 1
API_VERSION = 4
RESPONSE_TYPE

alias of FetchResponse_v4

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchRequest_v5(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 1
API_VERSION = 5
RESPONSE_TYPE

alias of FetchResponse_v5

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 1
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 1
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 1
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchResponse_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 1
API_VERSION = 3
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchResponse_v4(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 1
API_VERSION = 4
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.fetch.FetchResponse_v5(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 1
API_VERSION = 5
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.group module

class kafka.protocol.group.HeartbeatRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 12
API_VERSION = 0
RESPONSE_TYPE

alias of HeartbeatResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.HeartbeatRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 12
API_VERSION = 1
RESPONSE_TYPE

alias of HeartbeatResponse_v1

SCHEMA

alias of HeartbeatRequest_v0

class kafka.protocol.group.HeartbeatResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 12
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.HeartbeatResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 12
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.JoinGroupRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 11
API_VERSION = 0
RESPONSE_TYPE

alias of JoinGroupResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
UNKNOWN_MEMBER_ID = ''
class kafka.protocol.group.JoinGroupRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 11
API_VERSION = 1
RESPONSE_TYPE

alias of JoinGroupResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
UNKNOWN_MEMBER_ID = ''
class kafka.protocol.group.JoinGroupRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 11
API_VERSION = 2
RESPONSE_TYPE

alias of JoinGroupResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
UNKNOWN_MEMBER_ID = ''
class kafka.protocol.group.JoinGroupResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 11
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.JoinGroupResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 11
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.JoinGroupResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 11
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.LeaveGroupRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 13
API_VERSION = 0
RESPONSE_TYPE

alias of LeaveGroupResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.LeaveGroupRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 13
API_VERSION = 1
RESPONSE_TYPE

alias of LeaveGroupResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.LeaveGroupResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 13
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.LeaveGroupResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 13
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.MemberAssignment(*args, **kwargs)[source]

Bases: kafka.protocol.struct.Struct

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.ProtocolMetadata(*args, **kwargs)[source]

Bases: kafka.protocol.struct.Struct

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.SyncGroupRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 14
API_VERSION = 0
RESPONSE_TYPE

alias of SyncGroupResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.SyncGroupRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 14
API_VERSION = 1
RESPONSE_TYPE

alias of SyncGroupResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.SyncGroupResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 14
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.group.SyncGroupResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 14
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.legacy module

class kafka.protocol.legacy.KafkaProtocol[source]

Bases: object

Class to encapsulate all of the protocol encoding/decoding. This class does not have any state associated with it, it is purely for organization.

CONSUMER_METADATA_KEY = 10
FETCH_KEY = 1
METADATA_KEY = 3
OFFSET_COMMIT_KEY = 8
OFFSET_FETCH_KEY = 9
OFFSET_KEY = 2
PRODUCE_KEY = 0
classmethod decode_consumer_metadata_response(data)[source]

Decode bytes to a ConsumerMetadataResponse

Parameters:data – bytes to decode
classmethod decode_fetch_response(response)[source]

Decode FetchResponse struct to FetchResponsePayloads

Parameters:response – FetchResponse
classmethod decode_list_offset_response(response)[source]

Decode OffsetResponse_v2 into ListOffsetResponsePayloads

Parameters:response – OffsetResponse_v2

Returns: list of ListOffsetResponsePayloads

classmethod decode_message_set(messages)[source]
classmethod decode_metadata_response(response)[source]
classmethod decode_offset_commit_response(response)[source]

Decode OffsetCommitResponse to an OffsetCommitResponsePayload

Parameters:response – OffsetCommitResponse
classmethod decode_offset_fetch_response(response)[source]

Decode OffsetFetchResponse to OffsetFetchResponsePayloads

Parameters:response – OffsetFetchResponse
classmethod decode_offset_response(response)[source]

Decode OffsetResponse into OffsetResponsePayloads

Parameters:response – OffsetResponse

Returns: list of OffsetResponsePayloads

classmethod decode_produce_response(response)[source]

Decode ProduceResponse to ProduceResponsePayload

Parameters:response – ProduceResponse

Return: list of ProduceResponsePayload

classmethod encode_consumer_metadata_request(client_id, correlation_id, payloads)[source]

Encode a ConsumerMetadataRequest

Parameters:
  • client_id – string
  • correlation_id – int
  • payloads – string (consumer group)
classmethod encode_fetch_request(payloads=(), max_wait_time=100, min_bytes=4096)[source]

Encodes a FetchRequest struct

Parameters:
  • payloads – list of FetchRequestPayload
  • max_wait_time (int, optional) – ms to block waiting for min_bytes data. Defaults to 100.
  • min_bytes (int, optional) – minimum bytes required to return before max_wait_time. Defaults to 4096.

Return: FetchRequest

classmethod encode_list_offset_request(payloads=())[source]
classmethod encode_metadata_request(topics=(), payloads=None)[source]

Encode a MetadataRequest

Parameters:topics – list of strings
classmethod encode_offset_commit_request(group, payloads)[source]

Encode an OffsetCommitRequest struct

Parameters:
  • group – string, the consumer group you are committing offsets for
  • payloads – list of OffsetCommitRequestPayload
classmethod encode_offset_fetch_request(group, payloads, from_kafka=False)[source]

Encode an OffsetFetchRequest struct. The request is encoded using version 0 if from_kafka is false, indicating a request for Zookeeper offsets. It is encoded using version 1 otherwise, indicating a request for Kafka offsets.

Parameters:
  • group – string, the consumer group you are fetching offsets for
  • payloads – list of OffsetFetchRequestPayload
  • from_kafka – bool, default False, set True for Kafka-committed offsets
classmethod encode_offset_request(payloads=())[source]
classmethod encode_produce_request(payloads=(), acks=1, timeout=1000)[source]

Encode a ProduceRequest struct

Parameters:
  • payloads – list of ProduceRequestPayload
  • acks – How “acky” you want the request to be 1: written to disk by the leader 0: immediate response -1: waits for all replicas to be in sync
  • timeout – Maximum time (in ms) the server will wait for replica acks. This is _not_ a socket timeout

Returns: ProduceRequest

kafka.protocol.legacy.create_gzip_message(payloads, key=None, compresslevel=None)[source]

Construct a Gzipped Message containing multiple Messages

The given payloads will be encoded, compressed, and sent as a single atomic message to Kafka.

Parameters:
  • payloads – list(bytes), a list of payload to send be sent to Kafka
  • key – bytes, a key used for partition routing (optional)
kafka.protocol.legacy.create_message(payload, key=None)[source]

Construct a Message

Parameters:
  • payload – bytes, the payload to send to Kafka
  • key – bytes, a key used for partition routing (optional)
kafka.protocol.legacy.create_message_set(messages, codec=0, key=None, compresslevel=None)[source]

Create a message set using the given codec.

If codec is CODEC_NONE, return a list of raw Kafka messages. Otherwise, return a list containing a single codec-encoded message.

kafka.protocol.legacy.create_snappy_message(payloads, key=None)[source]

Construct a Snappy Message containing multiple Messages

The given payloads will be encoded, compressed, and sent as a single atomic message to Kafka.

Parameters:
  • payloads – list(bytes), a list of payload to send be sent to Kafka
  • key – bytes, a key used for partition routing (optional)

kafka.protocol.message module

class kafka.protocol.message.Message(value, key=None, magic=0, attributes=0, crc=0, timestamp=None)[source]

Bases: kafka.protocol.struct.Struct

CODEC_GZIP = 1
CODEC_LZ4 = 3
CODEC_MASK = 7
CODEC_SNAPPY = 2
HEADER_SIZE = 22
SCHEMA = <kafka.protocol.types.Schema object>
SCHEMAS = [<kafka.protocol.types.Schema object>, <kafka.protocol.types.Schema object>]
TIMESTAMP_TYPE_MASK = 8
classmethod decode(data)[source]
decompress()[source]
is_compressed()[source]
timestamp_type

0 for CreateTime; 1 for LogAppendTime; None if unsupported.

Value is determined by broker; produced messages should always set to 0 Requires Kafka >= 0.10 / message version >= 1

validate_crc()[source]
class kafka.protocol.message.MessageSet[source]

Bases: kafka.protocol.abstract.AbstractType

HEADER_SIZE = 12
ITEM = <kafka.protocol.types.Schema object>
classmethod decode(data, bytes_to_read=None)[source]

Compressed messages should pass in bytes_to_read (via message size) otherwise, we decode from data as Int32

classmethod encode(items)[source]
classmethod repr(messages)[source]
class kafka.protocol.message.PartialMessage[source]

Bases: str

kafka.protocol.metadata module

class kafka.protocol.metadata.MetadataRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

ALL_TOPICS = None
API_KEY = 3
API_VERSION = 0
RESPONSE_TYPE

alias of MetadataResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

ALL_TOPICS = -1
API_KEY = 3
API_VERSION = 1
NO_TOPICS = None
RESPONSE_TYPE

alias of MetadataResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

ALL_TOPICS = -1
API_KEY = 3
API_VERSION = 2
NO_TOPICS = None
RESPONSE_TYPE

alias of MetadataResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataRequest_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

ALL_TOPICS = -1
API_KEY = 3
API_VERSION = 3
NO_TOPICS = None
RESPONSE_TYPE

alias of MetadataResponse_v3

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataRequest_v4(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

ALL_TOPICS = -1
API_KEY = 3
API_VERSION = 4
NO_TOPICS = None
RESPONSE_TYPE

alias of MetadataResponse_v4

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 3
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 3
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 3
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataResponse_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 3
API_VERSION = 3
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.metadata.MetadataResponse_v4(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 3
API_VERSION = 4
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.offset module

class kafka.protocol.offset.OffsetRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 2
API_VERSION = 0
DEFAULTS = {'replica_id': -1}
RESPONSE_TYPE

alias of OffsetResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.offset.OffsetRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 2
API_VERSION = 1
DEFAULTS = {'replica_id': -1}
RESPONSE_TYPE

alias of OffsetResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.offset.OffsetRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 2
API_VERSION = 2
DEFAULTS = {'replica_id': -1}
RESPONSE_TYPE

alias of OffsetResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.offset.OffsetResetStrategy[source]

Bases: object

EARLIEST = -2
LATEST = -1
NONE = 0
class kafka.protocol.offset.OffsetResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 2
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.offset.OffsetResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 2
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.offset.OffsetResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 2
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.pickle module

kafka.protocol.produce module

class kafka.protocol.produce.ProduceRequest_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 0
API_VERSION = 0
RESPONSE_TYPE

alias of ProduceResponse_v0

SCHEMA = <kafka.protocol.types.Schema object>
expect_response()[source]
class kafka.protocol.produce.ProduceRequest_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 0
API_VERSION = 1
RESPONSE_TYPE

alias of ProduceResponse_v1

SCHEMA = <kafka.protocol.types.Schema object>
expect_response()[source]
class kafka.protocol.produce.ProduceRequest_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 0
API_VERSION = 2
RESPONSE_TYPE

alias of ProduceResponse_v2

SCHEMA = <kafka.protocol.types.Schema object>
expect_response()[source]
class kafka.protocol.produce.ProduceRequest_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Request

API_KEY = 0
API_VERSION = 3
RESPONSE_TYPE

alias of ProduceResponse_v3

SCHEMA = <kafka.protocol.types.Schema object>
expect_response()[source]
class kafka.protocol.produce.ProduceResponse_v0(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 0
API_VERSION = 0
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.produce.ProduceResponse_v1(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 0
API_VERSION = 1
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.produce.ProduceResponse_v2(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 0
API_VERSION = 2
SCHEMA = <kafka.protocol.types.Schema object>
class kafka.protocol.produce.ProduceResponse_v3(*args, **kwargs)[source]

Bases: kafka.protocol.api.Response

API_KEY = 0
API_VERSION = 3
SCHEMA = <kafka.protocol.types.Schema object>

kafka.protocol.struct module

class kafka.protocol.struct.Struct(*args, **kwargs)[source]

Bases: kafka.protocol.abstract.AbstractType

SCHEMA = <kafka.protocol.types.Schema object>
classmethod decode(data)[source]
classmethod encode(item)[source]

kafka.protocol.types module

class kafka.protocol.types.Array(*array_of)[source]

Bases: kafka.protocol.abstract.AbstractType

decode(data)[source]
encode(items)[source]
repr(list_of_items)[source]
class kafka.protocol.types.Boolean[source]

Bases: kafka.protocol.abstract.AbstractType

classmethod decode(data)[source]
classmethod encode(value)[source]
class kafka.protocol.types.Bytes[source]

Bases: kafka.protocol.abstract.AbstractType

classmethod decode(data)[source]
classmethod encode(value)[source]
class kafka.protocol.types.Int16[source]

Bases: kafka.protocol.abstract.AbstractType

classmethod decode(data)[source]
classmethod encode(value)[source]
class kafka.protocol.types.Int32[source]

Bases: kafka.protocol.abstract.AbstractType

classmethod decode(data)[source]
classmethod encode(value)[source]
class kafka.protocol.types.Int64[source]

Bases: kafka.protocol.abstract.AbstractType

classmethod decode(data)[source]
classmethod encode(value)[source]
class kafka.protocol.types.Int8[source]

Bases: kafka.protocol.abstract.AbstractType

classmethod decode(data)[source]
classmethod encode(value)[source]
class kafka.protocol.types.Schema(*fields)[source]

Bases: kafka.protocol.abstract.AbstractType

decode(data)[source]
encode(item)[source]
repr(value)[source]
class kafka.protocol.types.String(encoding='utf-8')[source]

Bases: kafka.protocol.abstract.AbstractType

decode(data)[source]
encode(value)[source]

Module contents