Skip to content

KafkaRoute

faststream.kafka.router.KafkaRoute #

KafkaRoute(call, *topics, publishers=(), batch=False, group_id=None, key_deserializer=None, value_deserializer=None, fetch_max_bytes=50 * 1024 * 1024, fetch_min_bytes=1, fetch_max_wait_ms=500, max_partition_fetch_bytes=1 * 1024 * 1024, auto_offset_reset='latest', auto_commit=True, auto_commit_interval_ms=5 * 1000, check_crcs=True, partition_assignment_strategy=(RoundRobinPartitionAssignor), max_poll_interval_ms=5 * 60 * 1000, rebalance_timeout_ms=None, session_timeout_ms=10 * 1000, heartbeat_interval_ms=3 * 1000, consumer_timeout_ms=200, max_poll_records=None, exclude_internal_topics=True, isolation_level='read_uncommitted', batch_timeout_ms=200, max_records=None, listener=None, pattern=None, partitions=(), dependencies=(), parser=None, decoder=None, middlewares=(), filter=default_filter, retry=False, no_ack=False, no_reply=False, title=None, description=None, include_in_schema=True)

Bases: SubscriberRoute

Class to store delayed KafkaBroker subscriber registration.

Source code in faststream/kafka/router.py
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
def __init__(
    self,
    call: Annotated[
        Union[
            Callable[..., "SendableMessage"],
            Callable[..., Awaitable["SendableMessage"]],
        ],
        Doc(
            "Message handler function "
            "to wrap the same with `@broker.subscriber(...)` way."
        ),
    ],
    *topics: Annotated[
        str,
        Doc("Kafka topics to consume messages from."),
    ],
    publishers: Annotated[
        Iterable[KafkaPublisher],
        Doc("Kafka publishers to broadcast the handler result."),
    ] = (),
    batch: Annotated[
        bool,
        Doc("Whether to consume messages in batches or not."),
    ] = False,
    group_id: Annotated[
        Optional[str],
        Doc(
            """
        Name of the consumer group to join for dynamic
        partition assignment (if enabled), and to use for fetching and
        committing offsets. If `None`, auto-partition assignment (via
        group coordinator) and offset commits are disabled.
        """
        ),
    ] = None,
    key_deserializer: Annotated[
        Optional[Callable[[bytes], Any]],
        Doc(
            "Any callable that takes a raw message `bytes` "
            "key and returns a deserialized one."
        ),
    ] = None,
    value_deserializer: Annotated[
        Optional[Callable[[bytes], Any]],
        Doc(
            "Any callable that takes a raw message `bytes` "
            "value and returns a deserialized value."
        ),
    ] = None,
    fetch_max_bytes: Annotated[
        int,
        Doc(
            """
        The maximum amount of data the server should
        return for a fetch request. This is not an absolute maximum, if
        the first message in the first non-empty partition of the fetch
        is larger than this value, the message will still be returned
        to ensure that the consumer can make progress. NOTE: consumer
        performs fetches to multiple brokers in parallel so memory
        usage will depend on the number of brokers containing
        partitions for the topic.
        """
        ),
    ] = 50 * 1024 * 1024,
    fetch_min_bytes: Annotated[
        int,
        Doc(
            """
        Minimum amount of data the server should
        return for a fetch request, otherwise wait up to
        `fetch_max_wait_ms` for more data to accumulate.
        """
        ),
    ] = 1,
    fetch_max_wait_ms: Annotated[
        int,
        Doc(
            """
        The maximum amount of time in milliseconds
        the server will block before answering the fetch request if
        there isn't sufficient data to immediately satisfy the
        requirement given by `fetch_min_bytes`.
        """
        ),
    ] = 500,
    max_partition_fetch_bytes: Annotated[
        int,
        Doc(
            """
        The maximum amount of data
        per-partition the server will return. The maximum total memory
        used for a request ``= #partitions * max_partition_fetch_bytes``.
        This size must be at least as large as the maximum message size
        the server allows or else it is possible for the producer to
        send messages larger than the consumer can fetch. If that
        happens, the consumer can get stuck trying to fetch a large
        message on a certain partition.
        """
        ),
    ] = 1 * 1024 * 1024,
    auto_offset_reset: Annotated[
        Literal["latest", "earliest", "none"],
        Doc(
            """
        A policy for resetting offsets on `OffsetOutOfRangeError` errors:

        * `earliest` will move to the oldest available message
        * `latest` will move to the most recent
        * `none` will raise an exception so you can handle this case
        """
        ),
    ] = "latest",
    auto_commit: Annotated[
        bool,
        Doc(
            """
        If `True` the consumer's offset will be
        periodically committed in the background.
        """
        ),
    ] = True,
    auto_commit_interval_ms: Annotated[
        int,
        Doc(
            """
        Milliseconds between automatic
        offset commits, if `auto_commit` is `True`."""
        ),
    ] = 5 * 1000,
    check_crcs: Annotated[
        bool,
        Doc(
            """
        Automatically check the CRC32 of the records
        consumed. This ensures no on-the-wire or on-disk corruption to
        the messages occurred. This check adds some overhead, so it may
        be disabled in cases seeking extreme performance.
        """
        ),
    ] = True,
    partition_assignment_strategy: Annotated[
        Sequence["AbstractPartitionAssignor"],
        Doc(
            """
        List of objects to use to
        distribute partition ownership amongst consumer instances when
        group management is used. This preference is implicit in the order
        of the strategies in the list. When assignment strategy changes:
        to support a change to the assignment strategy, new versions must
        enable support both for the old assignment strategy and the new
        one. The coordinator will choose the old assignment strategy until
        all members have been updated. Then it will choose the new
        strategy.
        """
        ),
    ] = (RoundRobinPartitionAssignor,),
    max_poll_interval_ms: Annotated[
        int,
        Doc(
            """
        Maximum allowed time between calls to
        consume messages in batches. If this interval
        is exceeded the consumer is considered failed and the group will
        rebalance in order to reassign the partitions to another consumer
        group member. If API methods block waiting for messages, that time
        does not count against this timeout.
        """
        ),
    ] = 5 * 60 * 1000,
    rebalance_timeout_ms: Annotated[
        Optional[int],
        Doc(
            """
        The maximum time server will wait for this
        consumer to rejoin the group in a case of rebalance. In Java client
        this behaviour is bound to `max.poll.interval.ms` configuration,
        but as ``aiokafka`` will rejoin the group in the background, we
        decouple this setting to allow finer tuning by users that use
        `ConsumerRebalanceListener` to delay rebalacing. Defaults
        to ``session_timeout_ms``
        """
        ),
    ] = None,
    session_timeout_ms: Annotated[
        int,
        Doc(
            """
        Client group session and failure detection
        timeout. The consumer sends periodic heartbeats
        (`heartbeat.interval.ms`) to indicate its liveness to the broker.
        If no hearts are received by the broker for a group member within
        the session timeout, the broker will remove the consumer from the
        group and trigger a rebalance. The allowed range is configured with
        the **broker** configuration properties
        `group.min.session.timeout.ms` and `group.max.session.timeout.ms`.
        """
        ),
    ] = 10 * 1000,
    heartbeat_interval_ms: Annotated[
        int,
        Doc(
            """
        The expected time in milliseconds
        between heartbeats to the consumer coordinator when using
        Kafka's group management feature. Heartbeats are used to ensure
        that the consumer's session stays active and to facilitate
        rebalancing when new consumers join or leave the group. The
        value must be set lower than `session_timeout_ms`, but typically
        should be set no higher than 1/3 of that value. It can be
        adjusted even lower to control the expected time for normal
        rebalances.
        """
        ),
    ] = 3 * 1000,
    consumer_timeout_ms: Annotated[
        int,
        Doc(
            """
        Maximum wait timeout for background fetching
        routine. Mostly defines how fast the system will see rebalance and
        request new data for new partitions.
        """
        ),
    ] = 200,
    max_poll_records: Annotated[
        Optional[int],
        Doc(
            """
        The maximum number of records returned in a
        single call by batch consumer. Has no limit by default.
        """
        ),
    ] = None,
    exclude_internal_topics: Annotated[
        bool,
        Doc(
            """
        Whether records from internal topics
        (such as offsets) should be exposed to the consumer. If set to True
        the only way to receive records from an internal topic is
        subscribing to it.
        """
        ),
    ] = True,
    isolation_level: Annotated[
        Literal["read_uncommitted", "read_committed"],
        Doc(
            """
        Controls how to read messages written
        transactionally.

        * `read_committed`, batch consumer will only return
        transactional messages which have been committed.

        * `read_uncommitted` (the default), batch consumer will
        return all messages, even transactional messages which have been
        aborted.

        Non-transactional messages will be returned unconditionally in
        either mode.

        Messages will always be returned in offset order. Hence, in
        `read_committed` mode, batch consumer will only return
        messages up to the last stable offset (LSO), which is the one less
        than the offset of the first open transaction. In particular any
        messages appearing after messages belonging to ongoing transactions
        will be withheld until the relevant transaction has been completed.
        As a result, `read_committed` consumers will not be able to read up
        to the high watermark when there are in flight transactions.
        Further, when in `read_committed` the seek_to_end method will
        return the LSO. See method docs below.
        """
        ),
    ] = "read_uncommitted",
    batch_timeout_ms: Annotated[
        int,
        Doc(
            """
        Milliseconds spent waiting if
        data is not available in the buffer. If 0, returns immediately
        with any records that are available currently in the buffer,
        else returns empty.
        """
        ),
    ] = 200,
    max_records: Annotated[
        Optional[int],
        Doc("Number of messages to consume as one batch."),
    ] = None,
    listener: Annotated[
        Optional["ConsumerRebalanceListener"],
        Doc(
            """
        Optionally include listener
           callback, which will be called before and after each rebalance
           operation.
           As part of group management, the consumer will keep track of
           the list of consumers that belong to a particular group and
           will trigger a rebalance operation if one of the following
           events trigger:

           * Number of partitions change for any of the subscribed topics
           * Topic is created or deleted
           * An existing member of the consumer group dies
           * A new member is added to the consumer group

           When any of these events are triggered, the provided listener
           will be invoked first to indicate that the consumer's
           assignment has been revoked, and then again when the new
           assignment has been received. Note that this listener will
           immediately override any listener set in a previous call
           to subscribe. It is guaranteed, however, that the partitions
           revoked/assigned
           through this interface are from topics subscribed in this call.
        """
        ),
    ] = None,
    pattern: Annotated[
        Optional[str],
        Doc(
            """
        Pattern to match available topics. You must provide either topics or pattern, but not both.
        """
        ),
    ] = None,
    partitions: Annotated[
        Optional[Iterable["TopicPartition"]],
        Doc(
            """
        A topic and partition tuple. You can't use 'topics' and 'partitions' in the same time.
        """
        ),
    ] = (),
    # broker args
    dependencies: Annotated[
        Iterable["Depends"],
        Doc("Dependencies list (`[Depends(),]`) to apply to the subscriber."),
    ] = (),
    parser: Annotated[
        Optional["CustomCallable"],
        Doc("Parser to map original **ConsumerRecord** object to FastStream one."),
    ] = None,
    decoder: Annotated[
        Optional["CustomCallable"],
        Doc("Function to decode FastStream msg bytes body to python objects."),
    ] = None,
    middlewares: Annotated[
        Iterable["SubscriberMiddleware[KafkaMessage]"],
        Doc("Subscriber middlewares to wrap incoming message processing."),
    ] = (),
    filter: Annotated[
        "Filter[KafkaMessage]",
        Doc(
            "Overload subscriber to consume various messages from the same source."
        ),
        deprecated(
            "Deprecated in **FastStream 0.5.0**. "
            "Please, create `subscriber` object and use it explicitly instead. "
            "Argument will be removed in **FastStream 0.6.0**."
        ),
    ] = default_filter,
    retry: Annotated[
        bool,
        Doc("Whether to `nack` message at processing exception."),
    ] = False,
    no_ack: Annotated[
        bool,
        Doc("Whether to disable **FastStream** autoacknowledgement logic or not."),
    ] = False,
    no_reply: Annotated[
        bool,
        Doc(
            "Whether to disable **FastStream** RPC and Reply To auto responses or not."
        ),
    ] = False,
    # AsyncAPI args
    title: Annotated[
        Optional[str],
        Doc("AsyncAPI subscriber object title."),
    ] = None,
    description: Annotated[
        Optional[str],
        Doc(
            "AsyncAPI subscriber object description. "
            "Uses decorated docstring as default."
        ),
    ] = None,
    include_in_schema: Annotated[
        bool,
        Doc("Whetever to include operation in AsyncAPI schema or not."),
    ] = True,
) -> None:
    super().__init__(
        call,
        *topics,
        publishers=publishers,
        group_id=group_id,
        key_deserializer=key_deserializer,
        value_deserializer=value_deserializer,
        fetch_max_wait_ms=fetch_max_wait_ms,
        fetch_max_bytes=fetch_max_bytes,
        fetch_min_bytes=fetch_min_bytes,
        max_partition_fetch_bytes=max_partition_fetch_bytes,
        auto_offset_reset=auto_offset_reset,
        auto_commit=auto_commit,
        auto_commit_interval_ms=auto_commit_interval_ms,
        check_crcs=check_crcs,
        partition_assignment_strategy=partition_assignment_strategy,
        max_poll_interval_ms=max_poll_interval_ms,
        rebalance_timeout_ms=rebalance_timeout_ms,
        session_timeout_ms=session_timeout_ms,
        heartbeat_interval_ms=heartbeat_interval_ms,
        consumer_timeout_ms=consumer_timeout_ms,
        max_poll_records=max_poll_records,
        exclude_internal_topics=exclude_internal_topics,
        isolation_level=isolation_level,
        max_records=max_records,
        batch_timeout_ms=batch_timeout_ms,
        batch=batch,
        listener=listener,
        pattern=pattern,
        partitions=partitions,
        # basic args
        dependencies=dependencies,
        parser=parser,
        decoder=decoder,
        middlewares=middlewares,
        filter=filter,
        no_reply=no_reply,
        # AsyncAPI args
        title=title,
        description=description,
        include_in_schema=include_in_schema,
        # FastDepends args
        retry=retry,
        no_ack=no_ack,
    )

args instance-attribute #

args = args

kwargs instance-attribute #

kwargs = kwargs

call instance-attribute #

call = call

publishers instance-attribute #

publishers = publishers