FastStream项目：实现Kafka RPC请求的完整指南-CSDN博客

本文链接：https://blog.csdn.net/gitblog_00362/article/details/148550359

FastStream项目：实现Kafka RPC请求的完整指南

引言

在现代分布式系统中，远程过程调用（RPC）是一种常见的通信模式。虽然Kafka本身并不原生支持RPC机制，但我们可以利用FastStream项目提供的工具来构建高效的Kafka RPC解决方案。本文将详细介绍如何使用FastStream实现Kafka RPC功能。

什么是Kafka RPC

RPC（远程过程调用）允许一个程序调用另一个地址空间（通常是另一台机器上）的过程或函数，就像调用本地函数一样简单。在Kafka环境中实现RPC意味着：

客户端发送请求消息到特定主题
服务端监听该主题并处理请求
服务端将响应发送到客户端指定的回复主题
客户端接收并处理响应

基础实现

服务端实现

首先，我们创建一个简单的FastStream服务端，它监听请求并返回响应：

from faststream import FastStream
from faststream.kafka import KafkaBroker

broker = KafkaBroker()
app = FastStream(broker)

@broker.subscriber("echo-topic")
async def echo_handler(msg: Any) -> Any:
    # 简单地将接收到的消息原样返回
    return msg

这个处理程序会自动将响应发送到请求消息中指定的reply_to主题。

客户端实现

客户端需要实现以下功能：

发送带有唯一标识的请求
监听响应主题
将响应与请求匹配

from asyncio import Future
from typing import Annotated
from faststream import FastStream, Context
from faststream.kafka import KafkaBroker, KafkaMessage

broker = KafkaBroker()
app = FastStream(broker)

@broker.subscriber("responses")
async def response_handler(
    msg: KafkaMessage,
    responses: Annotated[
        dict[str, Future[bytes]],
        Context("responses", initial=dict),
    ],
) -> None:
    # 通过correlation_id匹配响应与请求
    if (future := responses.pop(msg.correlation_id, None)):
        future.set_result(msg.body)

发送请求

@app.after_startup
async def send_request(
    responses: Annotated[
        dict[str, Future[bytes]],
        Context("responses", initial=dict),
    ],
) -> None:
    correlation_id = str(uuid4())
    future = responses[correlation_id] = Future[bytes]()

    await broker.publish(
        "echo", "echo-topic",
        reply_to="responses",
        correlation_id=correlation_id,
    )

    data: bytes = await future
    assert data == b"echo"  # 验证响应

封装可重用RPC工具类

为了便于在多个服务中使用，我们可以将上述功能封装成一个RPCWorker类：

from uuid import uuid4
from asyncio import Future, wait_for
from faststream.types import SendableMessage
from faststream.kafka import KafkaMessage, KafkaBroker

class RPCWorker:
    def __init__(self, broker: KafkaBroker, reply_topic: str) -> None:
        self.responses: dict[str, Future[bytes]] = {}
        self.broker = broker
        self.reply_topic = reply_topic
        
        # 设置响应处理器
        self.subscriber = broker.subscriber(reply_topic)
        self.subscriber(self._handle_responses)

    def _handle_responses(self, msg: KafkaMessage) -> None:
        """处理响应消息"""
        if (future := self.responses.pop(msg.correlation_id, None)):
            future.set_result(msg.body)

    async def request(
        self,
        data: SendableMessage,
        topic: str,
        timeout: float = 10.0,
    ) -> bytes:
        """发送RPC请求并等待响应"""
        correlation_id = str(uuid4())
        future = self.responses[correlation_id] = Future[bytes]()

        await self.broker.publish(
            data, topic,
            reply_to=self.reply_topic,
            correlation_id=correlation_id,
        )

        try:
            response: bytes = await wait_for(future, timeout=timeout)
        except TimeoutError:
            self.responses.pop(correlation_id, None)
            raise
        else:
            return response

使用示例

from faststream import FastStream
from faststream.kafka import KafkaBroker

broker = KafkaBroker()
worker = RPCWorker(broker, reply_topic="responses")
app = FastStream(broker)

@app.after_startup
async def send_request() -> None:
    data = await worker.request("echo", "echo-topic")
    assert data == "echo"

高级用法

延迟初始化

如果需要在应用启动后初始化RPCWorker，可以添加start方法：

class RPCWorker:
    async def start(self) -> None:
        """启动响应监听器"""
        self.broker.setup_subscriber(self.subscriber)
        await self.subscriber.start()

使用方式：

@app.after_startup
async def send_request() -> None:
    worker = RPCWorker(broker, reply_topic="responses")
    await worker.start()
    
    data = await worker.request("echo", "echo-topic")
    assert data == "echo"

完整类实现

from uuid import uuid4
from asyncio import Future, wait_for

class RPCWorker:
    responses: dict[str, Future[bytes]]

    def __init__(self, broker: KafkaBroker, reply_topic: str) -> None:
        self.responses = {}
        self.broker = broker
        self.reply_topic = reply_topic

        self.subscriber = broker.subscriber(reply_topic)
        self.subscriber(self._handle_responses)

    async def start(self) -> None:
        """启动响应监听器"""
        self.broker.setup_subscriber(self.subscriber)
        await self.subscriber.start()

    async def stop(self) -> None:
        """停止响应监听器"""
        await self.subscriber.close()

    def _handle_responses(self, msg: KafkaMessage) -> None:
        """处理响应消息"""
        if (future := self.responses.pop(msg.correlation_id, None)):
            future.set_result(msg.body)

    async def request(
        self,
        data: SendableMessage,
        topic: str,
        timeout: float = 10.0,
    ) -> bytes:
        """发送RPC请求并等待响应"""
        correlation_id = str(uuid4())
        future = self.responses[correlation_id] = Future[bytes]()

        await self.broker.publish(
            data, topic,
            reply_to=self.reply_topic,
            correlation_id=correlation_id,
        )

        try:
            response: bytes = await wait_for(future, timeout=timeout)
        except TimeoutError:
            self.responses.pop(correlation_id, None)
            raise
        else:
            return response