kafka

package
v0.21.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 23, 2026 License: MIT Imports: 20 Imported by: 0

Documentation

Overview

Package kafka provides a durable Kafka mutation publisher and consumer.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func EnsureTopics

func EnsureTopics(ctx context.Context, opts TopicOptions) error

func ReadDeadLetters

func ReadDeadLetters(ctx context.Context, selection DeadLetterRange) ([]*kgo.Record, error)

ReadDeadLetters uses explicit positions and never joins or advances a group. Missing/expired offsets fail instead of silently inspecting a different range.

func ValidateDeadLetterReplay

func ValidateDeadLetterReplay(records []*kgo.Record, store string) ([]queue.Mutation, error)

ValidateDeadLetterReplay validates the entire selection before the caller publishes anything. It preserves the business operation, including its Lua.

Types

type DeadLetterRange

type DeadLetterRange struct {
	Brokers   []string
	Topic     string
	Partition int32
	Offset    int64
	Count     int
}

type Handler

type Handler interface {
	// HandleBatch treats mutations as read-only and stops using the slice
	// before returning so the worker can reuse it for unresolved records.
	HandleBatch(ctx context.Context, mutations []queue.Mutation) []error
}

type Publisher

type Publisher struct {
	// contains filtered or unexported fields
}

func NewPublisher

func NewPublisher(opts PublisherOptions) (*Publisher, error)

func (*Publisher) Close

func (p *Publisher) Close()

func (*Publisher) Ping

func (p *Publisher) Ping(ctx context.Context) error

func (*Publisher) Publish

type PublisherOptions

type PublisherOptions struct {
	Store            string
	Topics           *TopicManager
	Brokers          []string
	Topic            string
	ClientOptions    []kgo.Opt
	Metrics          *sinkmetrics.Metrics
	MaxRecordBytes   int
	MaxBufferedBytes int
}

type TopicManager

type TopicManager struct {
	// contains filtered or unexported fields
}

TopicManager gates one store until its durable topic policy has been applied. Each store reconciles independently so an unavailable cluster does not prevent other stores from starting. The closed channel also publishes readiness safely.

func NewTopicManager

func NewTopicManager(options TopicOptions) *TopicManager

func (*TopicManager) Ping

func (m *TopicManager) Ping(ctx context.Context) error

func (*TopicManager) Run

func (m *TopicManager) Run(ctx context.Context)

func (*TopicManager) Wait

func (m *TopicManager) Wait(ctx context.Context) error

type TopicOptions

type TopicOptions struct {
	Brokers             []string
	Topics              []string
	Partitions          int
	ReplicationFactor   int
	Retention           time.Duration
	DeadLetterTopic     string
	DeadLetterRetention time.Duration
	MinInSyncReplicas   int
	MaxRecordBytes      int
}

type Worker

type Worker struct {
	// contains filtered or unexported fields
}

func NewWorker

func NewWorker(opts WorkerOptions) (*Worker, error)

func (*Worker) Close

func (w *Worker) Close()

func (*Worker) Ping

func (w *Worker) Ping(ctx context.Context) error

func (*Worker) Run

func (w *Worker) Run(ctx context.Context) error

type WorkerOptions

type WorkerOptions struct {
	Admission         *backpressure.Controller
	Memory            *capacity.Guard
	Topics            *TopicManager
	Brokers           []string
	Store             string
	Topic             string
	GroupID           string
	DeadLetterTopic   string
	Handler           Handler
	MaxPollRecords    int
	MaxRetryAttempts  int
	RetryBackoff      time.Duration
	MaxRetryBackoff   time.Duration
	ClientOptions     []kgo.Opt
	Metrics           *sinkmetrics.Metrics
	ProcessingTimeout time.Duration
	MaxRecordBytes    int
	ShutdownTimeout   time.Duration
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL