Why Go
- Case Studies
  
  Common problems companies solve with Go
- Use Cases
  
  Stories about how and why companies use Go
- Security
  
  How Go can help keep you secure by default
Learn
Docs
- Effective Go
  
  Tips for writing clear, performant, and idiomatic Go code
- Go User Manual
  
  A complete introduction to building software with Go
- Standard library
  
  Reference documentation for Go's standard library
- Release Notes
  
  Learn what's new in each Go release
- API
  
  Reference documentation for the pkg.go.dev API
Packages
Community
- Recorded Talks
  
  Videos from prior events
- Meetups
  
  Meet other local Go developers
- Conferences
  
  Learn and network with Go developers from around the world
- Go blog
  
  The Go project's official blog.
- Go project
  
  Get help and stay informed from Go
- Get connected

scraper

package

Go to main page

Versions in this module

v1

v1.33.0

Oct 16, 2026

v1.32.0

Oct 1, 2026

Changes in this version

+ func LanguagesFor(sc StudioScraper) []string

+ func NormalizeLanguage(s string) (string, error)

+ func Pace(ctx context.Context, delay time.Duration) bool

+ func PatternMatchesURL(pattern, rawURL string) bool

+ func ResolveLanguage(sc StudioScraper, requested string) (string, error)

+ func URLLooksUnhandled(s StudioScraper, rawURL string) bool

+ func WithPacer(ctx context.Context, p *Pacer) context.Context

+ func WorkerCount(opts ListOpts, def int) int

type ListOpts

+ Language string

+ type MultiLingual interface

+ Languages func() []string

+ type Pacer struct

+ func NewPacer(delay time.Duration) *Pacer

+ func PacerFrom(ctx context.Context) *Pacer

+ func (p *Pacer) Wait(ctx context.Context) bool

+ type URLRecognizer interface

+ RecognizesURL func(rawURL string) bool

v1.31.0

Sep 11, 2026

Changes in this version

type ListOpts

+ Cookie string

v1.30.1

Aug 25, 2026

Changes in this version

+ func PreferredStudioURL(studioURL string) string

+ type StudioURLCanonicalizer interface

+ PreferredStudioURL func(studioURL string) string

v1.30.0

Aug 20, 2026

v1.29.0

Aug 10, 2026

Changes in this version

+ func AbsentError(url string, err error) error

+ func Debugf(level int, format string, args ...any)

+ func HostMatches(rawURL string, hosts ...string) bool

+ func Paginate(ctx context.Context, opts ListOpts, siteID string, out chan<- SceneResult, ...)

+ func ParseError(url string, err error) error

+ func Register(s StudioScraper)

+ func ResetWarnDelayBelow()

+ func ResetWarnURLFallthrough()

+ func SetVerbose(level int)

+ func TransportError(url string, err error) error

+ func URLHasNonRootPath(rawURL string) bool

+ func Verbose() int

+ func WarnDelayBelow(siteID string, actual, recommended time.Duration)

+ func WarnURLFallthrough(siteID, rawURL string)

+ type FailureKind int

+ const FailureAbsent

+ const FailureParse

+ const FailureTransport

+ const FailureUnknown

+ func Classify(err error) FailureKind

+ func (k FailureKind) MissingData() bool

+ func (k FailureKind) String() string

+ type ListOpts struct

+ Delay time.Duration

+ KnownIDs map[string]bool

+ Workers int

+ type PageResult struct

+ Continue bool

+ Done bool

+ Scenes []models.Scene

+ Total int

+ type ResultKind int

+ const KindError

+ const KindScene

+ const KindStoppedEarly

+ const KindTotal

+ func (k ResultKind) String() string

+ type SceneResult struct

+ Err error

+ Kind ResultKind

+ Scene models.Scene

+ Total int

+ func Error(err error) SceneResult

+ func Progress(total int) SceneResult

+ func Scene(s models.Scene) SceneResult

+ func StoppedEarly() SceneResult

+ type ScrapeError struct

+ Err error

+ Kind FailureKind

+ URL string

+ func (e *ScrapeError) Error() string

+ func (e *ScrapeError) FailureKind() FailureKind

+ func (e *ScrapeError) Unwrap() error

+ type StudioScraper interface

+ ID func() string

+ ListScenes func(ctx context.Context, studioURL string, opts ListOpts) (<-chan SceneResult, error)

+ MatchesURL func(url string) bool

+ Patterns func() []string

+ func All() []StudioScraper

+ func ForID(id string) (StudioScraper, error)

+ func ForURL(url string) (StudioScraper, error)