replay

type Baseline struct {
	ID       string
	Role     rubric.Role
	Empty    float64
	Shuffled float64
	Err      error
	Warn     bool
}

type Bin struct {
	Lo, Hi       float64
	N            int
	MeanScore    float64
	PositiveRate float64
}

type Budget struct {
	Decider  jev.Decider
	LimitUSD float64
	// contains filtered or unexported fields
}

type CapRow struct {
	At                 float64
	MergedHeld, Merged int // merged history PRs
	GoodHeld, Good     int // known-good listed PRs
	// Caught counts known-bad listed PRs held, by kind: held, then total.
	Caught map[string][2]int
}

type CapSweep struct {
	Question string
	At       float64
	Rows     []CapRow
}

type Data struct {
	Store   *dataset.Store
	Offline bool
	Now     func() time.Time
}

type Edit struct {
	Name    string
	PR      func(*prstate.Snapshot) *prstate.Snapshot // nil leaves the PR as is
	Decider func(jev.Decider) jev.Decider             // nil asks as usual
}

type Estimate struct {
	PRs    int // labelled PRs that would be scored
	Calls  int
	Tokens int
	USD    float64 // at fallbackUSDPerMTok
}

type Failure struct {
	Number int
	Err    error
}

type Fit struct {
	OK        bool
	Why       string // when !OK
	ApproveAt float64
	DenyAt    float64
	// FalseApprove is the share of rejected history PRs approve_at lets in;
	// FalseDeny the share of merged and known-good PRs deny_at denies.
	FalseApprove, FalseDeny    float64
	FalseApproved, FalseDenied int
	ApproveRecall              float64 // share of merged history PRs approved
	// Precision at the fitted thresholds, on history. Reported only: it
	// depends on the base rate.
	ApprovePrecision, DenyPrecision float64
	NLabelled, NPositive, NNegative int // history PRs used
	NListedPositive                 int // known-good PRs in the false-deny rate
}

type FlipRate struct {
	Name          string
	N             int
	VerdictFlips  int
	QuestionFlips map[string]int
}

type HistoryOptions struct {
	Since  time.Time
	MaxPRs int
	Mode   LabelMode
	Data   Data
}

type Label string

type LabelMode string

type ListEntry struct {
	Kind, Owner, Repo string
	Number            int
	SHA               string
}

type Listed struct {
	Ref     string
	Kind    string
	Label   Label
	Verdict string
	Caps    []string
	Err     error
}

type LowPR struct {
	Ref   string
	Value float64
}

type Metrics struct {
	N                               int
	Approved, Denied, NeedsHuman    int
	ApprovePrecision, ApproveRecall float64
	DenyPrecision, DenyRecall       float64
	Disagreements                   []Point // approved negatives and denied positives
}

type Options struct {
	Owner, Repo   string
	Config        *config.Config
	Decider       jev.Decider
	BudgetUSD     float64
	History       HistoryOptions
	Rates         Rates // zero means DefaultRates
	Perturb       bool
	PerturbSample int // PRs to perturb; 0 means 10
	Concurrency   int // 0 means 4
	Progress      io.Writer
	Now           func() time.Time
	// Data is the local dataset. A zero Data reads everything from GitHub.
	Data Data
	// Rescore asks the model even when the dataset has its answer.
	Rescore bool
	// Lists also scores the dataset's listed PRs (known-bad and
	// known-good), outside the fit.
	Lists bool
}

type Point struct {
	PR         prstate.PR
	Number     int
	Title      string
	Label      Label
	Low        bool
	Reason     string
	Agg        float64
	Values     []verdict.Value
	Partial    bool
	LockSource bool
	AutoRun    bool
	Sensitive  bool
	TooLarge   bool
	Untrusted  bool
	// Listed marks a PR from a known-bad or known-good list. Known-good ones
	// count towards false denials only; known-bad ones towards neither rate.
	Listed bool
}

type PullStats struct {
	PRs, Labelled, Diffs, Failed int
	Cached                       int // already stored, so not read again
}

type QuestionReliability struct {
	ID   string
	ECE  float64 // expected calibration error over 10 equal-width bins
	N    int
	Bins []Bin // non-empty bins only
}

type Rates struct {
	MaxFalseApprove float64 // share of rejected history PRs approved
	MaxFalseDeny    float64 // share of merged and known-good PRs denied
}

type Report struct {
	Repo        string
	Labels      LabelMode
	Model       string
	Served      []string
	Samples     []Sample
	Points      []Point
	Failed      []Failure
	BudgetHit   bool
	Fit         Fit
	Current     Metrics // at the thresholds in config
	Fitted      *Metrics
	CurrentPol  verdict.Policy
	Reliability []QuestionReliability
	Probe       []Baseline
	Baseline    *FlipRate
	Edits       []FlipRate
	Shadow      *Shadow
	Listed      []Listed
	// ListedPoints are the listed PRs as points, for the false-deny rate
	// and the cap sweep.
	ListedPoints []Point
	Rates        Rates
	CapSweeps    []CapSweep
	VetoSweeps   []VetoSweep
	CostUSD      float64
	// Calibration is what --write stores, or nil with WhyNot saying why not.
	Calibration *config.Calibration
	WhyNot      string
}

type Sample struct {
	PR       prstate.PR
	Label    Label
	Reason   string
	Low      bool // low-confidence: reported but not used for the fit
	Created  time.Time
	Merged   time.Time // zero when closed unmerged
	Comments int       // review comments plus PR comments
}

type Shadow struct {
	N         int // labelled PRs with a shadow check run
	Agreed    int // approve on positive or deny on negative
	Decisive  int // labelled PRs where LGTM said approve or deny
	ByVerdict map[verdict.Verdict]ShadowGroup
}

type ShadowGroup struct {
	N                 int
	MedianTimeToMerge time.Duration
	MeanComments      float64
}

type VetoRow struct {
	At float64
	// Legit and Bad count denied, then total, by group: "history", or the
	// kind of a listed PR.
	Legit, Bad map[string][2]int
}

type VetoSweep struct {
	Question string
	Rows     []VetoRow
	Lowest   []LowPR
}