evaluation

type Config struct {
	AgentFilename  string   // Path to the agent configuration file
	EvalsDir       string   // Directory containing evaluation files
	JudgeModel     string   // Model for relevance checking (format: provider/model, optional)
	Concurrency    int      // Number of concurrent runs (0 = number of CPUs)
	TTYFd          int      // File descriptor for terminal size queries (e.g., int(os.Stdout.Fd()))
	Only           []string // Only run evaluations matching these patterns
	BaseImage      string   // Custom base Docker image for running evaluations
	KeepContainers bool     // If true, don't remove containers after evaluation (skip --rm)
}

type EvalCriteria struct {
	Relevance  []string `json:"relevance,omitempty"`   // Statements that should be true about the response
	WorkingDir string   `json:"working_dir,omitempty"` // Subdirectory under evals/working_dirs/
	Size       string   `json:"size,omitempty"`        // Expected response size: S, M, L, XL
}

type EvalRun struct {
	Name      string        `json:"name"`
	Timestamp time.Time     `json:"timestamp"`
	Duration  time.Duration `json:"duration"`
	Results   []Result      `json:"results"`
	Summary   Summary       `json:"summary"`
}

type EvalSession struct {
	session.Session
	Evals      EvalCriteria `json:"evals"`
	SourcePath string       `json:"-"` // Path to the source eval file (not serialized)
}

type Judge struct {
	// contains filtered or unexported fields
}

type Result struct {
	InputPath         string           `json:"input_path"`
	Title             string           `json:"title"`
	Question          string           `json:"question"`
	Response          string           `json:"response"`
	Cost              float64          `json:"cost"`
	OutputTokens      int64            `json:"output_tokens"`
	Size              string           `json:"size"`
	SizeExpected      string           `json:"size_expected"`
	ToolCallsScore    float64          `json:"tool_calls_score"`
	ToolCallsExpected float64          `json:"tool_calls_score_expected"`
	HandoffsMatch     bool             `json:"handoffs"`
	RelevancePassed   float64          `json:"relevance"`
	RelevanceExpected float64          `json:"relevance_expected"`
	FailedRelevance   []string         `json:"failed_relevance,omitempty"`
	Error             string           `json:"error,omitempty"`
	RawOutput         []map[string]any `json:"raw_output,omitempty"`
}

type Runner struct {
	Config
	// contains filtered or unexported fields
}

type Summary struct {
	TotalEvals      int     `json:"total_evals"`
	FailedEvals     int     `json:"failed_evals"`
	TotalCost       float64 `json:"total_cost"`
	SizesPassed     int     `json:"sizes_passed"`
	SizesTotal      int     `json:"sizes_total"`
	ToolsPassed     float64 `json:"tools_passed"`
	ToolsTotal      float64 `json:"tools_total"`
	HandoffsPassed  int     `json:"handoffs_passed"`
	HandoffsTotal   int     `json:"handoffs_total"`
	RelevancePassed float64 `json:"relevance_passed"`
	RelevanceTotal  float64 `json:"relevance_total"`
}