Std.Ai.Dataset.Synthetic
Types
type alias Spec row = { name : String , codec : Codec row , teacher : Provider , endpoint : Maybe Endpoint , teacherName : String , template : String , system : String , personas : List String , topics : List String , examples : List row , perCall : Int , target : Int , seed : Int , temperature : Float , maxTokens : Int , maxCalls : Maybe Int , budget : Maybe ( PriceBook, Money ) , dedupe : Maybe (row -> String) , filters : List ( String, row -> Bool ) , judge : Maybe { judge : Eval.Judge , rubric : String , show : row -> ( String, String ) } , journal : Maybe Journal }
type Generator row
type alias Journal = { setup : Task Error () , ask : String -> List String -> List Message -> Task Error Provider.ChatResponse -> Task Error ( Provider.ChatResponse, Bool ) , judge : Eval.Judge -> Int -> String -> String -> String -> Task Error Eval.Score }
type alias Call = { index : Int , persona : String , topic : String , seed : Int , messages : List Message }
type Stop = TargetReached | CallLimit | BudgetReached
type alias Report = { name : String , teacher : String , seed : Int , calls : Int , replayed : Int , judgeCalls : Int , generated : Int , kept : Int , unparsed : Int , duplicates : Int , filtered : Dict String Int , judgedOut : Int , promptTokens : Int , completionTokens : Int , cost : Maybe Money , stop : Stop }
type alias Generated row = { rows : List row , report : Report }
type alias State row = { kept : List row , seen : Set String , report : Report }
Values
defaultSystemPrompt : String
The system message of every teacher call unless `withSystemPrompt` sets
defaultTemplate : String
The user message of every teacher call unless `withTemplate` sets
generator : String -> Codec row -> Generator row
A generator for the dataset `name`, with rows in the codec's shape. It
parseRows : Codec row -> String -> ( List row, Int )
The rows in a teacher's answer: a JSON array of rows (or one row), in a
plan : Generator row -> List Call
Every call the run may make, in order. Pure: it calls nothing.
run : Generator row -> Task Error (Generated row)
Run the plan until the target, the call limit or the budget stops it.
teacherSlot : Endpoint
The teacher's model slot, `"teacher"`: the local pinned development model
toDataset : Generated row -> Dataset row
The kept rows as a dataset whose lineage names the generator, the
withCostBudget : PriceBook -> Money -> Generator row -> Generator row
Stop before a call once the teacher's spend, priced by the book, reaches
withDedupe : (row -> String) -> Generator row -> Generator row
Drop a row whose key repeats an earlier row's key. Without it, a row that
withExamples : List row -> Generator row -> Generator row
Rows shown to the teacher as examples of the format (few-shot).
withFilter : String -> (row -> Bool) -> Generator row -> Generator row
Keep only rows the predicate holds for; the rest count under `name`.
withJournal : Db -> Generator row -> Generator row
Journal the run's model calls in a database. A teacher call is kept in
withJudge : Eval.Judge -> String -> (row -> ( String, String )) -> Generator row -> Generator row
Keep only rows the judge passes against the rubric. `show` turns a row
withMaxCalls : Int -> Generator row -> Generator row
The most teacher calls the run makes.
withMaxTokens : Int -> Generator row -> Generator row
The most tokens one answer may have.
withPerCall : Int -> Generator row -> Generator row
How many rows each call asks for (at least 1).
withPersonas : List String -> Generator row -> Generator row
Who the examples are written for or by. Each call uses one.
withSeed : Int -> Generator row -> Generator row
The run's seed: it picks the first topic and persona and the sampling
withSystemPrompt : String -> Generator row -> Generator row
The system message of every call.
withTarget : Int -> Generator row -> Generator row
How many rows to keep.
withTeacher : String -> Provider -> Generator row -> Generator row
Use this provider as the teacher. `name` is what the report and the
withTeacherSlot : Endpoint -> Generator row -> Generator row
Use this slot as the teacher. The report records the slot's resolved mode
withTemperature : Float -> Generator row -> Generator row
The teacher's sampling temperature (default 0.7: varied rows, and still
withTemplate : String -> Generator row -> Generator row
The user message template (see `defaultTemplate`).
withTopics : List String -> Generator row -> Generator row
What the examples are about. Each call uses one.