← all modules

Std.Ai.Dataset

Std.Ai.Dataset

Types
  type alias Chat = { messages : List Message }
  type alias Instruction = { prompt : String , completion : String }
  type alias Preference = { prompt : String , chosen : String , rejected : String }
  type alias Stats = { scrub : Scrub.Report , duplicates : Int , filtered : Int }
  type Chunks row
  type Dataset row
  type alias RawLines = { lines : List String , next : Int , done : Bool }
  type Split = Train | Eval | Test
  type alias SplitSpec = { train : Int , eval : Int , test : Int , seed : Int }
  type Splits row
  type Limits
  type alias EmptyTurnPayload = { row : Int , turn : Int }
  type alias RoleOrderPayload = { row : Int , reason : String }
  type alias TooManyTokensPayload = { row : Int , tokens : Int , limit : Int }
  type alias TooFewTurnsPayload = { row : Int , turns : Int , limit : Int }
  type alias DuplicateRowPayload = { row : Int , firstRow : Int }
  type Issue = EmptyTurn EmptyTurnPayload | RoleOrder RoleOrderPayload | TooManyTokens TooManyTokensPayload | TooFewTurns TooFewTurnsPayload | TooManyTurns TooFewTurnsPayload | DuplicateRow DuplicateRowPayload
  type alias Report = { rows : Int , issues : List Issue , issueCount : Int , duplicates : Int , duplicateRate : Float , maxTokens : Int , meanTokens : Float , problems : List String , passed : Bool }
  type alias Acc = { rows : Int , issues : List Issue , issueCount : Int , duplicates : Int , maxTokens : Int , totalTokens : Int }
  type Folder
  type alias Manifest = { format : String , name : String , digest : String , file : String , rows : Int , train : Int , eval : Int , test : Int , splitWeights : List Int , splitSeed : Int , sources : List String , steps : List String , scrub : Dict String Int , duplicatesRemoved : Int , filtered : Int , schema : String , createdMs : Int , consentBasis : Maybe String , retention : Maybe String }
  type Version row
  type alias Tally = { rows : Int , train : Int , eval : Int , test : Int , stats : Stats }

Values
  blobPath : Version row -> String
      The path of the version's data file.
  chatCodec : Codec Chat
      `{"messages":[…]}`, the common chat fine-tuning format.
  countRows : Dataset row -> Task Error Int
      The number of rows, read in chunks.
  dedupeBounded : Int -> (row -> String) -> Dataset row -> Dataset row
      `dedupeBy` with the number of key digests held in memory before the set
  dedupeBy : (row -> String) -> Dataset row -> Dataset row
      Keep the first row for each key and drop the later ones. The set of keys
  dedupeMemoryKeys : Int
      How many key digests `dedupeBy` keeps in memory before it spills to disk:
  defaultSplit : SplitSpec
      80 / 10 / 10 with seed 0.
  describeIssue : Issue -> String
      An issue as one line, with no message text.
  digest : Version row -> String
      A version's digest: the SHA-256 of its data file, in lower-case hex.
  estimateTokens : String -> Int
      The token estimate used without a tokenizer: a quarter of the characters,
  filter : (row -> Bool) -> Dataset row -> Dataset row
      Keep the rows the predicate holds for. The lineage counts the rest.
  filterMap : (a -> Maybe b) -> Dataset a -> Dataset b
      Change rows and drop the ones that map to `Nothing`. The lineage counts
  folder : String -> String -> Folder
      Versions of the dataset `name` under `dir`: the data in
  freeze : Folder -> Codec row -> Splits row -> Task Error (Version row)
      Freeze a split dataset: write its canonical JSONL, take its SHA-256, and
  freezeJson : Folder -> Splits String -> Task Error (Version String)
      Freeze rows that are already JSON texts (as `fromJsonLines` reads them).
  fromCsv : (Dict String String -> Result Error row) -> String -> Dataset row
      Rows of a CSV file with a header line: each row is given to `decode` as
  fromJsonLines : String -> Dataset String
      Each non-blank line of a JSONL file as its canonical JSON text (keys
  fromJsonl : Codec row -> String -> Dataset row
      Rows of a JSONL file, one JSON object per line, decoded with the codec.
  fromList : List row -> Dataset row
      Rows held in memory.
  fromStore : Db -> Store a -> (a -> Maybe row) -> Dataset row
      Rows of a `Std.Db.Store` table, mapped to dataset rows (`Nothing` skips a
  fromVersion : Split -> Version row -> Dataset row
      One part of a frozen version, as a dataset that streams its data file.
  inspect : Limits -> Dataset Chat -> Task Error Report
      Check every conversation and report, without failing on the limits.
  instructionCodec : Codec Instruction
      `{"prompt":…,"completion":…}`.
  limits : Limits
      At most 4096 tokens per conversation, 2 to 64 turns (system messages do
  load : Folder -> String -> Codec row -> Task Error (Version row)
      Load a frozen version by its digest (with or without a `sha256:`
  loadJson : Folder -> String -> Task Error (Version String)
      `load` for a version frozen with `freezeJson`: each row is its canonical
  manifest : Version row -> Manifest
      A version's manifest.
  manifestFromJson : String -> Result Error Manifest
      Read a manifest back from its JSON.
  manifestJson : Manifest -> String
      The manifest as canonical JSON.
  map : (a -> b) -> Dataset a -> Dataset b
      Change every row.
  messageCodec : Codec Message
      `{"role":…,"content":…}`.
  part : Split -> Splits row -> Dataset row
      One part of a split dataset, as a dataset.
  preferenceCodec : Codec Preference
      `{"prompt":…,"chosen":…,"rejected":…}`.
  rows : Dataset row -> Task Error (List row)
      Every row, read into memory.
  scrub : Scrubber row -> Dataset row -> Dataset row
      Scrub every row (`Std.Ai.Dataset.Scrub`). The counts of what was
  sources : Dataset row -> List String
      The source descriptions the lineage records.
  split : SplitSpec -> (row -> String) -> Dataset row -> Splits row
      Assign each row to a part by the hash of its key: the same key and seed
  splitOf : SplitSpec -> String -> Split
      The part a key goes to. Pure: the first 28 bits of
  steps : Dataset row -> List String
      The steps applied, in order (`map`, `filter`, `dedupe`, `scrub`, …).
  toChat : (row -> Chat) -> Dataset row -> Dataset Chat
      Rows as chat records.
  toInstruction : (row -> Instruction) -> Dataset row -> Dataset Instruction
      Rows as instruction records.
  toJsonl : Codec row -> List row -> Result Error String
      Rows as JSONL text: one canonical JSON object per line, `\n` after each.
  unsplit : Dataset row -> Splits row
      Every row in `Train`.
  upTo : Int -> Dataset row -> Dataset row
      Only the first `n` rows.
  validate : Limits -> Dataset Chat -> Task Error Report
      `inspect`, then fail with `DatasetInvalid` (naming every limit broken)
  versionName : Version row -> String
      The dataset's name.
  versionPart : Split -> Version row -> Task Error (List row)
      Every row of one part of a frozen version.
  versionRows : Version row -> Task Error (List row)
      Every row of a frozen version, in file order.
  versions : Folder -> Task Error (List Manifest)
      The manifests of every version of a dataset, newest first.
  withConsentBasis : String -> Folder -> Folder
      The legal basis for using the data (for example `"contract"` or
  withEngineTokenizer : Endpoint -> Limits -> Limits
      Count tokens with the engine's own tokenizer: each message's content is
  withMaxDuplicateRate : Float -> Limits -> Limits
      The largest share of conversations that may repeat an earlier one
  withMaxIssues : Int -> Limits -> Limits
      The most issues (empty turns, role order, length, turns) allowed.
  withMaxTokens : Int -> Limits -> Limits
      The most tokens one conversation may have.
  withMaxTurns : Int -> Limits -> Limits
      The most user and assistant messages a conversation may have.
  withMemoryKeys : Int -> Limits -> Limits
      How many conversation digests the duplicate check keeps in memory before
  withMinRows : Int -> Limits -> Limits
      The fewest rows the dataset may have.
  withMinTurns : Int -> Limits -> Limits
      The fewest user and assistant messages a conversation may have.
  withRetention : String -> Folder -> Folder
      How long the data may be kept (for example `"P2Y"`), recorded in the
  withSource : String -> Dataset row -> Dataset row
      Replace the source description recorded in the lineage (for example
  withTokenizer : (String -> Task Error Int) -> Limits -> Limits
      Count tokens with the model's own tokenizer instead of the estimate. The
  writeJsonl : String -> Codec row -> Dataset row -> Task Error Int
      Write the rows to a JSONL file chunk by chunk (canonical lines). The file