← all modules

Std.Ai.Dataset.Scrub

Std.Ai.Dataset.Scrub

Types
  type Rule
  type alias Report = Dict String Int
  type Scrubber row

Values
  cardNumber : Rule
      Payment card numbers that pass the Luhn check (13 to 19 digits, with or
  countOf : String -> Report -> Int
      The count for one rule (0 when it replaced nothing).
  custom : String -> (String -> ( String, Int )) -> Rule
      Your own rule: a function from text to the scrubbed text and the number of
  defaults : List Rule
      Every built-in rule, in the order that keeps them apart: secrets first
  dropIf : String -> (row -> Bool) -> Scrubber row -> Scrubber row
      Drop every row the predicate holds for, counted under `name`. A dropped
  email : Rule
      Email addresses, replaced with `[EMAIL]`.
  emptyReport : Report
      The report with nothing replaced.
  forChat : List Rule -> Scrubber { messages : List Message }
      A scrubber for chat rows (`Std.Ai.Dataset.Chat`): every message's content.
  forInstruction : List Rule -> Scrubber { prompt : String, completion : String }
      A scrubber for instruction rows (`Std.Ai.Dataset.Instruction`).
  forPreference : List Rule -> Scrubber { prompt : String, chosen : String, rejected : String }
      A scrubber for preference rows (`Std.Ai.Dataset.Preference`).
  forString : List Rule -> Scrubber String
      A scrubber for plain strings.
  iban : Rule
      International bank account numbers that pass the mod-97 check (with or
  isCardNumber : String -> Bool
      Whether a string is a payment card number: 13 to 19 digits once spaces
  isIban : String -> Bool
      Whether a string is an IBAN: a country code, two check digits and 11 to 30
  merge : Report -> Report -> Report
      Two reports added together.
  pattern : String -> String -> Result Error Rule
      Your own pattern: every match of the RE2 regular expression is replaced
  phone : Rule
      Telephone numbers, replaced with `[PHONE]`: an international number
  ruleName : Rule -> String
      The name a rule reports under.
  scrubRow : Scrubber row -> row -> ( Maybe row, Report )
      Scrub one row: `Nothing` when a drop rule removed it.
  scrubRows : Scrubber row -> List row -> ( List row, Report )
      Scrub a list of rows: the rows kept, in order, and the summed report.
  scrubText : List Rule -> String -> ( String, Report )
      Apply rules to one string, in order.
  scrubber : List Rule -> (row -> List String) -> (List String -> row -> row) -> Scrubber row
      A scrubber for your own row type: `get` lists the row's text fields and
  secrets : Rule
      Secrets with a recognisable shape, replaced with `[SECRET]`: API keys with
  total : Report -> Int
      Every replacement and dropped row in a report.
  ukNino : Rule
      UK national insurance numbers (`QQ 12 34 56 C`, with or without spaces),
  usSsn : Rule
      US social security numbers written `123-45-6789`, replaced with `[SSN]`.