← all modules

Std.Ai.Device

Std.Ai.Device

Types
  type Placement = Auto | DeviceOnly | ServerOnly
  type Backend = Native | WebGpu | WasmCpu
  type alias RawSpec = { placement : String , model : String , adapter : String , context : Int , threads : String , gpuLayers : String , maxDownloadBytes : Int , backends : List String , metered : Bool }
  type Spec
  type alias OfferFile
  type alias Offer
  type alias BackendInfo
  type alias Capabilities
  type Reason = OperatorChoice | OnTheServer | OfferRefused String | NoBackend String | NoStorage | NotStored | PromptTooLong
  type Routing = OnDevice Backend | NeedsDownload Backend Int | OnServer Reason | Refused Failure
  type alias Ctx = { slot : String , offerJson : String , offer : Offer }
  type alias Runner = { device : Backend -> Ctx -> String -> String -> Task Error String , server : String -> String -> String -> Task Error String }
  type Status = ServerSide Reason | NotDownloaded { backend : Backend, bytes : Int } | Downloading { backend : Backend, doneBytes : Int, totalBytes : Int } | Ready { backend : Backend } | Unsupported Failure
  type alias Progress = { state : String , doneBytes : Int , totalBytes : Int }
  type alias Recorded
  type alias Recording
  type alias Prompt

Values
  backendFromText : String -> Maybe Backend
      The backend a text names.
  backendText : Backend -> String
      `native`, `webgpu` or `wasm-cpu`.
  capabilities : Task Error Capabilities
      What this device can do.
  decide : Capabilities -> Offer -> Int -> Routing
      The policy, for a device's capabilities, the server's offer and the
  decodeCapabilities : String -> Result Error Capabilities
      Read the capabilities a device reports.
  decodeOffer : String -> Result Error Offer
      Read an offer as the server sends it.
  decodeRecording : String -> Result Error Recording
      Read a recording back.
  encodeRecording : Recording -> String
      A recording as JSON.
  estimateTokens : String -> Int
      A token estimate for a request body: one token per three bytes. The
  evict : String -> Task Error ()
      Remove the slot's model and adapter from this device and unload them.
  placementText : Placement -> String
      `auto`, `device` or `server`.
  prepare : String -> Task Error Status
      Download, verify and load the slot's model and adapter on this device.
  provider : String -> Provider
      The slot as a `Provider`: on the device when `decide` says so, on the
  rawSpec : Spec -> RawSpec
      The spec as the runtime reads it.
  recordPrompts : String -> Provider -> List Prompt -> Task Error Recording
      Ask a provider every prompt, one after another, with the prompt's own
  replay : Recording -> Provider
      A provider that answers from a recording. A request that was not recorded
  spec : Placement -> Spec
      A spec with a placement. The device model defaults to the slot's local
  status : String -> Task Error Status
      The slot's status on this device. It downloads nothing. Poll it while
  statusOf : Routing -> { state : String , doneBytes : Int , totalBytes : Int } -> Status
      The status for a decision and the download state the runtime reports
  withAdapter : String -> Spec -> Spec
      The LoRA adapter the device applies: `registry:<name>@<alias>` (the
  withBackends : List Backend -> Spec -> Spec
      The backends a device may use, in order of preference. Default:
  withContext : Int -> Spec -> Spec
      The context window on the device, in tokens. A prompt longer than it runs
  withGpuLayers : Int -> Spec -> Spec
      How many of the model's layers the device engine offloads to its GPU.
  withMaxDownload : Int -> Spec -> Spec
      The largest download, in bytes, a device is offered (model and adapter
  withMetered : Bool -> Spec -> Spec
      Allow `prepare` to download on a metered connection. Default: `False`.
  withModel : String -> Spec -> Spec
      The model the device runs: a `Std.Ai.Local` reference
  withThreads : Int -> Spec -> Spec
      The most CPU threads the device engine uses, at least 1. Env: