← all modules

Std.Ai.Local

Std.Ai.Local

Types
  type ModelRef
  type Gpu = Auto | CpuOnly | AllLayers | Layers Int
  type alias NamedAdapter
  type alias Settings
  type Spec
  type alias RawSpec = { model : String , adapter : String , context : Int , gpu : String , parallel : Int , idleUnload : Int , embeddings : Bool , port : Int , draft : String , draftMax : Int , draftMin : Int , adapters : String }
  type alias Fit
  type alias RawFit = { accelerator : String , weightsBytes : Int , kvCacheBytes : Int , availableBytes : Int , gpuLayers : Int , contextTokens : Int , expectedTokensPerSecond : Int }
  type Engine
  type alias Status
  type alias Pulled
  type alias Provisioned
  type alias EngineInfo

Values
  devModel : ModelRef
      The default development model: Qwen2.5 0.5B Instruct (Apache-2.0), Q4_0,
  engines : () -> Task Error (List EngineInfo)
      Every engine recorded on this machine, including ones other programs
  fetch : ModelRef -> Task Error String
      Make a pinned model present and verified in the model store, and return
  file : { path : String , sha256 : String } -> ModelRef
      A GGUF file on disk, pinned by digest. The file is hashed before each
  fit : Spec -> Task Error Fit
      Predict whether the spec fits this host, without starting an engine. It
  fromReference : String -> ModelRef
      A model from its reference string, as an operator writes it in
  huggingFace : { repo : String , file : String , sha256 : String } -> ModelRef
      A GGUF file in a Hugging Face repository, pinned by digest.
  model : ModelRef -> Spec
      A spec for a model, with every setting at its default.
  namedAdapter : String -> ModelRef -> NamedAdapter
      A named adapter at scale 1.0, for `withAdapters`. A name is letters,
  provider : Engine -> Provider
      The engine as a `Provider`. A call to an engine that was unloaded for
  provision : () -> Task Error Provisioned
      Install the pinned `llama-server` build for this host now, so a host with
  pull : String -> String -> Task Error Pulled
      Download a GGUF file from a Hugging Face repository whose digest you do
  reference : ModelRef -> String
      The reference string: `hf:<org>/<repo>/<file>#sha256=<hex>` or
  settings : Spec -> Settings
      Read back what a spec holds.
  start : Spec -> Task Error Engine
      Start the engine now: fetch and verify the engine build and the model,
  status : Engine -> Task Error Status
      The engine's current status.
  stop : Engine -> Task Error ()
      Stop the engine. A later call through its provider starts it again.
  stopSlot : String -> Task Error Int
      Stop the recorded engines of a slot (`""` stops all of them) and return how
  withAdapter : ModelRef -> Spec -> Spec
      Load a LoRA adapter with the model: a GGUF adapter file pinned by digest,
  withAdapters : List NamedAdapter -> Spec -> Spec
      Load several LoRA adapters on one engine, each pinned by digest (a
  withContext : Int -> Spec -> Spec
      The context in tokens, shared by the engine's parallel slots. Default:
  withDraft : ModelRef -> Spec -> Spec
      Speed up answers with a draft model (speculative decoding): a smaller model
  withDraftTokens : { max : Int, min : Int } -> Spec -> Spec
      How many tokens the draft model drafts per step: at most `max` (above 0)
  withEmbeddings : Spec -> Spec
      Serve `/v1/embeddings` (mean pooling) for `Provider.embed`. llama.cpp then
  withGpu : Gpu -> Spec -> Spec
      Where the layers go. Default: `Auto`.
  withIdleUnload : Int -> Spec -> Spec
      Stop the engine after this many seconds with no call; the next call starts
  withParallel : Int -> Spec -> Spec
      How many requests the engine serves at once. Default: 2.
  withPort : Int -> Spec -> Spec
      The port on 127.0.0.1. Default: `0`, a free port.