← all modules

Std.Ai.Endpoint

Std.Ai.Endpoint

Types
  type Mode = Local | Remote
  type alias RawLocal = { model : String , adapter : String , context : Int , gpu : String , parallel : Int , idleUnload : Int , embeddings : Bool , port : Int , draft : String , draftMax : Int , draftMin : Int , adapters : String }
  type alias Config = { slot : String , mode : String , url : String , hasKey : Bool , key : Secret , model : String , adapter : String , timeoutMs : Int , maxInFlight : Int , queue : Int , queueWaitMs : Int , retries : Int , hasLocal : Bool , local : RawLocal , hasResolver : Bool , resolverName : String , resolve : String -> Bool -> Task Error Adapter , adapterRefreshS : Int , auth : String , idToken : String -> Task Error Secret , hasDevice : Bool , device : Device.RawSpec }
  type alias Adapter
  type alias Resolver
  type Endpoint
  type alias Setting
  type alias Description

Values
  describe : Endpoint -> Task Error Description
      Resolve the slot without starting anything, and report each setting's
  endpoint : String -> Endpoint
      A slot, named by a lower-case literal: `"chat"`, `"embed"`, `"judge"`. Its
  fromRegistry : Resolver -> Endpoint -> Endpoint
      Let the slot serve registry adapters. With it, an adapter named
  inspect : Endpoint -> Task Error Description
      The slot's settings and where each came from, like `describe`, without
  provider : Endpoint -> Provider
      The slot as a `Provider`. It is pure: the first call resolves the slot and
  slotName : Endpoint -> String
      The slot's name: `"chat"` for `Endpoint.endpoint "chat"`.
  tokenize : Endpoint -> String -> Task Error Int
      Count the tokens of a text with the model's own tokenizer: the engine's
  warm : Endpoint -> Task Error Description
      Resolve the slot and, for a local slot, start its engine now instead of on
  withAdapter : String -> Endpoint -> Endpoint
      The adapter to answer with, sent as the request's model name (how vLLM and
  withAdapterRefresh : Int -> Endpoint -> Endpoint
      How often, in seconds, a slot with a registry adapter checks whether its
  withDevice : Device.Spec -> Endpoint -> Endpoint
      Let devices run this slot: a Sky.Spa client's `Device.provider "<slot>"`
  withGoogleAuth : Endpoint -> Endpoint
      Call a private Cloud Run service (a `Std.Ai.Serve` GPU tier) as this
  withKey : Secret -> Endpoint -> Endpoint
      The bearer key a remote server wants. Env: `<PREFIX>_AI_<SLOT>_KEY`.
  withLocal : LocalEngine.Spec -> Endpoint -> Endpoint
      The managed local engine this slot runs when it is not remote. Its context,
  withMaxInFlight : Int -> Endpoint -> Endpoint
      Calls in flight at once. Env: `<PREFIX>_AI_<SLOT>_MAX_INFLIGHT`. Default:
  withMode : Mode -> Endpoint -> Endpoint
      Force local or remote. Env: `<PREFIX>_AI_<SLOT>_MODE` (`local` or
  withModel : String -> Endpoint -> Endpoint
      Remote: the served model name. Local: a model reference
  withQueue : Int -> Endpoint -> Endpoint
      Calls that may wait for a free place. Env: `<PREFIX>_AI_<SLOT>_QUEUE`.
  withQueueWait : Int -> Endpoint -> Endpoint
      The longest a call waits in the queue, in milliseconds. Env:
  withRetry : Int -> Endpoint -> Endpoint
      Retries for an overloaded server or a connection that never reached it.
  withTimeout : Int -> Endpoint -> Endpoint
      The longest a call may take, in milliseconds (queue wait excluded). Env:
  withUrl : String -> Endpoint -> Endpoint
      The OpenAI-compatible base URL, ending `/v1`. Env: