← all modules

Std.Ai.Serve

Std.Ai.Serve
  Std.Ai.Serve — the GPU serving tier: one base model and many fine-tuned

Types
  type Engine = Vllm
  type alias TierSettings = { engine : Engine , base : Train.Snapshot , baseName : String , adapters : List String , maxAdapters : Int , maxModelLength : Int , gpuFraction : Float , bind : String , token : Maybe Secret , pollS : Int , registry : Maybe Store , engineUrl : String , modelsDir : String }
  type Tier
  type alias Setting = { key : String , name : String , value : String , source : String }
  type alias RawTier = { engine : String , bind : String , engineUrl : String , baseId : String , baseName : String , adapters : List String , maxAdapters : Int , pollS : Int , modelsDir : String , hasToken : Bool , token : Secret }
  type alias RunTier = { engine : String , bind : String , engineUrl : String , baseId : String , baseName : String , adapters : List String , maxAdapters : Int , pollS : Int , modelsDir : String , hasToken : Bool , token : Secret , resolve : String -> Task Error TierAdapter }
  type alias TierAdapter = { ref : String , label : String , servedName : String , baseId : String , digest : String , peft : String , config : String }
  type alias TargetSettings = { project : String , region : String , service : String , tierImage : String , engineImage : String , minInstances : Int , maxInstances : Int , serviceAccount : String , invokers : List String , tokenSecret : String , registryUrl : String , timeoutSeconds : Int , labels : List ( String, String ) }
  type Target
  type alias Rendered = { serviceJson : String , knativeYaml : String , dockerfile : String }
  type alias Deployment = { url : String , revision : String , region : String , service : String }
  type alias RawTarget = { project : String , region : String , image : String }

Values
  adapterFrom : Store -> String -> Task Error TierAdapter
      Resolve `<name>@<alias>` or `<name>@v<n>` through a store's pointer
  baseIdOf : Tier -> String
      The base's id, `<org>/<repo>@<commit>`: what an adapter must have been
  basePath : String -> Tier -> String
      Where the base's files are staged below the AI home
  cloudRun : { project : String , region : String , service : String } -> Target
      Google Cloud Run with one NVIDIA L4, scaling to zero: `{ project,
  defaultEngineImage : String
      vLLM's OpenAI server image, pinned by digest: vllm/vllm-openai
  deploy : Target -> Tier -> Task Error Deployment
      Deploy a tier: check both images exist, create the service (or update it
  describe : Tier -> Task Error (List Setting)
      The tier's settings, each from the operator's variable, else the code,
  devBase : Train.Snapshot
      The development base's original weights: Qwen2.5 0.5B Instruct
  dockerfile : String
      The tier image's Dockerfile: the static linux/amd64 tier binary (`app`,
  dockerfileWith : List String -> String
      The tier image's Dockerfile for a binary that takes arguments: the
  knativeYaml : Target -> Tier -> String
      The same service as Knative YAML (`gcloud run services replace`).
  render : Target -> Tier -> Rendered
      Render a tier for a target.
  run : Tier -> Task Error ()
      Run the tier: stage the base's files below the AI home (`SKY_AI_HOME`;
  serviceJson : Target -> Tier -> String
      The Cloud Run service for a tier (Admin API v2), pure. One instance holds
  serviceName : Target -> String
      The service's full resource name.
  settings : Tier -> TierSettings
      What the tier holds.
  targetSettings : Target -> TargetSettings
      Where a target deploys.
  teardown : Target -> Task Error ()
      Delete the target's service. A service that is already gone counts as
  tier : Engine -> Train.Snapshot -> Tier
      A tier on a base model's pinned files (the original 16-bit weights vLLM
  tierBaseImage : String
      The tier image's base: distroless static, nonroot, linux/amd64, pinned by
  vllmArgs : String -> Tier -> List String
      vLLM's arguments for the tier, with the base staged below the AI home
  withAdapters : List String -> Tier -> Tier
      The registry aliases the tier keeps loaded: `"support-lora@prod"`.
  withAliasPolling : Int -> Tier -> Tier
      How often, in seconds, the tier reads its aliases' pointers. Env:
  withBaseName : String -> Tier -> Tier
      The name requests use for the base itself.
  withBind : String -> Tier -> Tier
      Where the tier listens. Env: `SKY_AI_SERVE_BIND`. Default:
  withEngineImage : String -> Target -> Target
      vLLM's image, pinned by digest. Default: `defaultEngineImage`.
  withEngineUrl : String -> Tier -> Tier
      Where vLLM listens. Env: `SKY_AI_SERVE_ENGINE_URL`. Default:
  withGpuMemoryFraction : Float -> Tier -> Tier
      The share of GPU memory vLLM may use (`--gpu-memory-utilization`).
  withInvokers : List String -> Target -> Target
      Who may call the service (`roles/run.invoker`): the app's service
  withLabels : List ( String, String ) -> Target -> Target
      Labels for the service, beside Sky's own (`sky-managed`,
  withMaxAdapters : Int -> Tier -> Tier
      Adapters resident at once (vLLM's `--max-loras`), at least one per
  withMaxInstances : Int -> Target -> Target
      The most instances (each with one L4). Default: 1.
  withMaxModelLength : Int -> Tier -> Tier
      vLLM's `--max-model-len`. Default: 4096.
  withMinInstances : Int -> Target -> Target
      Instances kept running when idle. Default: 0 (scale to zero: nothing is
  withModelsDir : String -> Tier -> Tier
      The directory the tier shares with vLLM (the base and the adapters).
  withRegistry : Store -> Tier -> Tier
      The registry's store, whose pointer objects the tier reads. Default: the
  withRegistryUrl : String -> Target -> Target
      The registry's store location the tier reads (`gs://bucket/prefix`).
  withServiceAccount : String -> Target -> Target
      The service account the tier runs as: it needs read access to the
  withTierImage : String -> Target -> Target
      The tier's image (yours, built by `sky ai image build --tier`), pinned
  withTimeoutSeconds : Int -> Target -> Target
      The longest a request may take, in seconds. Default: 300.
  withToken : Secret -> Tier -> Tier
      The service token callers send as `Authorization: Bearer`. Env:
  withTokenSecret : String -> Target -> Target
      The Secret Manager secret holding the service token (its latest