feat: alow to show model metrics in playground

Signed-off-by: Evzen Gasta <[email protected]>

Author: gastoner

packages/backend/src/assets/inference-images.json

@@ -3,8 +3,8 @@
     "default": "ghcr.io/containers/whispercpp@sha256:6c529656529da7aba851b6ab2d0653f23b77b9ca3c11cb46db47ff6dbd6d5e7c"
   },
   "llamacpp": {
-    "default": "ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat@sha256:20734e9d60f047d27e4c9cf6a3b663e0627d48bd06d0a73b968f9d81c82de2f1",
-    "cuda": "ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat-cuda@sha256:798acced911527254601d0e39a90c5a29ecad82755f28594bea9a587ea9e6043",
-    "vulkan": "ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat-vulkan@sha256:22e11661fe66ace7c30b419703305b803eb937da10e19c23cb6767f03578256c"
+    "default": "ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat@sha256:2eb6cd7a4c4f76e54eeb88465281f4ff2a8f7b7e49db893a579a8f6842f69eb1",
+    "cuda": "ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat-cuda@sha256:e4b57e52c31b379b4a73f8e9536bc130fdea665d88dbd05643350295b3402a2f",
+    "vulkan": "ghcr.io/containers/podman-desktop-extension-ai-lab-playground-images/ai-lab-playground-chat-vulkan@sha256:6a93b247099643f4f8c78ee9896c2ce4e9a455af114a69be09c16ad36aa51fd2"
   }
 }

packages/backend/src/managers/playgroundV2Manager.ts

@@ -208,7 +208,7 @@ export class PlaygroundV2Manager implements Disposable {
       throw new Error(
         `modelId '${conversation.modelId}' is not available on the inference server, valid model ids are: ${server.models.map(model => model.id).join(', ')}.`,
       );
-
+      console.error(options)
     this.#conversationRegistry.submit(conversation.id, {
       content: userInput,
       options: options,
@@ -303,7 +303,7 @@ export class PlaygroundV2Manager implements Disposable {
       id: messageId,
       timestamp: start,
     } as PendingChat);
-
+    console.log(stream)
     for await (const chunk of stream) {
       this.#conversationRegistry.appendChoice(conversationId, messageId, {
         content: chunk.choices[0]?.delta?.content ?? '',

packages/frontend/src/pages/Playground.svelte

@@ -54,6 +54,7 @@ $: {
   } else {
     sendEnabled = true;
   }
+  console.error(conversation)
 }
 $: server = $inferenceServers.find(is => conversation && is.models.map(mi => mi.id).includes(conversation?.modelId));
 function askPlayground(): void {
@@ -64,6 +65,7 @@ function askPlayground(): void {
       temperature,
       max_tokens,
       top_p,
+      stream_options: {include_usage: true},
     })
     .then(token => {
       cancellationTokenId = token;
@@ -263,6 +265,50 @@ function handleOnClick(): void {
               </div>
             </svelte:fragment>
           </ContentDetailsLayout>
+
+          <ContentDetailsLayout detailsTitle="Model Metrics" detailsLabel="model metrics">
+            <svelte:fragment slot="details">
+              <div
+                class="bg-[var(--pd-content-card-inset-bg)] text-[var(--pd-content-card-text)] w-full rounded-md p-4">
+                <div class="flex flex-col space-y-4" aria-label="metrics">
+                  <div class="flex flex-row">
+                    <div class="w-full">
+                      Lorem
+                      <div class="flex flex-row">
+                        Ipsum
+                      </div>
+                    </div>
+                    <Tooltip left>
+                      <Fa class="text-[var(--pd-content-card-icon)]" icon={faCircleInfo} />
+                      <svelte:fragment slot="tip">
+                        <div class="inline-block py-2 px-4 rounded-md" aria-label="tooltip">
+                          What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output
+                          more random, while lower values like 0.2 will make it more focused and deterministic.
+                        </div>
+                      </svelte:fragment>
+                    </Tooltip>
+                  </div>
+                  <div class="flex flex-row">
+                    <div class="w-full">
+                      Lorem1
+                      <div class="flex flex-row">
+                        Ipsum2
+                      </div>
+                    </div>
+                    <Tooltip left>
+                      <Fa class="text-[var(--pd-content-card-icon)]" icon={faCircleInfo} />
+                      <svelte:fragment slot="tip">
+                        <div class="inline-block py-2 px-4 rounded-md" aria-label="tooltip">
+                          What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output
+                          more random, while lower values like 0.2 will make it more focused and deterministic.
+                        </div>
+                      </svelte:fragment>
+                    </Tooltip>
+                  </div>
+                </div>
+              </div>
+            </svelte:fragment>
+          </ContentDetailsLayout>
         </div>
         {#if errorMsg}
           <div class="text-[var(--pd-input-field-error-text)] p-2">{errorMsg}</div>

packages/shared/src/models/IModelOptions.ts

@@ -1,5 +1,10 @@
+interface StreamOptions {
+  include_usage?: boolean;
+}
+
 export interface ModelOptions {
   temperature?: number;
   max_tokens?: number;
   top_p?: number;
+  stream_options?: StreamOptions,
 }