Skip to content

Commit 95bbf92

Browse files
davidmotsonDavid Motsonashvilirachelsaundersdaymxn
authored
Initial implementation to API spec (#6607)
Added support for querying imagen models to generate images both in gcs and inline. Documentation incoming in a separate PR for readability --------- Co-authored-by: David Motsonashvili <[email protected]> Co-authored-by: rachelsaunders <[email protected]> Co-authored-by: Daymon <[email protected]>
1 parent e125fb7 commit 95bbf92

27 files changed

+965
-111
lines changed

‎firebase-vertexai/CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Unreleased
22
* [fixed] Added support for new values sent by the server for `FinishReason` and `BlockReason`.
33
* [changed] Added support for modality-based token count. (#6658)
4+
* [feature] Added support for generating images with Imagen models.
45

56
# 16.1.0
67
* [changed] Internal improvements to correctly handle empty model responses.
@@ -65,3 +66,4 @@
6566
* [feature] Added support for `responseMimeType` in `GenerationConfig`.
6667
* [changed] Renamed `GoogleGenerativeAIException` to `FirebaseVertexAIException`.
6768
* [changed] Updated the KDocs for various classes and functions.
69+

‎firebase-vertexai/api.txt

Lines changed: 125 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@ package com.google.firebase.vertexai {
2525
method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app);
2626
method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(com.google.firebase.FirebaseApp app = Firebase.app, String location);
2727
method public static com.google.firebase.vertexai.FirebaseVertexAI getInstance(String location);
28+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName);
29+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null);
30+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null);
31+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public com.google.firebase.vertexai.ImagenModel imagenModel(String modelName, com.google.firebase.vertexai.type.ImagenGenerationConfig? generationConfig = null, com.google.firebase.vertexai.type.ImagenSafetySettings? safetySettings = null, com.google.firebase.vertexai.type.RequestOptions requestOptions = com.google.firebase.vertexai.type.RequestOptions());
2832
property public static final com.google.firebase.vertexai.FirebaseVertexAI instance;
2933
field public static final com.google.firebase.vertexai.FirebaseVertexAI.Companion Companion;
3034
}
@@ -55,6 +59,10 @@ package com.google.firebase.vertexai {
5559
method public com.google.firebase.vertexai.Chat startChat(java.util.List<com.google.firebase.vertexai.type.Content> history = emptyList());
5660
}
5761

62+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenModel {
63+
method public suspend Object? generateImages(String prompt, kotlin.coroutines.Continuation<? super com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>>);
64+
}
65+
5866
}
5967

6068
package com.google.firebase.vertexai.java {
@@ -86,6 +94,17 @@ package com.google.firebase.vertexai.java {
8694
method public com.google.firebase.vertexai.java.GenerativeModelFutures from(com.google.firebase.vertexai.GenerativeModel model);
8795
}
8896

97+
@com.google.firebase.vertexai.type.PublicPreviewAPI public abstract class ImagenModelFutures {
98+
method public static final com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
99+
method public abstract com.google.common.util.concurrent.ListenableFuture<com.google.firebase.vertexai.type.ImagenGenerationResponse<com.google.firebase.vertexai.type.ImagenInlineImage>> generateImages(String prompt);
100+
method public abstract com.google.firebase.vertexai.ImagenModel getImageModel();
101+
field public static final com.google.firebase.vertexai.java.ImagenModelFutures.Companion Companion;
102+
}
103+
104+
public static final class ImagenModelFutures.Companion {
105+
method public com.google.firebase.vertexai.java.ImagenModelFutures from(com.google.firebase.vertexai.ImagenModel model);
106+
}
107+
89108
}
90109

91110
package com.google.firebase.vertexai.type {
@@ -163,6 +182,9 @@ package com.google.firebase.vertexai.type {
163182
property public final String? role;
164183
}
165184

185+
public final class ContentBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
186+
}
187+
166188
public final class ContentKt {
167189
method public static com.google.firebase.vertexai.type.Content content(String? role = "user", kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.Content.Builder,kotlin.Unit> init);
168190
}
@@ -376,6 +398,104 @@ package com.google.firebase.vertexai.type {
376398
property public final android.graphics.Bitmap image;
377399
}
378400

401+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenAspectRatio {
402+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio.Companion Companion;
403+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_16x9;
404+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio LANDSCAPE_4x3;
405+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_3x4;
406+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio PORTRAIT_9x16;
407+
field public static final com.google.firebase.vertexai.type.ImagenAspectRatio SQUARE_1x1;
408+
}
409+
410+
public static final class ImagenAspectRatio.Companion {
411+
}
412+
413+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationConfig {
414+
ctor public ImagenGenerationConfig(String? negativePrompt = null, Integer? numberOfImages = 1, com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio = null, com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat = null, Boolean? addWatermark = null);
415+
method public Boolean? getAddWatermark();
416+
method public com.google.firebase.vertexai.type.ImagenAspectRatio? getAspectRatio();
417+
method public com.google.firebase.vertexai.type.ImagenImageFormat? getImageFormat();
418+
method public String? getNegativePrompt();
419+
method public Integer? getNumberOfImages();
420+
property public final Boolean? addWatermark;
421+
property public final com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio;
422+
property public final com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat;
423+
property public final String? negativePrompt;
424+
property public final Integer? numberOfImages;
425+
field public static final com.google.firebase.vertexai.type.ImagenGenerationConfig.Companion Companion;
426+
}
427+
428+
public static final class ImagenGenerationConfig.Builder {
429+
ctor public ImagenGenerationConfig.Builder();
430+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig build();
431+
field public Boolean? addWatermark;
432+
field public com.google.firebase.vertexai.type.ImagenAspectRatio? aspectRatio;
433+
field public com.google.firebase.vertexai.type.ImagenImageFormat? imageFormat;
434+
field public String? negativePrompt;
435+
field public Integer? numberOfImages;
436+
}
437+
438+
public static final class ImagenGenerationConfig.Companion {
439+
method public com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder builder();
440+
}
441+
442+
public final class ImagenGenerationConfigKt {
443+
method @com.google.firebase.vertexai.type.PublicPreviewAPI public static com.google.firebase.vertexai.type.ImagenGenerationConfig imagenGenerationConfig(kotlin.jvm.functions.Function1<? super com.google.firebase.vertexai.type.ImagenGenerationConfig.Builder,kotlin.Unit> init);
444+
}
445+
446+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenGenerationResponse<T> {
447+
method public String? getFilteredReason();
448+
method public java.util.List<T> getImages();
449+
property public final String? filteredReason;
450+
property public final java.util.List<T> images;
451+
}
452+
453+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenImageFormat {
454+
method public Integer? getCompressionQuality();
455+
method public String getMimeType();
456+
property public final Integer? compressionQuality;
457+
property public final String mimeType;
458+
field public static final com.google.firebase.vertexai.type.ImagenImageFormat.Companion Companion;
459+
}
460+
461+
public static final class ImagenImageFormat.Companion {
462+
method public com.google.firebase.vertexai.type.ImagenImageFormat jpeg(Integer? compressionQuality = null);
463+
method public com.google.firebase.vertexai.type.ImagenImageFormat png();
464+
}
465+
466+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenInlineImage {
467+
method public android.graphics.Bitmap asBitmap();
468+
method public byte[] getData();
469+
method public String getMimeType();
470+
property public final byte[] data;
471+
property public final String mimeType;
472+
}
473+
474+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenPersonFilterLevel {
475+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ADULT;
476+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel ALLOW_ALL;
477+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel BLOCK_ALL;
478+
field public static final com.google.firebase.vertexai.type.ImagenPersonFilterLevel.Companion Companion;
479+
}
480+
481+
public static final class ImagenPersonFilterLevel.Companion {
482+
}
483+
484+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetyFilterLevel {
485+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_LOW_AND_ABOVE;
486+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_MEDIUM_AND_ABOVE;
487+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_NONE;
488+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel BLOCK_ONLY_HIGH;
489+
field public static final com.google.firebase.vertexai.type.ImagenSafetyFilterLevel.Companion Companion;
490+
}
491+
492+
public static final class ImagenSafetyFilterLevel.Companion {
493+
}
494+
495+
@com.google.firebase.vertexai.type.PublicPreviewAPI public final class ImagenSafetySettings {
496+
ctor public ImagenSafetySettings(com.google.firebase.vertexai.type.ImagenSafetyFilterLevel safetyFilterLevel, com.google.firebase.vertexai.type.ImagenPersonFilterLevel personFilterLevel);
497+
}
498+
379499
public final class InlineDataPart implements com.google.firebase.vertexai.type.Part {
380500
ctor public InlineDataPart(byte[] inlineData, String mimeType);
381501
method public byte[] getInlineData();
@@ -413,8 +533,8 @@ package com.google.firebase.vertexai.type {
413533
}
414534

415535
public final class PromptBlockedException extends com.google.firebase.vertexai.type.FirebaseVertexAIException {
416-
method public com.google.firebase.vertexai.type.GenerateContentResponse getResponse();
417-
property public final com.google.firebase.vertexai.type.GenerateContentResponse response;
536+
method public com.google.firebase.vertexai.type.GenerateContentResponse? getResponse();
537+
property public final com.google.firebase.vertexai.type.GenerateContentResponse? response;
418538
}
419539

420540
public final class PromptFeedback {
@@ -427,6 +547,9 @@ package com.google.firebase.vertexai.type {
427547
property public final java.util.List<com.google.firebase.vertexai.type.SafetyRating> safetyRatings;
428548
}
429549

550+
@kotlin.RequiresOptIn(level=kotlin.RequiresOptIn.Level.ERROR, message="This API is part of an experimental public preview and may change in " + "backwards-incompatible ways without notice.") @kotlin.annotation.Retention(kotlin.annotation.AnnotationRetention.BINARY) public @interface PublicPreviewAPI {
551+
}
552+
430553
public final class RequestOptions {
431554
ctor public RequestOptions();
432555
ctor public RequestOptions(long timeoutInMillis = 180.seconds.inWholeMilliseconds);

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/FirebaseVertexAI.kt

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ import com.google.firebase.auth.internal.InternalAuthProvider
2424
import com.google.firebase.inject.Provider
2525
import com.google.firebase.vertexai.type.Content
2626
import com.google.firebase.vertexai.type.GenerationConfig
27+
import com.google.firebase.vertexai.type.ImagenGenerationConfig
28+
import com.google.firebase.vertexai.type.ImagenSafetySettings
2729
import com.google.firebase.vertexai.type.InvalidLocationException
30+
import com.google.firebase.vertexai.type.PublicPreviewAPI
2831
import com.google.firebase.vertexai.type.RequestOptions
2932
import com.google.firebase.vertexai.type.SafetySetting
3033
import com.google.firebase.vertexai.type.Tool
@@ -79,6 +82,37 @@ internal constructor(
7982
)
8083
}
8184

85+
/**
86+
* Instantiates a new [ImagenModel] given the provided parameters.
87+
*
88+
* @param modelName The name of the model to use, for example `"imagen-3.0-generate-001"`.
89+
* @param generationConfig The configuration parameters to use for image generation.
90+
* @param safetySettings The safety bounds the model will abide by during image generation.
91+
* @param requestOptions Configuration options for sending requests to the backend.
92+
* @return The initialized [ImagenModel] instance.
93+
*/
94+
@JvmOverloads
95+
@PublicPreviewAPI
96+
public fun imagenModel(
97+
modelName: String,
98+
generationConfig: ImagenGenerationConfig? = null,
99+
safetySettings: ImagenSafetySettings? = null,
100+
requestOptions: RequestOptions = RequestOptions(),
101+
): ImagenModel {
102+
if (location.trim().isEmpty() || location.contains("/")) {
103+
throw InvalidLocationException(location)
104+
}
105+
return ImagenModel(
106+
"projects/${firebaseApp.options.projectId}/locations/${location}/publishers/google/models/${modelName}",
107+
firebaseApp.options.apiKey,
108+
generationConfig,
109+
safetySettings,
110+
requestOptions,
111+
appCheckProvider.get(),
112+
internalAuthProvider.get(),
113+
)
114+
}
115+
82116
public companion object {
83117
/** The [FirebaseVertexAI] instance for the default [FirebaseApp] */
84118
@JvmStatic

‎firebase-vertexai/src/main/kotlin/com/google/firebase/vertexai/GenerativeModel.kt

Lines changed: 5 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,12 @@
1717
package com.google.firebase.vertexai
1818

1919
import android.graphics.Bitmap
20-
import android.util.Log
2120
import com.google.firebase.appcheck.interop.InteropAppCheckTokenProvider
2221
import com.google.firebase.auth.internal.InternalAuthProvider
2322
import com.google.firebase.vertexai.common.APIController
23+
import com.google.firebase.vertexai.common.AppCheckHeaderProvider
2424
import com.google.firebase.vertexai.common.CountTokensRequest
2525
import com.google.firebase.vertexai.common.GenerateContentRequest
26-
import com.google.firebase.vertexai.common.HeaderProvider
2726
import com.google.firebase.vertexai.type.Content
2827
import com.google.firebase.vertexai.type.CountTokensResponse
2928
import com.google.firebase.vertexai.type.FinishReason
@@ -38,12 +37,9 @@ import com.google.firebase.vertexai.type.SerializationException
3837
import com.google.firebase.vertexai.type.Tool
3938
import com.google.firebase.vertexai.type.ToolConfig
4039
import com.google.firebase.vertexai.type.content
41-
import kotlin.time.Duration
42-
import kotlin.time.Duration.Companion.seconds
4340
import kotlinx.coroutines.flow.Flow
4441
import kotlinx.coroutines.flow.catch
4542
import kotlinx.coroutines.flow.map
46-
import kotlinx.coroutines.tasks.await
4743

4844
/**
4945
* Represents a multimodal model (like Gemini), capable of generating content based on various input
@@ -57,10 +53,8 @@ internal constructor(
5753
private val tools: List<Tool>? = null,
5854
private val toolConfig: ToolConfig? = null,
5955
private val systemInstruction: Content? = null,
60-
private val controller: APIController
56+
private val controller: APIController,
6157
) {
62-
63-
@JvmOverloads
6458
internal constructor(
6559
modelName: String,
6660
apiKey: String,
@@ -84,42 +78,8 @@ internal constructor(
8478
modelName,
8579
requestOptions,
8680
"gl-kotlin/${KotlinVersion.CURRENT} fire/${BuildConfig.VERSION_NAME}",
87-
object : HeaderProvider {
88-
override val timeout: Duration
89-
get() = 10.seconds
90-
91-
override suspend fun generateHeaders(): Map<String, String> {
92-
val headers = mutableMapOf<String, String>()
93-
if (appCheckTokenProvider == null) {
94-
Log.w(TAG, "AppCheck not registered, skipping")
95-
} else {
96-
val token = appCheckTokenProvider.getToken(false).await()
97-
98-
if (token.error != null) {
99-
Log.w(TAG, "Error obtaining AppCheck token", token.error)
100-
}
101-
// The Firebase App Check backend can differentiate between apps without App Check, and
102-
// wrongly configured apps by verifying the value of the token, so it always needs to be
103-
// included.
104-
headers["X-Firebase-AppCheck"] = token.token
105-
}
106-
107-
if (internalAuthProvider == null) {
108-
Log.w(TAG, "Auth not registered, skipping")
109-
} else {
110-
try {
111-
val token = internalAuthProvider.getAccessToken(false).await()
112-
113-
headers["Authorization"] = "Firebase ${token.token!!}"
114-
} catch (e: Exception) {
115-
Log.w(TAG, "Error getting Auth token ", e)
116-
}
117-
}
118-
119-
return headers
120-
}
121-
}
122-
)
81+
AppCheckHeaderProvider(TAG, appCheckTokenProvider, internalAuthProvider),
82+
),
12383
)
12484

12585
/**
@@ -247,7 +207,7 @@ internal constructor(
247207
generationConfig?.toInternal(),
248208
tools?.map { it.toInternal() },
249209
toolConfig?.toInternal(),
250-
systemInstruction?.copy(role = "system")?.toInternal()
210+
systemInstruction?.copy(role = "system")?.toInternal(),
251211
)
252212

253213
private fun constructCountTokensRequest(vararg prompt: Content) =

0 commit comments

Comments
 (0)