Merge f9a38a7 into aad3da8

Author: dlarocque

firebase-ai/CHANGELOG.md

@@ -1,4 +1,6 @@
 # Unreleased
+* [feature] **Breaking Change**: Add support for Grounding with Google Search (#7042).
+    * **Action Required:** Update all references of `groundingAttributions`, `webSearchQueries`, `retrievalQueries` in `GroundingMetadata` to be non-optional.
 
 
 # 16.2.0

firebase-ai/api.txt

@@ -171,10 +171,12 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.CitationMetadata? getCitationMetadata();
     method public com.google.firebase.ai.type.Content getContent();
     method public com.google.firebase.ai.type.FinishReason? getFinishReason();
+    method public com.google.firebase.ai.type.GroundingMetadata? getGroundingMetadata();
     method public java.util.List<com.google.firebase.ai.type.SafetyRating> getSafetyRatings();
     property public final com.google.firebase.ai.type.CitationMetadata? citationMetadata;
     property public final com.google.firebase.ai.type.Content content;
     property public final com.google.firebase.ai.type.FinishReason? finishReason;
+    property public final com.google.firebase.ai.type.GroundingMetadata? groundingMetadata;
     property public final java.util.List<com.google.firebase.ai.type.SafetyRating> safetyRatings;
   }
 
@@ -398,6 +400,48 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.GenerativeBackend vertexAI(String location = "us-central1");
   }
 
+  public final class GoogleSearch {
+    ctor public GoogleSearch();
+  }
+
+  @Deprecated public final class GroundingAttribution {
+    ctor @Deprecated public GroundingAttribution(com.google.firebase.ai.type.Segment segment, Float? confidenceScore);
+    method @Deprecated public Float? getConfidenceScore();
+    method @Deprecated public com.google.firebase.ai.type.Segment getSegment();
+    property @Deprecated public final Float? confidenceScore;
+    property @Deprecated public final com.google.firebase.ai.type.Segment segment;
+  }
+
+  public final class GroundingChunk {
+    ctor public GroundingChunk(com.google.firebase.ai.type.WebGroundingChunk? web);
+    method public com.google.firebase.ai.type.WebGroundingChunk? getWeb();
+    property public final com.google.firebase.ai.type.WebGroundingChunk? web;
+  }
+
+  public final class GroundingMetadata {
+    ctor public GroundingMetadata(java.util.List<java.lang.String> webSearchQueries, com.google.firebase.ai.type.SearchEntryPoint? searchEntryPoint, java.util.List<java.lang.String> retrievalQueries, @Deprecated java.util.List<com.google.firebase.ai.type.GroundingAttribution> groundingAttribution, java.util.List<com.google.firebase.ai.type.GroundingChunk> groundingChunks, java.util.List<com.google.firebase.ai.type.GroundingSupport> groundingSupports);
+    method @Deprecated public java.util.List<com.google.firebase.ai.type.GroundingAttribution> getGroundingAttribution();
+    method public java.util.List<com.google.firebase.ai.type.GroundingChunk> getGroundingChunks();
+    method public java.util.List<com.google.firebase.ai.type.GroundingSupport> getGroundingSupports();
+    method public java.util.List<java.lang.String> getRetrievalQueries();
+    method public com.google.firebase.ai.type.SearchEntryPoint? getSearchEntryPoint();
+    method public java.util.List<java.lang.String> getWebSearchQueries();
+    property @Deprecated public final java.util.List<com.google.firebase.ai.type.GroundingAttribution> groundingAttribution;
+    property public final java.util.List<com.google.firebase.ai.type.GroundingChunk> groundingChunks;
+    property public final java.util.List<com.google.firebase.ai.type.GroundingSupport> groundingSupports;
+    property public final java.util.List<java.lang.String> retrievalQueries;
+    property public final com.google.firebase.ai.type.SearchEntryPoint? searchEntryPoint;
+    property public final java.util.List<java.lang.String> webSearchQueries;
+  }
+
+  public final class GroundingSupport {
+    ctor public GroundingSupport(com.google.firebase.ai.type.Segment segment, java.util.List<java.lang.Integer> groundingChunkIndices);
+    method public java.util.List<java.lang.Integer> getGroundingChunkIndices();
+    method public com.google.firebase.ai.type.Segment getSegment();
+    property public final java.util.List<java.lang.Integer> groundingChunkIndices;
+    property public final com.google.firebase.ai.type.Segment segment;
+  }
+
   public final class HarmBlockMethod {
     method public int getOrdinal();
     property public final int ordinal;
@@ -897,6 +941,26 @@ package com.google.firebase.ai.type {
     method public com.google.firebase.ai.type.Schema str(String? description = null, boolean nullable = false, com.google.firebase.ai.type.StringFormat? format = null, String? title = null);
   }
 
+  public final class SearchEntryPoint {
+    ctor public SearchEntryPoint(String renderedContent, String? sdkBlob);
+    method public String getRenderedContent();
+    method public String? getSdkBlob();
+    property public final String renderedContent;
+    property public final String? sdkBlob;
+  }
+
+  public final class Segment {
+    ctor public Segment(int startIndex, int endIndex, int partIndex, String text);
+    method public int getEndIndex();
+    method public int getPartIndex();
+    method public int getStartIndex();
+    method public String getText();
+    property public final int endIndex;
+    property public final int partIndex;
+    property public final int startIndex;
+    property public final String text;
+  }
+
   public final class SerializationException extends com.google.firebase.ai.type.FirebaseAIException {
   }
 
@@ -935,11 +999,13 @@ package com.google.firebase.ai.type {
 
   public final class Tool {
     method public static com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
+    method public static com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
     field public static final com.google.firebase.ai.type.Tool.Companion Companion;
   }
 
   public static final class Tool.Companion {
     method public com.google.firebase.ai.type.Tool functionDeclarations(java.util.List<com.google.firebase.ai.type.FunctionDeclaration> functionDeclarations);
+    method public com.google.firebase.ai.type.Tool googleSearch(com.google.firebase.ai.type.GoogleSearch googleSearch = com.google.firebase.ai.type.GoogleSearch());
   }
 
   public final class ToolConfig {
@@ -987,5 +1053,15 @@ package com.google.firebase.ai.type {
   @Deprecated public static final class Voices.Companion {
   }
 
+  public final class WebGroundingChunk {
+    ctor public WebGroundingChunk(String? uri, String? title, String? domain);
+    method public String? getDomain();
+    method public String? getTitle();
+    method public String? getUri();
+    property public final String? domain;
+    property public final String? title;
+    property public final String? uri;
+  }
+
 }
 

firebase-ai/src/main/kotlin/com/google/firebase/ai/type/Candidate.kt

@@ -33,13 +33,15 @@ import kotlinx.serialization.json.JsonNames
  * @property safetyRatings A list of [SafetyRating]s describing the generated content.
  * @property citationMetadata Metadata about the sources used to generate this content.
  * @property finishReason The reason the model stopped generating content, if it exist.
+ * @property groundingMetadata Metadata returned to the client when grounding is enabled.
  */
 public class Candidate
 internal constructor(
   public val content: Content,
   public val safetyRatings: List<SafetyRating>,
   public val citationMetadata: CitationMetadata?,
-  public val finishReason: FinishReason?
+  public val finishReason: FinishReason?,
+  public val groundingMetadata: GroundingMetadata?
 ) {
 
   @Serializable
@@ -48,48 +50,22 @@ internal constructor(
     val finishReason: FinishReason.Internal? = null,
     val safetyRatings: List<SafetyRating.Internal>? = null,
     val citationMetadata: CitationMetadata.Internal? = null,
-    val groundingMetadata: GroundingMetadata? = null,
+    val groundingMetadata: GroundingMetadata.Internal? = null
   ) {
     internal fun toPublic(): Candidate {
       val safetyRatings = safetyRatings?.mapNotNull { it.toPublic() }.orEmpty()
       val citations = citationMetadata?.toPublic()
       val finishReason = finishReason?.toPublic()
+      val groundingMetadata = groundingMetadata?.toPublic()
 
       return Candidate(
         this.content?.toPublic() ?: content("model") {},
         safetyRatings,
         citations,
-        finishReason
+        finishReason,
+        groundingMetadata
       )
     }
-
-    @Serializable
-    internal data class GroundingMetadata(
-      @SerialName("web_search_queries") val webSearchQueries: List<String>?,
-      @SerialName("search_entry_point") val searchEntryPoint: SearchEntryPoint?,
-      @SerialName("retrieval_queries") val retrievalQueries: List<String>?,
-      @SerialName("grounding_attribution") val groundingAttribution: List<GroundingAttribution>?,
-    ) {
-
-      @Serializable
-      internal data class SearchEntryPoint(
-        @SerialName("rendered_content") val renderedContent: String?,
-        @SerialName("sdk_blob") val sdkBlob: String?,
-      )
-
-      @Serializable
-      internal data class GroundingAttribution(
-        val segment: Segment,
-        @SerialName("confidence_score") val confidenceScore: Float?,
-      ) {
-
-        @Serializable
-        internal data class Segment(
-          @SerialName("start_index") val startIndex: Int,
-          @SerialName("end_index") val endIndex: Int,
-        )
-      }
-    }
   }
 }
 
@@ -317,3 +293,203 @@ public class FinishReason private constructor(public val name: String, public va
     public val MALFORMED_FUNCTION_CALL: FinishReason = FinishReason("MALFORMED_FUNCTION_CALL", 9)
   }
 }
+
+/**
+ * Metadata returned to the client when grounding is enabled.
+ *
+ * If using Grounding with Google Search, you are required to comply with the "Grounding with Google
+ * Search" usage requirements for your chosen API provider:
+ * [Gemini Developer
+ * API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or
+ * Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section
+ * within the Service Specific Terms).
+ *
+ * @property webSearchQueries The list of web search queries that the model performed to gather the
+ * grounding information. These can be used to allow users to explore the search results themselves.
+ * @property searchEntryPoint Google search entry point for web searches. This contains an HTML/CSS
+ * snippet that **must** be embedded in an app to display a Google Search Entry point for follow-up
+ * web searches related to the model's "Grounded Response".
+ * @property groundingChunks The list of [GroundingChunk] classes. Each chunk represents a piece of
+ * retrieved content that the model used to ground its response.
+ * @property groundingSupports The list of [GroundingSupport] objects. Each object details how
+ * specific segments of the model's response are supported by the `groundingChunks`.
+ */
+public class GroundingMetadata(
+  public val webSearchQueries: List<String>,
+  public val searchEntryPoint: SearchEntryPoint?,
+  public val retrievalQueries: List<String>,
+  @Deprecated("Use groundingChunks instead")
+  public val groundingAttribution: List<GroundingAttribution>,
+  public val groundingChunks: List<GroundingChunk>,
+  public val groundingSupports: List<GroundingSupport>,
+) {
+  @Serializable
+  internal data class Internal(
+    val webSearchQueries: List<String>?,
+    val searchEntryPoint: SearchEntryPoint.Internal?,
+    val retrievalQueries: List<String>?,
+    @Deprecated("Use groundingChunks instead")
+    val groundingAttribution: List<GroundingAttribution.Internal>?,
+    val groundingChunks: List<GroundingChunk.Internal>?,
+    val groundingSupports: List<GroundingSupport.Internal>?,
+  ) {
+    internal fun toPublic() =
+      GroundingMetadata(
+        webSearchQueries = webSearchQueries.orEmpty(),
+        searchEntryPoint = searchEntryPoint?.toPublic(),
+        retrievalQueries = retrievalQueries.orEmpty(),
+        groundingAttribution = groundingAttribution?.map { it.toPublic() }.orEmpty(),
+        groundingChunks = groundingChunks?.map { it.toPublic() }.orEmpty(),
+        groundingSupports = groundingSupports?.map { it.toPublic() }.orEmpty().filterNotNull()
+      )
+  }
+}
+
+/**
+ * Represents a Google Search entry point.
+ *
+ * @property renderedContent An HTML/CSS snippet that can be embedded in your app. To ensure proper
+ * rendering, it's recommended to display this content within a `WebView`.
+ * @property sdkBlob A blob of data for the client SDK to render the search entry point.
+ */
+public class SearchEntryPoint(
+  public val renderedContent: String,
+  public val sdkBlob: String?,
+) {
+  @Serializable
+  internal data class Internal(
+    val renderedContent: String?,
+    val sdkBlob: String?,
+  ) {
+    internal fun toPublic(): SearchEntryPoint {
+      // If rendered content is null, the user must not display the grounded result. If they do,
+      // they violate the service terms. To prevent this from happening, throw an exception.
+      if (renderedContent == null) {
+        throw SerializationException("renderedContent is null, should be a string")
+      }
+      return SearchEntryPoint(renderedContent = renderedContent, sdkBlob = sdkBlob)
+    }
+  }
+}
+
+/**
+ * Represents a chunk of retrieved data that supports a claim in the model's response. This is part
+ * of the grounding information provided when grounding is enabled.
+ *
+ * @property web Contains details if the grounding chunk is from a web source.
+ */
+public class GroundingChunk(
+  public val web: WebGroundingChunk?,
+) {
+  @Serializable
+  internal data class Internal(
+    val web: WebGroundingChunk.Internal?,
+  ) {
+    internal fun toPublic() = GroundingChunk(web = web?.toPublic())
+  }
+}
+
+/**
+ * A grounding chunk from the web.
+ *
+ * @property uri The URI of the retrieved web page.
+ * @property title The title of the retrieved web page.
+ * @property domain The domain of the original URI from which the content was retrieved. This is
+ * only populated when using the Vertex AI Gemini API.
+ */
+public class WebGroundingChunk(
+  public val uri: String?,
+  public val title: String?,
+  public val domain: String?
+) {
+  @Serializable
+  internal data class Internal(val uri: String?, val title: String?, val domain: String?) {
+    internal fun toPublic() = WebGroundingChunk(uri = uri, title = title, domain = domain)
+  }
+}
+
+/**
+ * Provides information about how a specific segment of the model's response is supported by the
+ * retrieved grounding chunks.
+ *
+ * @property segment Specifies the segment of the model's response content that this grounding
+ * support pertains to.
+ * @property groundingChunkIndices A list of indices that refer to specific [GroundingChunk] classes
+ * within the [GroundingMetadata.groundingChunks] array. These referenced chunks are the sources
+ * that support the claim made in the associated `segment` of the response. For example, an array
+ * `[1, 3, 4]` means that `groundingChunks[1]`, `groundingChunks[3]`, `groundingChunks[4]` are the
+ * retrieved content supporting this part of the response.
+ */
+public class GroundingSupport(
+  public val segment: Segment,
+  public val groundingChunkIndices: List<Int>,
+) {
+  @Serializable
+  internal data class Internal(
+    val segment: Segment.Internal?,
+    val groundingChunkIndices: List<Int>?,
+  ) {
+    internal fun toPublic(): GroundingSupport? {
+      if (segment == null) {
+        return null
+      }
+      return GroundingSupport(
+        segment = segment.toPublic(),
+        groundingChunkIndices = groundingChunkIndices.orEmpty(),
+      )
+    }
+  }
+}
+
+@Deprecated("Use GroundingChunk instead")
+public class GroundingAttribution(
+  public val segment: Segment,
+  public val confidenceScore: Float?,
+) {
+  @Deprecated("Use GroundingChunk instead")
+  @Serializable
+  internal data class Internal(
+    val segment: Segment.Internal,
+    val confidenceScore: Float?,
+  ) {
+    internal fun toPublic() =
+      GroundingAttribution(segment = segment.toPublic(), confidenceScore = confidenceScore)
+  }
+}
+
+/**
+ * Represents a specific segment within a [Content] object, often used to pinpoint the exact
+ * location of text or data that grounding information refers to.
+ *
+ * @property partIndex The zero-based index of the [Part] object within the `parts` array of its
+ * parent [Content] object. This identifies which part of the content the segment belongs to.
+ * @property startIndex The zero-based start index of the segment within the specified [Part],
+ * measured in UTF-8 bytes. This offset is inclusive, starting from 0 at the beginning of the part's
+ * content.
+ * @property endIndex The zero-based end index of the segment within the specified [Part], measured
+ * in UTF-8 bytes. This offset is exclusive, meaning the character at this index is not included in
+ * the segment.
+ * @property text The text corresponding to the segment from the response.
+ */
+public class Segment(
+  public val startIndex: Int,
+  public val endIndex: Int,
+  public val partIndex: Int,
+  public val text: String,
+) {
+  @Serializable
+  internal data class Internal(
+    val startIndex: Int?,
+    val endIndex: Int?,
+    val partIndex: Int?,
+    val text: String?,
+  ) {
+    internal fun toPublic() =
+      Segment(
+        startIndex = startIndex ?: 0,
+        endIndex = endIndex ?: 0,
+        partIndex = partIndex ?: 0,
+        text = text ?: ""
+      )
+  }
+}