[docs] Update with the latest changes and use compilable examples

docs/connecting.asciidoc

@@ -1,9 +1,7 @@
 [[connecting]]
 == Connecting
 
-beta[]
-
-The Java client is structured around three main components:
+The {java-client} is structured around three main components:
 
 * **API client classes**. These provide strongly typed data structures and 
 methods for {es} APIs. Since the {es} API is large, it is structured in feature 
@@ -18,16 +16,7 @@ This code snippet creates and wires together these three components:
 
 ["source","java"]
 --------------------------------------------------
-// Create the low-level client
-RestClient restClient = RestClient.builder(
-    new HttpHost("localhost", 9200)).build();
-
-// Create the transport with a Jackson mapper
-Transport transport = new RestClientTransport(
-    restClient, new JacksonJsonpMapper());
-
-// And create the API client
-ElasticsearchClient client = new ElasticsearchClient(transport);
+include-tagged::{doc-tests}/ConnectingTest.java[create-client]
 --------------------------------------------------
 
 Authentication is managed by the <<java-rest-low>>. For further details on 
@@ -46,16 +35,5 @@ concise DSL-like code. This pattern is explained in more detail in
 
 ["source","java"]
 --------------------------------------------------
-SearchResponse<Product> search = client.search(s -> s
-    .index("products")
-    .query(q -> q
-        .term(t -> t
-            .field("name")
-            .value("bicycle")
-    )),
-    Product.class);
-
-for (Hit<AppData> hit: search.hits().hits()) {
-    processAppData(hit.source());
-}
---------------------------------------------------
+include-tagged::{doc-tests}/ConnectingTest.java[first-request]
+--------------------------------------------------

docs/index-local.asciidoc

@@ -0,0 +1,4 @@
+// Allow building docs locally without a checkout of the Elasticsearch repo
+:elasticsearch-root: {docdir}/local/elasticsearch
+
+include::index.asciidoc[]

docs/index.asciidoc

@@ -3,9 +3,18 @@
 :branch: master
 include::{asciidoc-dir}/../../shared/attributes.asciidoc[]
 
+:version: 7.16.0
+:java-client: Java API Client
+:doc-tests: {docdir}/../java-client/src/test/java/co/elastic/clients/documentation
+
 include::introduction.asciidoc[]
 include::installation.asciidoc[]
 include::connecting.asciidoc[]
 include::migrate.asciidoc[]
 include::api-conventions.asciidoc[]
-include::{elasticsearch-root}/docs/java-rest/low-level/index.asciidoc[]
+
+:version!:
+:java-client!:
+:doc-tests!:
+
+include::{elasticsearch-root}/docs/java-rest/low-level/index.asciidoc[]

docs/installation.asciidoc

@@ -1,12 +1,10 @@
 [[installation]]
 == Installation
 
-beta[]
-
 Requirements:
 
 * Java 8 or later.
-* Optionally, a JSON object mapping library to allow seamless integration of 
+* A JSON object mapping library to allow seamless integration of
   your application classes with the Elasticsearch API. The Java client has 
   support for https://github.com/FasterXML/jackson[Jackson] or a 
   http://json-b.net/[JSON-B] library like 
@@ -23,10 +21,10 @@ available at https://snapshots.elastic.co/maven/.
 [[gradle]]
 === Installation in a Gradle project by using Jackson
 
-["source","groovy",subs="attributes"]
+["source","groovy",subs="attributes+"]
 --------------------------------------------------
 dependencies {
-    implementation 'co.elastic.clients:elasticsearch-java:7.15.0'
+    implementation 'co.elastic.clients:elasticsearch-java:{version}'
     implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3'
 }
 --------------------------------------------------
@@ -38,15 +36,15 @@ dependencies {
 In the `pom.xml` of your project, add the following repository definition and 
 dependencies:
 
-["source","xml",subs="attributes"]
+["source","xml",subs="attributes+"]
 --------------------------------------------------
 <project>
 
   <dependencies>
     <dependency>
       <groupId>co.elastic.clients</groupId>
       <artifactId>elasticsearch-java</artifactId>
-      <version>7.15.0</version>
+      <version>{version}</version>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
@@ -56,4 +54,4 @@ dependencies:
   </dependencies>
 
 </project>
---------------------------------------------------
+--------------------------------------------------

docs/introduction.asciidoc

@@ -1,8 +1,6 @@
 [[introduction]]
 == Introduction
 
-beta[]
-
 This is the documentation for the official Java API Client for {es}. The client 
 provides strongly typed requests and responses for all {es} APIs.
 
@@ -20,29 +18,32 @@ provides strongly typed requests and responses for all {es} APIs.
   retries, node discovery, and so on.
 
 [discrete]
-=== Beta status
-
-The Java API Client is a new product and is still in beta. What this means in 
-concrete terms:
-
-* Although the general structure will not change, some renaming may happen to 
-  improve ease of use.
-* Some APIs may not be implemented yet. The code for APIs is generated from the 
-  https://github.com/elastic/elasticsearch-specification[{es} API specification] 
-  where complete coverage is targeted for the GA release of the Java client.
-* Some complex API data types are not yet fully supported. This revolves mostly 
-  around union types where the JSON API accepts many different shapes for some 
-  property. These types are currently represented as raw `JsonValue` objects.
-* Some built-in types like dates are represented as primitive strings. 
-  Additional support for standard JDK types will be added for the GA release.
+=== Main changes since version 7.15
+
+Version 7.15 of the {java-client} was a beta release. We learned a lot from the
+initial feedback and this led to a few breaking changes in this version:
+
+* the `_core` package that contains top-level {es} APIs has been renamed to `core`.
+* property setters for lists and maps have been revisited.
+* all API data types are now supported, and there are no more raw `JsonValue` properties.
+* the supporting JSON framework and transport classes have been refactored and improved.
+  However the high level APIs used by applications haven't changed so there should be
+  no of very little impact on existing application code.
+
+This version of the {java-client} supports all {es} APIs are implemented except the
+Vector tile search API and the Find structure API.
+
+While this version is considered stable, most of the code is generated from the
+https://github.com/elastic/elasticsearch-specification[{es} API specification] where
+some of the less often used {es} APIs still need to be fully validated. Changes in
+the specification that may impact application code will be clearly identified in later
+versions of the Java API client.
 
 [discrete]
 === Compatibility
 
-The main branch targets the next major release (8.0), the 7.x branch targets the 
-next minor release for the 7.x series. Support is still incomplete as the API 
-code is generated from the {es} Specification that is also still a work in 
-progress.
+The `main` branch targets the next major release (8.0), the `7.x` branch targets the
+next minor release for the `7.x` series.
 
 The {es} Java client is forward compatible; meaning that the client supports 
 communicating with greater or equal minor versions of {es}. {es} language 

docs/local/README.md

@@ -0,0 +1 @@
+This directory contains stubs that allow building the Java client docs in isolation following the instructions in "[Building documentation for a local repo](https://github.com/elastic/docs#building-documentation)".

docs/local/elasticsearch/docs/java-rest/low-level/index.asciidoc

@@ -0,0 +1,4 @@
+[[java-rest-low]]
+== Java Low Level REST Client
+
+This is a stub for the Java Low Level REST Client.

docs/migrate.asciidoc

@@ -39,21 +39,5 @@ The code below shows how to initialize both clients with the same HTTP client:
 
 ["source","java"]
 --------------------------------------------------
-// Create the low-level client
-RestClientBuilder httpClientBuilder = RestClient.builder(
-    new HttpHost("localhost", 9200)
-);
-
-// Create the HLRC
-RestHighLevelClient hlrc = new RestHighLevelClient(httpClientBuilder);
-
-// Create the new Java Client with the same low level client
-Transport transport = new RestClientTransport(
-    hlrc.getLowLevelClient(),
-    new JacksonJsonpMapper()
-);
-
-ElasticsearchClient esClient = new ElasticsearchClient(transport);
-
-// hlrc and esClient share the same httpClient
+include-tagged::{doc-tests}/MigrateHlrcTest.java[migrate]
 --------------------------------------------------

java-client/src/test/java/co/elastic/clients/documentation/ApiConventionsTest.java

@@ -21,6 +21,7 @@
 
 import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient;
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
+import co.elastic.clients.elasticsearch._types.NodeStatistics;
 import co.elastic.clients.elasticsearch._types.SortOrder;
 import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
@@ -31,6 +32,7 @@
 import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
 import co.elastic.clients.transport.ElasticsearchTransport;
 import co.elastic.clients.transport.TransportException;
+import co.elastic.clients.util.ApiTypeHelper;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -234,6 +236,25 @@ public void collections() {
 
     }
 
+    @Test
+    public void optionalCollections() {
+        //tag::optional-collections
+        NodeStatistics stats = NodeStatistics.of(b -> b
+            .total(1)
+            .failed(0)
+            .successful(1)
+        );
+
+        // The `failures` list was not provided.
+        // - it's not null
+        assertNotNull(stats.failures());
+        // - it's empty
+        assertEquals(0, stats.failures().size());
+        // - and if needed we can know it was actually not defined
+        assertFalse(ApiTypeHelper.isDefined(stats.failures()));
+        //end::optional-collections
+    }
+
     private void doSomething(Object... o) {
 
     }