Support advancing LSQT

[sammefford]

This new functionality has been added to the REST API, so we can add it to the clients.

[sammefford]

Added two methods to TemporalDocumentManager

 /**
  *  Enables Last Stable Query Time (LSQT) on the named collection and
  *  advances the LSQT for the collection to the maximum system start time.
  *  When LSQT is enabled on the temporal collection, you can use the
  *  systemTime argument on many of the other TemporalDocumentManager methods.
  *
  *  The system time is returned in ISO 8601 format like all MarkLogic
  *  timestamps.  It can be parsed by
  *  DatatypeConverter.parseDateTime but will lose precision since
  *  java.util.Calendar only supports millisecond precision.
  *
  *  Requires a user with the "rest-admin" privilege.
  *
  *  For details on how to use LSQT, see Last Stable Query Time (LSQT) and
  *  Application-controlled System Time in the Temporal Developer's Guide.
  *
  *  @param temporalCollection the name of the temporal collection existing in
  *    the database into which this document should be written
  *  @param lag the milliseconds behind the maximum system start time to set LSQT
  *  @return the temporal system time
  */
 public String advanceLsqt(String temporalCollection);
 public String advanceLsqt(String temporalCollection, long lag);

[georgeajit]

  @Test
  public void testAdvancingLSQT() throws Exception {
      try {
          System.out.println("Inside testAdvancingLSQT");
          ConnectedRESTQA.disableAutomationOnTemporalCollection(dbName, temporalLsqtCollectionName, true);

          String docId = "javaSingleJSONDoc.json";
          String afterLSQTAdvance = null;

          Calendar firstInsertTime = DatatypeConverter.parseDateTime("2010-01-01T00:00:01");
          JSONDocumentManager docMgr = writerClient.newJSONDocumentManager();

          JacksonDatabindHandle<ObjectNode> handle = getJSONDocumentHandle(
                  "2001-01-01T00:00:00", "2011-12-31T23:59:59", "999 Skyway Park - JSON",
                  docId);
          TemporalDescriptor desc = docMgr.write(docId, null, handle, null, null, temporalLsqtCollectionName, firstInsertTime);
          // Verify permissions for LSQT advance
          String permnExceptMsg = "User is not allowed to advanceLsqt resource at temporal/collections/" + temporalLsqtCollectionName;
          String extMsg = null;
          try {
              JSONDocumentManager docMgr1 = readerClient.newJSONDocumentManager();
              docMgr1.advanceLsqt(temporalLsqtCollectionName);
          }
          catch (ForbiddenUserException ex) {
              extMsg = ex.getMessage();
              System.out.println("Permissions exception message for LSQT advance is " + extMsg);
          }
          assertTrue("Expected exception message incorrect for LSQT advance user permission", extMsg.contains(permnExceptMsg));

          QueryManager queryMgrLSQT = adminClient.newQueryManager();
          StructuredQueryBuilder sqbLSQT = queryMgrLSQT.newStructuredQueryBuilder();

          Calendar queryTimeLSQT = DatatypeConverter.parseDateTime("2007-01-01T00:00:01");
          StructuredQueryDefinition periodQueryLSQT = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT, 0, new String[] {});

          long startLSQT = 1;
          JSONDocumentManager docMgrQy = adminClient.newJSONDocumentManager();
          String WithoutAdvaceSetExceptMsg = "Timestamp 2007-01-01T00:00:01-08:00 provided is greater than LSQT 1601-01-01T00:00:00Z";
          String actualNoAdvanceMsg = null;
          DocumentPage termQueryResultsLSQT = null;
          try {
              termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT);
          }
          catch(Exception ex) {
              actualNoAdvanceMsg = ex.getMessage();
              System.out.println("Exception message for LSQT without advance set is " + actualNoAdvanceMsg);
          }
          assertTrue("Expected exception message not available for LSQT advance user permission", actualNoAdvanceMsg.contains(WithoutAdvaceSetExceptMsg));

          // Set the Advance manually.
          docMgr.advanceLsqt(temporalLsqtCollectionName);
          termQueryResultsLSQT = docMgrQy.search(periodQueryLSQT, startLSQT);

          assertTrue("LSQT Query results (Total Pages) before advance is incorrect", termQueryResultsLSQT.getTotalPages() == 0);
          assertTrue("LSQT Query results (Size) before advance is incorrect", termQueryResultsLSQT.size() == 0);

          // After Advance of the LSQT, query again with new query time greater than LSQT

          afterLSQTAdvance = desc.getTemporalSystemTime();
          Calendar queryTimeLSQT2 = DatatypeConverter.parseDateTime(afterLSQTAdvance);
          queryTimeLSQT2.add(Calendar.YEAR, 10);
          docMgrQy = adminClient.newJSONDocumentManager();
          docMgrQy.setMetadataCategories(Metadata.ALL); // Get all meta-data
          StructuredQueryDefinition periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT2, 0, new String[] {});

          String excepMsgGrtr = "Timestamp 2020-01-01T00:00:01-08:00 provided is greater than LSQT 2010-01-01T08:00:01Z";
          String actGrMsg = null;
          DocumentPage termQueryResultsLSQT2 = null;
          try {
              termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT);
          }
          catch(Exception ex) {
              actGrMsg = ex.getMessage();
          }
          assertTrue("Expected exception message not available for LSQT advance user permission", actGrMsg.contains(excepMsgGrtr));

          // Query again with query time less than LSQT. 10 minutes less than the LSQT
          Calendar lessTime = DatatypeConverter.parseDateTime("2009-01-01T00:00:01");

          periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, lessTime, 0, new String[] {});
          termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT);

          System.out.println("LSQT Query results (Total Pages) after advance " + termQueryResultsLSQT2.getTotalPages());
          System.out.println("LSQT Query results (Size) after advance " + termQueryResultsLSQT2.size());
          assertTrue("LSQT Query results (Total Pages) after advance is incorrect", termQueryResultsLSQT2.getTotalPages() == 0);
          assertTrue("LSQT Query results (Size) after advance is incorrect", termQueryResultsLSQT2.size() == 0);

          // Query again with query time equal to LSQT.
          queryTimeLSQT2 = DatatypeConverter.parseDateTime(afterLSQTAdvance);
          periodQueryLSQT2 = sqbLSQT.temporalLsqtQuery(temporalLsqtCollectionName, queryTimeLSQT2, 0, new String[] {});
          termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT);

          System.out.println("LSQT Query results (Total Pages) after advance " + termQueryResultsLSQT2.getTotalPages());
          System.out.println("LSQT Query results (Size) after advance " + termQueryResultsLSQT2.size());
          assertTrue("LSQT Query results (Total Pages) after advance is incorrect", termQueryResultsLSQT2.getTotalPages() == 1);
          assertTrue("LSQT Query results (Size) after advance is incorrect", termQueryResultsLSQT2.size() == 1);

          while (termQueryResultsLSQT2.hasNext()) {
              DocumentRecord record = termQueryResultsLSQT2.next();
              System.out.println("URI = " + record.getUri());
              StringHandle resultHandleOfLSQT2 = new StringHandle();
              record.getContent(resultHandleOfLSQT2);
              String strResOfLSQT2 = resultHandleOfLSQT2.get();

              System.out.println("Result of LSQT Query 2 is " + strResOfLSQT2);
          }

          // Verify that the document was inserted
          JacksonDatabindHandle<ObjectNode> recordHandle = new JacksonDatabindHandle<>(ObjectNode.class);
          DocumentMetadataHandle metadataHandle = new DocumentMetadataHandle();
          docMgr.read(docId, metadataHandle, recordHandle);
          DocumentPage readResults = docMgr.read(docId);

          System.out.println("Number of results = " + readResults.size());
          assertEquals("Wrong number of results", 1, readResults.size());

          DocumentRecord record = readResults.next();
          System.out.println("URI after insert = " + record.getUri());
          assertEquals("Document uri wrong after insert", docId, record.getUri());
          System.out.println("Content = " + recordHandle.toString());

          // Make sure System start time was what was set ("2010-01-01T00:00:01")
          if (record.getFormat() != Format.JSON) {
              assertFalse("Invalid document format: " + record.getFormat(), true);
          } else {
              JsonFactory factory = new JsonFactory();
              ObjectMapper mapper = new ObjectMapper(factory);
              TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {};

              Map<String, Object> docObject = mapper.readValue(recordHandle.toString(), typeRef);

              @SuppressWarnings("unchecked")
              Map<String, Object> validNode = (HashMap<String, Object>) (docObject.get(systemNodeName));

              String systemStartDate = (String) validNode.get(systemStartERIName);
              String systemEndDate = (String) validNode.get(systemEndERIName);
              System.out.println("systemStartDate = " + systemStartDate);
              System.out.println("systemEndDate = " + systemEndDate);

              assertTrue("System start date check failed", (systemStartDate.contains("2010-01-01T00:00:01")));
              assertTrue("System end date check failed", (systemEndDate.contains("9999-12-31T11:59:59")));

              // Validate collections
              Iterator<String> resCollections = metadataHandle.getCollections().iterator();
              while (resCollections.hasNext()) {
                  String collection = resCollections.next();
                  System.out.println("Collection = " + collection);

                  if (!collection.equals(docId)
                          && !collection.equals(insertCollectionName)
                          && !collection.equals(temporalLsqtCollectionName)
                          && !collection.equals(latestCollectionName)) {
                      assertFalse("Collection not what is expected: " + collection, true);
                  }
              }

              // Validate permissions
              DocumentPermissions permissions = metadataHandle.getPermissions();
              System.out.println("Permissions: " + permissions);

              String actualPermissions = getDocumentPermissionsString(permissions);
              System.out.println("actualPermissions: " + actualPermissions);

              assertTrue("Document permissions difference in size value",
                      actualPermissions.contains("size:3"));

              assertTrue("Document permissions difference in rest-reader permission",
                      actualPermissions.contains("rest-reader:[READ]"));
              // Split up rest-writer:[READ, EXECUTE, UPDATE] string
              String[] writerPerms = actualPermissions.split("rest-writer:\\[")[1].split("\\]")[0].split(",");

              assertTrue("Document permissions difference in rest-writer permission - first permission",
                      writerPerms[0].contains("UPDATE") || writerPerms[1].contains("UPDATE") || writerPerms[2].contains("UPDATE"));
              assertTrue("Document permissions difference in rest-writer permission - second permission",
                      writerPerms[0].contains("EXECUTE") || writerPerms[1].contains("EXECUTE") || writerPerms[2].contains("EXECUTE"));
              assertTrue("Document permissions difference in rest-writer permission - third permission",
                      writerPerms[0].contains("READ") || writerPerms[1].contains("READ") || writerPerms[2].contains("READ"));

              // Split up temporal-admin=[READ, UPDATE] string
              String[] temporalAdminPerms = actualPermissions.split("temporal-admin:\\[")[1].split("\\]")[0].split(",");

              assertTrue("Document permissions difference in temporal-admin permission - first permission",
                      temporalAdminPerms[0].contains("UPDATE") || temporalAdminPerms[1].contains("UPDATE"));
              assertTrue("Document permissions difference in rest-writer permission - second permission",
                      temporalAdminPerms[0].contains("READ") || temporalAdminPerms[1].contains("READ"));
          }

          // =============================================================================
          // Check update works
          // =============================================================================
          Calendar updateTime = DatatypeConverter.parseDateTime(afterLSQTAdvance);
          // Advance the system time for update. To be greater than LSQT time.
          updateTime.add(Calendar.DAY_OF_MONTH, 5);
          JacksonDatabindHandle<ObjectNode> handleUpd = getJSONDocumentHandle("2003-01-01T00:00:00", "2008-12-31T23:59:59",
                  "1999 Skyway Park - Updated - JSON", docId);
          docMgr.setMetadataCategories(Metadata.ALL);
          DocumentMetadataHandle mh = setMetadata(true);

          desc = docMgr.write(docId, mh, handleUpd, null, null, temporalLsqtCollectionName, updateTime);
          // Validate the advance from desc
          docMgr.advanceLsqt(temporalLsqtCollectionName);
          afterLSQTAdvance = desc.getTemporalSystemTime();
          System.out.println("LSQT on collection after update and manual advance is " + afterLSQTAdvance);
          assertTrue("LSQT Advance incorrect", desc.getTemporalSystemTime().trim().contains("2010-01-06T00:00:01-08:00"));

          // Verify that the document was updated 
          // Make sure there are 1 documents in latest collection
          QueryManager queryMgr = readerClient.newQueryManager();
          StructuredQueryBuilder sqb = queryMgr.newStructuredQueryBuilder();
          StructuredQueryDefinition termQuery = sqb.collection(latestCollectionName);
          long start = 1;
          DocumentPage termQueryResults = docMgr.search(termQuery, start);
          System.out.println("Number of results = " + termQueryResults.getTotalSize());
          assertEquals("Wrong number of results", 1, termQueryResults.getTotalSize());

          // Document URIs in latest collection must be the same as the one as the
          // original documents
          while (termQueryResults.hasNext()) {
              record = termQueryResults.next();
              String uri = record.getUri();
              System.out.println("URI = " + uri);
              assertTrue("URIs are not what is expected", uri.equals(docId));
          }

          // Make sure there are 4 documents in jsonDocId collection
          queryMgr = readerClient.newQueryManager();
          sqb = queryMgr.newStructuredQueryBuilder();
          termQuery = sqb.collection(docId);

          start = 1;
          termQueryResults = docMgr.search(termQuery, start);
          System.out.println("Number of results = " + termQueryResults.getTotalSize());
          assertEquals("Wrong number of results", 4, termQueryResults.getTotalSize());

          // Make sure there are 4 documents in temporal collection
          queryMgr = readerClient.newQueryManager();
          sqb = queryMgr.newStructuredQueryBuilder();
          termQuery = sqb.collection(temporalLsqtCollectionName);

          start = 1;
          termQueryResults = docMgr.search(termQuery, start);
          System.out.println("Number of results = " + termQueryResults.getTotalSize());
          assertEquals("Wrong number of results", 5, termQueryResults.getTotalSize());

          // Issue a period range search to make sure update went fine.
          StructuredQueryBuilder.Axis axis = sqbLSQT.axis(axisValidName);
          StructuredQueryBuilder.Period period = sqbLSQT.period("2003-01-01T00:00:00", "2009-12-31T23:59:59");

          periodQueryLSQT2 = sqbLSQT.temporalPeriodRange(axis, StructuredQueryBuilder.TemporalOperator.ALN_CONTAINS, period, new String[] {});
          termQueryResultsLSQT2 = docMgrQy.search(periodQueryLSQT2, startLSQT);
          assertTrue("CTS Period range query returned incorrect results", termQueryResultsLSQT2.getTotalSize() == 1);

          while (termQueryResultsLSQT2.hasNext()) {
              DocumentRecord recordContains = termQueryResultsLSQT2.next();
              System.out.println("URI = " + recordContains.getUri());

              JacksonDatabindHandle<ObjectNode> recordContainsHandle = new JacksonDatabindHandle<>(
                      ObjectNode.class);
              recordContains.getContent(recordContainsHandle);
              String docContents = recordContainsHandle.toString();
              System.out.println("Content = " + docContents);
              assertTrue("CTS Period range query returned incorrect results",docContents.contains("\"javaValidStartERI\":\"2001-01-01T00:00:00\",\"javaValidEndERI\":\"2011-12-31T23:59:59\""));
          }
      }
    catch (Exception ex) {
        System.out.println("Exception thrown from testAdvacingLSQT method " + ex.getMessage() );
    }
    finally {
        ConnectedRESTQA.updateTemporalCollectionForLSQT(dbName, temporalLsqtCollectionName, true);
    }
  }

and

// Disable automation for a LSQT enabled DB on a collection. Tests need to manually advance LSQT.
  public static void disableAutomationOnTemporalCollection(String dbName, String collectionName, boolean enable)
          throws Exception {
        ObjectMapper mapper = new ObjectMapper();
        ObjectNode rootNode = mapper.createObjectNode();
        rootNode.put("lsqt-enabled", enable);

        // Set automation value to false
        ObjectNode automation = mapper.createObjectNode();
        automation.put("enabled", false);

        rootNode.set("automation", automation);

        System.out.println(rootNode.toString());

        DefaultHttpClient client = new DefaultHttpClient();
        client.getCredentialsProvider().setCredentials(
            new AuthScope(host_name, getAdminPort()),
            new UsernamePasswordCredentials("admin", "admin"));

        HttpPut put = new HttpPut("http://" + host_name + ":" + admin_port + "/manage/v2/databases/" + dbName + "/temporal/collections/lsqt/properties?collection=" + collectionName);

        put.addHeader("Content-type", "application/json");
        put.addHeader("accept", "application/json");
        put.setEntity(new StringEntity(rootNode.toString()));

        HttpResponse response = client.execute(put);
        HttpEntity respEntity = response.getEntity();
        if (response.getStatusLine().getStatusCode() == 400) {
          HttpEntity entity = response.getEntity();
          String responseString = EntityUtils.toString(entity, "UTF-8");
          System.out.println(responseString);
        }
        else if (respEntity != null) {
          // EntityUtils to get the response content
          String content = EntityUtils.toString(respEntity);
          System.out.println(content);

          System.out.println("Temporal collection: " + collectionName + " created");
          System.out.println("==============================================================");
        }
        else {
          System.out.println("No Proper Response");
        }
        client.getConnectionManager().shutdown();
      }