Enhancements in CRC32 and CRC32C Checksum implementations

Author: joviegas

.changes/next-release/feature-AWSSDKforJavav2-3676574.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS SDK for Java v2",
+    "contributor": "",
+    "description": "The SDK now defaults to Java built-in CRC32 and CRC32C(if it's Java 9+) implementations, resulting in improved performance."
+}

core/checksums/src/main/java/software/amazon/awssdk/checksums/SdkChecksum.java

@@ -18,8 +18,8 @@
 import java.nio.ByteBuffer;
 import java.util.zip.Checksum;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
-import software.amazon.awssdk.checksums.internal.Crc32CChecksum;
 import software.amazon.awssdk.checksums.internal.Crc32Checksum;
+import software.amazon.awssdk.checksums.internal.Crc32cProvider;
 import software.amazon.awssdk.checksums.internal.Crc64NvmeChecksum;
 import software.amazon.awssdk.checksums.internal.Md5Checksum;
 import software.amazon.awssdk.checksums.internal.Sha1Checksum;
@@ -39,7 +39,7 @@ public interface SdkChecksum extends Checksum {
     static SdkChecksum forAlgorithm(ChecksumAlgorithm algorithm) {
         switch (algorithm.algorithmId()) {
             case "CRC32C":
-                return new Crc32CChecksum();
+                return Crc32cProvider.create();
             case "CRC32":
                 return new Crc32Checksum();
             case "SHA1":

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/ConstructorCache.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.checksums.internal;
+
+import java.lang.ref.WeakReference;
+import java.lang.reflect.Constructor;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Optional;
+import java.util.WeakHashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.utils.ClassLoaderHelper;
+import software.amazon.awssdk.utils.Logger;
+
+/**
+ * A cache that stores classes and their constructors by class name and class loader.
+ * <p>
+ * This cache uses weak references to both class loaders and classes, allowing them to be garbage collected
+ * when no longer needed. It provides methods to retrieve the zero-argument constructor for a class,
+ * based on the current thread's context class loader or the system class loader.
+ * <p>
+ * If a class or its zero-argument constructor cannot be found, an empty result is returned.
+ */
+@SdkInternalApi
+public final class ConstructorCache {
+    private static final Logger log = Logger.loggerFor(ConstructorCache.class);
+
+    /**
+     * Cache storing classes by class name and class loader.
+     * Uses weak references to allow garbage collection when not needed.
+     */
+    private final Map<String, Map<ClassLoader, Optional<WeakReference<Class<?>>>>> classesByClassName =
+        new ConcurrentHashMap<>();
+
+    /**
+     * Retrieve the class for the given class name from the context or system class loader.
+     * Returns an empty result if the class is not found.
+     */
+    private Optional<Class<?>> getClass(String className) {
+        Map<ClassLoader, Optional<WeakReference<Class<?>>>> classesByClassLoader =
+            classesByClassName.computeIfAbsent(className, k -> Collections.synchronizedMap(new WeakHashMap<>()));
+
+        ClassLoader classLoader = ClassLoaderHelper.contextClassLoader();
+        Optional<WeakReference<Class<?>>> classRef = classesByClassLoader.computeIfAbsent(classLoader, k -> {
+            try {
+                Class<?> clazz = classLoader.loadClass(className);
+                return Optional.of(new WeakReference<>(clazz));
+            } catch (ClassNotFoundException e) {
+                return Optional.empty();
+            }
+        });
+        return classRef.map(WeakReference::get);
+    }
+
+    /**
+     * Retrieve the zero-argument constructor for the given class name.
+     * Returns an empty result if no such constructor is found.
+     */
+    public Optional<Constructor<?>> getConstructor(String className) {
+        return getClass(className).flatMap(clazz -> {
+            try {
+                return Optional.of(clazz.getConstructor());
+            } catch (NoSuchMethodException e) {
+                log.debug(() -> "Classloader contains " + className + ", but without a zero-arg constructor.", e);
+                return Optional.empty();
+            }
+        });
+    }
+}

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32CChecksum.java

@@ -15,64 +15,50 @@
 
 package software.amazon.awssdk.checksums.internal;
 
-import static software.amazon.awssdk.utils.NumericUtils.longToByte;
 
-import java.util.Arrays;
-import java.util.zip.Checksum;
+import java.util.zip.CRC32;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.checksums.SdkChecksum;
-import software.amazon.awssdk.crt.checksums.CRC32;
-import software.amazon.awssdk.utils.ClassLoaderHelper;
 
-/**
- * Implementation of {@link SdkChecksum} to calculate an CRC32 checksum.
- */
 @SdkInternalApi
-public class Crc32Checksum extends BaseCrcChecksum {
-    private static final String CRT_CLASSPATH_FOR_CRC32 = "software.amazon.awssdk.crt.checksums.CRC32";
-    private static final ThreadLocal<Boolean> IS_CRT_AVAILABLE = ThreadLocal.withInitial(Crc32Checksum::isCrtAvailable);
+public final class Crc32Checksum implements SdkChecksum {
+    private final CrcCombineOnMarkChecksum crc32;
 
-    /**
-     * Creates CRT Based Crc32 checksum if Crt classpath for Crc32 is loaded, else create Sdk Implemented Crc32.
-     */
     public Crc32Checksum() {
-        super(createChecksum());
+        // Delegates to CrcCombineOnMarkChecksum with CRC32
+        this.crc32 = new CrcCombineOnMarkChecksum(
+            new CRC32(),
+            SdkCrc32Checksum::combine
+        );
     }
 
-    private static Checksum createChecksum() {
-        if (IS_CRT_AVAILABLE.get()) {
-            return new CRC32();
-        }
-
-        // TODO: use Java implementation
-        return SdkCrc32Checksum.create();
+    @Override
+    public byte[] getChecksumBytes() {
+        return crc32.getChecksumBytes();
     }
 
-    private static boolean isCrtAvailable() {
-        try {
-            ClassLoaderHelper.loadClass(CRT_CLASSPATH_FOR_CRC32, false);
-        } catch (ClassNotFoundException e) {
-            return false;
-        }
-
-        return true;
+    @Override
+    public void mark(int readLimit) {
+        crc32.mark(readLimit);
     }
 
     @Override
-    public byte[] getChecksumBytes() {
-        return Arrays.copyOfRange(longToByte(getChecksum().getValue()), 4, 8);
+    public void update(int b) {
+        crc32.update(b);
     }
 
     @Override
-    public Checksum cloneChecksum(Checksum checksum) {
-        if (checksum instanceof CRC32) {
-            return (Checksum) ((CRC32) checksum).clone();
-        }
+    public void update(byte[] b, int off, int len) {
+        crc32.update(b, off, len);
+    }
 
-        if (checksum instanceof SdkCrc32Checksum) {
-            return (Checksum) ((SdkCrc32Checksum) checksum).clone();
-        }
+    @Override
+    public long getValue() {
+        return crc32.getValue();
+    }
 
-        throw new IllegalStateException("Unsupported checksum");
+    @Override
+    public void reset() {
+        crc32.reset();
     }
 }

core/checksums/src/main/java/software/amazon/awssdk/checksums/internal/Crc32Checksum.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.checksums.internal;
+
+import java.util.zip.Checksum;
+import software.amazon.awssdk.annotations.SdkInternalApi;
+import software.amazon.awssdk.checksums.SdkChecksum;
+import software.amazon.awssdk.crt.checksums.CRC32C;
+
+/**
+ * Utility class to provide different implementations of CRC32C checksum. This class supports the use of: 1. Java-based CRC32C
+ * (Java 9+ when available) 2. CRT-based CRC32C (when available) 3. SDK-based CRC32C (as fallback)
+ */
+@SdkInternalApi
+public final class Crc32cProvider {
+
+
+    // Class paths for different CRC32C implementations
+    private static final String CRT_CRC32C_CLASS_PATH = "software.amazon.awssdk.crt.checksums.CRC32C";
+    private static final String JAVA_CRC32C_CLASS_PATH = "java.util.zip.CRC32C";
+    private static final ConstructorCache CONSTRUCTOR_CACHE = new ConstructorCache();
+
+    // Private constructor to prevent instantiation
+    private Crc32cProvider() {
+    }
+
+    /**
+     * Creates an instance of the SDK-based CRC32C checksum as a fallback.
+     *
+     * @return An SdkChecksum instance.
+     */
+    static SdkChecksum createSdkBasedCrc32C() {
+        SdkCrc32CChecksum sdkChecksum = SdkCrc32CChecksum.create();
+        return new CrcCloneOnMarkChecksum(sdkChecksum, checksumToClone -> ((SdkCrc32CChecksum) checksumToClone).clone());
+    }
+
+    /**
+     * Tries to create a Java 9-based CRC32C checksum.
+     * If it's not available, it tries to create a CRT-based checksum.
+     * If both are not available, it falls back to an SDK-based CRC32C checksum.
+     *
+     * @return An instance of {@link SdkChecksum}, based on the first available option.
+     */
+    public static SdkChecksum create() {
+        SdkChecksum checksum = createJavaCrc32C();
+        if (checksum == null) {
+            checksum = createCrtCrc32C();
+        }
+        return checksum != null ? checksum : createSdkBasedCrc32C();
+    }
+
+    static SdkChecksum createCrtCrc32C() {
+        return CONSTRUCTOR_CACHE.getConstructor(CRT_CRC32C_CLASS_PATH).map(constructor -> {
+            try {
+                return new CrcCloneOnMarkChecksum((Checksum) constructor.newInstance(), checksumToClone ->
+                    (Checksum) ((CRC32C) checksumToClone).clone());
+            } catch (ClassCastException | ReflectiveOperationException e) {
+                throw new IllegalStateException("Failed to instantiate " + JAVA_CRC32C_CLASS_PATH, e);
+            }
+        }).orElse(null);
+    }
+
+    static SdkChecksum createJavaCrc32C() {
+        return CONSTRUCTOR_CACHE.getConstructor(JAVA_CRC32C_CLASS_PATH).map(constructor -> {
+            try {
+                return new CrcCombineOnMarkChecksum((Checksum) constructor.newInstance(), SdkCrc32CChecksum::combine);
+            } catch (ClassCastException | ReflectiveOperationException e) {
+                throw new IllegalStateException("Failed to instantiate " + JAVA_CRC32C_CLASS_PATH, e);
+            }
+        }).orElse(null);
+    }
+
+}