benchmark - JMH: reduced time, improved filtering

app/build.gradle.kts

@@ -50,8 +50,8 @@ tasks.register<JavaExec>("runCbmCgl") {
     group = "application"
     classpath = sourceSets["main"].runtimeClasspath
     mainClass = "de.uni_marburg.powersort.benchmark.CbmCgl"
-    // Results in `-Xmx8g` being passed to JVM
-    maxHeapSize = "8g"
+    // Results in `-Xmx4g` being passed to JVM
+    maxHeapSize = "4g"
 }
 tasks.register<JavaExec>("runCbmCompetition") {
     description = "Run Custom Benchmark (CBM) with Powersort competition lists"
@@ -84,9 +84,15 @@ jmh {
     // https://github.com/openjdk/jmh/blob/1be779e0b4af174b67abf8080d1b76dda570d88d/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java#L148-L152
     forceGC = true
 
+    // If human output is saved, it won't be written to stdout while running the benchmark!
+    //humanOutputFile = project.file("${project.layout.buildDirectory.get()}/reports/jmh/human.txt")
+
+    resultsFile = project.file("${project.layout.buildDirectory.get()}/reports/jmh/results.txt")
+    resultFormat = "CSV"
+
     excludes = listOf(
         // To skip JmhCgl or JmhCompetition, uncomment it below.
 //        "de.uni_marburg.powersort.benchmark.JmhCgl.benchmark",
-//        "de.uni_marburg.powersort.benchmark.JmhCompetition.benchmark",
+        "de.uni_marburg.powersort.benchmark.JmhCompetition.benchmark",
     )
 }

app/src/jmh/java/de/uni_marburg/powersort/benchmark/JmhBase.java

@@ -13,28 +13,45 @@ import org.openjdk.jmh.annotations.OutputTimeUnit;
 import org.openjdk.jmh.annotations.Scope;
 import org.openjdk.jmh.annotations.Setup;
 import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
 import org.openjdk.jmh.annotations.Timeout;
 import org.openjdk.jmh.annotations.Warmup;
 
 import java.util.concurrent.TimeUnit;
 
-// TODO: The parameters are way too low. Use for debugging only!
 /*
  * Benchmark parameters
  */
-@Fork(value = 1, jvmArgsAppend = {"-Xms8g", "-Xmx8g"})
-@Warmup(iterations = 1)
-@Measurement(iterations = 6)
-@Timeout(time = 15, timeUnit = TimeUnit.SECONDS)
-@BenchmarkMode(Mode.AverageTime)
+
+@Fork(value = 1, jvmArgsAppend = {"-Xms4g", "-Xmx4g"})
+@Threads(1)
+@Timeout(time = 10, timeUnit = TimeUnit.SECONDS)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 /*
- * Benchmark state parameters
- *
- * Quote from JMH:
- * State objects naturally encapsulate the state on which benchmark is working on.
+ * Benchmark mode and related parameters
  */
+
+// AverageTime: "Average time per operation."
+// - "This mode is time-based, and it will run until the iteration time expires."
+//@BenchmarkMode(Mode.AverageTime)
+//@Warmup(iterations = 6, time = 1, timeUnit = TimeUnit.SECONDS)
+//@Measurement(iterations = 6, time = 1, timeUnit = TimeUnit.SECONDS)
+
+// SingleShotTime: "Time per single operation"
+// - "More warmup/measurement iterations are generally required."
+// - "Timers overhead might be significant if benchmarks are small;"
+@BenchmarkMode(Mode.SingleShotTime)
+@Warmup(iterations = 12)
+@Measurement(iterations = 6)
+
+/*
+ * State parameters
+ *
+ */
+
+// "State objects naturally encapsulate the state on which benchmark is working on."
 @State(Scope.Benchmark)
+
 public class JmhBase {
     DataEnum getDataEnum(){
         return null;

app/src/jmh/java/de/uni_marburg/powersort/benchmark/JmhCgl.java

@@ -16,9 +16,14 @@ import org.openjdk.jmh.annotations.State;
  */
 @State(Scope.Benchmark)
 public class JmhCgl extends JmhBase {
+    // Either all or a selection of input lists.
     @Param()
+    //@Param({"ASCENDING_RUNS", "ASCENDING_RUNS_WITH_OVERLAP"})
     CglEnum dataEnum;
+
+    // Either all or a selection of sort implementations.
     @Param()
+    //@Param({"TIM_SORT", "FASTER_FINN_SORT"})
     SortEnum sortEnum;
 
     @Override

app/src/main/java/de/uni_marburg/powersort/benchmark/Filter.java

@@ -19,11 +19,11 @@ public class Filter {
     public static boolean isFiltered(DataEnum d, SortEnum s) {
         // To skip some of the inputs for all sort algorithms, uncomment them here.
 //        if (List.of(
-//                CglEnum.RANDOM_INTEGERS_300M,
-//                CglEnum.ASCENDING_INTEGERS_300M,
-//                CglEnum.DESCENDING_INTEGERS_250M,
-//                CglEnum.ASCENDING_RUNS_1M,
-//                CglEnum.ASCENDING_RUNS_WITH_OVERLAP_1M
+//                CglEnum.RANDOM_INTEGERS,
+//                CglEnum.ASCENDING_INTEGERS,
+//                CglEnum.DESCENDING_INTEGERS,
+//                CglEnum.ASCENDING_RUNS,
+//                CglEnum.ASCENDING_RUNS_WITH_OVERLAP
 //        ).contains(d)) {
 //            return true;
 //        }
@@ -54,11 +54,11 @@ public class Filter {
 
         if (s == SortEnum.QUICK_SORT) {
             return List.of(
-                    CglEnum.RANDOM_INTEGERS_300M,
-                    CglEnum.ASCENDING_INTEGERS_300M,
-                    CglEnum.DESCENDING_INTEGERS_250M,
-                    CglEnum.ASCENDING_RUNS_1M,
-                    CglEnum.ASCENDING_RUNS_WITH_OVERLAP_1M,
+                    CglEnum.RANDOM_INTEGERS,
+                    CglEnum.ASCENDING_INTEGERS,
+                    CglEnum.DESCENDING_INTEGERS,
+                    CglEnum.ASCENDING_RUNS,
+                    CglEnum.ASCENDING_RUNS_WITH_OVERLAP,
 
                     CompetitionEnum.COMPETITION_34,
                     CompetitionEnum.COMPETITION_35,
@@ -109,25 +109,25 @@ public class Filter {
 
         if (s == SortEnum.MERGE_SORT) {
             return List.of(
-                    CglEnum.RANDOM_INTEGERS_300M,
-                    CglEnum.ASCENDING_INTEGERS_300M,
-                    CglEnum.DESCENDING_INTEGERS_250M
+                    CglEnum.RANDOM_INTEGERS,
+                    CglEnum.ASCENDING_INTEGERS,
+                    CglEnum.DESCENDING_INTEGERS
             ).contains(d);
         }
 
         if (s == SortEnum.BUBBLE_SORT) {
             return List.of(
-                    CglEnum.DESCENDING_INTEGERS_250M,
-                    CglEnum.ASCENDING_RUNS_1M,
-                    CglEnum.ASCENDING_RUNS_WITH_OVERLAP_1M
+                    CglEnum.DESCENDING_INTEGERS,
+                    CglEnum.ASCENDING_RUNS,
+                    CglEnum.ASCENDING_RUNS_WITH_OVERLAP
             ).contains(d);
         }
 
         // TODO: Remove this once performance of FinnSort improved
         if (s == SortEnum.FINN_SORT) {
             return List.of(
-                    CglEnum.DESCENDING_INTEGERS_250M,
-                    CglEnum.ASCENDING_RUNS_1M,
+                    CglEnum.DESCENDING_INTEGERS,
+                    CglEnum.ASCENDING_RUNS,
 
                     CompetitionEnum.COMPETITION_83,
                     CompetitionEnum.COMPETITION_85,
@@ -173,6 +173,8 @@ public class Filter {
         // TODO: Remove this once performance of ASort improved
         if (s == SortEnum.ASORT) {
             return List.of(
+                    CglEnum.ASCENDING_RUNS,
+                    CglEnum.ASCENDING_RUNS_WITH_OVERLAP,
                     CompetitionEnum.COMPETITION_173,
                     CompetitionEnum.COMPETITION_174
             ).contains(d);

app/src/main/java/de/uni_marburg/powersort/data/CglEnum.java

@@ -4,11 +4,11 @@ package de.uni_marburg.powersort.data;
  * Enumeration of Custom Generated Lists (CGL).
  */
 public enum CglEnum implements DataEnum {
-    RANDOM_INTEGERS_300M,
-    ASCENDING_INTEGERS_300M,
-    DESCENDING_INTEGERS_250M,
-    ASCENDING_RUNS_1M,
-    ASCENDING_RUNS_WITH_OVERLAP_1M;
+    RANDOM_INTEGERS,
+    ASCENDING_INTEGERS,
+    DESCENDING_INTEGERS,
+    ASCENDING_RUNS,
+    ASCENDING_RUNS_WITH_OVERLAP;
 
     @Override
     public ObjectSupplier getObjectSupplier() {
@@ -16,18 +16,22 @@ public enum CglEnum implements DataEnum{
         // final long seed = 3651660232967549736L; // System.nanoTime() ++ Math.random()
         final long seed = 140506881906827520L; // (long) 'P' * (long) 'O' *(long) 'W' * (long) 'E' * (long) 'R' * (long) 'S' * (long) 'O' * (long) 'R' * (long) 'T';
 
-        int longListSize = 300_000_000; // Any larger: Out of Heap Space. TODO GRADLE config
-        int middleListSize = 250_000_000;
-        int runs = 1_000;
-        int runLength = 1_000;
+        int listSize = 66_000_000;
+        int runs = 3010;
+        int runLength = 3010;
+
+        // Constant factors
+        double a = 0.96;
+        double b = 0.25;
+        double c = 0.81;
 
         return switch (this) {
-            case RANDOM_INTEGERS_300M -> new RandomIntegers(longListSize, seed);
-            case ASCENDING_INTEGERS_300M -> new AscendingIntegers(longListSize);
-            case DESCENDING_INTEGERS_250M -> new DescendingIntegers(middleListSize);
-            case ASCENDING_RUNS_1M -> AscendingRuns.newAscendingRuns(runs, runLength, -runLength);
-            case ASCENDING_RUNS_WITH_OVERLAP_1M ->
-                    AscendingRuns.newAscendingRuns(runs, runLength, (int) (-0.5 * runLength));
+            case RANDOM_INTEGERS -> new RandomIntegers(listSize, seed);
+            case ASCENDING_INTEGERS -> new AscendingIntegers((int) (a * listSize));
+            case DESCENDING_INTEGERS -> new DescendingIntegers((int) (b * listSize));
+            case ASCENDING_RUNS -> AscendingRuns.newAscendingRuns(runs, runLength, -1 * runLength);
+            case ASCENDING_RUNS_WITH_OVERLAP ->
+                    AscendingRuns.newAscendingRuns((int) (c * runs), (int) (c * runLength), (int) (-0.5 * c * runLength));
         };
     }
 }

app/src/main/java/de/uni_marburg/powersort/sort/SortEnum.java

@@ -7,8 +7,8 @@ import de.uni_marburg.powersort.sort.dpqs.DualPivotQuicksort;
 
 public enum SortEnum {
     TIM_SORT,
-    ASORT,
     FASTER_FINN_SORT,
+    ASORT,
     FINN_SORT,
     DPQS,
     QUICK_SORT,