Improve benchmarks, add bicoloring (take 2, less memory-hungry) (#189)

* Improve benchmarks, add bicoloring

* Better deps specification

* Less memory-hungry

Author: gdalle

.github/workflows/Benchmark.yml

@@ -35,7 +35,7 @@ jobs:
                 echo $PATH
                 ls -l ~/.julia/bin
                 mkdir results
-                benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune
+                benchpkg ${{ steps.extract-package-name.outputs.package_name }} --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --url=${{ github.event.repository.clone_url }} --bench-on="${{github.event.repository.default_branch}}" --output-dir=results/ --tune --add StableRNGs
             - name: Create markdown table from benchmarks
               run: |
                 benchpkgtable ${{ steps.extract-package-name.outputs.package_name }} --mode="time,memory" --rev="${{github.event.repository.default_branch}},${{github.event.pull_request.head.sha}}" --input-dir=results/ --ratio > table.md

benchmark/benchmarks.jl

@@ -2,29 +2,44 @@ using BenchmarkTools
 using LinearAlgebra
 using SparseMatrixColorings
 using SparseArrays
+using StableRNGs
 
 SUITE = BenchmarkGroup()
 
 for structure in [:nonsymmetric, :symmetric],
-    partition in (structure == :nonsymmetric ? [:column, :row] : [:column]),
-    decompression in (structure == :nonsymmetric ? [:direct] : [:direct, :substitution]),
+    partition in (structure == :nonsymmetric ? [:column, :row, :bidirectional] : [:column]),
+    decompression in (
+        if (structure == :nonsymmetric && partition in [:column, :row])
+            [:direct]
+        else
+            [:direct, :substitution]
+        end
+    ),
     n in [10^3, 10^5],
     p in [2 / n, 5 / n, 10 / n]
 
     problem = ColoringProblem(; structure, partition)
-    algo = GreedyColoringAlgorithm(; decompression)
+    algo = GreedyColoringAlgorithm(; decompression, postprocessing=true)
 
-    SUITE[:coloring][structure][partition][decompression]["n=$n"]["p=$p"] = @benchmarkable coloring(
-        A, $problem, $algo
-    ) setup = ( #
-        A = sparse(Symmetric(sprand($n, $n, $p)))
-    )
+    # use several random matrices to reduce variance
+    nb_samples = 5
+    As = [sparse(Symmetric(sprand(StableRNG(i), Bool, n, n, p))) for i in 1:nb_samples]
+    results = [coloring(A, problem, algo) for A in As]
+    Bs = [compress(A, result) for (A, result) in zip(As, results)]
 
-    SUITE[:decompress][structure][partition][decompression]["n=$n"]["p=$p"] = @benchmarkable decompress(
-        B, result
-    ) setup = ( #
-        A = sparse(Symmetric(sprand($n, $n, $p)));
-        result = coloring(A, $problem, $algo);
-        B = compress(A, result)
-    )
+    SUITE[:coloring][structure][partition][decompression]["n=$n"]["p=$p"] = @benchmarkable begin
+        for A in $As
+            coloring(A, $problem, $algo)
+        end
+    end
+
+    SUITE[:decompress][structure][partition][decompression]["n=$n"]["p=$p"] = @benchmarkable begin
+        for (B, result) in zip($Bs, $results)
+            if B isa AbstractMatrix
+                decompress(B, result)
+            elseif B isa Tuple
+                decompress(B[1], B[2], result)
+            end
+        end
+    end
 end