Add averaging to nearest neighbor benchmark to produce smoother plots

Orthtree/benchmark/Orthtree/nearest_neighbor.cpp

@@ -29,6 +29,8 @@ using std::chrono::microseconds;
 
 int main(int argc, char **argv) {
 
+  int num_runs = 100;
+
   size_t k = 10;
 
   // Set output file
@@ -36,72 +38,90 @@ int main(int argc, char **argv) {
   file.open((argc > 1) ? argv[1] : "../nearest_neighbor_benchmark.csv");
 
   // Add header for CSV
-  file << "Number of Points,Octree,kDTree,Naive \n";
+  file << "Number of Points,Octree,kDTree \n";
 
   // Perform tests for various dataset sizes
   for (size_t num_points = 100; num_points < 100000; num_points *= 1.05) {
 
-    // Create a collection of the right number of points
+    // We want the average of several runs for each point count, for cleaner results
-    auto points = generate<Kernel>(num_points);
+    float octreeAverage = 0;
+    float kdtreeAverage = 0;
+    float naiveAverage = 0;
 
-    // Create a search point
+    // Repeat the tests, generating a new point set for each run
-    auto search_point = *(generate<Kernel>().points().end() - 1);
+    for (int i = 0; i < num_runs; ++i) {
 
-    // Build the kd tree from the point set
+      // Create a collection of the right number of points
-    Kdtree kdtree(points.points().begin(), points.points().end());
+      auto points = generate<Kernel>(num_points);
-    kdtree.build();
 
-    // Time how long it takes to find neighbors using the kd tree
+      // Create a search point
-    auto kdtreeTime = bench<microseconds>(
+      auto search_point = *(generate<Kernel>().points().end() - 1);
-            [&] {
-              Kd_tree_search search(kdtree, search_point, k);
-            }
-    );
 
-    // Time how long it takes to find neighbors using a naive approach
+      // Build the kd tree from the point set
-    auto naiveTime = bench<microseconds>(
+      Kdtree kdtree(points.points().begin(), points.points().end());
-            [&] {
+      kdtree.build();
-
-              std::vector<Point> nearest_neighbors;
-
-              // Iterate over every point
-              for (auto &point : points.points()) {
-
-                // Find out how this point ranks in comparison with other points we've saved
-                auto iter = nearest_neighbors.begin();
-                for (; iter < nearest_neighbors.end() &&
-                       CGAL::squared_distance(point, search_point) <
-                       CGAL::squared_distance(*iter, search_point);
-                       iter++) {}
-
-                // Add the point to the list (it'll usually be at the end)
-                nearest_neighbors.insert(iter, point);
-
-                // Never keep more than k neighbors
-                if (nearest_neighbors.size() > k)
-                  nearest_neighbors.resize(k);
 
+      // Time how long it takes to find neighbors using the kd tree
+      auto kdtreeTime = bench<microseconds>(
+              [&] {
+                Kd_tree_search search(kdtree, search_point, k);
               }
+      );
 
-            }
+      // Time how long it takes to find neighbors using a naive approach
-    );
+//    auto naiveTime = bench<microseconds>(
+//            [&] {
+//
+//              std::vector<Point> nearest_neighbors;
+//
+//              // Iterate over every point
+//              for (auto &point : points.points()) {
+//
+//                // Find out how this point ranks in comparison with other points we've saved
+//                auto iter = nearest_neighbors.begin();
+//                for (; iter < nearest_neighbors.end() &&
+//                       CGAL::squared_distance(point, search_point) <
+//                       CGAL::squared_distance(*iter, search_point);
+//                       iter++) {}
+//
+//                // Add the point to the list (it'll usually be at the end)
+//                nearest_neighbors.insert(iter, point);
+//
+//                // Never keep more than k neighbors
+//                if (nearest_neighbors.size() > k)
+//                  nearest_neighbors.resize(k);
+//
+//              }
+//
+//            }
+//    );
 
-    // Build the octree from points (this had to be done second because it rearranges the point set)
+      // Build the octree from points (this had to be done second because it rearranges the point set)
-    Octree octree(points, points.point_map());
+      Octree octree(points, points.point_map());
-    octree.refine();
+      octree.refine();
 
-    // Time how long it takes to find neighbors using the octree
+      // Time how long it takes to find neighbors using the octree
-    auto octreeTime = bench<microseconds>(
+      auto octreeTime = bench<microseconds>(
-            [&] {
+              [&] {
-              std::vector<Point> nearest_neighbors;
+                std::vector<Point> nearest_neighbors;
-              octree.nearest_neighbors(search_point, k, std::back_inserter(nearest_neighbors));
+                octree.nearest_neighbors(search_point, k, std::back_inserter(nearest_neighbors));
-            }
+              }
-    );
+      );
+
+      // Incorporate our results into the average
+      octreeAverage += (float) octreeTime.count() / (float) num_runs;
+      kdtreeAverage += (float) kdtreeTime.count() / (float) num_runs;
+//      naiveAverage += (float) naiveTime.count() / (float) num_runs;
+
+      // A simple progress indication
+      std::cout << ".";
+    }
 
     file << num_points << ",";
-    file << octreeTime.count() << ",";
+    file << octreeAverage << ",";
-    file << kdtreeTime.count() << ",";
+    file << kdtreeAverage << ",";
-    file << naiveTime.count() << "\n";
+//    file << naiveAverage << ",";
+    file << "\n";
 
     std::cout << num_points << std::endl;
 

Orthtree/benchmark/Orthtree/plot_nearest_neighbor_benchmark.p

@@ -14,4 +14,4 @@ set ylabel "Time (us)"
 set key autotitle columnhead
 
 set datafile separator ","
-plot for [col=2:4] 'nearest_neighbor_benchmark.csv' using 1:col
+plot for [col=2:10] 'nearest_neighbor_benchmark.csv' using 1:col

Orthtree/benchmark/Orthtree/util.h

@@ -26,19 +26,20 @@ CGAL::Point_set_3<typename Kernel::Point_3> generate(size_t num_points = 1) {
 }
 
 template<typename Time_unit>
-Time_unit bench(const std::function<void(void)>& f) {
+Time_unit bench(const std::function<void(void)> &f, size_t repetitions = 1) {
 
-    // Start the timer
+  // Start the timer
-    auto start = std::chrono::high_resolution_clock::now();
+  auto start = std::chrono::high_resolution_clock::now();
 
-    // Run the function being benchmarked
+  // Run the function being benchmarked as many times as requested
+  for (int i = 0; i < repetitions; ++i)
     f();
 
-    // End the timer
+  // End the timer
-    auto end = std::chrono::high_resolution_clock::now();
+  auto end = std::chrono::high_resolution_clock::now();
 
-    // Return the elapsed time
+  // Return the elapsed time
-    return std::chrono::duration_cast<Time_unit>(end - start);
+  return std::chrono::duration_cast<Time_unit>(end - start) / repetitions;
 }
 
 #endif //OCTREE_UTIL_H