From 8757d4ac078e6e9efdf03f5c62b85866b56e33a4 Mon Sep 17 00:00:00 2001 From: Stephen Kiazyk Date: Tue, 23 Sep 2014 16:19:08 -0400 Subject: [PATCH] Added the ability to select kernel when running benchmarks --- .../Surface_mesh_shortest_path/benchmark.py | 7 +- .../benchmark_shortest_paths.cpp | 174 ++++++++++++------ .../compileBenchmarks.py | 122 +++++++----- .../run_benchmarks.sh | 3 +- .../simpleModels.txt | 3 + 5 files changed, 203 insertions(+), 106 deletions(-) create mode 100644 Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/simpleModels.txt diff --git a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark.py b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark.py index 93addc44412..6f27beb2146 100644 --- a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark.py +++ b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark.py @@ -6,7 +6,7 @@ class TestConfig: Describes a benchmark test case to run over a set of models """ - def __init__(self, testName, testModels, numTrials, numSources, numQueries, randSeed): + def __init__(self, testName, testModels, numTrials, numSources, numQueries, randSeed, kernel="epick"): """ testName - the name of the test as it should appear in the documentation file testModels - a list of .off model files to run the test on @@ -21,9 +21,10 @@ class TestConfig: self.numSources = numSources; self.numQueries = numQueries; self.randSeed = randSeed; + self.kernel = kernel; def __str__(self): - return "%s : #Trials = %d, #Sources = %d, #Queries = %d" % (self.testName, self.numTrials, self.numSources, self.numQueries); + return "%s : Kernel = %s #Trials = %d, #Sources = %d, #Queries = %d" % (self.testName, self.kernel, self.numTrials, self.numSources, self.numQueries); def read_all_lines(file): f = open(file, "r"); @@ -55,7 +56,7 @@ def run_benchmarks(testConfig, outputFile): for model in testConfig.testModels: sys.stdout.write("Model = %s ... " % model); sys.stdout.flush(); - result = subprocess.call(['./benchmark_shortest_paths.exe', '-p', model.strip(), '-r', str(testConfig.randSeed), '-t', str(testConfig.numTrials), '-n', str(testConfig.numSources), '-q', str(testConfig.numQueries)], stdout=outputFile); + result = subprocess.call(['./benchmark_shortest_paths.exe', '-k', testConfig.kernel, '-p', model.strip(), '-r', str(testConfig.randSeed), '-t', str(testConfig.numTrials), '-n', str(testConfig.numSources), '-q', str(testConfig.numQueries)], stdout=outputFile); if result == 0: sys.stdout.write("Done.\n"); else: diff --git a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark_shortest_paths.cpp b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark_shortest_paths.cpp index 9701a43d611..6bc958d1052 100644 --- a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark_shortest_paths.cpp +++ b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/benchmark_shortest_paths.cpp @@ -1,43 +1,42 @@ +#include +#include +#include +#include + +#include +#include +#include + #include #include +#include +#include #include #include #include -#include -#include - #include #include -#include -#include - -#include -#include -#include +#include +#include #define UNUSED(X) (void)sizeof(X) -typedef CGAL::Exact_predicates_inexact_constructions_kernel Kernel; -typedef CGAL::Polyhedron_3 Polyhedron_3; -typedef CGAL::Surface_mesh_shortest_path_traits Traits; -typedef Traits::Barycentric_coordinate Barycentric_coordinate; -typedef Traits::Construct_barycentric_coordinate Construct_barycentric_coordinate; -typedef CGAL::Surface_mesh_shortest_path Surface_mesh_shortest_path; -typedef boost::graph_traits GraphTraits; -typedef GraphTraits::vertex_descriptor vertex_descriptor; -typedef GraphTraits::vertex_iterator vertex_iterator; -typedef GraphTraits::halfedge_descriptor halfedge_descriptor; -typedef GraphTraits::halfedge_iterator halfedge_iterator; -typedef GraphTraits::face_descriptor face_descriptor; -typedef GraphTraits::face_iterator face_iterator; - +namespace po = boost::program_options; typedef boost::timer::nanosecond_type nanosecond_type; +enum Kernel_type +{ + KERNEL_UNKNOWN, + KERNEL_IPICK, + KERNEL_EPICK, + KERNEL_EPECK, +}; + template class SampledValue { @@ -148,16 +147,43 @@ void print_results(std::ostream& stream, const std::string& filename, const Benc #endif } -Barycentric_coordinate random_coordinate(CGAL::Random& rand) +template +typename Traits::Barycentric_coordinate random_coordinate(CGAL::Random& rand) { - Construct_barycentric_coordinate construct_barycentric_coordinate; - Traits::FT u = rand.uniform_01(); - Traits::FT v = rand.uniform_real(Traits::FT(0.0), Traits::FT(1.0) - u); - return construct_barycentric_coordinate(u, v, Traits::FT(1.0) - u - v); + typedef typename Traits::FT FT; + typename Traits::Construct_barycentric_coordinate construct_barycentric_coordinate; + FT u = rand.uniform_real(FT(0.0), FT(1.0)); + FT v = rand.uniform_real(FT(0.0), FT(1.0) - u); + return construct_barycentric_coordinate(u, v, FT(1.0) - u - v); } -void run_benchmarks(CGAL::Random& rand, size_t numTrials, size_t numSources, size_t numQueries, Polyhedron_3& polyhedron, Benchmark_data& outData) +template +void run_benchmarks(CGAL::Random& rand, size_t numTrials, size_t numSources, size_t numQueries, const std::string& polyhedronFile, Benchmark_data& outData) { + typedef CGAL::Polyhedron_3 Polyhedron_3; + typedef CGAL::Surface_mesh_shortest_path_traits Traits; + typedef typename Traits::Barycentric_coordinate Barycentric_coordinate; + typedef CGAL::Surface_mesh_shortest_path Surface_mesh_shortest_path; + typedef typename Surface_mesh_shortest_path::Face_location Face_location; + typedef typename Surface_mesh_shortest_path::FT FT; + typedef boost::graph_traits GraphTraits; + typedef typename GraphTraits::face_descriptor face_descriptor; + typedef typename GraphTraits::face_iterator face_iterator; + + std::ifstream inFile(polyhedronFile.c_str()); + + if (!inFile) + { + throw std::runtime_error(std::string("Model file \"") + polyhedronFile + std::string("\" does not exist.")); + } + + Polyhedron_3 polyhedron; + + inFile >> polyhedron; + inFile.close(); + + CGAL::set_halfedgeds_items_id(polyhedron); + boost::timer::cpu_timer timer; outData.reset(); @@ -180,13 +206,13 @@ void run_benchmarks(CGAL::Random& rand, size_t numTrials, size_t numSources, siz for (size_t i = 0; i < numTrials; ++i) { - std::vector sourcePoints; + std::vector sourcePoints; while (sourcePoints.size() < numSources) { face_descriptor sourceFace = allFaces[rand.get_int(0, allFaces.size())]; - Barycentric_coordinate sourceLocation = random_coordinate(rand); - sourcePoints.push_back(Surface_mesh_shortest_path::Face_location(sourceFace, sourceLocation)); + Barycentric_coordinate sourceLocation = random_coordinate(rand); + sourcePoints.push_back(Face_location(sourceFace, sourceLocation)); } timer.start(); @@ -204,10 +230,10 @@ void run_benchmarks(CGAL::Random& rand, size_t numTrials, size_t numSources, siz for (size_t j = 0; j < numQueries; ++j) { face_descriptor sourceFace = allFaces[rand.get_int(0, allFaces.size())]; - Barycentric_coordinate sourceLocation = random_coordinate(rand); + Barycentric_coordinate sourceLocation = random_coordinate(rand); timer.start(); - Traits::FT distance = shortestPaths.shortest_distance_to_source_points(sourceFace, sourceLocation).first; + FT distance = shortestPaths.shortest_distance_to_source_points(sourceFace, sourceLocation).first; timer.stop(); UNUSED(distance); @@ -218,6 +244,26 @@ void run_benchmarks(CGAL::Random& rand, size_t numTrials, size_t numSources, siz } } +Kernel_type parse_kernel_type(const std::string& s) +{ + if (boost::iequals(s, "ipick")) + { + return KERNEL_IPICK; + } + else if (boost::iequals(s, "epick")) + { + return KERNEL_EPICK; + } + else if (boost::iequals(s, "epeck")) + { + return KERNEL_EPECK; + } + else + { + return KERNEL_UNKNOWN; + } +} + int main(int argc, char* argv[]) { namespace po = boost::program_options; @@ -227,9 +273,10 @@ int main(int argc, char* argv[]) ("help,h", "Display help message") ("polyhedron,p", po::value(), "Polyhedron input file") ("randomseed,r", po::value()->default_value(0), "Randomization seed value") - ("trials,t", po::value()->default_value(10), "Number of trials to run") - ("numpoints,n", po::value()->default_value(10), "Number of source points per trial") - ("queries,q", po::value()->default_value(10), "Number of queries to run per trial") + ("trials,t", po::value()->default_value(20), "Number of trials to run") + ("numpoints,n", po::value()->default_value(1), "Number of source points per trial") + ("queries,q", po::value()->default_value(100), "Number of queries to run per trial") + ("kernel,k", po::value()->default_value("epick"), "Kernel to use. One of \'ipick\', \'epick\', \'epeck\'") ; po::variables_map vm; @@ -242,34 +289,39 @@ int main(int argc, char* argv[]) } else if (vm.count("polyhedron")) { - Polyhedron_3 polyhedron; - - std::ifstream inFile(vm["polyhedron"].as().c_str()); + Benchmark_data results; + + CGAL::Random rand(vm["randomseed"].as()); + size_t numTrials = vm["trials"].as(); + size_t numPoints = vm["numpoints"].as(); + size_t numQueries = vm["queries"].as(); + std::string polyhedronFile = vm["polyhedron"].as(); - if (!inFile) + try { - std::cout << "Model file \"" << vm["polyhedron"].as().c_str() << "\" does not exist." << std::endl; - return 1; + switch (parse_kernel_type(vm["kernel"].as())) + { + case KERNEL_IPICK: + run_benchmarks >(rand, numTrials, numPoints, numQueries, polyhedronFile, results); + break; + case KERNEL_EPICK: + run_benchmarks(rand, numTrials, numPoints, numQueries, polyhedronFile, results); + break; + case KERNEL_EPECK: + run_benchmarks(rand, numTrials, numPoints, numQueries, polyhedronFile, results); + break; + default: + std::cerr << "Invalid kernel type: \"" << vm["kernel"].as() << "\"" << std::endl; + return EXIT_FAILURE; + } + } + catch (std::exception& e) + { + std::cerr << "Runtime error: " << e.what() << std::endl; + return EXIT_FAILURE; } - inFile >> polyhedron; - inFile.close(); - - CGAL::set_halfedgeds_items_id(polyhedron); - - Benchmark_data results; - - CGAL::Random rand(vm["randomseed"].as()); - - run_benchmarks( - rand, - vm["trials"].as(), - vm["numpoints"].as(), - vm["queries"].as(), - polyhedron, - results); - - print_results(std::cout, vm["polyhedron"].as(), results); + print_results(std::cout, polyhedronFile, results); } else { diff --git a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/compileBenchmarks.py b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/compileBenchmarks.py index d6208e86921..63c0f9b9ffd 100644 --- a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/compileBenchmarks.py +++ b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/compileBenchmarks.py @@ -25,6 +25,7 @@ import random; import re; import benchmark; import os; +import argparse; def make_table_file_name(tableFileBase, numSources): return tableFileBase + '_' + str(numSources) + ".txt"; @@ -37,7 +38,7 @@ def make_figure_file_name(figureFileBase, modelFile): def print_to_datafiles(config, dataFileName, modelInfo): file = open(dataFileName, "a"); - file.write("%d %d %s %s %s\n" % (config.numSources, modelInfo.get("num vertices", 0), modelInfo.get('construction', '0'), modelInfo.get('query', '0'), modelInfo.get('memory (peak)', '0'))); + file.write("%d %d %s %s %s\n" % (config.numSources, int(modelInfo.get("num vertices", 0)), modelInfo.get('construction', '0'), modelInfo.get('query', '0'), modelInfo.get('memory (peak)', '0'))); file.close(); def print_table(infoSet, config, outFile): @@ -55,32 +56,73 @@ def print_table(infoSet, config, outFile): outFile.write("\n"); outFile.write('\n'); -# Specify a file to output to, and optionally a random seed to ensure consistent tests are run -if len(sys.argv) <= 5: - print("Usage: python %s " % sys.argv[0]); - sys.exit(0); - -sampleRange = reversed([1] + list(range(5, 55, 5))); - -testModels = benchmark.read_all_lines(sys.argv[1]); +parser = argparse.ArgumentParser(description="Run benchmarks on multiple model files"); + +parser.add_argument('-r', '--range', type=str, default=['1'], nargs='*', help="Can specify multiple single values, or ranges. Format is '#' or '#,#' or '#,#,#' for a single value, a range of values, or a range values of values with a specified skip"); +parser.add_argument('-f', '--modelsfile', '--modelsfiles', type=str, nargs='*', help="a file containing a list of models to test on, one per line"); +parser.add_argument('-m', '--model', '--models', type=str, nargs='*'); +parser.add_argument('-s', '--randseed', type=int, help="Random seed for tests."); +parser.add_argument('-d', '--datafilebase', type=str, default='_modeldata', help="Base name for temporary data files (existing files will be overwritten)."); +parser.add_argument('-t', '--tablefilebase', type=str, help="Base name for generated user manual tables (will be of the form \"_#.txt\", leave blank to suppress generation)"); +parser.add_argument('-o', '--plotfilebase', type=str, help="Base name for generated plot figures (will be of the form \"_.png\", leave blank to suppress generation)"); +parser.add_argument('-k', '--kernel', choices=['ipick', 'epick', 'epeck'], default='epick', help="Geometry kernel to use for the benchmark."); + +programArgs = parser.parse_args(); + +setOfSamples = set(); + +if programArgs.range: + for r in programArgs.range: + toks = r.split(','); + if len(toks) == 1: + setOfSamples.add(int(toks[0])); + elif len(toks) == 2: + setOfSamples.update(range(int(toks[0]), int(toks[1]))); + elif len(toks) == 3: + setOfSamples.update(range(int(toks[0]), int(toks[1]), int(toks[2]))); + else: + raise "Invalid range: " + r; + +sampleRange = reversed(sorted(list(setOfSamples))); + +testModels = []; + +if programArgs.modelsfile: + for modelsFile in programArgs.modelsfile: + testModels.extend(benchmark.read_all_lines(modelsFile)); + +if programArgs.model: + for model in programArgs.model: + testModels.append(model); + +if len(testModels) == 0: + print("Error, must specify at least one model to benchmark"); + sys.exit(1); + +rand = random.Random(programArgs.randseed); + +dataFileBase = programArgs.datafilebase; +tableFileBase = programArgs.tablefilebase; +plotFileBase = programArgs.plotfilebase; + +kernel = programArgs.kernel; + +if tableFileBase == None and plotFileBase == None: + print("Error, must specify either a table output or figure output file"); + sys.exit(1); if not benchmark.prepare_program(): + print("Error, could not compile program"); sys.exit(1); -rand = random.Random(int(sys.argv[5])); - -dataFileBase = sys.argv[2]; -tableFileBase = sys.argv[3]; -figureFileBase = sys.argv[4]; - for model in testModels: - modelFileName = make_model_data_file_name(dataFileBase, model); - if os.path.exists(modelFileName): - os.remove(modelFileName); + modelDataFileName = make_model_data_file_name(dataFileBase, model); + if os.path.exists(modelDataFileName): + os.remove(modelDataFileName); for numSources in sampleRange: testname = "1 Source Point" if numSources == 1 else ("%d Source Points" % numSources); - config = benchmark.TestConfig(testname, testModels, 20, numSources, 100, rand.randint(0, 65536)); + config = benchmark.TestConfig(testname, testModels, 20, numSources, 100, rand.randint(0, 65536), kernel); infoSet = {}; infoFile = tempfile.TemporaryFile(); benchmark.run_benchmarks(config, infoFile); @@ -90,26 +132,24 @@ for numSources in sampleRange: infoFile.close(); for k in infoSet.keys(): print_to_datafiles(config, make_model_data_file_name(dataFileBase, k), infoSet[k]); - tableFile = open(make_table_file_name(tableFileBase, numSources), "w"); - print_table(infoSet, config, tableFile); - tableFile.close(); + if tableFileBase != None: + tableFile = open(make_table_file_name(tableFileBase, numSources), "w"); + print_table(infoSet, config, tableFile); + tableFile.close(); -for runParams in [('query', 4, "Average Query Time"), ('construction', 3, "Average Construction Time"), ('memory', 5, "Peak Memory Usage")]: - plotCommands = []; - - for k in testModels: - plotCommands.append('"%s" using 1:%d with lines title "%s"' % (make_model_data_file_name(dataFileBase, k), runParams[1], os.path.basename(k))); - - plotCommand = "plot " + ', '.join(plotCommands); - - plotCommandFile = tempfile.TemporaryFile(); - plotCommandFile.write('set terminal png size 1280,960;\n'); - plotCommandFile.write('set output "%s";\n' % make_figure_file_name(figureFileBase, runParams[0])); - plotCommandFile.write('set xlabel "Number of Source Points";\n'); - plotCommandFile.write('set ylabel "%s";\n' % runParams[2]); - plotCommandFile.write(plotCommand + ";\n"); - plotCommandFile.write('unset output;\n'); - plotCommandFile.write('quit\n'); - plotCommandFile.seek(0); - - subprocess.call(['gnuplot'], stdin=plotCommandFile); +if plotFileBase != None: + for runParams in [('query', 4, "Average Query Time"), ('construction', 3, "Average Construction Time"), ('memory', 5, "Peak Memory Usage")]: + plotCommands = []; + for k in testModels: + plotCommands.append('"%s" using 1:%d with lines title "%s"' % (make_model_data_file_name(dataFileBase, k), runParams[1], os.path.basename(k))); + plotCommand = "plot " + ', '.join(plotCommands); + plotCommandFile = tempfile.TemporaryFile(); + plotCommandFile.write('set terminal png size 1280,960;\n'); + plotCommandFile.write('set output "%s";\n' % make_figure_file_name(plotFileBase, runParams[0])); + plotCommandFile.write('set xlabel "Number of Source Points";\n'); + plotCommandFile.write('set ylabel "%s";\n' % runParams[2]); + plotCommandFile.write(plotCommand + ";\n"); + plotCommandFile.write('unset output;\n'); + plotCommandFile.write('quit\n'); + plotCommandFile.seek(0); + subprocess.call(['gnuplot'], stdin=plotCommandFile); diff --git a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/run_benchmarks.sh b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/run_benchmarks.sh index fc2fe99b1de..a6ca605744d 100755 --- a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/run_benchmarks.sh +++ b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/run_benchmarks.sh @@ -6,4 +6,5 @@ if [ ! -a CMakeLists.txt ] cgal_create_CMakeLists -b program_options:timer -c Core fi -python compileBenchmarks.py testModels.txt _modeldata benchmark_table benchmark_plot 6062699 +python compileBenchmarks.py -f testModels.txt -d _modeldata -t benchmark_table -o benchmark_plot -s 6062699 -r 1 5,55,5 +#python compileBenchmarks.py -k epeck -f simpleModels.txt -d _modeldata -t epeck_table -s 9894710 -r 1 diff --git a/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/simpleModels.txt b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/simpleModels.txt new file mode 100644 index 00000000000..543491e8da2 --- /dev/null +++ b/Surface_mesh_shortest_path/benchmark/Surface_mesh_shortest_path/simpleModels.txt @@ -0,0 +1,3 @@ +data/cube.off +data/cross.off +data/star.off \ No newline at end of file