dev/alsk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

thesis version

Browse Source
This commit is contained in:
Alexis Pereda
2021-05-10 18:14:13 +02:00
commit b688da651b
191 changed files with 35833 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

402
lib/celero/Archive.cpp Normal file
View File

@ -0,0 +1,402 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <assert.h>
#include <celero/Archive.h>
#include <celero/Benchmark.h>
#include <celero/FileReader.h>
#include <celero/PimplImpl.h>
#include <algorithm>
#include <chrono>
#include <fstream>
#include <iostream>
#include <map>
#include <vector>
using namespace celero;
///
/// Structure to assist with archiving data during runtime and to a file.
///
struct ArchiveEntry
{
ArchiveEntry()
: GroupName(),
RunName(),
ExperimentValue(0),
ExperimentValueScale(0),
FirstRanDate(0),
TotalSamplesCollected(0),
AverageBaseline(0),
MinBaseline(0),
MinBaseline_TimeSinceEpoch(0),
MinStats(),
MaxBaseline(0),
MaxBaseline_TimeSinceEpoch(0),
MaxStats(),
CurrentBaseline(0),
CurrentBaseline_TimeSinceEpoch(0),
CurrentStats(),
Failure(false)
{
}
static void WriteHeader(std::ostream& str)
{
str << "GroupName,RunName,Failure,ExperimentValue,ExperimentValueScale,FirstRanDate,TotalSamplesCollected,AverageBaseline,";
str << "MinBaseline,MinBaselineTimeSinceEpoch,";
str << "MinStatSize,MinStatMean,MinStatVariance,MinStatStandardDeviation,MinStatSkewness,MinStatKurtosis,";
str << "MinStatMin,MinStatMax,";
str << "MaxBaseline,MaxBaselineTimeSinceEpoch,";
str << "MaxStatSize,MaxStatMean,MaxStatVariance,MaxStatStandardDeviation,MaxStatSkewness,MaxStatKurtosis,";
str << "MaxStatMin,MaxStatMax,";
str << "CurrentBaseline,CurrentBaselineTimeSinceEpoch,";
str << "CurrentStatSize,CurrentStatMean,CurrentStatVariance,CurrentStatStandardDeviation,CurrentStatSkewness,CurrentStatKurtosis,";
str << "CurrentStatMin,CurrentStatMax" << std::endl;
}
struct Stat
{
Stat() : Size(0), Mean(0), Variance(0), StandardDeviation(0), Skewness(0), Kurtosis(0), Min(0), Max(0)
{
}
Stat& operator=(const celero::Statistics<int64_t>& s)
{
this->Size = s.getSize();
this->Mean = s.getMean();
this->Variance = s.getVariance();
this->StandardDeviation = s.getStandardDeviation();
this->Skewness = s.getSkewness();
this->Kurtosis = s.getKurtosis();
this->Min = s.getMin();
this->Max = s.getMax();
return *this;
}
uint64_t Size;
double Mean;
double Variance;
double StandardDeviation;
double Skewness;
double Kurtosis;
uint64_t Min;
uint64_t Max;
};
std::string GroupName;
std::string RunName;
/// The data set size, if one was specified.
int64_t ExperimentValue;
double ExperimentValueScale;
uint64_t FirstRanDate;
uint32_t TotalSamplesCollected;
double AverageBaseline;
double MinBaseline;
uint64_t MinBaseline_TimeSinceEpoch;
Stat MinStats;
double MaxBaseline;
uint64_t MaxBaseline_TimeSinceEpoch;
Stat MaxStats;
double CurrentBaseline;
uint64_t CurrentBaseline_TimeSinceEpoch;
Stat CurrentStats;
bool Failure;
};
///
/// Overload operator<< to allow for easy output of result data to a human-readable text file.
///
std::ostream& operator<<(std::ostream& str, ArchiveEntry::Stat const& data)
{
str << data.Size << ",";
str << data.Mean << ",";
str << data.Variance << ",";
str << data.StandardDeviation << ",";
str << data.Skewness << ",";
str << data.Kurtosis << ",";
str << data.Min << ",";
str << data.Max;
return str;
}
///
/// Overload operator<< to allow for easy output of result data to a human-readable text file.
///
std::ostream& operator<<(std::ostream& str, ArchiveEntry const& data)
{
str << data.GroupName << ",";
str << data.RunName << ",";
str << data.Failure << ",";
str << data.ExperimentValue << ",";
str << data.ExperimentValueScale << ",";
str << data.FirstRanDate << ",";
str << data.TotalSamplesCollected << ",";
str << data.AverageBaseline << ",";
str << data.MinBaseline << ",";
str << data.MinBaseline_TimeSinceEpoch << ",";
str << data.MinStats << ",";
str << data.MaxBaseline << ",";
str << data.MaxBaseline_TimeSinceEpoch << ",";
str << data.MaxStats << ",";
str << data.CurrentBaseline << ",";
str << data.CurrentBaseline_TimeSinceEpoch << ",";
str << data.CurrentStats << std::endl;
return str;
}
///
/// Overload operator>> to allow for easy input of result data from a text file.
///
std::istream& operator>>(std::istream& str, ArchiveEntry::Stat& data)
{
// Use FieldReader to classify commas as whitespace.
str.imbue(std::locale(std::locale(), new celero::FieldReader));
str >> data.Size;
str >> data.Mean;
str >> data.Variance;
str >> data.StandardDeviation;
str >> data.Skewness;
str >> data.Kurtosis;
str >> data.Min;
str >> data.Max;
return str;
}
///
/// Overload operator>> to allow for easy input of result data from a text file.
///
std::istream& operator>>(std::istream& str, ArchiveEntry& data)
{
// Use FieldReader to classify commas as whitespace.
str.imbue(std::locale(std::locale(), new celero::FieldReader));
str >> data.GroupName;
str >> data.RunName;
str >> data.Failure;
str >> data.ExperimentValue;
str >> data.ExperimentValueScale;
str >> data.FirstRanDate;
str >> data.TotalSamplesCollected;
str >> data.AverageBaseline;
str >> data.MinBaseline;
str >> data.MinBaseline_TimeSinceEpoch;
str >> data.MinStats;
str >> data.MaxBaseline;
str >> data.MaxBaseline_TimeSinceEpoch;
str >> data.MaxStats;
str >> data.CurrentBaseline;
str >> data.CurrentBaseline_TimeSinceEpoch;
str >> data.CurrentStats;
return str;
}
///
/// \class Impl
///
class celero::Archive::Impl
{
public:
Impl() : results(), fileName()
{
}
/// Return milliseconds since epoch.
uint64_t now() const
{
return static_cast<uint64_t>(
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count());
}
void readExistingResults()
{
// Read in existing results?
std::ifstream is;
is.open(this->fileName, std::fstream::in);
if((is.is_open() == true) && (is.good() == true) && (is.fail() == false))
{
// Throw away the header.
is.ignore(std::numeric_limits<std::streamsize>::max(), '\n');
// Read in existing results.
while((is.eof() == false) && (is.tellg() >= 0))
{
ArchiveEntry r;
is >> r;
if(r.GroupName.empty() == false)
{
this->results.push_back(r);
}
}
// Close the file for reading.
is.close();
}
}
std::vector<ArchiveEntry> results;
std::string fileName;
};
Archive::Archive() : pimpl()
{
}
Archive::~Archive()
{
}
Archive& Archive::Instance()
{
static Archive singleton;
return singleton;
}
void Archive::setFileName(const std::string& x)
{
if(x.empty() == false)
{
this->pimpl->fileName = x;
this->pimpl->readExistingResults();
}
}
void Archive::add(std::shared_ptr<celero::ExperimentResult> x)
{
if(x == nullptr)
{
return;
}
const auto found = std::find_if(std::begin(this->pimpl->results), std::end(this->pimpl->results), [x](const ArchiveEntry& r) -> bool {
return (r.GroupName == x->getExperiment()->getBenchmark()->getName()) && (r.RunName == x->getExperiment()->getName())
&& (r.ExperimentValue == x->getProblemSpaceValue());
});
if(found != std::end(this->pimpl->results))
{
if(x->getFailure() == true)
{
return;
}
found->CurrentBaseline = x->getBaselineMeasurement();
found->CurrentBaseline_TimeSinceEpoch = this->pimpl->now();
found->CurrentStats = *x->getTimeStatistics();
if(found->Failure || found->CurrentBaseline <= found->MinBaseline)
{
found->MinBaseline = found->CurrentBaseline;
found->MinBaseline_TimeSinceEpoch = found->CurrentBaseline_TimeSinceEpoch;
found->MinStats = found->CurrentStats;
}
if(found->Failure || found->CurrentBaseline >= found->MaxBaseline)
{
found->MaxBaseline = found->CurrentBaseline;
found->MaxBaseline_TimeSinceEpoch = found->CurrentBaseline_TimeSinceEpoch;
found->MaxStats = found->CurrentStats;
}
// This is not good IEEE math.
if(found->Failure == false)
{
found->AverageBaseline =
((found->AverageBaseline * found->TotalSamplesCollected) + found->CurrentBaseline) / (found->TotalSamplesCollected + 1);
}
else
{
found->AverageBaseline = found->CurrentBaseline;
}
found->TotalSamplesCollected++;
}
else
{
ArchiveEntry r;
const auto experiment = x->getExperiment();
if(experiment != nullptr)
{
r.GroupName = experiment->getBenchmark()->getName();
r.RunName = experiment->getName();
r.Failure = x->getFailure();
r.FirstRanDate = this->pimpl->now();
r.AverageBaseline = x->getBaselineMeasurement();
r.ExperimentValue = x->getProblemSpaceValue();
r.ExperimentValueScale = x->getProblemSpaceValueScale();
r.TotalSamplesCollected = x->getFailure() ? 0 : 1;
r.CurrentBaseline = x->getBaselineMeasurement();
r.CurrentBaseline_TimeSinceEpoch = r.FirstRanDate;
r.CurrentStats = *x->getTimeStatistics();
r.MaxBaseline = x->getBaselineMeasurement();
r.MaxBaseline_TimeSinceEpoch = r.FirstRanDate;
r.MaxStats = *x->getTimeStatistics();
r.MinBaseline = x->getBaselineMeasurement();
r.MinBaseline_TimeSinceEpoch = r.FirstRanDate;
r.MinStats = *x->getTimeStatistics();
this->pimpl->results.push_back(r);
}
}
this->save();
}
void Archive::save()
{
if(this->pimpl->fileName.empty() == false)
{
// Get ready to write out new results.
// We will write all known results every time, replacing file contents.
std::ofstream os;
os.open(this->pimpl->fileName.c_str(), std::fstream::out);
if(os.is_open() == true)
{
ArchiveEntry::WriteHeader(os);
for(auto& i : this->pimpl->results)
{
os << i;
}
os.flush();
os.close();
}
else
{
std::cerr << "Celero: Could not open result output file: \"" << this->pimpl->fileName << "\"" << std::endl;
}
}
}

100
lib/celero/Archive.h Normal file
View File

@ -0,0 +1,100 @@
#ifndef H_CELERO_ARCHIVE_H
#define H_CELERO_ARCHIVE_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Experiment.h>
#include <celero/ExperimentResult.h>
#include <string>
namespace celero
{
///
/// \class Archive
///
/// \author John Farrier
///
class CELERO_EXPORT Archive
{
public:
///
/// Singleton
///
static Archive& Instance();
///
/// Specify a file name for a results output file.
///
/// \param x The name of the output file in which to store Celero's results.
///
void setFileName(const std::string& x);
///
/// Adds or updates a result which will be saved to a results archive file.
///
/// This should re-save on every new result so that the output can be monitored externally.
///
void add(std::shared_ptr<celero::ExperimentResult> x);
///
/// Saves all current results to a results archive file.
///
/// Will overwrite all existing data and refresh with new data.
///
void save();
private:
///
/// Default Constructor
///
Archive();
///
/// Non-copyable.
/// Visual studio 2012 does not support "delete" here.
///
Archive(Archive&) = delete;
///
/// Default Destructor
///
~Archive();
///
/// Non-assignable.
/// Visual studio 2012 does not support "delete" here.
///
Archive& operator=(const Archive&)
{
return *this;
}
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

124
lib/celero/Benchmark.cpp Normal file
View File

@ -0,0 +1,124 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/PimplImpl.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <cassert>
using namespace celero;
class Benchmark::Impl
{
public:
Impl() : stats(), name(), baseline(), experiments()
{
}
explicit Impl(const std::string& x) : stats(), name(x), baseline(), experiments()
{
}
Impl(const Benchmark& other) : stats(), name(other.pimpl->name), baseline(), experiments()
{
}
void copy(const Benchmark& other)
{
stats = other.pimpl->stats;
name = other.pimpl->name;
baseline = other.pimpl->baseline;
experiments = other.pimpl->experiments;
}
Statistics<int64_t> stats;
/// Group name
std::string name;
std::shared_ptr<Experiment> baseline;
std::vector<std::shared_ptr<Experiment>> experiments;
};
Benchmark::Benchmark() : pimpl()
{
}
Benchmark::Benchmark(const std::string& name) : pimpl(name)
{
}
Benchmark::Benchmark(const Benchmark& other) : pimpl(other)
{
}
Benchmark::~Benchmark()
{
}
Benchmark& Benchmark::operator=(const Benchmark& other)
{
if(&other != this)
{
this->pimpl->copy(other);
}
return *this;
}
std::string Benchmark::getName() const
{
return this->pimpl->name;
}
void Benchmark::setBaseline(std::shared_ptr<Experiment> x)
{
this->pimpl->baseline = x;
}
std::shared_ptr<Experiment> Benchmark::getBaseline() const
{
return this->pimpl->baseline;
}
void Benchmark::addExperiment(std::shared_ptr<Experiment> x)
{
this->pimpl->experiments.push_back(x);
}
std::shared_ptr<Experiment> Benchmark::getExperiment(size_t x)
{
if(x < this->pimpl->experiments.size())
{
return this->pimpl->experiments[x];
}
return nullptr;
}
std::shared_ptr<Experiment> Benchmark::getExperiment(const std::string& x)
{
return *std::find_if(std::begin(this->pimpl->experiments), std::end(this->pimpl->experiments),
[x](decltype(*std::begin(this->pimpl->experiments)) i) -> bool { return (i->getName() == x); });
}
size_t Benchmark::getExperimentSize() const
{
return this->pimpl->experiments.size();
}

112
lib/celero/Benchmark.h Normal file
View File

@ -0,0 +1,112 @@
#ifndef H_CELERO_BENCHMARK_H
#define H_CELERO_BENCHMARK_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Experiment.h>
#include <functional>
#include <memory>
#include <string>
namespace celero
{
///
/// \class Benchmark
///
/// \author John Farrier
///
class CELERO_EXPORT Benchmark
{
public:
///
/// \brief Default constructor
///
Benchmark();
///
/// \brief Overloaded constructor.
///
/// \param name Name of the test group.
///
explicit Benchmark(const std::string& name);
///
///
///
explicit Benchmark(const Benchmark& other);
///
/// \brief Default destructor.
///
~Benchmark();
///
/// Assignment Operator
///
Benchmark& operator=(const Benchmark& other);
///
/// The name to group all experiment under.
///
std::string getName() const;
///
///
///
void setBaseline(std::shared_ptr<Experiment> x);
///
/// Gets the baseline case associated this benchmark.
///
std::shared_ptr<Experiment> getBaseline() const;
///
///
///
void addExperiment(std::shared_ptr<Experiment> x);
///
/// Gets the test case associated with the given experiment index.
///
std::shared_ptr<Experiment> getExperiment(size_t experimentIndex);
///
/// Gets the test case associated with the given experiment name.
///
std::shared_ptr<Experiment> getExperiment(const std::string& experimentName);
///
/// Returns the total number of experiments per benchmark.
///
size_t getExperimentSize() const;
private:
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

52
lib/celero/Callbacks.cpp Normal file
View File

@ -0,0 +1,52 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Callbacks.h>
#include <algorithm>
#include <vector>
using namespace celero;
std::vector<std::function<void(std::shared_ptr<Experiment>)>> ExperimentFunctions;
std::vector<std::function<void(std::shared_ptr<celero::ExperimentResult>)>> ExperimentResultFunctions;
void celero::impl::ExperimentComplete(std::shared_ptr<Experiment> x)
{
for(auto& i : ExperimentFunctions)
{
i(x);
}
}
void celero::impl::ExperimentResultComplete(std::shared_ptr<celero::ExperimentResult> x)
{
for(auto& i : ExperimentResultFunctions)
{
i(x);
}
}
void celero::AddExperimentCompleteFunction(std::function<void(std::shared_ptr<Experiment>)> x)
{
ExperimentFunctions.push_back(x);
}
void celero::AddExperimentResultCompleteFunction(std::function<void(std::shared_ptr<celero::ExperimentResult>)> x)
{
ExperimentResultFunctions.push_back(x);
}

65
lib/celero/Callbacks.h Normal file
View File

@ -0,0 +1,65 @@
#ifndef H_CELERO_CALLBACKS_H
#define H_CELERO_CALLBACKS_H
///
/// \namespace celero
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018, 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Ideas from Nick Brunn's Hayai (https://github.com/nickbruun/hayai) were used and I likely owe him a beer.
///
/// Special thanks to the band "3" for providing the development soundtrack.
///
/// "Iterations" refers to how many loops of the test function are measured as a time.
/// For very fast code, many iterations would help amoratize measurement error.
///
/// "Samples" refers to how many sets of "iterations" will be performed. Each "sample" is
/// a single measurement. Set to 0 to have Celero decide how many samples are required
/// for a minimally significant answer.
///
/// It is highly encouraged to only run this code compiled in a "Release" mode to use all available optimizations.
///
#include <celero/Experiment.h>
#include <celero/Export.h>
#include <functional>
#include <memory>
namespace celero
{
///
/// \brief Add a function to call when a experiment is completed.
///
/// This will be called at the end of a complete experiment (benchmark + experiment results.)
///
CELERO_EXPORT void AddExperimentCompleteFunction(std::function<void(std::shared_ptr<celero::Experiment>)> x);
///
/// \brief Add a function to call when a experiment is completed.
///
/// This will be called at the end of every benchmark or user experiment upon completion.
///
CELERO_EXPORT void AddExperimentResultCompleteFunction(std::function<void(std::shared_ptr<celero::ExperimentResult>)> x);
namespace impl
{
CELERO_EXPORT void ExperimentComplete(std::shared_ptr<Experiment> x);
CELERO_EXPORT void ExperimentResultComplete(std::shared_ptr<celero::ExperimentResult> x);
} // namespace impl
} // namespace celero
#endif

289
lib/celero/Celero.cpp Normal file
View File

@ -0,0 +1,289 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Archive.h>
#include <celero/Benchmark.h>
#include <celero/Callbacks.h>
#include <celero/Celero.h>
#include <celero/CommandLine.h>
#include <celero/Console.h>
#include <celero/Distribution.h>
#include <celero/Exceptions.h>
#include <celero/Executor.h>
#include <celero/JUnit.h>
#include <celero/Print.h>
#include <celero/ResultTable.h>
#include <celero/TestVector.h>
#include <celero/UserDefinedMeasurement.h>
#include <celero/Utilities.h>
#include <cassert>
#include <cmath>
#include <fstream>
#include <iostream>
#include <set>
using namespace celero;
std::shared_ptr<celero::Benchmark> celero::RegisterTest(const char* groupName, const char* benchmarkName, const uint64_t samples,
const uint64_t iterations, const uint64_t threads,
std::shared_ptr<celero::Factory> experimentFactory, const double target)
{
if(groupName != nullptr && benchmarkName != nullptr)
{
auto bm = celero::TestVector::Instance()[groupName];
if(bm == nullptr)
{
bm = std::make_shared<Benchmark>(groupName);
celero::TestVector::Instance().push_back(bm);
}
auto p = std::make_shared<Experiment>(bm);
p->setIsBaselineCase(false);
p->setName(benchmarkName);
p->setSamples(samples);
p->setIterations(iterations);
p->setThreads(threads);
p->setFactory(experimentFactory);
p->setBaselineTarget(target);
bm->addExperiment(p);
return bm;
}
return nullptr;
}
std::shared_ptr<celero::Benchmark> celero::RegisterBaseline(const char* groupName, const char* benchmarkName, const uint64_t samples,
const uint64_t iterations, const uint64_t threads,
std::shared_ptr<celero::Factory> experimentFactory)
{
if(groupName != nullptr && benchmarkName != nullptr)
{
auto bm = celero::TestVector::Instance()[groupName];
if(bm == nullptr)
{
bm = std::make_shared<Benchmark>(groupName);
celero::TestVector::Instance().push_back(bm);
}
auto p = std::make_shared<Experiment>(bm);
p->setIsBaselineCase(true);
p->setName(benchmarkName);
p->setSamples(samples);
p->setIterations(iterations);
p->setThreads(threads);
p->setFactory(experimentFactory);
p->setBaselineTarget(1.0);
bm->setBaseline(p);
return bm;
}
return nullptr;
}
void celero::Run(int argc, char** argv)
{
#ifdef _DEBUG
std::cout << "Celero is running in Debug. Results are for debugging only as any measurements made while in Debug are likely not representative "
"of non-debug results."
<< std::endl
<< std::endl;
#endif
cmdline::parser args;
args.add("list", 'l', "Prints a list of all available benchmarks.");
args.add<std::string>("group", 'g', "Runs a specific group of benchmarks.", false, "");
args.add<std::string>("outputTable", 't', "Saves a results table to the named file.", false, "");
args.add<std::string>("junit", 'j', "Saves a JUnit XML-formatted file to the named file.", false, "");
args.add<std::string>("archive", 'a', "Saves or updates a result archive file.", false, "");
args.add<uint64_t>("distribution", 'd', "Builds a file to help characterize the distribution of measurements and exits.", false, 0);
args.add<bool>("catchExceptions", 'e', "Allows Celero to catch exceptions and continue processing following benchmarks.", false, true);
args.parse_check(argc, argv);
if(args.exist("list") == true)
{
auto& tests = celero::TestVector::Instance();
std::vector<std::string> testNames;
for(auto i = size_t(0); i < tests.size(); i++)
{
auto bm = celero::TestVector::Instance()[i];
testNames.push_back(bm->getName());
}
std::sort(std::begin(testNames), std::end(testNames));
std::cout << "Avaliable tests:" << std::endl;
for(auto i : testNames)
{
std::cout << "\t" << i << std::endl;
}
return;
}
// Initial output
std::cout << "Celero" << std::endl;
// Disable dynamic CPU frequency scaling
celero::timer::CachePerformanceFrequency(false);
// Shall we build a distribution?
auto intArgument = args.get<uint64_t>("distribution");
if(intArgument > 0)
{
RunDistribution(intArgument);
}
// Has a result output file been specified?
auto mustCloseFile = false;
auto argument = args.get<std::string>("outputTable");
if(argument.empty() == false)
{
std::cout << "Writing results to: " << argument << std::endl;
celero::ResultTable::Instance().setFileName(argument);
celero::AddExperimentResultCompleteFunction([](std::shared_ptr<celero::ExperimentResult> p) { celero::ResultTable::Instance().add(p); });
mustCloseFile = true;
}
// Has a result output file been specified?
argument = args.get<std::string>("archive");
if(argument.empty() == false)
{
std::cout << "Archiving results to: " << argument << std::endl;
celero::Archive::Instance().setFileName(argument);
celero::AddExperimentResultCompleteFunction([](std::shared_ptr<celero::ExperimentResult> p) { celero::Archive::Instance().add(p); });
}
// Has a JUnit output file been specified?
argument = args.get<std::string>("junit");
if(argument.empty() == false)
{
std::cout << "Writing JUnit results to: " << argument << std::endl;
celero::JUnit::Instance().setFileName(argument);
celero::AddExperimentResultCompleteFunction([](std::shared_ptr<celero::ExperimentResult> p) { celero::JUnit::Instance().add(p); });
}
// Has a flag to catch exceptions or not been specified?
if(args.exist("catchExceptions") == true)
{
ExceptionSettings::SetCatchExceptions(args.get<bool>("catchExceptions"));
}
// Has a run group been specified?
argument = args.get<std::string>("group");
// Collect all user-defined fields
std::set<std::string> userDefinedFields;
auto collectFromBenchmark = [&](std::shared_ptr<Benchmark> bmark) {
// Collect from baseline
auto baselineExperiment = bmark->getBaseline();
if(baselineExperiment != nullptr)
{
auto test = baselineExperiment->getFactory()->Create();
UserDefinedMeasurementCollector udmCollector(test);
for(const auto& fieldName : udmCollector.getFields(test))
{
userDefinedFields.insert(fieldName);
}
}
// Collect from all experiments
const auto experimentSize = bmark->getExperimentSize();
for(size_t i = 0; i < experimentSize; i++)
{
auto e = bmark->getExperiment(i);
assert(e != nullptr);
auto test = baselineExperiment->getFactory()->Create();
UserDefinedMeasurementCollector udmCollector(test);
for(const auto& fieldName : udmCollector.getFields(test))
{
userDefinedFields.insert(fieldName);
}
}
};
if(argument.empty() == false)
{
auto bmark = celero::TestVector::Instance()[argument];
collectFromBenchmark(bmark);
}
else
{
for(size_t i = 0; i < celero::TestVector::Instance().size(); i++)
{
auto bmark = celero::TestVector::Instance()[i];
collectFromBenchmark(bmark);
}
}
std::vector<std::string> userDefinedFieldsOrder(userDefinedFields.begin(), userDefinedFields.end());
Printer::get().initialize(userDefinedFieldsOrder);
Printer::get().TableBanner();
const auto startTime = celero::timer::GetSystemTime();
if(argument.empty() == false)
{
executor::Run(argument);
}
else
{
executor::RunAll();
}
const auto totalTime = celero::timer::ConvertSystemTime(celero::timer::GetSystemTime() - startTime);
if(mustCloseFile == true)
{
celero::ResultTable::Instance().closeFile();
}
// Final output.
auto hours = std::to_string(static_cast<int>(totalTime) / 3600);
auto minutes = std::to_string((static_cast<int>(totalTime) % 3600) / 60);
auto seconds = std::to_string(fmod(totalTime, 60.0));
if(hours.length() < 2)
{
hours = std::string("0") + hours;
}
if(minutes.length() < 2)
{
minutes = std::string("0") + minutes;
}
if(fmod(totalTime, 60.0) < 10.0)
{
seconds = std::string("0") + seconds;
}
std::cout << "Completed in " << hours << ":" << minutes << ":" << seconds << std::endl;
}

315
lib/celero/Celero.h Normal file
View File

@ -0,0 +1,315 @@
#ifndef H_CELERO_CELERO_H
#define H_CELERO_CELERO_H
///
/// \namespace celero
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Special thanks to the bands "3" and "Coheed and Cambria" for providing the development soundtrack.
///
/// "Iterations" refers to how many loops of the test function are measured as a time.
/// For very fast code, many iterations would help amoratize measurement error.
///
/// "Samples" refers to how many sets of "Iterations" will be performed. Each "sample" is
/// a single measurement.
///
/// It is highly encouraged to only run this code compiled in a "Release" mode to use all available optimizations.
///
#ifdef WIN32
#include <process.h>
#endif
#include <celero/Benchmark.h>
#include <celero/GenericFactory.h>
#include <celero/TestFixture.h>
#include <celero/ThreadTestFixture.h>
#include <celero/UserDefinedMeasurementCollector.h>
#include <celero/UserDefinedMeasurementTemplate.h>
#include <celero/Utilities.h>
namespace celero
{
///
/// \brief Adds a new test to the list of tests to be executed.
///
/// All tests must be registered prior to calling celer::Run().
///
/// \param groupName The name of the Test Group. Used for retrieving the associated baseline.
/// \param benchmarkName A unique name for a specific test within a Test Group.
/// \param samples The total number of times to execute the Test. (Each test contains iterations.)
/// \param iterations The total number of iterations per Test.
/// \param threads The total number of threads per Test sample.
/// \param experimentFactory The factory implementation for the test.
///
/// \returns a pointer to a Benchmark instance representing the given test.
///
CELERO_EXPORT std::shared_ptr<celero::Benchmark> RegisterTest(const char* groupName, const char* benchmarkName, const uint64_t samples,
const uint64_t iterations, const uint64_t threads,
std::shared_ptr<celero::Factory> experimentFactory, const double target = -1);
///
/// \brief Adds a new test baseline to the list of test baseliness to be executed.
///
/// All test baselines must be registered prior to calling celer::Run().
///
/// \param groupName The name of the Test Group that the baseline is associated with.
/// \param benchmarkName A unique name for a specific test baseline within a Test Group.
/// \param samples The total number of times to execute the Test baseline. (Each sample contains one or more iterations.)
/// \param iterations The total number of iterations per Test baseline sample.
/// \param threads The total number of threads per Test baseline.
/// \param experimentFactory The factory implementation for the test baseline.
///
/// \returns a pointer to a Benchmark instance representing the given test.
///
CELERO_EXPORT std::shared_ptr<Benchmark> RegisterBaseline(const char* groupName, const char* benchmarkName, const uint64_t samples,
const uint64_t iterations, const uint64_t threads,
std::shared_ptr<Factory> experimentFactory);
///
/// \brief Builds a distribution of total system measurement error.
///
/// The result vector contains microseconds for each trivial timer sample.
/// The purpose is to be able to characterize the generic distribution of results
/// on a given system.
///
/// This is just an attempt to characterize the distribution, not quantify it.
///
CELERO_EXPORT std::vector<uint64_t> BuildDistribution(uint64_t numberOfSamples, uint64_t iterationsPerSample);
///
/// \brief The main test executor.
///
CELERO_EXPORT void Run(int argc, char** argv);
} // namespace celero
///
/// \define CELERO_MAIN
///
/// \brief A macro to build the most basic main() required to run the benchmark tests.
///
#define CELERO_MAIN \
int main(int argc, char** argv) \
{ \
celero::Run(argc, argv); \
return 0; \
}
///
/// \define BENCHMARK_CLASS_NAME
///
/// \brief A macro to build a class name based on the test groupo and benchmark names.
///
#define BENCHMARK_CLASS_NAME(groupName, benchmarkName) CeleroUserBenchmark##_##groupName##_##benchmarkName
///
/// \define BENCHMARK_IMPL
///
/// A macro to create a class of a unique name which can be used to register and execute a benchmark test.
///
#define BENCHMARK_IMPL(groupName, benchmarkName, fixtureName, samples, iterations, threads) \
class BENCHMARK_CLASS_NAME(groupName, benchmarkName) : public fixtureName \
{ \
public: \
BENCHMARK_CLASS_NAME(groupName, benchmarkName)() : fixtureName() \
{ \
} \
\
protected: \
virtual void UserBenchmark() override; \
\
private: \
static const std::shared_ptr<::celero::Benchmark> info; \
}; \
\
const std::shared_ptr<::celero::Benchmark> BENCHMARK_CLASS_NAME(groupName, benchmarkName)::info = \
::celero::RegisterTest(#groupName, #benchmarkName, samples, iterations, threads, \
std::make_shared<::celero::GenericFactory<BENCHMARK_CLASS_NAME(groupName, benchmarkName)>>()); \
\
void BENCHMARK_CLASS_NAME(groupName, benchmarkName)::UserBenchmark()
///
/// \define BENCHMARK_TEST_IMPL
///
/// A macro to create a class of a unique name which can be used to register and execute a benchmark test.
///
#define BENCHMARK_TEST_IMPL(groupName, benchmarkName, fixtureName, samples, iterations, threads, target) \
class BENCHMARK_CLASS_NAME(groupName, benchmarkName) : public fixtureName \
{ \
public: \
BENCHMARK_CLASS_NAME(groupName, benchmarkName)() : fixtureName() \
{ \
} \
\
protected: \
virtual void UserBenchmark() override; \
\
private: \
static const std::shared_ptr<::celero::Benchmark> info; \
}; \
\
const std::shared_ptr<::celero::Benchmark> BENCHMARK_CLASS_NAME(groupName, benchmarkName)::info = \
::celero::RegisterTest(#groupName, #benchmarkName, samples, iterations, threads, \
std::make_shared<::celero::GenericFactory<BENCHMARK_CLASS_NAME(groupName, benchmarkName)>>(), target); \
\
void BENCHMARK_CLASS_NAME(groupName, benchmarkName)::UserBenchmark()
///
/// \define BENCHMARK_F
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark containing a test fixture.
///
/// Using the BENCHMARK_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BENCHMARK_F(groupName, benchmarkName, fixtureName, samples, iterations) \
BENCHMARK_IMPL(groupName, benchmarkName, fixtureName, samples, iterations, 1)
///
/// \define BENCHMARK_T
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark containing a threaded test fixture.
///
/// Using the BENCHMARK_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BENCHMARK_T(groupName, benchmarkName, fixtureName, samples, iterations, threads) \
BENCHMARK_IMPL(groupName, benchmarkName, fixtureName, samples, iterations, threads)
///
/// \define BENCHMARK_TEST_F
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark containing a test fixture.
///
/// Using the BENCHMARK_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BENCHMARK_TEST_F(groupName, benchmarkName, fixtureName, samples, iterations, target) \
BENCHMARK_TEST_IMPL(groupName, benchmarkName, fixtureName, samples, iterations, 1, target)
///
/// \define BENCHMARK_TEST_T
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark containing a threaded test fixture.
///
/// Using the BENCHMARK_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BENCHMARK_TEST_T(groupName, benchmarkName, fixtureName, samples, iterations, threads, target) \
BENCHMARK_TEST_IMPL(groupName, benchmarkName, fixtureName, samples, iterations, threads, target)
///
/// \define BENCHMARK
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark.
///
/// Using the BENCHMARK_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BENCHMARK(groupName, benchmarkName, samples, iterations) \
BENCHMARK_IMPL(groupName, benchmarkName, ::celero::TestFixture, samples, iterations, 1)
///
/// \define BENCHMARK
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark.
///
/// Using the BENCHMARK_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BENCHMARK_TEST(groupName, benchmarkName, samples, iterations, target) \
BENCHMARK_TEST_IMPL(groupName, benchmarkName, ::celero::TestFixture, samples, iterations, 1, target)
///
/// \define BASELINE_CLASS_NAME
///
/// \brief A macro to build a class name based on the test group and baseline names.
///
#define BASELINE_CLASS_NAME(groupName, baselineName) CeleroUserBaseline##_##groupName##_##baselineName
///
/// \define BASELINE_IMPL
///
/// A macro to create a class of a unique name which can be used to register and execute a baseline benchmark test.
///
#define BASELINE_IMPL(groupName, baselineName, fixtureName, samples, iterations, threads, useconds) \
class BASELINE_CLASS_NAME(groupName, baselineName) : public fixtureName \
{ \
public: \
BASELINE_CLASS_NAME(groupName, baselineName)() : fixtureName() \
{ \
} \
\
protected: \
virtual void UserBenchmark() override; \
virtual uint64_t HardCodedMeasurement() const override \
{ \
return uint64_t(useconds); \
} \
\
private: \
static const std::shared_ptr<::celero::Benchmark> info; \
}; \
\
const std::shared_ptr<::celero::Benchmark> BASELINE_CLASS_NAME(groupName, baselineName)::info = \
::celero::RegisterBaseline(#groupName, #baselineName, samples, iterations, threads, \
std::make_shared<::celero::GenericFactory<BASELINE_CLASS_NAME(groupName, baselineName)>>()); \
\
void BASELINE_CLASS_NAME(groupName, baselineName)::UserBenchmark()
///
/// \define BASELINE_F
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark containing a test fixture.
///
/// Using the BASELINE_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BASELINE_F(groupName, baselineName, fixtureName, samples, iterations) \
BASELINE_IMPL(groupName, baselineName, fixtureName, samples, iterations, 1, 0)
///
/// \define BASELINE_T
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark containing a threaded test fixture.
///
/// Using the BASELINE_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BASELINE_T(groupName, baselineName, fixtureName, samples, iterations, threads) \
BASELINE_IMPL(groupName, baselineName, fixtureName, samples, iterations, threads, 0)
///
/// \define BASELINE
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark.
///
/// Using the BASELINE_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BASELINE(groupName, baselineName, samples, iterations) \
BASELINE_IMPL(groupName, baselineName, ::celero::TestFixture, samples, iterations, 1, 0)
///
/// \define BASELINE_FIXED
///
/// \brief A macro to place in user code to define a UserBenchmark function for a benchmark with a hard-coded timing.
///
/// This will NOT perform any timing measurments but will instead use the number of microseconds passed in as the measured time.
///
/// Using the BASELINE_ macro, this effectivly fills in a class's UserBenchmark() function.
///
#define BASELINE_FIXED(groupName, baselineName, samples, iterations, useconds) \
BASELINE_IMPL(groupName, baselineName, ::celero::TestFixture, samples, iterations, 1, useconds)
#define BASELINE_FIXED_F(groupName, baselineName, fixtureName, samples, iterations, useconds) \
BASELINE_IMPL(groupName, baselineName, fixtureName, samples, iterations, 1, useconds)
#define BASELINE_FIXED_T(groupName, baselineName, fixtureName, samples, iterations, threads, useconds) \
BASELINE_IMPL(groupName, baselineName, fixtureName, samples, iterations, threads, useconds)
#endif

942
lib/celero/CommandLine.h Normal file
View File

@ -0,0 +1,942 @@
/*
Copyright (c) 2009, Hideyuki Tanaka
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the <organization> nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY <copyright holder> ''AS IS'' AND ANY
EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <copyright holder> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#pragma once
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <iostream>
#include <map>
#include <sstream>
#include <stdexcept>
#include <string>
#include <typeinfo>
#include <vector>
#ifndef WIN32
#include <cxxabi.h>
#endif
namespace cmdline
{
namespace detail
{
template <typename Target, typename Source, bool Same>
class lexical_cast_t
{
public:
static Target cast(const Source &arg)
{
Target ret;
std::stringstream ss;
if(!(ss << arg && ss >> ret && ss.eof()))
throw std::bad_cast();
return ret;
}
};
template <typename Target, typename Source>
class lexical_cast_t<Target, Source, true>
{
public:
static Target cast(const Source &arg)
{
return arg;
}
};
template <typename Source>
class lexical_cast_t<std::string, Source, false>
{
public:
static std::string cast(const Source &arg)
{
std::ostringstream ss;
ss << arg;
return ss.str();
}
};
template <typename Target>
class lexical_cast_t<Target, std::string, false>
{
public:
static Target cast(const std::string &arg)
{
Target ret;
std::istringstream ss(arg);
if(!(ss >> ret && ss.eof()))
throw std::bad_cast();
return ret;
}
};
template <typename T1, typename T2>
struct is_same
{
static const bool value = false;
};
template <typename T>
struct is_same<T, T>
{
static const bool value = true;
};
template <typename Target, typename Source>
Target lexical_cast(const Source &arg)
{
return lexical_cast_t<Target, Source, detail::is_same<Target, Source>::value>::cast(arg);
}
#ifdef WIN32
static inline std::string demangle(const std::string &)
{
return std::string();
}
#else
static inline std::string demangle(const std::string &name)
{
int status = 0;
char *p = abi::__cxa_demangle(name.c_str(), 0, 0, &status);
std::string ret(p);
free(p);
return ret;
}
#endif
template <class T>
std::string readable_typename()
{
return demangle(typeid(T).name());
}
template <class T>
std::string default_value(T def)
{
return detail::lexical_cast<std::string>(def);
}
template <>
inline std::string readable_typename<std::string>()
{
return "string";
}
} // namespace detail
//-----
class cmdline_error : public std::exception
{
public:
cmdline_error(const std::string &msg) : msg(msg)
{
}
~cmdline_error() throw()
{
}
const char *what() const throw()
{
return msg.c_str();
}
private:
std::string msg;
};
template <class T>
struct default_reader
{
T operator()(const std::string &str)
{
return detail::lexical_cast<T>(str);
}
};
template <class T>
struct range_reader
{
range_reader(const T &low, const T &high) : low(low), high(high)
{
}
T operator()(const std::string &s) const
{
T ret = default_reader<T>()(s);
if(!(ret >= low && ret <= high))
throw cmdline::cmdline_error("range_error");
return ret;
}
private:
T low, high;
};
template <class T>
range_reader<T> range(const T &low, const T &high)
{
return range_reader<T>(low, high);
}
template <class T>
struct oneof_reader
{
T operator()(const std::string &s)
{
T ret = default_reader<T>()(s);
if(std::find(alt.begin(), alt.end(), ret) == alt.end())
throw cmdline_error("");
return ret;
}
void add(const T &v)
{
alt.push_back(v);
}
private:
std::vector<T> alt;
};
template <class T>
oneof_reader<T> oneof(T a1)
{
oneof_reader<T> ret;
ret.add(a1);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
ret.add(a5);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
ret.add(a5);
ret.add(a6);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
ret.add(a5);
ret.add(a6);
ret.add(a7);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7, T a8)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
ret.add(a5);
ret.add(a6);
ret.add(a7);
ret.add(a8);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7, T a8, T a9)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
ret.add(a5);
ret.add(a6);
ret.add(a7);
ret.add(a8);
ret.add(a9);
return ret;
}
template <class T>
oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7, T a8, T a9, T a10)
{
oneof_reader<T> ret;
ret.add(a1);
ret.add(a2);
ret.add(a3);
ret.add(a4);
ret.add(a5);
ret.add(a6);
ret.add(a7);
ret.add(a8);
ret.add(a9);
ret.add(a10);
return ret;
}
//-----
class parser
{
public:
~parser()
{
for(std::map<std::string, option_base *>::iterator p = options.begin(); p != options.end(); p++)
delete p->second;
}
void add(const std::string &name, char short_name = 0, const std::string &desc = "")
{
if(options.count(name))
throw cmdline_error("multiple definition: " + name);
options[name] = new option_without_value(name, short_name, desc);
ordered.push_back(options[name]);
}
template <class T>
void add(const std::string &name, char short_name = 0, const std::string &desc = "", bool need = true, const T def = T())
{
add(name, short_name, desc, need, def, default_reader<T>());
}
template <class T, class F>
void add(const std::string &name, char short_name = 0, const std::string &desc = "", bool need = true, const T def = T(), F reader = F())
{
if(options.count(name))
throw cmdline_error("multiple definition: " + name);
options[name] = new option_with_value_with_reader<T, F>(name, short_name, need, def, desc, reader);
ordered.push_back(options[name]);
}
void footer(const std::string &f)
{
ftr = f;
}
void set_program_name(const std::string &name)
{
prog_name = name;
}
bool exist(const std::string &name) const
{
if(options.count(name) == 0)
throw cmdline_error("there is no flag: --" + name);
return options.find(name)->second->has_set();
}
template <class T>
const T &get(const std::string &name) const
{
if(options.count(name) == 0)
throw cmdline_error("there is no flag: --" + name);
const option_with_value<T> *p = dynamic_cast<const option_with_value<T> *>(options.find(name)->second);
if(p == NULL)
throw cmdline_error("type mismatch flag '" + name + "'");
return p->get();
}
const std::vector<std::string> &rest() const
{
return others;
}
bool parse(const std::string &arg)
{
std::vector<std::string> args;
std::string buf;
bool in_quote = false;
for(std::string::size_type i = 0; i < arg.length(); i++)
{
if(arg[i] == '\"')
{
in_quote = !in_quote;
continue;
}
if(arg[i] == ' ' && !in_quote)
{
args.push_back(buf);
buf = "";
continue;
}
if(arg[i] == '\\')
{
i++;
if(i >= arg.length())
{
errors.push_back("unexpected occurrence of '\\' at end of string");
return false;
}
}
buf += arg[i];
}
if(in_quote)
{
errors.push_back("quote is not closed");
return false;
}
if(buf.length() > 0)
args.push_back(buf);
for(size_t i = 0; i < args.size(); i++)
std::cout << "\"" << args[i] << "\"" << std::endl;
return parse(args);
}
bool parse(const std::vector<std::string> &args)
{
int argc = static_cast<int>(args.size());
std::vector<const char *> argv(static_cast<size_t>(argc));
for(int i = 0; i < argc; i++)
argv[i] = args[i].c_str();
return parse(argc, &argv[0]);
}
bool parse(int argc, const char *const argv[])
{
errors.clear();
others.clear();
if(argc < 1)
{
errors.push_back("argument number must be longer than 0");
return false;
}
if(prog_name == "")
prog_name = argv[0];
std::map<char, std::string> lookup;
for(std::map<std::string, option_base *>::iterator p = options.begin(); p != options.end(); p++)
{
if(p->first.length() == 0)
continue;
char initial = p->second->short_name();
if(initial)
{
if(lookup.count(initial) > 0)
{
lookup[initial] = "";
errors.push_back(std::string("short option '") + initial + "' is ambiguous");
return false;
}
else
lookup[initial] = p->first;
}
}
for(int i = 1; i < argc; i++)
{
if(strncmp(argv[i], "--", 2) == 0)
{
const char *p = strchr(argv[i] + 2, '=');
if(p)
{
std::string name(argv[i] + 2, p);
std::string val(p + 1);
set_option(name, val);
}
else
{
std::string name(argv[i] + 2);
if(options.count(name) == 0)
{
errors.push_back("undefined option: --" + name);
continue;
}
if(options[name]->has_value())
{
if(i + 1 >= argc)
{
errors.push_back("option needs value: --" + name);
continue;
}
else
{
i++;
set_option(name, argv[i]);
}
}
else
{
set_option(name);
}
}
}
else if(strncmp(argv[i], "-", 1) == 0)
{
if(!argv[i][1])
continue;
char last = argv[i][1];
for(int j = 2; argv[i][j]; j++)
{
last = argv[i][j];
if(lookup.count(argv[i][j - 1]) == 0)
{
errors.push_back(std::string("undefined short option: -") + argv[i][j - 1]);
continue;
}
if(lookup[argv[i][j - 1]] == "")
{
errors.push_back(std::string("ambiguous short option: -") + argv[i][j - 1]);
continue;
}
set_option(lookup[argv[i][j - 1]]);
}
if(lookup.count(last) == 0)
{
errors.push_back(std::string("undefined short option: -") + last);
continue;
}
if(lookup[last] == "")
{
errors.push_back(std::string("ambiguous short option: -") + last);
continue;
}
if(i + 1 < argc && options[lookup[last]]->has_value())
{
set_option(lookup[last], argv[i + 1]);
i++;
}
else
{
set_option(lookup[last]);
}
}
else
{
others.push_back(argv[i]);
}
}
for(std::map<std::string, option_base *>::iterator p = options.begin(); p != options.end(); p++)
{
if(!p->second->valid())
{
errors.push_back("need option: --" + std::string(p->first));
}
}
return errors.size() == 0;
}
void parse_check(const std::string &arg)
{
if(!options.count("help"))
add("help", '?', "print this message");
check(0, parse(arg));
}
void parse_check(const std::vector<std::string> &args)
{
if(!options.count("help"))
add("help", '?', "print this message");
check(static_cast<int>(args.size()), parse(args));
}
void parse_check(int argc, char *argv[])
{
if(!options.count("help"))
add("help", '?', "print this message");
check(argc, parse(argc, argv));
}
std::string error() const
{
return errors.size() > 0 ? errors[0] : "";
}
std::string error_full() const
{
std::ostringstream oss;
for(size_t i = 0; i < errors.size(); i++)
oss << errors[i] << std::endl;
return oss.str();
}
std::string usage() const
{
std::ostringstream oss;
oss << "usage: " << prog_name << " ";
for(size_t i = 0; i < ordered.size(); i++)
{
if(ordered[i]->must())
oss << ordered[i]->short_description() << " ";
}
oss << "[options] ... " << ftr << std::endl;
oss << "options:" << std::endl;
size_t max_width = 0;
for(size_t i = 0; i < ordered.size(); i++)
{
max_width = std::max(max_width, ordered[i]->name().length());
}
for(size_t i = 0; i < ordered.size(); i++)
{
if(ordered[i]->short_name())
{
oss << " -" << ordered[i]->short_name() << ", ";
}
else
{
oss << " ";
}
oss << "--" << ordered[i]->name();
for(size_t j = ordered[i]->name().length(); j < max_width + 4; j++)
oss << ' ';
oss << ordered[i]->description() << std::endl;
}
return oss.str();
}
private:
void check(int argc, bool ok)
{
if((argc == 1 && !ok) || exist("help"))
{
std::cerr << usage();
exit(0);
}
if(!ok)
{
std::cerr << error() << std::endl << usage();
exit(1);
}
}
void set_option(const std::string &name)
{
if(options.count(name) == 0)
{
errors.push_back("undefined option: --" + name);
return;
}
if(!options[name]->set())
{
errors.push_back("option needs value: --" + name);
return;
}
}
void set_option(const std::string &name, const std::string &value)
{
if(options.count(name) == 0)
{
errors.push_back("undefined option: --" + name);
return;
}
if(!options[name]->set(value))
{
errors.push_back("option value is invalid: --" + name + "=" + value);
return;
}
}
class option_base
{
public:
virtual ~option_base()
{
}
virtual bool has_value() const = 0;
virtual bool set() = 0;
virtual bool set(const std::string &value) = 0;
virtual bool has_set() const = 0;
virtual bool valid() const = 0;
virtual bool must() const = 0;
virtual const std::string &name() const = 0;
virtual char short_name() const = 0;
virtual const std::string &description() const = 0;
virtual std::string short_description() const = 0;
};
class option_without_value : public option_base
{
public:
option_without_value(const std::string &name, char short_name, const std::string &desc)
: nam(name), snam(short_name), desc(desc), has(false)
{
}
~option_without_value()
{
}
bool has_value() const
{
return false;
}
bool set()
{
has = true;
return true;
}
bool set(const std::string &)
{
return false;
}
bool has_set() const
{
return has;
}
bool valid() const
{
return true;
}
bool must() const
{
return false;
}
const std::string &name() const
{
return nam;
}
char short_name() const
{
return snam;
}
const std::string &description() const
{
return desc;
}
std::string short_description() const
{
return "--" + nam;
}
private:
std::string nam;
char snam;
std::string desc;
bool has;
};
template <class T>
class option_with_value : public option_base
{
public:
option_with_value(const std::string &name, char short_name, bool need, const T &def, const std::string &desc)
: nam(name), snam(short_name), need(need), has(false), def(def), actual(def)
{
this->desc = full_description(desc);
}
~option_with_value()
{
}
const T &get() const
{
return actual;
}
bool has_value() const
{
return true;
}
bool set()
{
return false;
}
bool set(const std::string &value)
{
try
{
actual = read(value);
has = true;
}
catch(const std::exception &)
{
return false;
}
return true;
}
bool has_set() const
{
return has;
}
bool valid() const
{
if(need && !has)
return false;
return true;
}
bool must() const
{
return need;
}
const std::string &name() const
{
return nam;
}
char short_name() const
{
return snam;
}
const std::string &description() const
{
return desc;
}
std::string short_description() const
{
return "--" + nam + "=" + detail::readable_typename<T>();
}
protected:
std::string full_description(const std::string &description)
{
return description + " (" + detail::readable_typename<T>() + (need ? "" : " [=" + detail::default_value<T>(def) + "]") + ")";
}
virtual T read(const std::string &s) = 0;
std::string nam;
char snam;
bool need;
std::string desc;
bool has;
T def;
T actual;
};
template <class T, class F>
class option_with_value_with_reader : public option_with_value<T>
{
public:
option_with_value_with_reader(const std::string &name, char short_name, bool need, const T def, const std::string &desc, F reader)
: option_with_value<T>(name, short_name, need, def, desc), reader(reader)
{
}
private:
T read(const std::string &s)
{
return reader(s);
}
F reader;
};
std::map<std::string, option_base *> options;
std::vector<option_base *> ordered;
std::string ftr;
std::string prog_name;
std::vector<std::string> others;
std::vector<std::string> errors;
};
} // namespace cmdline

253
lib/celero/Console.cpp Normal file
View File

@ -0,0 +1,253 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Console.h>
using namespace celero;
#ifdef WIN32
#include <Windows.h>
#include <stdio.h>
#else
#include <iostream>
#endif
#ifdef WIN32
auto WinColor() -> decltype(GetStdHandle(STD_OUTPUT_HANDLE))
{
auto h = GetStdHandle(STD_OUTPUT_HANDLE);
CONSOLE_SCREEN_BUFFER_INFO csbiInfo;
GetConsoleScreenBufferInfo(h, &csbiInfo);
return h;
}
#endif
void Red()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED);
#else
std::cout << "\033[49m\033[31m";
#endif
}
void RedBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;31m";
#endif
}
void Green()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_GREEN);
#else
std::cout << "\033[49m\033[32m";
#endif
}
void GreenBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_GREEN | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;32m";
#endif
}
void Blue()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_BLUE);
#else
std::cout << "\033[49m\033[34m";
#endif
}
void BlueBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_BLUE | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;34m";
#endif
}
void Cyan()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_BLUE | FOREGROUND_GREEN);
#else
std::cout << "\033[49m\033[36m";
#endif
}
void CyanBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;36m";
#endif
}
void Yellow()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED | FOREGROUND_GREEN);
#else
std::cout << "\033[49m\033[33m";
#endif
}
void YellowBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;33m";
#endif
}
void White()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);
#else
std::cout << "\033[49m\033[37m";
#endif
}
void WhiteBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;37m";
#endif
}
void WhiteOnRed()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, BACKGROUND_RED | FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE);
#else
std::cout << "\033[41m\033[37m";
#endif
}
void WhiteOnRedBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, BACKGROUND_RED | FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
#else
std::cout << "\033[41m\033[1;37m";
#endif
}
void PurpleBold()
{
#ifdef WIN32
auto h = WinColor();
SetConsoleTextAttribute(h, FOREGROUND_RED | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
#else
std::cout << "\033[49m\033[1;38m";
#endif
}
void Default()
{
#ifdef WIN32
White();
#else
std::cout << "\033[0m";
#endif
}
void celero::console::SetConsoleColor(const celero::console::ConsoleColor x)
{
switch(x)
{
case console::ConsoleColor::Red:
Red();
break;
case console::ConsoleColor::Red_Bold:
RedBold();
break;
case console::ConsoleColor::Green:
Green();
break;
case console::ConsoleColor::Green_Bold:
GreenBold();
break;
case console::ConsoleColor::Blue:
Blue();
break;
case console::ConsoleColor::Blue_Bold:
BlueBold();
break;
case console::ConsoleColor::Cyan:
Cyan();
break;
case console::ConsoleColor::Cyan_Bold:
CyanBold();
break;
case console::ConsoleColor::Yellow:
Yellow();
break;
case console::ConsoleColor::Yellow_Bold:
YellowBold();
break;
case console::ConsoleColor::White:
White();
break;
case console::ConsoleColor::White_Bold:
WhiteBold();
break;
case console::ConsoleColor::WhiteOnRed:
WhiteOnRed();
break;
case console::ConsoleColor::WhiteOnRed_Bold:
WhiteOnRedBold();
break;
case console::ConsoleColor::Purple_Bold:
PurpleBold();
break;
case console::ConsoleColor::Default:
default:
Default();
break;
}
}

65
lib/celero/Console.h Normal file
View File

@ -0,0 +1,65 @@
#ifndef H_CELERO_CONSOLE_H
#define H_CELERO_CONSOLE_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
namespace celero
{
///
/// \namespace console
///
/// \author John farrier
///
namespace console
{
///
/// \enum ConsoleColor
///
/// \author John farrier
///
enum class ConsoleColor
{
Default,
Red,
Red_Bold,
Green,
Green_Bold,
Blue,
Blue_Bold,
Cyan,
Cyan_Bold,
Yellow,
Yellow_Bold,
White,
White_Bold,
WhiteOnRed,
WhiteOnRed_Bold,
Purple_Bold
};
///
/// Set the color of std::out on the console.
///
CELERO_EXPORT void SetConsoleColor(const celero::console::ConsoleColor x);
} // namespace console
} // namespace celero
#endif

165
lib/celero/Distribution.cpp Normal file
View File

@ -0,0 +1,165 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Distribution.h>
#include <celero/Print.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <array>
#include <cmath>
#include <fstream>
#include <iostream>
#include <map>
using namespace celero;
std::vector<uint64_t> celero::BuildDistribution(uint64_t numberOfSamples, uint64_t iterationsPerSample)
{
std::vector<uint64_t> measurements;
while(numberOfSamples--)
{
// Dummy variable
auto dummy = uint64_t(0);
auto cps = iterationsPerSample;
// Get the starting time.
const auto startTime = celero::timer::GetSystemTime();
while(cps--)
{
celero::DoNotOptimizeAway(dummy++);
}
const auto endTime = celero::timer::GetSystemTime();
measurements.push_back(endTime - startTime);
}
return measurements;
}
void celero::RunDistribution(uint64_t intArgument)
{
std::vector<double> series1Normalized(intArgument);
std::vector<double> series2Normalized(intArgument);
std::vector<double> series3Normalized(intArgument);
std::vector<double> series4Normalized(intArgument);
const auto series1 = celero::BuildDistribution(intArgument, uint64_t(64));
const auto series2 = celero::BuildDistribution(intArgument, uint64_t(256));
const auto series3 = celero::BuildDistribution(intArgument, uint64_t(1024));
const auto series4 = celero::BuildDistribution(intArgument, uint64_t(4096));
if(series1.empty() == true || series2.empty() == true || series3.empty() == true || series4.empty() == true)
{
return;
}
std::array<std::map<double, uint64_t>, 4> histograms;
// Find the global max for all tests:
auto maxVal = std::max(*(std::max_element(std::begin(series1), std::end(series1))), *(std::max_element(std::begin(series2), std::end(series2))));
maxVal = std::max(maxVal, *(std::max_element(std::begin(series3), std::end(series3))));
maxVal = std::max(maxVal, *(std::max_element(std::begin(series4), std::end(series4))));
// Find the global min for all tests:
auto minVal = std::min(*(std::min_element(std::begin(series1), std::end(series1))), *(std::min_element(std::begin(series2), std::end(series2))));
minVal = std::min(minVal, *(std::min_element(std::begin(series3), std::end(series3))));
minVal = std::min(minVal, *(std::min_element(std::begin(series4), std::end(series4))));
// Normalize all vectors:
auto normalize = [minVal, maxVal](uint64_t val) -> double {
if(val >= minVal)
{
if(val <= maxVal)
{
const auto delta = maxVal - minVal;
val -= minVal;
return static_cast<double>(val) / static_cast<double>(delta);
}
return static_cast<double>(maxVal);
}
return static_cast<double>(minVal);
};
std::transform(std::begin(series1), std::end(series1), std::begin(series1Normalized),
[normalize](const uint64_t val) -> double { return normalize(val); });
std::transform(std::begin(series2), std::end(series2), std::begin(series2Normalized),
[normalize](const uint64_t val) -> double { return normalize(val); });
std::transform(std::begin(series3), std::end(series3), std::begin(series3Normalized),
[normalize](const uint64_t val) -> double { return normalize(val); });
std::transform(std::begin(series4), std::end(series4), std::begin(series4Normalized),
[normalize](const uint64_t val) -> double { return normalize(val); });
// Build histograms of each of the series:
std::for_each(std::begin(series1Normalized), std::end(series1Normalized),
[&histograms](const double val) { histograms[0][static_cast<int>(val * 1024)]++; });
std::for_each(std::begin(series2Normalized), std::end(series2Normalized),
[&histograms](const double val) { histograms[1][static_cast<int>(val * 1024)]++; });
std::for_each(std::begin(series3Normalized), std::end(series3Normalized),
[&histograms](const double val) { histograms[2][static_cast<int>(val * 1024)]++; });
std::for_each(std::begin(series4Normalized), std::end(series4Normalized),
[&histograms](const double val) { histograms[3][static_cast<int>(val * 1024)]++; });
// Find the maximum length of all histograms:
auto maxLen = size_t(0);
maxLen = std::max(maxLen, histograms[0].size());
maxLen = std::max(maxLen, histograms[1].size());
maxLen = std::max(maxLen, histograms[2].size());
maxLen = std::max(maxLen, histograms[3].size());
// Write out a CSV file that contains all four series:
std::ofstream os;
os.open("celeroDistribution.csv");
os << "64,,256,,1024,,4096,," << std::endl;
for(size_t i = 0; i < maxLen; ++i)
{
for(size_t j = 0; j < histograms.size(); j++)
{
if(i < histograms[j].size())
{
auto element = std::begin(histograms[j]);
for(size_t k = 0; k < i; k++)
{
++element;
}
os << element->first << "," << element->second << ",";
}
else
{
os << ",,";
}
}
os << std::endl;
}
os.close();
}

39
lib/celero/Distribution.h Normal file
View File

@ -0,0 +1,39 @@
#ifndef H_CELERO_DISTRIBUTION_H
#define H_CELERO_DISTRIBUTION_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <cstdint>
#include <vector>
namespace celero
{
///
/// Collects results from Celero for analysis of a hard-coded internal trivial measurement case.
///
CELERO_EXPORT std::vector<uint64_t> BuildDistribution(uint64_t numberOfSamples, uint64_t iterationsPerSample);
///
/// Builds a .csv file to help determine Celero's measurement distribution.
///
CELERO_EXPORT void RunDistribution(uint64_t iterationsPerSample);
}
#endif

251
lib/celero/Exceptions.cpp Normal file
View File

@ -0,0 +1,251 @@
///
/// \author Peter Azmanov
///
/// \copyright Copyright 2016, 2017, 2018 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Exceptions.h>
#include <celero/Console.h>
#include <celero/TestFixture.h>
#ifdef WIN32
#include <Windows.h>
#endif // WIN32
#include <iomanip>
#include <iostream>
//
// Macros and general logics below taken from Google Test code,
// see gtest/internal/gtest-port.h
// gtest/src/gtest.cc
//
#ifndef CELERO_HAS_EXCEPTIONS
// The user didn't tell us whether exceptions are enabled, so we need
// to figure it out.
#if defined(_MSC_VER) || defined(__BORLANDC__)
// MSVC's and C++Builder's implementations of the STL use the _HAS_EXCEPTIONS
// macro to enable exceptions, so we'll do the same.
// Assumes that exceptions are enabled by default.
#ifndef _HAS_EXCEPTIONS
#define _HAS_EXCEPTIONS 1
#endif // _HAS_EXCEPTIONS
#define CELERO_HAS_EXCEPTIONS _HAS_EXCEPTIONS
#elif defined(__GNUC__) && __EXCEPTIONS
// gcc defines __EXCEPTIONS to 1 iff exceptions are enabled.
#define CELERO_HAS_EXCEPTIONS 1
#elif defined(__SUNPRO_CC)
// Sun Pro CC supports exceptions. However, there is no compile-time way of
// detecting whether they are enabled or not. Therefore, we assume that
// they are enabled unless the user tells us otherwise.
#define CELERO_HAS_EXCEPTIONS 1
#elif defined(__IBMCPP__) && __EXCEPTIONS
// xlC defines __EXCEPTIONS to 1 iff exceptions are enabled.
#define CELERO_HAS_EXCEPTIONS 1
#elif defined(__HP_aCC)
// Exception handling is in effect by default in HP aCC compiler. It has to
// be turned of by +noeh compiler option if desired.
#define CELERO_HAS_EXCEPTIONS 1
#else
// For other compilers, we assume exceptions are disabled to be
// conservative.
#define CELERO_HAS_EXCEPTIONS 0
#endif // defined(_MSC_VER) || defined(__BORLANDC__)
#endif // CELERO_HAS_EXCEPTIONS
// Determine whether the compiler supports Microsoft's Structured Exception
// Handling. This is supported by several Windows compilers but generally
// does not exist on any other system.
#ifndef CELERO_HAS_SEH
// The user didn't tell us, so we need to figure it out.
#if defined(_MSC_VER) || defined(__BORLANDC__)
// These two compilers are known to support SEH.
#define CELERO_HAS_SEH 1
#else
// Assume no SEH.
#define CELERO_HAS_SEH 0
#endif
#endif // CELERO_HAS_SEH
namespace celero
{
bool ExceptionSettings::GetCatchExceptions()
{
return ExceptionSettings::instance().catchExceptions;
}
void ExceptionSettings::SetCatchExceptions(bool x)
{
ExceptionSettings::instance().catchExceptions = x;
}
ExceptionSettings& ExceptionSettings::instance()
{
static ExceptionSettings settings;
return settings;
}
#if CELERO_HAS_SEH
int HandleSEH(DWORD exceptionCode)
{
// see https://support.microsoft.com/en-us/kb/185294
const DWORD cppExceptionCode = 0xe06d7363;
if((exceptionCode == EXCEPTION_BREAKPOINT) || (exceptionCode == cppExceptionCode))
{
return EXCEPTION_EXECUTE_HANDLER;
}
return EXCEPTION_CONTINUE_SEARCH;
}
const char* ExceptionCodeToStr(DWORD exceptionCode)
{
switch(exceptionCode)
{
case EXCEPTION_ACCESS_VIOLATION:
return "EXCEPTION_ACCESS_VIOLATION";
case EXCEPTION_ARRAY_BOUNDS_EXCEEDED:
return "EXCEPTION_ARRAY_BOUNDS_EXCEEDED";
case EXCEPTION_BREAKPOINT:
return "EXCEPTION_BREAKPOINT";
case EXCEPTION_DATATYPE_MISALIGNMENT:
return "EXCEPTION_DATATYPE_MISALIGNMENT";
case EXCEPTION_FLT_DENORMAL_OPERAND:
return "EXCEPTION_FLT_DENORMAL_OPERAND";
case EXCEPTION_FLT_DIVIDE_BY_ZERO:
return "EXCEPTION_FLT_DIVIDE_BY_ZERO";
case EXCEPTION_FLT_INEXACT_RESULT:
return "EXCEPTION_FLT_INEXACT_RESULT";
case EXCEPTION_FLT_INVALID_OPERATION:
return "EXCEPTION_FLT_INVALID_OPERATION";
case EXCEPTION_FLT_OVERFLOW:
return "EXCEPTION_FLT_OVERFLOW";
case EXCEPTION_FLT_STACK_CHECK:
return "EXCEPTION_FLT_STACK_CHECK";
case EXCEPTION_FLT_UNDERFLOW:
return "EXCEPTION_FLT_UNDERFLOW";
case EXCEPTION_GUARD_PAGE:
return "EXCEPTION_GUARD_PAGE";
case EXCEPTION_ILLEGAL_INSTRUCTION:
return "EXCEPTION_ILLEGAL_INSTRUCTION";
case EXCEPTION_IN_PAGE_ERROR:
return "EXCEPTION_IN_PAGE_ERROR";
case EXCEPTION_INT_DIVIDE_BY_ZERO:
return "EXCEPTION_INT_DIVIDE_BY_ZERO";
case EXCEPTION_INT_OVERFLOW:
return "EXCEPTION_INT_OVERFLOW";
case EXCEPTION_INVALID_DISPOSITION:
return "EXCEPTION_INVALID_DISPOSITION";
case EXCEPTION_INVALID_HANDLE:
return "EXCEPTION_INVALID_HANDLE";
case EXCEPTION_NONCONTINUABLE_EXCEPTION:
return "EXCEPTION_NONCONTINUABLE_EXCEPTION";
case EXCEPTION_PRIV_INSTRUCTION:
return "EXCEPTION_PRIV_INSTRUCTION";
case EXCEPTION_SINGLE_STEP:
return "EXCEPTION_SINGLE_STEP";
case EXCEPTION_STACK_OVERFLOW:
return "EXCEPTION_STACK_OVERFLOW";
case STATUS_UNWIND_CONSOLIDATE:
return "STATUS_UNWIND_CONSOLIDATE";
default:
return "Unknown exception code.";
}
}
#endif // CELERO_HAS_SEH
std::pair<bool, uint64_t> RunAndCatchSEHExc(TestFixture& test, uint64_t threads, uint64_t calls,
const celero::TestFixture::ExperimentValue& experimentValue)
{
#if CELERO_HAS_SEH
__try
{
return std::make_pair(true, test.run(threads, calls, experimentValue));
}
__except(HandleSEH(GetExceptionCode()))
{
const auto exceptionCode = GetExceptionCode();
celero::console::SetConsoleColor(celero::console::ConsoleColor::Red);
std::cout << "SEH exception " << ExceptionCodeToStr(exceptionCode) << std::endl;
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
return std::make_pair(false, 0);
}
#else // CELERO_HAS_SEH
return std::make_pair(true, test.run(threads, calls, experimentValue));
#endif // CELERO_HAS_SEH
}
std::pair<bool, uint64_t> RunAndCatchExc(TestFixture& test, uint64_t threads, uint64_t calls,
const celero::TestFixture::ExperimentValue& experimentValue)
{
if(ExceptionSettings::GetCatchExceptions() == true)
{
#if CELERO_HAS_EXCEPTIONS
try
{
return RunAndCatchSEHExc(test, threads, calls, experimentValue);
}
catch(const std::exception& e)
{
celero::console::SetConsoleColor(celero::console::ConsoleColor::Red);
std::cout << "C++ exception \"" << e.what() << "\"" << std::endl;
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
}
catch(...)
{
celero::console::SetConsoleColor(celero::console::ConsoleColor::Red);
std::cout << "Unknown C++ exception" << std::endl;
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
}
return std::make_pair(false, 0);
#else // CELERO_HAS_EXCEPTIONS
return RunAndCatchSEHExc(test, threads, calls, experimentValue);
#endif // CELERO_HAS_EXCEPTIONS
}
else
{
return std::make_pair(true, test.run(threads, calls, experimentValue));
}
}
} // namespace celero

64
lib/celero/Exceptions.h Normal file
View File

@ -0,0 +1,64 @@
#ifndef H_CELERO_EXCEPTIONS_H
#define H_CELERO_EXCEPTIONS_H
///
/// \author Peter Azmanov
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/TestFixture.h>
#include <cstdint>
#include <utility>
namespace celero
{
///
/// A Singleton storing exception settings (currently only one flag)
///
class ExceptionSettings
{
public:
///
/// Get a flag indicating whether Celero should catch exceptions or not
///
static bool GetCatchExceptions();
///
/// Set a flag indicating whether Celero should catch exceptions or not
///
static void SetCatchExceptions(bool catchExceptions);
private:
static ExceptionSettings& instance();
private:
bool catchExceptions{true};
};
///
/// Run test and catch SEH exceptions if they are supported by compiler
///
std::pair<bool, uint64_t> RunAndCatchSEHExc(TestFixture& test, uint64_t threads, uint64_t calls,
const celero::TestFixture::ExperimentValue& experimentValue);
///
/// Run test and catch all exceptions we can
///
std::pair<bool, uint64_t> RunAndCatchExc(TestFixture& test, uint64_t threads, uint64_t calls,
const celero::TestFixture::ExperimentValue& experimentValue);
} // namespace celero
#endif

399
lib/celero/Executor.cpp Normal file
View File

@ -0,0 +1,399 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/Callbacks.h>
#include <celero/Celero.h>
#include <celero/Console.h>
#include <celero/Exceptions.h>
#include <celero/Executor.h>
#include <celero/Print.h>
#include <celero/TestVector.h>
#include <celero/UserDefinedMeasurementCollector.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <cassert>
#include <iostream>
using namespace celero;
///
/// A local function to figure out how many iterations and samples are required when the user doesn't specify any.
///
bool AdjustSampleAndIterationSize(std::shared_ptr<celero::ExperimentResult> r)
{
if((r->getExperiment()->getSamples() == 0) || (r->getExperiment()->getIterations() == 0))
{
// The smallest test should take at least 10x as long as our timer's resolution.
// I chose "2x" arbitrarily.
const auto minTestTime = static_cast<int64_t>(celero::timer::CachePerformanceFrequency(true) * 1e6) * 2;
// Compute a good number to use for iterations and set the sample size to 30.
auto test = r->getExperiment()->getFactory()->Create();
auto testTime = int64_t(0);
auto testIterations = int64_t(1);
while(testTime < minTestTime)
{
const auto runResult = RunAndCatchExc(*test, r->getExperiment()->getThreads(), testIterations, r->getProblemSpaceValue());
if(runResult.first == false)
{
return false; // something bad happened
}
testTime = runResult.second;
if(testTime < minTestTime)
{
testIterations *= 2;
}
}
const auto iterations = static_cast<uint64_t>(std::max(static_cast<double>(testIterations), 1000000.0 / testTime));
auto experiment = r->getExperiment();
if(experiment->getIterations() == 0)
{
experiment->setIterations(std::max(iterations, uint64_t(30)));
}
if(experiment->getSamples() == 0)
{
experiment->setSamples(30);
}
r->setProblemSpaceValue(r->getProblemSpaceValue(), r->getProblemSpaceValueScale(), iterations);
}
return true;
}
///
/// A local function to support running an individual user-defined function for measurement.
///
bool ExecuteProblemSpace(std::shared_ptr<celero::ExperimentResult> r)
{
// Define a small internal function object to use to uniformly execute the tests.
auto testRunner = [r](const bool record, std::shared_ptr<UserDefinedMeasurementCollector> udmCollector) {
auto test = r->getExperiment()->getFactory()->Create();
const auto runResult = RunAndCatchExc(*test, r->getExperiment()->getThreads(), r->getProblemSpaceIterations(), r->getProblemSpaceValue());
if(runResult.first == false)
{
// something bad happened
return false;
}
const auto testTime = runResult.second;
// Save test results
if(record == true)
{
r->getTimeStatistics()->addSample(testTime);
r->getExperiment()->incrementTotalRunTime(testTime);
if(udmCollector != nullptr)
{
udmCollector->collect(test);
}
}
return true;
};
if(r->getExperiment()->getSamples() > 0)
{
// make a first pass to maybe cache instructions/data or other kinds of fist-run-only costs
if(testRunner(false, nullptr) == false)
{
r->setFailure(true);
return false;
}
auto udmCollector = std::make_shared<UserDefinedMeasurementCollector>(r->getExperiment()->getFactory()->Create());
for(auto i = r->getExperiment()->getSamples(); i > 0; --i)
{
if(testRunner(true, udmCollector) == false)
{
r->setFailure(true);
return false;
}
}
r->setUserDefinedMeasurements(udmCollector);
r->setComplete(true);
}
else
{
std::cerr << "Celero: Test \"" << r->getExperiment()->getBenchmark()->getName() << "::" << r->getExperiment()->getName()
<< "\" must have at least 1 sample." << std::endl;
return false;
}
return true;
}
void executor::RunAll()
{
executor::RunAllBaselines();
executor::RunAllExperiments();
}
void executor::RunAllBaselines()
{
// Run through all the tests in ascending order.
for(size_t i = 0; i < celero::TestVector::Instance().size(); i++)
{
auto bmark = celero::TestVector::Instance()[i];
executor::RunBaseline(bmark);
}
}
void executor::RunAllExperiments()
{
// Run through all the tests in ascending order.
for(size_t i = 0; i < celero::TestVector::Instance().size(); i++)
{
auto bmark = celero::TestVector::Instance()[i];
executor::RunExperiments(bmark);
}
}
void executor::RunBaseline(std::shared_ptr<Benchmark> bmark)
{
if(bmark == nullptr)
{
return;
}
auto baselineExperiment = bmark->getBaseline();
if(baselineExperiment != nullptr)
{
// Populate the problem space with a test fixture instantiation.
{
const auto testValues = baselineExperiment->getFactory()->Create()->getExperimentValues();
const auto valueResultScale = baselineExperiment->getFactory()->Create()->getExperimentValueResultScale();
for(auto i : testValues)
{
if(i.Iterations > 0)
{
baselineExperiment->addProblemSpace(i.Value, static_cast<double>(valueResultScale), i.Iterations);
}
else
{
baselineExperiment->addProblemSpace(i.Value, static_cast<double>(valueResultScale), baselineExperiment->getIterations());
}
}
// Add a single default problem space if none was specified.
// This is needed to get the result size later.
if(baselineExperiment->getResultSize() == 0)
{
baselineExperiment->addProblemSpace(static_cast<int64_t>(TestFixture::Constants::NoProblemSpaceValue));
}
}
for(size_t i = 0; i < baselineExperiment->getResultSize(); i++)
{
auto r = baselineExperiment->getResult(i);
assert(r != nullptr);
Printer::get().TableRowExperimentHeader(r->getExperiment());
// Do a quick sample, if necessary, and adjust sample and iteration sizes, if necessary.
if(AdjustSampleAndIterationSize(r) == true)
{
// Describe the beginning of the run.
Printer::get().TableRowProblemSpaceHeader(r);
if(ExecuteProblemSpace(r))
{
// Describe the end of the run.
Printer::get().TableResult(r);
}
}
else
{
r->setFailure(true);
}
celero::impl::ExperimentResultComplete(r);
}
celero::impl::ExperimentComplete(baselineExperiment);
}
else
{
std::cerr << "No Baseline case defined for \"" + bmark->getName() + "\". Exiting.";
std::exit(EXIT_FAILURE);
}
}
void executor::RunExperiments(std::shared_ptr<Benchmark> bmark)
{
if(bmark == nullptr)
{
return;
}
const auto experimentSize = bmark->getExperimentSize();
for(size_t i = 0; i < experimentSize; i++)
{
auto e = bmark->getExperiment(i);
assert(e != nullptr);
executor::Run(e);
}
}
void executor::Run(std::shared_ptr<Experiment> e)
{
if(e == nullptr)
{
return;
}
auto bmark = e->getBenchmark();
auto baseline = bmark->getBaseline();
if(baseline->getResultSize() == 0 || baseline->getResult(0)->getComplete() == false)
{
if(baseline->getResultSize() != 0 && baseline->getResult(0)->getFailure())
{
Printer::get().TableRowExperimentHeader(e.get());
Printer::get().TableRowFailure("Baseline failure, skip");
// Add result output failed result
e->addProblemSpace(0);
auto r = e->getResult(0);
r->setFailure(true);
celero::impl::ExperimentResultComplete(r);
return;
}
executor::RunBaseline(bmark);
}
// Populate the problem space with a fake test fixture instantiation.
{
auto factory = e->getFactory();
if(factory == nullptr)
{
return;
}
auto factoryCreate = factory->Create();
if(factoryCreate == nullptr)
{
return;
}
const auto testValues = factoryCreate->getExperimentValues();
factoryCreate = factory->Create();
const auto valueResultScale = factoryCreate->getExperimentValueResultScale();
for(auto i : testValues)
{
if(i.Iterations > 0)
{
e->addProblemSpace(i.Value, valueResultScale, i.Iterations);
}
else
{
e->addProblemSpace(i.Value, valueResultScale, e->getIterations());
}
}
// Add a single default problem space if none was specified.
// This is needed to get the result size later.
if(e->getResultSize() == 0)
{
e->addProblemSpace(0);
}
}
// Result size will grow based on the problem spaces added above.
for(size_t i = 0; i < e->getResultSize(); i++)
{
auto r = e->getResult(i);
Printer::get().TableRowExperimentHeader(r->getExperiment());
// Do a quick sample, if necessary, and adjust sample and iteration sizes, if necessary.
const auto adjustSuccess = AdjustSampleAndIterationSize(r);
if(adjustSuccess == true)
{
// Describe the beginning of the run.
Printer::get().TableRowProblemSpaceHeader(r);
if(ExecuteProblemSpace(r))
{
// Describe the end of the run.
Printer::get().TableResult(r);
}
}
else
{
r->setFailure(true);
}
celero::impl::ExperimentResultComplete(r);
}
celero::impl::ExperimentComplete(e);
}
void executor::Run(std::shared_ptr<Benchmark> bmark)
{
executor::RunBaseline(bmark);
executor::RunExperiments(bmark);
}
void executor::Run(const std::string& benchmarkName)
{
auto bmark = celero::TestVector::Instance()[benchmarkName];
if(bmark != nullptr)
{
executor::Run(bmark);
}
}
void executor::Run(const std::string& benchmarkName, const std::string& experimentName)
{
auto bmark = celero::TestVector::Instance()[benchmarkName];
if(bmark != nullptr)
{
auto e = bmark->getExperiment(experimentName);
assert(e != nullptr);
executor::Run(e);
}
}

82
lib/celero/Executor.h Normal file
View File

@ -0,0 +1,82 @@
#ifndef H_CELERO_EXECUTOR_H
#define H_CELERO_EXECUTOR_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/Export.h>
#include <memory>
#include <string>
namespace celero
{
namespace executor
{
///
/// Run all baselines and experiments registered within the final application.
///
CELERO_EXPORT void RunAll();
///
/// Run all baselines (but not experiments) registered within the final application.
///
CELERO_EXPORT void RunAllBaselines();
///
/// Run a specific benchmark's baseline.
///
CELERO_EXPORT void RunBaseline(std::shared_ptr<Benchmark> x);
///
/// Run all experiments registered within the final application.
///
CELERO_EXPORT void RunAllExperiments();
///
/// Run all experiments within a specific benchmark.
///
CELERO_EXPORT void RunExperiments(std::shared_ptr<Benchmark> x);
///
/// Run a specific benchmark.
///
CELERO_EXPORT void Run(std::shared_ptr<Benchmark> x);
///
/// Run a specific experiment.
///
/// If the baseline is not complete for the given experiment, it will be executed first.
///
CELERO_EXPORT void Run(std::shared_ptr<Experiment> x);
///
/// Run a specific benchmark with the specified name.
///
CELERO_EXPORT void Run(const std::string& group);
///
/// Run a specific benchmark with the specified name and one specific experiment within it.
///
/// If the baseline is not complete for the given experiment, it will be executed first.
///
CELERO_EXPORT void Run(const std::string& group, const std::string& experiment);
}
}
#endif

315
lib/celero/Experiment.cpp Normal file
View File

@ -0,0 +1,315 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/Experiment.h>
#include <celero/Factory.h>
#include <celero/PimplImpl.h>
#include <celero/TestFixture.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <cassert>
using namespace celero;
class Experiment::Impl
{
public:
Impl() :
results(),
benchmark(),
factory(),
name(),
baselineUnit(0),
baselineTarget(0),
samples(0),
iterations(0),
threads(1),
totalRunTime(0),
isBaselineCase(false)
{
}
Impl(std::weak_ptr<Benchmark> bm, const std::string& n, const uint64_t s, const uint64_t c, const uint64_t t, const double pBaselineTarget) :
results(),
benchmark(bm),
factory(),
name(n),
baselineUnit(0),
baselineTarget(pBaselineTarget),
samples(s),
iterations(c),
threads(t),
totalRunTime(0),
isBaselineCase(false)
{
}
explicit Impl(std::weak_ptr<Benchmark> bm) :
results(),
benchmark(bm),
factory(),
name(),
baselineUnit(0),
baselineTarget(0),
samples(0),
iterations(0),
threads(1),
totalRunTime(0),
isBaselineCase(false)
{
}
/// There is one result for each problem space value.
/// In the event there are not any problem spaces, there shal be a single result.
std::vector<std::shared_ptr<celero::ExperimentResult>> results;
/// The owning benchmark object which groups together all experiments.
std::weak_ptr<Benchmark> benchmark;
/// The factory to associate with this benchmark.
std::shared_ptr<Factory> factory;
/// The name of this experiment.
std::string name;
/// The number of microseconds per test (which makes up one baseline unit).
double baselineUnit{0};
/// Used to pass/fail benchmarks when outputting JUnit.
double baselineTarget{0};
/// Test samples to complete.
uint64_t samples{0};
/// Iterations per test run. (Size of each sample.)
uint64_t iterations{0};
/// Threads per test run.
uint64_t threads{0};
/// The best run time for this test
uint64_t totalRunTime{0};
bool isBaselineCase{false};
};
Experiment::Experiment() : pimpl()
{
}
Experiment::Experiment(std::weak_ptr<Benchmark> benchmark) : pimpl(benchmark)
{
}
Experiment::Experiment(std::weak_ptr<Benchmark> benchmark, const std::string& name, uint64_t samples, uint64_t iterations, uint64_t threads,
double baselineTarget) :
pimpl(benchmark, name, samples, iterations, threads, baselineTarget)
{
}
Experiment::Experiment(const Experiment&)
{
}
Experiment::~Experiment()
{
}
std::shared_ptr<Benchmark> Experiment::getBenchmark()
{
return this->pimpl->benchmark.lock();
}
void Experiment::setName(const std::string& x)
{
this->pimpl->name = x;
}
std::string Experiment::getName() const
{
return this->pimpl->name;
}
void Experiment::setSamples(uint64_t x)
{
this->pimpl->samples = x;
}
uint64_t Experiment::getSamples() const
{
return this->pimpl->samples;
}
void Experiment::setIterations(uint64_t x)
{
this->pimpl->iterations = x;
}
uint64_t Experiment::getIterations() const
{
return this->pimpl->iterations;
}
void Experiment::setThreads(uint64_t x)
{
this->pimpl->threads = x;
}
uint64_t Experiment::getThreads() const
{
return this->pimpl->threads;
}
Experiment::operator std::string() const
{
auto output = this->getShort();
if(this->getSamples() > 0)
{
output += " -- " + std::to_string(this->getSamples());
if(this->getSamples() == 1)
{
output += " run, ";
}
else
{
output += " samples, ";
}
}
else
{
output += " -- Auto Run, ";
}
output += std::to_string(this->getIterations());
if(this->getIterations() == 1)
{
output += " iteration per run,";
}
else
{
output += " iterations per run,";
}
if(this->getThreads() == 1)
{
output += " thread per run.";
}
else
{
output += " threads per run.";
}
return output;
}
std::string Experiment::getShort() const
{
auto bm = this->pimpl->benchmark.lock();
if(bm != nullptr)
{
return bm->getName() + "." + this->getName();
}
return this->getName();
}
void Experiment::setBaselineTarget(double x)
{
this->pimpl->baselineTarget = x;
}
double Experiment::getBaselineTarget() const
{
return this->pimpl->baselineTarget;
}
void Experiment::incrementTotalRunTime(const uint64_t x)
{
this->pimpl->totalRunTime += x;
}
uint64_t Experiment::getTotalRunTime() const
{
return this->pimpl->totalRunTime;
}
void Experiment::setIsBaselineCase(bool x)
{
this->pimpl->isBaselineCase = x;
}
bool Experiment::getIsBaselineCase() const
{
return this->pimpl->isBaselineCase;
}
void Experiment::setFactory(std::shared_ptr<Factory> x)
{
this->pimpl->factory = x;
}
std::shared_ptr<Factory> Experiment::getFactory() const
{
return this->pimpl->factory;
}
void Experiment::addProblemSpace(int64_t x, double scale, uint64_t iterations)
{
auto r = std::make_shared<celero::ExperimentResult>(this);
r->setProblemSpaceValue(x, scale, iterations);
this->pimpl->results.push_back(r);
}
size_t Experiment::getResultSize()
{
if(this->pimpl->results.empty() == true)
{
auto defaultResult = std::make_shared<celero::ExperimentResult>(this);
defaultResult->setProblemSpaceValue(static_cast<int64_t>(TestFixture::Constants::NoProblemSpaceValue), 1.0, this->getIterations());
this->pimpl->results.push_back(defaultResult);
}
return this->pimpl->results.size();
}
std::shared_ptr<celero::ExperimentResult> Experiment::getResult(size_t x)
{
// get the result OR thrown an exception if the result list is empty;
return this->pimpl->results.at(x);
}
std::shared_ptr<celero::ExperimentResult> Experiment::getResultByValue(int64_t x)
{
std::shared_ptr<celero::ExperimentResult> r;
const auto found = std::find_if(std::begin(this->pimpl->results), std::end(this->pimpl->results),
[x](std::shared_ptr<celero::ExperimentResult> i) -> bool { return (i->getProblemSpaceValue() == x); });
if(found != std::end(this->pimpl->results))
{
r = (*found);
}
return r;
}

197
lib/celero/Experiment.h Normal file
View File

@ -0,0 +1,197 @@
#ifndef H_CELERO_EXPERIMENT_H
#define H_CELERO_EXPERIMENT_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/ExperimentResult.h>
#include <celero/Factory.h>
#include <celero/Statistics.h>
#include <string>
namespace celero
{
class Benchmark;
///
/// \class Experiment
///
/// \author John Farrier
///
class CELERO_EXPORT Experiment
{
public:
///
///
///
explicit Experiment(std::weak_ptr<celero::Benchmark> benchmark);
///
///
///
explicit Experiment(std::weak_ptr<celero::Benchmark> benchmark, const std::string& name, uint64_t samples, uint64_t iterations,
uint64_t threads,
double baselineTarget);
///
/// \brief Default destructor.
///
~Experiment();
///
/// Gets a pointer to the owning Benchmark object.
///
std::shared_ptr<celero::Benchmark> getBenchmark();
///
///
///
void setName(const std::string& x);
///
///
///
std::string getName() const;
///
///
///
void setSamples(uint64_t x);
///
///
///
uint64_t getSamples() const;
///
///
///
void setIterations(uint64_t x);
///
///
///
uint64_t getIterations() const;
///
///
///
void setThreads(uint64_t x);
///
///
///
uint64_t getThreads() const;
///
///
///
operator std::string() const;
///
///
///
std::string getShort() const;
///
///
///
void setBaselineTarget(double x);
///
///
///
double getBaselineTarget() const;
///
///
///
void incrementTotalRunTime(const uint64_t x);
///
///
///
uint64_t getTotalRunTime() const;
///
/// Used to set a flag indicating that this is a Baseline case, not a benchmark case.
///
void setIsBaselineCase(bool x);
///
/// Used to get a flag indicating that this is a Baseline case, not a benchmark case.
///
bool getIsBaselineCase() const;
///
/// Sets the factory used to create this experiment's test fixtures.
///
void setFactory(std::shared_ptr<celero::Factory> x);
///
/// Gets the factory used to create this experiment's test fixtures.
///
std::shared_ptr<celero::Factory> getFactory() const;
///
/// \param x Can be interpreted in any way be the test fixture (i.e. index into an array, etc.)
/// \param scale Used to format unit results.
/// \param iterations Override the default iterations with the number here when greater than zero.
///
void addProblemSpace(int64_t x, double scale = 1.0, uint64_t iterations = 0);
///
/// There is one result for each problem space.
///
size_t getResultSize();
///
/// Get an ExperimentResult at a given index.
///
std::shared_ptr<celero::ExperimentResult> getResult(size_t x);
///
/// Get the ExperimentResult for the given problem space value.
///
std::shared_ptr<celero::ExperimentResult> getResultByValue(int64_t x);
private:
///
/// Hide the default constructor
///
Experiment();
///
/// Hide the copy constructor
///
explicit Experiment(const celero::Experiment&);
///
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

206
lib/celero/ExperimentResult.cpp Normal file
View File

@ -0,0 +1,206 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/Experiment.h>
#include <celero/ExperimentResult.h>
#include <celero/PimplImpl.h>
#include <celero/Statistics.h>
#include <celero/Timer.h>
#include <celero/UserDefinedMeasurementCollector.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <cassert>
using namespace celero;
class ExperimentResult::Impl
{
public:
Impl()
{
}
explicit Impl(Experiment* const p) : parent(p)
{
}
/// Track statistics related to execution time about this specific experiment.
Statistics<int64_t> statsTime;
Statistics<int64_t> statsRAM;
std::shared_ptr<UserDefinedMeasurementCollector> udmCollector;
int64_t problemSpaceValue{0};
double problemSpaceValueScale{1.0};
uint64_t problemSpaceIterations{0};
/// A pointer back to our owning Experiment parent.
Experiment* parent{nullptr};
/// A "completed" flag.
bool complete{false};
/// A "failure" flag.
bool failure{false};
};
ExperimentResult::ExperimentResult()
{
}
ExperimentResult::ExperimentResult(Experiment* x) : pimpl(x)
{
}
ExperimentResult::~ExperimentResult()
{
}
Experiment* ExperimentResult::getExperiment() const
{
return this->pimpl->parent;
}
void ExperimentResult::setProblemSpaceValue(int64_t x, double scale, uint64_t iterations)
{
this->pimpl->problemSpaceValue = x;
this->pimpl->problemSpaceValueScale = scale;
this->pimpl->problemSpaceIterations = iterations;
}
int64_t ExperimentResult::getProblemSpaceValue() const
{
return this->pimpl->problemSpaceValue;
}
double ExperimentResult::getProblemSpaceValueScale() const
{
return this->pimpl->problemSpaceValueScale;
}
uint64_t ExperimentResult::getProblemSpaceIterations() const
{
return this->pimpl->problemSpaceIterations;
}
Statistics<int64_t>* ExperimentResult::getTimeStatistics()
{
return &this->pimpl->statsTime;
}
void ExperimentResult::addRunTimeSample(const uint64_t runTime)
{
this->pimpl->statsTime.addSample(runTime);
}
uint64_t ExperimentResult::getRunTime() const
{
return this->pimpl->statsTime.getMin();
}
double ExperimentResult::getUsPerCall() const
{
if(this->pimpl->failure == false)
{
return static_cast<double>(this->pimpl->statsTime.getMin()) / static_cast<double>(this->pimpl->problemSpaceIterations);
}
return 0.0;
}
double ExperimentResult::getCallsPerSecond() const
{
if(this->pimpl->failure == false)
{
return 1.0 / (this->getUsPerCall() * celero::UsToSec);
}
return 0.0;
}
double ExperimentResult::getUnitsPerSecond() const
{
return (this->pimpl->problemSpaceValueScale > 0.0)
? ((this->pimpl->problemSpaceValue * this->pimpl->problemSpaceIterations / this->pimpl->problemSpaceValueScale)
/ (this->pimpl->statsTime.getMin() * celero::UsToSec))
: 0.0;
}
double ExperimentResult::getBaselineMeasurement() const
{
if(this->pimpl->parent->getIsBaselineCase() == false)
{
const auto bm = this->pimpl->parent->getBenchmark();
if(bm != nullptr)
{
const auto baselineExperiment = bm->getBaseline();
if(baselineExperiment != nullptr)
{
const auto baselineResult = baselineExperiment->getResultByValue(this->getProblemSpaceValue());
if(baselineResult != nullptr)
{
const auto baselineResultUs = baselineResult->getUsPerCall();
// Prevent possible divide by zero.
if(baselineResultUs > 0)
{
return this->getUsPerCall() / baselineResult->getUsPerCall();
}
}
}
}
return -1.0;
}
return 1.0;
}
void ExperimentResult::setComplete(bool x)
{
this->pimpl->complete = x;
}
bool ExperimentResult::getComplete() const
{
return this->pimpl->complete;
}
void ExperimentResult::setFailure(bool x)
{
this->pimpl->failure = x;
}
bool ExperimentResult::getFailure() const
{
return this->pimpl->failure;
}
void ExperimentResult::setUserDefinedMeasurements(std::shared_ptr<UserDefinedMeasurementCollector> x)
{
this->pimpl->udmCollector = x;
}
std::shared_ptr<UserDefinedMeasurementCollector> ExperimentResult::getUserDefinedMeasurements() const
{
return this->pimpl->udmCollector;
}

161
lib/celero/ExperimentResult.h Normal file
View File

@ -0,0 +1,161 @@
#ifndef H_CELERO_EXPERIMENTRESULT_H
#define H_CELERO_EXPERIMENTRESULT_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <celero/Pimpl.h>
#include <celero/Statistics.h>
#include <string>
namespace celero
{
class Experiment;
class UserDefinedMeasurementCollector;
///
/// \class ExperimentResult
///
/// \author John Farrier
///
class CELERO_EXPORT ExperimentResult
{
public:
explicit ExperimentResult(Experiment* x);
~ExperimentResult();
///
/// Gets a pointer to the owning experiment.
///
Experiment* getExperiment() const;
///
///
///
void setProblemSpaceValue(int64_t x, double scale = 1.0, uint64_t iterations = 0);
///
///
///
int64_t getProblemSpaceValue() const;
///
///
///
double getProblemSpaceValueScale() const;
///
///
///
uint64_t getProblemSpaceIterations() const;
///
///
///
Statistics<int64_t>* getTimeStatistics();
///
/// Adds a run time sample during experiment execution.
///
void addRunTimeSample(const uint64_t x);
///
/// Returns the best run time sample observed.
///
uint64_t getRunTime() const;
///
/// \brief Get the number of computed microseconds per iteration (i.e. a single call to the code under test.)
///
/// A call is defined as one iteration of one execution of the code under test.
///
double getUsPerCall() const;
///
/// \brief Get the number of times the code under test could be called per second.
///
/// A call is defined as one iteration of one execution of the code under test.
///
double getCallsPerSecond() const;
///
/// \brief Get the processing speed in units per second.
///
/// A call is defined as one iteration of one execution of the code under test.
///
double getUnitsPerSecond() const;
///
/// Calculate this experiments baseline value.
///
/// If this IS a baseline experiment, the function will return 1.0;
/// Returns -1 on error.
///
double getBaselineMeasurement() const;
///
/// Sets a flag indicating if this result is complete.
///
void setComplete(bool x);
///
/// Gets a flag indicating if this result is complete.
///
bool getComplete() const;
///
/// Sets a flag indicating if failure happened during evaluation.
///
void setFailure(bool x);
///
/// Gets a flag indicating if failure happened during evaluation.
///
bool getFailure() const;
///
///
///
void setUserDefinedMeasurements(std::shared_ptr<UserDefinedMeasurementCollector> x);
///
///
///
std::shared_ptr<UserDefinedMeasurementCollector> getUserDefinedMeasurements() const;
private:
///
/// Disable default constructor
///
ExperimentResult();
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

40
lib/celero/Export.h Normal file
View File

@ -0,0 +1,40 @@
#ifndef H_CELERO_EXPORT_H
#define H_CELERO_EXPORT_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#ifdef CELERO_STATIC
#define CELERO_EXPORT
#define CELERO_EXPORT_C
#else
#ifdef WIN32
#if defined CELERO_EXPORTS
#define CELERO_EXPORT _declspec(dllexport)
#define CELERO_EXPORT_C extern "C" _declspec(dllexport)
#else
#define CELERO_EXPORT _declspec(dllimport)
#define CELERO_EXPORT_C extern "C" _declspec(dllimport)
#endif
#else
#define CELERO_EXPORT
#define CELERO_EXPORT_C extern "C"
#endif
#endif
#endif

59
lib/celero/Factory.h Normal file
View File

@ -0,0 +1,59 @@
#ifndef H_CELERO_FACTORY_H
#define H_CELERO_FACTORY_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <celero/TestFixture.h>
#include <memory>
namespace celero
{
///
/// \class Factory
///
/// \author John Farrier
///
/// Pure Virtual Base class for benchmarks.
///
class CELERO_EXPORT Factory
{
public:
///
/// \brief Default Constructor
///
Factory()
{
}
///
/// \brief Virtual Destructor
///
virtual ~Factory()
{
}
///
/// \brief Pure virtual function.
///
virtual std::shared_ptr<TestFixture> Create() = 0;
};
}
#endif

50
lib/celero/FileReader.h Normal file
View File

@ -0,0 +1,50 @@
#ifndef H_CELERO_FILEREADER_H
#define H_CELERO_FILEREADER_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <locale>
namespace celero
{
///
/// \struct FileReader
///
/// A helper struct to aid in reading CSV files.
///
/// Classify commas as whitespace.
///
struct FieldReader : std::ctype<char>
{
FieldReader() : std::ctype<char>(FieldReader::GetTable())
{
}
static std::ctype_base::mask const* GetTable()
{
static std::vector<std::ctype_base::mask> rc(table_size, std::ctype_base::mask());
rc[','] = std::ctype_base::space;
rc['\n'] = std::ctype_base::space;
rc['\r'] = std::ctype_base::space;
return &rc[0];
}
};
}
#endif

60
lib/celero/GenericFactory.h Normal file
View File

@ -0,0 +1,60 @@
#ifndef H_CELERO_GENERICFACTORY_H
#define H_CELERO_GENERICFACTORY_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <celero/Factory.h>
namespace celero
{
///
/// \class GenericFactory
///
/// \author John farrier
///
template <class T>
class GenericFactory : public Factory
{
public:
///
/// \brief Default Constructor
///
GenericFactory() : Factory()
{
}
///
/// \brief Virtual Destructor
///
virtual ~GenericFactory()
{
}
///
/// \brief Overload the pure virtual base class function.
///
virtual std::shared_ptr<TestFixture> Create()
{
return std::make_shared<T>();
}
};
}
#endif

149
lib/celero/JUnit.cpp Normal file
View File

@ -0,0 +1,149 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <assert.h>
#include <celero/Benchmark.h>
#include <celero/JUnit.h>
#include <celero/PimplImpl.h>
#include <celero/Timer.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <fstream>
#include <iostream>
#include <map>
#include <tuple>
#include <vector>
using namespace celero;
///
/// \struct celero::JUnit::Impl
///
class celero::JUnit::Impl
{
public:
std::string fileName;
/// Store the test case size, measured baseline, objective baseline, and total run time in seconds.
std::map<std::string, std::vector<std::shared_ptr<celero::ExperimentResult>>> results;
double totalTime = {0.0};
};
JUnit& JUnit::Instance()
{
static JUnit singleton;
return singleton;
}
void JUnit::setFileName(const std::string& x)
{
assert(x.empty() == false);
this->pimpl->fileName = x;
}
void JUnit::add(std::shared_ptr<celero::ExperimentResult> x)
{
this->pimpl->results[x->getExperiment()->getBenchmark()->getName()].push_back(x);
this->save();
}
void JUnit::save()
{
std::ofstream ofs;
ofs.open(this->pimpl->fileName);
if(ofs.is_open() == true)
{
const auto os = &ofs;
*os << "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>" << std::endl;
for(auto i : this->pimpl->results)
{
uint64_t testSuiteTime = 0;
size_t testSuiteFailures = 0;
size_t testSuiteErrors = 0;
const auto runs = i.second;
for(auto j : runs)
{
if(j->getFailure())
{
testSuiteErrors++;
continue;
}
else if((j->getExperiment()->getBaselineTarget() > 0.0) && (j->getBaselineMeasurement() > j->getExperiment()->getBaselineTarget()))
{
testSuiteFailures++;
}
testSuiteTime += j->getRunTime();
}
*os << "<testsuite errors=\"" << testSuiteErrors << "\" ";
*os << "tests=\"" << i.second.size() << "\" ";
*os << "time=\"" << celero::timer::ConvertSystemTime(testSuiteTime) << "\" ";
*os << "failures=\"" << testSuiteFailures << "\" ";
*os << "name=\"" << i.first << "\">" << std::endl;
for(auto j : runs)
{
*os << "\t<testcase ";
*os << "time=\"" << celero::timer::ConvertSystemTime(j->getFailure() ? 0 : j->getRunTime()) << "\" ";
*os << "name=\"" << j->getExperiment()->getName() << "#" << j->getProblemSpaceValue() << "\"";
// Compare measured to objective
if(j->getFailure())
{
// Error
*os << ">" << std::endl;
*os << "\t\t<error ";
*os << "type=\"exception\"";
*os << "/>" << std::endl;
*os << "\t</testcase>" << std::endl;
}
else if((j->getExperiment()->getBaselineTarget() > 0.0) && (j->getBaselineMeasurement() > j->getExperiment()->getBaselineTarget()))
{
// Failure
*os << ">" << std::endl;
*os << "\t\t<failure ";
*os << "type=\"Performance objective not met.\" ";
*os << "message=\"Measurement of " << j->getBaselineMeasurement() << " exceeds objective baseline of "
<< j->getExperiment()->getBaselineTarget() << "\" ";
*os << "/>" << std::endl;
*os << "\t</testcase>" << std::endl;
}
else
{
// Success
*os << "/>" << std::endl;
}
}
*os << "</testsuite>" << std::endl;
}
ofs.close();
}
}

73
lib/celero/JUnit.h Normal file
View File

@ -0,0 +1,73 @@
#ifndef H_CELERO_JUNIT_H
#define H_CELERO_JUNIT_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Experiment.h>
#include <celero/Pimpl.h>
#include <string>
namespace celero
{
///
/// \class JUnit
///
/// \author John Farrier
///
class JUnit
{
public:
///
/// Singleton
///
static JUnit& Instance();
///
/// Specify a file name for a results output file.
///
/// \param x The name of the output file in which to store Celero's results.
///
void setFileName(const std::string& x);
///
/// Add a new result to the JUnit output XML.
///
/// This should re-save on every new result so that the output can be monitored externally.
///
void add(std::shared_ptr<celero::ExperimentResult> x);
///
/// Save the JUnit (XUnit) formatted file to the given file name.
///
void save();
private:
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

392
lib/celero/Memory.cpp Normal file
View File

@ -0,0 +1,392 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Memory.h>
#include <sstream>
#ifdef WIN32
#include <Windows.h>
#include <Psapi.h>
#elif defined(__APPLE__)
#include <sys/param.h>
#include <sys/sysctl.h>
#include <sys/types.h>
#include <unistd.h>
#include <array>
#else
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/resource.h>
#include <sys/sysinfo.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>
#endif
///
/// References:
/// http://blogs.microsoft.co.il/sasha/2016/01/05/windows-process-memory-usage-demystified/
/// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366770(v=vs.85).aspx
/// https://msdn.microsoft.com/en-us/library/windows/desktop/ms684877(v=vs.85).aspx
/// http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system
/// http://nadeausoftware.com/articles/2012/07/c_c_tip_how_get_process_resident_set_size_physical_memory_use
/// https://stackoverflow.com/questions/669438/how-to-get-memory-usage-at-run-time-in-c
/// https://stackoverflow.com/questions/2513505/how-to-get-available-memory-c-g
///
using namespace celero;
#ifdef WIN32
#else
constexpr int64_t Kilobytes2Bytes{1024};
namespace celero
{
namespace impl
{
int ParseLine(char* line)
{
const auto i = strlen(line);
while(*line < '0' || *line > '9')
{
line++;
}
line[i - 3] = '\0';
return atoi(line);
}
} // namespace impl
} // namespace celero
#endif
celero::RAMReport::operator std::string()
{
std::stringstream ss;
ss << "System Total: " << this->RamSystemTotal << std::endl;
ss << "System Available: " << this->RamSystemAvailable << std::endl;
ss << "System Used: " << this->RamSystemUsed << std::endl;
ss << "System UsedByCurrentProcess: " << this->RamSystemUsedByCurrentProcess << std::endl;
ss << "Physical Total: " << this->RamPhysicalTotal << std::endl;
ss << "Physical Available: " << this->RamPhysicalAvailable << std::endl;
ss << "Physical Used: " << this->RamPhysicalUsed << std::endl;
ss << "Physical UsedByCurrentProcess: " << this->RamPhysicalUsedByCurrentProcess << std::endl;
ss << "Physical UsedByCurrentProcessPeak: " << this->RamPhysicalUsedByCurrentProcessPeak << std::endl;
ss << "Virtual Total: " << this->RamVirtualTotal << std::endl;
ss << "Virtual Available: " << this->RamVirtualAvailable << std::endl;
ss << "Virtual Used: " << this->RamVirtualUsed << std::endl;
ss << "Virtual UsedByCurrentProcess: " << this->RamVirtualUsedByCurrentProcess << std::endl;
return ss.str();
}
celero::RAMReport celero::RAMReport::operator-(const RAMReport& x)
{
celero::RAMReport r;
r.RamSystemTotal = this->RamSystemTotal - x.RamSystemTotal;
r.RamSystemAvailable = this->RamSystemAvailable - x.RamSystemAvailable;
r.RamSystemUsed = this->RamSystemUsed - x.RamSystemUsed;
r.RamSystemUsedByCurrentProcess = this->RamSystemUsedByCurrentProcess - x.RamSystemUsedByCurrentProcess;
r.RamPhysicalTotal = this->RamPhysicalTotal - x.RamPhysicalTotal;
r.RamPhysicalAvailable = this->RamPhysicalAvailable - x.RamPhysicalAvailable;
r.RamPhysicalUsed = this->RamPhysicalUsed - x.RamPhysicalUsed;
r.RamPhysicalUsedByCurrentProcess = this->RamPhysicalUsedByCurrentProcess - x.RamPhysicalUsedByCurrentProcess;
r.RamPhysicalUsedByCurrentProcessPeak = this->RamPhysicalUsedByCurrentProcessPeak - x.RamPhysicalUsedByCurrentProcessPeak;
r.RamVirtualTotal = this->RamVirtualTotal - x.RamVirtualTotal;
r.RamVirtualAvailable = this->RamVirtualAvailable - x.RamVirtualAvailable;
r.RamVirtualUsed = this->RamVirtualUsed - x.RamVirtualUsed;
r.RamVirtualUsedByCurrentProcess = this->RamVirtualUsedByCurrentProcess - x.RamVirtualUsedByCurrentProcess;
return r;
}
int64_t celero::GetRAMSystemTotal()
{
#ifdef WIN32
MEMORYSTATUSEX memInfo;
memInfo.dwLength = sizeof(MEMORYSTATUSEX);
GlobalMemoryStatusEx(&memInfo);
return static_cast<int64_t>(memInfo.ullTotalPhys) + static_cast<int64_t>(memInfo.ullTotalVirtual);
#elif defined(__unix__) || defined(__unix) || defined(unix)
// Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM
// return static_cast<int64_t>(sysconf(_SC_PHYS_PAGES)) * static_cast<int64_t>(sysconf(_SC_PAGE_SIZE));
struct sysinfo memInfo;
sysinfo(&memInfo);
int64_t total = memInfo.totalram;
total += memInfo.totalswap;
total += memInfo.totalhigh;
return total * static_cast<int64_t>(memInfo.mem_unit);
#elif defined(__APPLE__)
int mib[2];
mib[0] = CTL_HW;
mib[1] = HW_MEMSIZE;
int64_t memInfo{0};
auto len = sizeof(memInfo);
if(sysctl(mib, 2, &memInfo, &len, nullptr, 0) == 0)
{
return memInfo;
}
return -1;
#else
return -1;
#endif
}
int64_t celero::GetRAMSystemAvailable()
{
#ifdef WIN32
MEMORYSTATUSEX memInfo;
memInfo.dwLength = sizeof(MEMORYSTATUSEX);
GlobalMemoryStatusEx(&memInfo);
return static_cast<int64_t>(memInfo.ullAvailPhys) + static_cast<int64_t>(memInfo.ullAvailVirtual);
#else
return celero::GetRAMSystemTotal() - celero::GetRAMSystemUsed();
#endif
}
int64_t celero::GetRAMSystemUsed()
{
#ifdef WIN32
return celero::GetRAMSystemTotal() - celero::GetRAMSystemAvailable();
#elif defined(__APPLE__)
int mib[2];
mib[0] = CTL_HW;
mib[1] = HW_MEMSIZE;
std::array<int64_t, 2> memInfo{{0, 0}};
auto len = sizeof(memInfo[0]);
if(sysctl(mib, 2, &memInfo[0], &len, nullptr, 0) == 0)
{
if(sysctl(mib, 2, &memInfo[1], &len, nullptr, 0) == 0)
{
return memInfo[0] + memInfo[1];
}
}
return -1;
#else
struct sysinfo memInfo;
sysinfo(&memInfo);
int64_t total = memInfo.totalram - memInfo.freeram;
total += memInfo.totalswap - memInfo.freeswap;
total += memInfo.totalhigh - memInfo.freehigh;
return total * static_cast<int64_t>(memInfo.mem_unit);
#endif
}
int64_t celero::GetRAMSystemUsedByCurrentProcess()
{
#ifdef WIN32
PROCESS_MEMORY_COUNTERS_EX pmc;
GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PPROCESS_MEMORY_COUNTERS>(&pmc), sizeof(pmc));
return static_cast<int64_t>(pmc.WorkingSetSize);
#else
return celero::GetRAMPhysicalUsedByCurrentProcess() + celero::GetRAMVirtualUsedByCurrentProcess();
#endif
}
int64_t celero::GetRAMPhysicalTotal()
{
#ifdef WIN32
MEMORYSTATUSEX memInfo;
memInfo.dwLength = sizeof(MEMORYSTATUSEX);
GlobalMemoryStatusEx(&memInfo);
return static_cast<int64_t>(memInfo.ullTotalPhys);
#elif defined(__APPLE__)
return -1;
#else
struct sysinfo memInfo;
sysinfo(&memInfo);
return memInfo.totalram * memInfo.mem_unit;
#endif
}
int64_t celero::GetRAMPhysicalAvailable()
{
#ifdef WIN32
MEMORYSTATUSEX memInfo;
memInfo.dwLength = sizeof(MEMORYSTATUSEX);
GlobalMemoryStatusEx(&memInfo);
return static_cast<int64_t>(memInfo.ullAvailPhys);
#else
return celero::GetRAMPhysicalTotal() - celero::GetRAMPhysicalUsed();
#endif
}
int64_t celero::GetRAMPhysicalUsed()
{
#ifdef WIN32
return celero::GetRAMPhysicalTotal() - celero::GetRAMPhysicalAvailable();
#elif defined(__APPLE__)
struct rusage rusage;
getrusage(RUSAGE_SELF, &rusage);
return (size_t)rusage.ru_maxrss;
#else
struct sysinfo memInfo;
sysinfo(&memInfo);
return (static_cast<int64_t>(memInfo.totalram) - static_cast<int64_t>(memInfo.freeram)) * static_cast<int64_t>(memInfo.mem_unit);
#endif
}
int64_t celero::GetRAMPhysicalUsedByCurrentProcess()
{
#ifdef WIN32
PROCESS_MEMORY_COUNTERS_EX pmc;
GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PPROCESS_MEMORY_COUNTERS>(&pmc), sizeof(pmc));
return static_cast<int64_t>(pmc.WorkingSetSize);
#else
constexpr int BufferSize{128};
int64_t result = 0;
auto file = fopen("/proc/self/status", "r");
char line[BufferSize];
while(fgets(line, BufferSize, file) != nullptr)
{
if(strncmp(line, "VmRSS:", 6) == 0)
{
result += celero::impl::ParseLine(line) * Kilobytes2Bytes;
}
}
fclose(file);
return static_cast<int64_t>(result);
#endif
}
int64_t celero::GetRAMPhysicalUsedByCurrentProcessPeak()
{
#if defined(_WIN32)
PROCESS_MEMORY_COUNTERS pmc;
GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));
return static_cast<int64_t>(pmc.PeakWorkingSetSize);
#elif defined(__APPLE__) && defined(__MACH__)
struct rusage rusage;
getrusage(RUSAGE_SELF, &rusage);
return static_cast<int64_t>(rusage.ru_maxrss);
#else
constexpr int BufferSize{128};
int64_t result = 0;
auto file = fopen("/proc/self/status", "r");
char line[BufferSize];
while(fgets(line, BufferSize, file) != nullptr)
{
if(strncmp(line, "VmHWM:", 6) == 0)
{
result += celero::impl::ParseLine(line) * Kilobytes2Bytes;
}
}
fclose(file);
return static_cast<int64_t>(result);
#endif
}
int64_t celero::GetRAMVirtualTotal()
{
#ifdef WIN32
MEMORYSTATUSEX memInfo;
memInfo.dwLength = sizeof(MEMORYSTATUSEX);
GlobalMemoryStatusEx(&memInfo);
return memInfo.ullTotalPageFile;
#elif defined(__APPLE__)
return -1;
#else
struct sysinfo memInfo;
sysinfo(&memInfo);
return static_cast<int64_t>(memInfo.totalswap) * static_cast<int64_t>(memInfo.mem_unit);
#endif
}
int64_t celero::GetRAMVirtualAvailable()
{
#ifdef WIN32
MEMORYSTATUSEX memInfo;
memInfo.dwLength = sizeof(MEMORYSTATUSEX);
GlobalMemoryStatusEx(&memInfo);
return memInfo.ullTotalPageFile;
#else
return celero::GetRAMVirtualTotal() - celero::GetRAMVirtualUsed();
#endif
}
int64_t celero::GetRAMVirtualUsed()
{
#ifdef WIN32
return celero::GetRAMVirtualTotal() - celero::GetRAMVirtualAvailable();
#elif defined(__APPLE__)
return -1;
#else
struct sysinfo memInfo;
sysinfo(&memInfo);
const int64_t total = memInfo.totalswap - memInfo.freeswap;
return total * static_cast<int64_t>(memInfo.mem_unit);
#endif
}
int64_t celero::GetRAMVirtualUsedByCurrentProcess()
{
#ifdef WIN32
PROCESS_MEMORY_COUNTERS_EX pmc;
GetProcessMemoryInfo(GetCurrentProcess(), reinterpret_cast<PPROCESS_MEMORY_COUNTERS>(&pmc), sizeof(pmc));
return pmc.PrivateUsage;
#else
// Verified Correct.
constexpr int BufferSize{128};
int64_t result = 0;
FILE* file = fopen("/proc/self/status", "r");
char line[BufferSize];
while(fgets(line, BufferSize, file) != NULL)
{
if(strncmp(line, "VmSize:", 7) == 0)
{
result = celero::impl::ParseLine(line) * Kilobytes2Bytes;
break;
}
}
fclose(file);
return result;
#endif
}
celero::RAMReport celero::GetRAMReport()
{
celero::RAMReport r;
r.RamSystemTotal = GetRAMSystemTotal();
r.RamSystemAvailable = GetRAMSystemAvailable();
r.RamSystemUsed = GetRAMSystemUsed();
r.RamSystemUsedByCurrentProcess = GetRAMSystemUsedByCurrentProcess();
r.RamPhysicalTotal = GetRAMPhysicalTotal();
r.RamPhysicalAvailable = GetRAMPhysicalAvailable();
r.RamPhysicalUsed = GetRAMPhysicalUsed();
r.RamPhysicalUsedByCurrentProcess = GetRAMPhysicalUsedByCurrentProcess();
r.RamVirtualTotal = GetRAMVirtualTotal();
r.RamVirtualAvailable = GetRAMVirtualAvailable();
r.RamVirtualUsed = GetRAMVirtualUsed();
r.RamVirtualUsedByCurrentProcess = GetRAMVirtualUsedByCurrentProcess();
return r;
}

142
lib/celero/Memory.h Normal file
View File

@ -0,0 +1,142 @@
#ifndef H_CELERO_MEORY_H
#define H_CELERO_MEORY_H
///
/// \author John Farrier
///
/// \copyright Copyright 2018 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <cstdint>
#include <string>
namespace celero
{
///
/// \struct RAMReport
///
/// Contans all Memory measurements (in bytes)
///
struct RAMReport
{
int64_t RamSystemTotal{0};
int64_t RamSystemAvailable{0};
int64_t RamSystemUsed{0};
int64_t RamSystemUsedByCurrentProcess{0};
int64_t RamPhysicalTotal{0};
int64_t RamPhysicalAvailable{0};
int64_t RamPhysicalUsed{0};
int64_t RamPhysicalUsedByCurrentProcess{0};
int64_t RamPhysicalUsedByCurrentProcessPeak{0};
int64_t RamVirtualTotal{0};
int64_t RamVirtualAvailable{0};
int64_t RamVirtualUsed{0};
int64_t RamVirtualUsedByCurrentProcess{0};
operator std::string();
celero::RAMReport operator-(const celero::RAMReport& x);
};
// ----------------------------------------------------------------
// Physical + Virtual Memory
CELERO_EXPORT int64_t GetRAMSystemTotal();
CELERO_EXPORT int64_t GetRAMSystemAvailable();
CELERO_EXPORT int64_t GetRAMSystemUsed();
///
/// The sum of the physical RAM used by the current process and the virtual RAM used by the current process.
///
CELERO_EXPORT int64_t GetRAMSystemUsedByCurrentProcess();
// ----------------------------------------------------------------
// Physical Memory
///
/// The total physical RAM, in bytes.
/// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366770(v=vs.85).aspx
///
CELERO_EXPORT int64_t GetRAMPhysicalTotal();
///
/// The total physical RAM available to the current process, in bytes.
///
/// On Windows, this is defined as "This is the amount of physical memory that can be immediately reused without having to write its contents to
/// disk first. It is the sum of the size of the standby, free, and zero lists."
/// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366770(v=vs.85).aspx
///
CELERO_EXPORT int64_t GetRAMPhysicalAvailable();
///
/// The total amount of physical RAM minus the amount of physical RAM which is available.
///
CELERO_EXPORT int64_t GetRAMPhysicalUsed();
///
/// On Windows, this is defined by the Working Set Size. The working set size is defined as "The working set of a process is the set of pages in
/// the virtual address space of the process that are currently resident in physical memory. The working set contains only pageable memory
/// allocations; nonpageable memory allocations such as Address Windowing Extensions (AWE) or large page allocations are not included in the
/// working set."
/// https://msdn.microsoft.com/en-us/library/windows/desktop/ms684877(v=vs.85).aspx
/// https://msdn.microsoft.com/en-us/library/windows/desktop/cc441804(v=vs.85).aspx
///
CELERO_EXPORT int64_t GetRAMPhysicalUsedByCurrentProcess();
///
///
///
CELERO_EXPORT int64_t GetRAMPhysicalUsedByCurrentProcessPeak();
// ----------------------------------------------------------------
// Virtual Memory
///
/// The total amount of Virtual RAM (page file size).
///
/// On Windows, this is defined by the amount of page file that the current process has access to. It is not the total available on the system.
/// From the Windows documentation: "The current committed memory limit for the system or the current process, whichever is smaller, in bytes. To
/// get the system-wide committed memory limit, call GetPerformanceInfo."
///
CELERO_EXPORT int64_t GetRAMVirtualTotal();
///
/// The amount of non-physical memory (page file) available.
///
/// On Windows, this is defined by the amount of page file that the current process has access to. It is not the total available on the system.
/// From the Windows documentation: "The maximum amount of memory the current process can commit, in bytes. This value is equal to or smaller than
/// the system-wide available commit value."
///
CELERO_EXPORT int64_t GetRAMVirtualAvailable();
///
/// The total virtual RAM minus the available virtual RAM.
///
CELERO_EXPORT int64_t GetRAMVirtualUsed();
///
/// On Windows, this is defined as the commit charge. "The Commit Charge value in bytes for this process. Commit Charge is the total amount of
/// memory that the memory manager has committed for a running process."
/// https://msdn.microsoft.com/en-us/library/windows/desktop/ms684877(v=vs.85).aspx
///
CELERO_EXPORT int64_t GetRAMVirtualUsedByCurrentProcess();
///
/// Returns a RAMReport class containing all RAM measurements.
///
CELERO_EXPORT celero::RAMReport GetRAMReport();
} // namespace celero
#endif

65
lib/celero/Pimpl.h Normal file
View File

@ -0,0 +1,65 @@
#ifndef H_CELERO_PIMPL_H
#define H_CELERO_PIMPL_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <memory>
namespace celero
{
///
/// \class Pimpl
///
/// \author Herb Sutter
/// \author John Farrier
///
/// Classes using this must overload the assignment operator.
/// Original code by Herb Sutter. Adapted for more primitive compilers by John Farrier.
///
template <typename T>
class Pimpl
{
public:
Pimpl();
// template<typename ...Args> Pimpl( Args&& ... );
template <typename Arg1>
Pimpl(Arg1&&);
template <typename Arg1, typename Arg2>
Pimpl(Arg1&&, Arg2&&);
template <typename Arg1, typename Arg2, typename Arg3>
Pimpl(Arg1&&, Arg2&&, Arg3&&);
template <typename Arg1, typename Arg2, typename Arg3, typename Arg4>
Pimpl(Arg1&&, Arg2&&, Arg3&&, Arg4&&);
template <typename Arg1, typename Arg2, typename Arg3, typename Arg4, typename Arg5>
Pimpl(Arg1&&, Arg2&&, Arg3&&, Arg4&&, Arg5&&);
template <typename Arg1, typename Arg2, typename Arg3, typename Arg4, typename Arg5, typename Arg6>
Pimpl(Arg1&&, Arg2&&, Arg3&&, Arg4&&, Arg5&&, Arg6&&);
~Pimpl();
T* operator->();
const T* operator->() const;
T& operator*();
private:
std::unique_ptr<T> _pimpl;
};
} // namespace celero
#endif

98
lib/celero/PimplImpl.h Normal file
View File

@ -0,0 +1,98 @@
#ifndef H_CELERO_PIMPLIMPL_H
#define H_CELERO_PIMPLIMPL_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <memory>
#include <utility>
namespace celero
{
template <typename T>
Pimpl<T>::Pimpl() : _pimpl(new T())
{
}
template <typename T>
template <typename Arg1>
Pimpl<T>::Pimpl(Arg1&& arg1) : _pimpl(new T(std::forward<Arg1>(arg1)))
{
}
template <typename T>
template <typename Arg1, typename Arg2>
Pimpl<T>::Pimpl(Arg1&& arg1, Arg2&& arg2) : _pimpl(new T(std::forward<Arg1>(arg1), std::forward<Arg2>(arg2)))
{
}
template <typename T>
template <typename Arg1, typename Arg2, typename Arg3>
Pimpl<T>::Pimpl(Arg1&& arg1, Arg2&& arg2, Arg3&& arg3)
: _pimpl(new T(std::forward<Arg1>(arg1), std::forward<Arg2>(arg2), std::forward<Arg3>(arg3)))
{
}
template <typename T>
template <typename Arg1, typename Arg2, typename Arg3, typename Arg4>
Pimpl<T>::Pimpl(Arg1&& arg1, Arg2&& arg2, Arg3&& arg3, Arg4&& arg4)
: _pimpl(new T(std::forward<Arg1>(arg1), std::forward<Arg2>(arg2), std::forward<Arg3>(arg3), std::forward<Arg4>(arg4)))
{
}
template <typename T>
template <typename Arg1, typename Arg2, typename Arg3, typename Arg4, typename Arg5>
Pimpl<T>::Pimpl(Arg1&& arg1, Arg2&& arg2, Arg3&& arg3, Arg4&& arg4, Arg5&& arg5)
: _pimpl(
new T(std::forward<Arg1>(arg1), std::forward<Arg2>(arg2), std::forward<Arg3>(arg3), std::forward<Arg4>(arg4), std::forward<Arg5>(arg5)))
{
}
template <typename T>
template <typename Arg1, typename Arg2, typename Arg3, typename Arg4, typename Arg5, typename Arg6>
Pimpl<T>::Pimpl(Arg1&& arg1, Arg2&& arg2, Arg3&& arg3, Arg4&& arg4, Arg5&& arg5, Arg6&& arg6)
: _pimpl(new T(std::forward<Arg1>(arg1), std::forward<Arg2>(arg2), std::forward<Arg3>(arg3), std::forward<Arg4>(arg4),
std::forward<Arg5>(arg5), std::forward<Arg6>(arg6)))
{
}
template <typename T>
Pimpl<T>::~Pimpl()
{
}
template <typename T>
T* Pimpl<T>::operator->()
{
return _pimpl.get();
}
template <typename T>
const T* Pimpl<T>::operator->() const
{
return _pimpl.get();
}
template <typename T>
T& Pimpl<T>::operator*()
{
return *_pimpl.get();
}
}
#endif

337
lib/celero/Print.cpp Normal file
View File

@ -0,0 +1,337 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/Console.h>
#include <celero/Print.h>
#include <celero/TestVector.h>
#include <celero/Timer.h>
#include <celero/UserDefinedMeasurementCollector.h>
#include <celero/Utilities.h>
#include <algorithm>
#include <chrono>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <unordered_map>
using namespace celero;
enum PrintConstants : size_t
{
ColumnSeperatorWidth = 3,
DoubleDecimals = 5,
NumberOfColumns = 8,
ColumnWidth = 15
};
///
/// http://stackoverflow.com/questions/14765155/how-can-i-easily-format-my-data-table-in-c
/// Center-aligns string within a field of width w. Pads with blank spaces to enforce alignment.
///
std::string PrintCenter(const std::string& s, const size_t w = PrintConstants::ColumnWidth)
{
std::stringstream ss;
std::stringstream spaces;
// count excess room to pad
auto padding = w - s.size();
for(size_t i = 0; i < padding / 2; ++i)
{
spaces << " ";
}
// format with padding
ss << spaces.str() << s << spaces.str();
// if odd #, add 1 space
if((padding > 0) && (padding % 2 != 0))
{
ss << " ";
}
ss << " | ";
return ss.str();
}
///
/// http://stackoverflow.com/questions/14765155/how-can-i-easily-format-my-data-table-in-c
/// Convert double to string with specified number of places after the decimal and left padding.
///
std::string PrintColumn(const double x, const size_t decDigits = PrintConstants::DoubleDecimals, const size_t width = PrintConstants::ColumnWidth)
{
std::stringstream ss;
ss << std::fixed << std::right;
// fill space around displayed #
ss.fill(' ');
// set width around displayed #
ss.width(width);
// set # places after decimal
ss.precision(decDigits);
ss << x;
return ss.str();
}
///
/// http://stackoverflow.com/questions/14765155/how-can-i-easily-format-my-data-table-in-c
/// Convert double to string with specified number of places after the decimal.
///
std::string PrintColumn(const int64_t x, const size_t width = PrintConstants::ColumnWidth)
{
std::stringstream ss;
ss << std::fixed;
// fill space around displayed #
ss.fill(' ');
// set width around displayed #
ss.width(width);
ss << x << " | ";
return ss.str();
}
///
/// http://stackoverflow.com/questions/14765155/how-can-i-easily-format-my-data-table-in-c
/// Convert double to string with specified number of places after the decimal.
///
std::string PrintColumn(const uint64_t x, const size_t width = PrintConstants::ColumnWidth)
{
std::stringstream ss;
ss << std::fixed;
// fill space around displayed #
ss.fill(' ');
// set width around displayed #
ss.width(width);
ss << x << " | ";
return ss.str();
}
///
/// http://stackoverflow.com/questions/14765155/how-can-i-easily-format-my-data-table-in-c
/// Convert double to string with specified number of places after the decimal.
///
std::string PrintStrColumnAligned(const std::string& x, const size_t width = PrintConstants::ColumnWidth, bool alignLeft = true)
{
std::stringstream ss;
ss << std::fixed << (alignLeft ? std::left : std::right);
// fill space around displayed #
ss.fill(' ');
// set width around displayed #
ss.width(width);
if(x.length() > width)
{
// Truncate
std::string xTrunc = x;
xTrunc = xTrunc.substr(0, width);
ss << xTrunc << " | ";
}
else
{
ss << x << " | ";
}
return ss.str();
}
std::string PrintColumn(const std::string& x, const size_t width = PrintConstants::ColumnWidth)
{
return PrintStrColumnAligned(x, width);
}
std::string PrintColumnRight(const std::string& x, const size_t width = PrintConstants::ColumnWidth)
{
return PrintStrColumnAligned(x, width, false);
}
std::string PrintHRule(const size_t additionalColumns = 0)
{
std::stringstream ss;
std::string column{":"};
while(column.length() < PrintConstants::ColumnWidth)
{
column += "-";
}
std::string firstColumn = column + ":|";
column += "-:|";
ss << "|" << firstColumn;
for(size_t i = 0; i < PrintConstants::NumberOfColumns + additionalColumns - 1; ++i)
{
ss << column;
}
ss << std::endl;
return ss.str();
}
namespace celero
{
void Printer::Console(const std::string& x)
{
std::cout << "Celero: " << x << std::endl;
}
void Printer::TableBanner()
{
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
std::cout << "|" << PrintCenter("Group") << PrintCenter("Experiment") << PrintCenter("Prob. Space") << PrintCenter("Samples")
<< PrintCenter("Iterations") << PrintCenter("Baseline") << PrintCenter("us/Iteration") << PrintCenter("Iterations/sec");
for(size_t i = PrintConstants::NumberOfColumns; i < this->columnWidths.size(); ++i)
{
std::cout << PrintCenter(this->userDefinedColumns[i - PrintConstants::NumberOfColumns], this->columnWidths[i]);
}
std::cout << "\n";
std::cout << PrintHRule(this->userDefinedColumns.size());
}
void Printer::TableRowExperimentHeader(Experiment* x)
{
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
std::cout << "|" << PrintColumn(x->getBenchmark()->getName()) << PrintColumn(x->getName());
}
void Printer::TableRowFailure(const std::string& msg)
{
std::cout << PrintColumnRight("-") << PrintColumnRight("-") << PrintColumnRight("-");
for(size_t i = PrintConstants::NumberOfColumns; i < this->columnWidths.size(); ++i)
{
std::cout << PrintColumnRight("-", this->columnWidths[i]);
}
celero::console::SetConsoleColor(celero::console::ConsoleColor::Red);
std::cout << msg << std::endl;
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
}
void Printer::TableRowProblemSpaceHeader(std::shared_ptr<celero::ExperimentResult> x)
{
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
if(x->getProblemSpaceValue() == static_cast<int64_t>(TestFixture::Constants::NoProblemSpaceValue))
{
std::cout << PrintColumnRight("Null");
}
else
{
std::cout << PrintColumn(x->getProblemSpaceValue());
}
std::cout << PrintColumn(x->getExperiment()->getSamples()) << PrintColumn(x->getProblemSpaceIterations());
}
void Printer::TableRowHeader(std::shared_ptr<celero::ExperimentResult> x)
{
TableRowExperimentHeader(x->getExperiment());
TableRowProblemSpaceHeader(x);
}
void Printer::TableResult(std::shared_ptr<celero::ExperimentResult> x)
{
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
celero::console::ConsoleColor temp_color;
// Slower than Baseline
if(x->getBaselineMeasurement() > 1.0)
{
temp_color = celero::console::ConsoleColor::Yellow;
}
else if(x->getBaselineMeasurement() < 1.0)
{
temp_color = celero::console::ConsoleColor::Green;
}
else
{
temp_color = celero::console::ConsoleColor::Cyan;
}
celero::console::SetConsoleColor(temp_color);
std::cout << PrintColumn(x->getBaselineMeasurement());
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
std::cout << " | ";
celero::console::SetConsoleColor(temp_color);
std::cout << PrintColumn(x->getUsPerCall());
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
std::cout << " | ";
celero::console::SetConsoleColor(temp_color);
std::cout << PrintColumn(x->getCallsPerSecond(), 2);
celero::console::SetConsoleColor(celero::console::ConsoleColor::Default);
std::cout << " | ";
std::unordered_map<std::string, double> udmValues;
auto udmCollector = x->getUserDefinedMeasurements();
for(const auto& entry : udmCollector->getAggregateValues())
{
udmValues[entry.first] = entry.second;
}
for(size_t i = 0; i < this->userDefinedColumns.size(); ++i)
{
const auto& fieldName = this->userDefinedColumns[i];
if(udmValues.find(fieldName) == udmValues.end())
{
std::cout << PrintCenter("---", this->columnWidths[i + PrintConstants::NumberOfColumns]);
}
else
{
std::cout << PrintColumn(udmValues.at(fieldName), 2, this->columnWidths[i + PrintConstants::NumberOfColumns]);
}
}
std::cout << "\n";
}
void Printer::initialize(std::vector<std::string> userDefinedColumnsIn)
{
this->userDefinedColumns = userDefinedColumnsIn;
this->columnWidths.clear();
this->columnWidths.resize(PrintConstants::NumberOfColumns, PrintConstants::ColumnWidth);
for(const auto& name : this->userDefinedColumns)
{
this->columnWidths.push_back(std::max(name.size() + 2, (size_t)PrintConstants::ColumnWidth));
}
}
} // namespace celero

65
lib/celero/Print.h Normal file
View File

@ -0,0 +1,65 @@
#ifndef H_CELERO_PRINT_H
#define H_CELERO_PRINT_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Experiment.h>
#include <memory>
#include <string>
namespace celero
{
///
/// \class Printer
///
/// \author John farrier
///
class Printer
{
public:
///
/// Singleton implementation.
static Printer& get()
{
static Printer p;
return p;
}
///
/// Initialize the Printer object with specific user-defined columns.
///
void initialize(std::vector<std::string> userDefinedColumns);
void Console(const std::string& x);
void TableBanner();
void TableRowExperimentHeader(Experiment* x);
void TableRowFailure(const std::string& msg);
void TableRowProblemSpaceHeader(std::shared_ptr<celero::ExperimentResult> x);
void TableRowHeader(std::shared_ptr<celero::ExperimentResult> x);
void TableResult(std::shared_ptr<celero::ExperimentResult> x);
private:
Printer() = default;
std::vector<std::string> userDefinedColumns;
std::vector<size_t> columnWidths;
};
} // namespace celero
#endif

125
lib/celero/ResultTable.cpp Normal file
View File

@ -0,0 +1,125 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/PimplImpl.h>
#include <celero/ResultTable.h>
#include <assert.h>
#include <fstream>
#include <iostream>
#include <sstream>
using namespace celero;
///
/// \class Impl
///
class celero::ResultTable::Impl
{
public:
Impl() : precision(5)
{
}
~Impl()
{
closeFile();
}
void closeFile()
{
if(this->ofs.is_open() == true)
{
this->ofs.close();
}
}
void setFileName(const std::string& x)
{
if(this->ofs.is_open() == true)
{
this->ofs.close();
}
this->ofs.open(x);
// Print the header for the table.
if(this->ofs.is_open() == true)
{
this->ofs << "Group,Experiment,Problem "
"Space,Samples,Iterations,Failure,Baseline,";
this->ofs << "us/Iteration,Iterations/sec,Min (us),Mean (us),Max "
"(us),Variance,Standard Deviation,Skewness,Kurtosis,Z Score"
<< std::endl;
}
}
std::string format(double x)
{
std::stringstream ss;
ss << std::fixed;
ss.precision(this->precision);
ss << x;
return ss.str();
}
std::ofstream ofs;
const size_t precision;
};
ResultTable::ResultTable() : pimpl()
{
}
ResultTable::~ResultTable()
{
}
ResultTable& ResultTable::Instance()
{
static ResultTable singleton;
return singleton;
}
void ResultTable::setFileName(const std::string& x)
{
assert(x.empty() == false);
this->pimpl->setFileName(x);
}
void ResultTable::closeFile()
{
this->pimpl->closeFile();
}
void ResultTable::add(std::shared_ptr<celero::ExperimentResult> x)
{
if(this->pimpl->ofs.is_open() == true)
{
this->pimpl->ofs << x->getExperiment()->getBenchmark()->getName() << "," << x->getExperiment()->getName() << "," << x->getProblemSpaceValue()
<< "," << x->getExperiment()->getSamples() << "," << x->getProblemSpaceIterations() << "," << x->getFailure() << ",";
this->pimpl->ofs << x->getBaselineMeasurement() << "," << x->getUsPerCall() << "," << x->getCallsPerSecond() << ","
<< x->getTimeStatistics()->getMin() << "," << x->getTimeStatistics()->getMean() << "," << x->getTimeStatistics()->getMax()
<< "," << x->getTimeStatistics()->getVariance() << "," << x->getTimeStatistics()->getStandardDeviation() << ","
<< x->getTimeStatistics()->getSkewness() << "," << x->getTimeStatistics()->getKurtosis() << ","
<< x->getTimeStatistics()->getZScore() << std::endl;
}
}

87
lib/celero/ResultTable.h Normal file
View File

@ -0,0 +1,87 @@
#ifndef H_CELERO_RESULTTABLE_H
#define H_CELERO_RESULTTABLE_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Experiment.h>
#include <celero/Pimpl.h>
#include <string>
namespace celero
{
///
/// \class ResultTable
///
/// \author John Farrier
///
class ResultTable
{
public:
///
/// Singleton
///
static ResultTable& Instance();
///
/// Specify a file name for a results output file.
///
/// \param x The name of the output file in which to store Celero's results.
///
void setFileName(const std::string& x);
///
/// Force the output file (if any) to close
///
void closeFile();
///
/// Add a new result to the result table.
///
/// This should re-save on every new result so that the output can be monitored externally.
///
void add(std::shared_ptr<celero::ExperimentResult> x);
///
///
///
void save();
private:
///
/// Default Constructor
///
ResultTable();
///
/// Default Destructor
///
~ResultTable();
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

252
lib/celero/Statistics.h Normal file
View File

@ -0,0 +1,252 @@
#ifndef H_CELERO_STATISTICS_H
#define H_CELERO_STATISTICS_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <limits>
namespace celero
{
///
/// \class Statistics
///
/// \author John Farrier
///
/// Sources:
/// http://www.johndcook.com/skewness_kurtosis.html
/// http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance
/// http://prod.sandia.gov/techlib/access-control.cgi/2008/086212.pdf
/// http://en.wikipedia.org/wiki/Kurtosis
///
template <typename T = int64_t>
class Statistics
{
static_assert(std::is_arithmetic<T>::value, "Statistics requres an arithmetic type.");
public:
///
/// \brief Default constructor
///
Statistics() = default;
///
///
///
Statistics(const Statistics& other) :
sampleSize(other.sampleSize),
M1(other.M1),
M2(other.M2),
M3(other.M3),
M4(other.M4),
min(other.min),
max(other.max)
{
}
///
///
///
~Statistics() = default;
///
///
///
Statistics operator+(const Statistics<T>& other)
{
Statistics<T> combined;
combined.sampleSize = this->sampleSize + other.sampleSize;
const auto delta = other.M1 - this->M1;
const auto delta2 = delta * delta;
const auto delta3 = delta * delta2;
const auto delta4 = delta2 * delta2;
combined.M1 = (this->sampleSize * this->M1 + other.sampleSize * other.M1) / combined.sampleSize;
combined.M2 = this->M2 + other.M2 + delta2 * this->sampleSize * other.sampleSize / combined.sampleSize;
combined.M3 =
this->M3 + other.M3
+ delta3 * this->sampleSize * other.sampleSize * (this->sampleSize - other.sampleSize) / (combined.sampleSize * combined.sampleSize);
combined.M3 += 3.0 * delta * (this->sampleSize * other.M2 - other.sampleSize * this->M2) / combined.sampleSize;
combined.M4 = this->M4 + other.M4
+ delta4 * this->sampleSize * other.sampleSize
* (this->sampleSize * this->sampleSize - this->sampleSize * other.sampleSize + other.sampleSize * other.sampleSize)
/ (combined.sampleSize * combined.sampleSize * combined.sampleSize);
combined.M4 += 6.0 * delta2 * (this->sampleSize * this->sampleSize * other.M2 + other.sampleSize * other.sampleSize * this->M2)
/ (combined.sampleSize * combined.sampleSize)
+ 4.0 * delta * (this->sampleSize * other.M3 - other.sampleSize * this->M3) / combined.sampleSize;
combined.min = std::min(this->min, other.min);
combined.max = std::max(this->max, other.max);
return combined;
}
Statistics& operator+=(const Statistics& other)
{
const auto combined = *this + other;
*this = combined;
return *this;
}
Statistics& operator=(const Statistics& other)
{
this->sampleSize = other.sampleSize;
this->M1 = other.M1;
this->M2 = other.M2;
this->M3 = other.M3;
this->M4 = other.M4;
this->min = other.min;
this->max = other.max;
return *this;
}
///
/// Resets all accumulated statistics.
///
void reset()
{
this->sampleSize = 0;
this->M1 = 0;
this->M2 = 0;
this->M3 = 0;
this->M4 = 0;
this->min = std::numeric_limits<decltype(this->min)>::max();
this->max = std::numeric_limits<decltype(this->max)>::min();
}
///
/// Adds a statistical sample.
///
void addSample(T x)
{
const auto n1 = this->sampleSize;
this->sampleSize++;
const auto delta = x - this->M1;
const auto delta_n = delta / this->sampleSize;
const auto delta_n2 = delta_n * delta_n;
const auto term1 = delta * delta_n * n1;
this->M1 += delta_n;
this->M4 += term1 * delta_n2 * (this->sampleSize * this->sampleSize - 3 * this->sampleSize + 3) + 6 * delta_n2 * this->M2
- 4 * delta_n * this->M3;
this->M3 += term1 * delta_n * (this->sampleSize - 2) - 3 * delta_n * this->M2;
this->M2 += term1;
this->min = std::min(this->min, x);
this->max = std::max(this->max, x);
}
size_t getSize() const
{
return this->sampleSize;
}
double getMean() const
{
return this->M1;
}
double getVariance() const
{
if(this->sampleSize > 1)
{
return this->M2 / (this->sampleSize - 1);
}
return 0.0;
}
double getStandardDeviation() const
{
return std::sqrt(this->getVariance());
}
double getSkewness() const
{
if(this->sampleSize > 2)
{
return sqrt(this->sampleSize) * this->M3 / pow(this->M2, 1.5);
}
return 0.0;
}
double getKurtosis() const
{
if(this->sampleSize > 3)
{
if(this->M2 != 0)
{
return this->sampleSize * this->M4 / (this->M2 * this->M2) - 3.0;
}
}
return 0.0;
}
///
/// Computed as (mean - hypothesis)/standard_deviation
///
/// Here, the hypothesis is our minimum value.
///
double getZScore() const
{
const auto sd = this->getStandardDeviation();
if(sd != 0.0)
{
return (this->getMean() - static_cast<double>(this->getMin())) / sd;
}
return 0.0;
}
T getMin() const
{
return this->min;
}
T getMax() const
{
return this->max;
}
private:
size_t sampleSize{0};
double M1{0.0};
double M2{0.0};
double M3{0.0};
double M4{0.0};
T min{std::numeric_limits<T>::max()};
T max{std::numeric_limits<T>::min()};
};
} // namespace celero
#endif

145
lib/celero/TestFixture.cpp Normal file
View File

@ -0,0 +1,145 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <assert.h>
#include <celero/TestFixture.h>
#include <celero/UserDefinedMeasurement.h>
#include <algorithm>
#include <iostream>
using namespace celero;
TestFixture::TestFixture()
{
}
TestFixture::~TestFixture()
{
}
void TestFixture::onExperimentStart(const celero::TestFixture::ExperimentValue&)
{
}
void TestFixture::onExperimentEnd()
{
}
void TestFixture::setUp(const celero::TestFixture::ExperimentValue&)
{
}
void TestFixture::setExperimentIterations(uint64_t x)
{
this->experimentIterations = x;
}
uint64_t TestFixture::getExperimentIterations() const
{
return this->experimentIterations;
}
void TestFixture::setExperimentTime(uint64_t x)
{
this->experimentTime = x;
}
uint64_t TestFixture::getExperimentTime() const
{
return this->experimentTime;
}
void TestFixture::tearDown()
{
}
uint64_t TestFixture::run(const uint64_t, const uint64_t iterations, const celero::TestFixture::ExperimentValue& experimentValue)
{
// This function constitutes one sample consisting of several iterations for a single experiment value.
if(this->HardCodedMeasurement() == 0)
{
uint64_t totalTime = 0;
// Set up the testing fixture.
this->setUp(experimentValue);
this->setExperimentIterations(iterations);
// Run the test body for each iterations.
auto iterationCounter = iterations;
// Get the starting time.
const auto startTime = celero::timer::GetSystemTime();
// Count down to zero
// Iterations are used when the benchmarks are very fast.
// Do not start/stop the timer inside this loop.
// The purpose of the loop is to help counter timer quantization/errors.
while(iterationCounter--)
{
this->onExperimentStart(experimentValue);
this->UserBenchmark();
this->onExperimentEnd();
}
// See how long it took.
totalTime += celero::timer::GetSystemTime() - startTime;
this->setExperimentTime(totalTime);
// Tear down the testing fixture.
this->tearDown();
// Return the duration in microseconds for the given problem size.
return totalTime;
}
return this->HardCodedMeasurement();
}
void TestFixture::UserBenchmark()
{
}
uint64_t TestFixture::HardCodedMeasurement() const
{
return uint64_t(0);
}
std::vector<std::shared_ptr<UserDefinedMeasurement>> TestFixture::getUserDefinedMeasurements() const
{
return {};
}
std::vector<std::string> TestFixture::getUserDefinedMeasurementNames() const
{
std::vector<std::string> names;
const auto udms = this->getUserDefinedMeasurements();
if(udms.empty() == false)
{
for(const auto udm : udms)
{
names.emplace_back(udm->getName());
}
}
return names;
}

216
lib/celero/TestFixture.h Normal file
View File

@ -0,0 +1,216 @@
#ifndef H_CELERO_TESTFIXTURE_H
#define H_CELERO_TESTFIXTURE_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <celero/Timer.h>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <string>
#include <vector>
// This must be included last.
#include <celero/ThreadLocal.h>
namespace celero
{
class Benchmark;
class UserDefinedMeasurement;
///
/// \class TestFixture
///
/// \author John Farrier
///
class CELERO_EXPORT TestFixture
{
public:
///
/// Default Constructor.
///
TestFixture();
///
/// Virtual destructor for inheritance.
///
virtual ~TestFixture();
enum class Constants : int64_t
{
#ifdef _MSC_VER
#if(_MSC_VER < 1900)
NoProblemSpaceValue = -9223372036854775807
#else
NoProblemSpaceValue = std::numeric_limits<int64_t>::min()
#endif
#else
NoProblemSpaceValue = std::numeric_limits<int64_t>::min()
#endif
};
///
/// \class ExperimentValue
///
/// You can derive from this type to add your own information to the experiment value set.
///
class ExperimentValue
{
public:
ExperimentValue() = default;
ExperimentValue(int64_t v) : Value(v){};
ExperimentValue(int64_t v, int64_t i) : Value(v), Iterations(i){};
virtual ~ExperimentValue() = default;
/// An arbitrary integer value which can be used or translated for use by the test fixture.
int64_t Value{0};
/// The number of iterations to do with this test value. 0 (default) indicates that the default number of iterations set up for the test
/// case should be used.
int64_t Iterations{0};
};
///
/// Allows a test fixture to supply values to use for experiments.
///
/// This is used to create multiple runs of the same experiment
/// and varrying the data set size, for example. The second value
/// of the pair is an optional override for the number of iterations
/// to be used. If zero is specified, then the default number of
/// iterations is used.
///
/// It is only guaranteed that the constructor is called prior to this function being called.
///
virtual std::vector<celero::TestFixture::ExperimentValue> getExperimentValues() const
{
return std::vector<celero::TestFixture::ExperimentValue>();
};
///
/// Provide a units result scale of each experiment value.
///
/// If the value is greater than 0 then additional statistic value will be printed
/// in output - [ xxxx units/sec ]. For example for measure speed of
/// file IO operations method might return 1024 * 1024 to get megabytes
/// per second.
///
/// It is only guaranteed that the constructor is called prior to this function being called.
///
virtual double getExperimentValueResultScale() const
{
return 1.0;
};
///
/// Allows the text fixture to run code that will be executed once immediately before the benchmark.
///
/// Unlike setUp, the evaluation of this function IS included in the total experiment execution
/// time.
///
/// \param x The value for the experiment. This can be ignored if the test does not utilize experiment values.
///
virtual void onExperimentStart(const celero::TestFixture::ExperimentValue& x);
///
/// Allows the text fixture to run code that will be executed once immediately after the benchmark.
/// Unlike tearDown, the evaluation of this function IS included in the total experiment execution
/// time.
///
virtual void onExperimentEnd();
///
/// Set up the test fixture before benchmark execution.
///
/// This code is NOT included in the benchmark timing.
/// It is executed once before all iterations are executed and between each Sample.
/// Your experiment should NOT rely on "setUp" methods to be called before EACH experiment run, only between each sample.
///
/// \param x The celero::TestFixture::ExperimentValue for the experiment. This can be ignored if the test does not utilize experiment values.
///
virtual void setUp(const celero::TestFixture::ExperimentValue& x);
///
/// Internal to Celero
///
void setExperimentTime(uint64_t x);
///
/// Valid inside tearDown().
///
uint64_t getExperimentTime() const;
///
/// Internal to Celero
///
void setExperimentIterations(uint64_t x);
///
/// Valid inside tearDown().
///
uint64_t getExperimentIterations() const;
///
/// Called after test completion to destroy the fixture.
///
/// This code is NOT included in the benchmark timing.
/// It is executed once after all iterations are executed and between each Sample.
/// Your experiment should NOT rely on "tearDown" methods to be called after EACH experiment run, only between each sample.
///
virtual void tearDown();
///
///
/// \param threads The number of working threads.
/// \param iterations The number of times to loop over the UserBenchmark function.
/// \param experimentValue The experiment value to pass in setUp function.
///
/// \return Returns the number of microseconds the run took.
///
virtual uint64_t run(uint64_t threads, uint64_t iterations, const celero::TestFixture::ExperimentValue& experimentValue);
///
/// \brief If you want to use user-defined measurements, override this method to return them
///
/// This method must return a vector of pointers, one per type of user-defined measurement that you want to measure.
///
virtual std::vector<std::shared_ptr<UserDefinedMeasurement>> getUserDefinedMeasurements() const;
///
/// \brief Returns the names of all user-defined measurements in this fixture.
///
std::vector<std::string> getUserDefinedMeasurementNames() const;
protected:
/// Executed for each operation the benchmarking test is run.
virtual void UserBenchmark();
///
/// Only used for baseline cases. Used to define a hard-coded execution time vs. actually making a measurement.
///
virtual uint64_t HardCodedMeasurement() const;
private:
uint64_t experimentIterations{0};
uint64_t experimentTime{0};
};
} // namespace celero
#endif

96
lib/celero/TestVector.cpp Normal file
View File

@ -0,0 +1,96 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/PimplImpl.h>
#include <celero/TestVector.h>
#include <algorithm>
#include <mutex>
#include <vector>
using namespace celero;
///
/// \class Impl
///
class TestVector::Impl
{
public:
Impl() : testVectorMutex(), testVector()
{
}
~Impl()
{
}
mutable std::mutex testVectorMutex;
std::vector<std::shared_ptr<Benchmark>> testVector;
};
TestVector::TestVector() : pimpl()
{
}
TestVector::~TestVector()
{
}
TestVector& TestVector::Instance()
{
static TestVector singleton;
return singleton;
}
void TestVector::push_back(std::shared_ptr<Benchmark> x)
{
std::lock_guard<std::mutex> mutexLock(this->pimpl->testVectorMutex);
this->pimpl->testVector.push_back(x);
}
void TestVector::clear()
{
this->pimpl->testVector.clear();
}
size_t TestVector::size() const
{
std::lock_guard<std::mutex> mutexLock(this->pimpl->testVectorMutex);
return this->pimpl->testVector.size();
}
std::shared_ptr<Benchmark> TestVector::operator[](size_t x)
{
std::lock_guard<std::mutex> mutexLock(this->pimpl->testVectorMutex);
return this->pimpl->testVector[x];
}
std::shared_ptr<Benchmark> TestVector::operator[](const std::string& x)
{
std::lock_guard<std::mutex> mutexLock(this->pimpl->testVectorMutex);
const auto found = std::find_if(std::begin(this->pimpl->testVector), std::end(this->pimpl->testVector),
[x](std::shared_ptr<Benchmark> const& bmark) -> bool { return (bmark->getName() == x); });
if(found != std::end(this->pimpl->testVector))
{
return *found;
}
return nullptr;
}

67
lib/celero/TestVector.h Normal file
View File

@ -0,0 +1,67 @@
#ifndef H_CELERO_TESTVECTOR_H
#define H_CELERO_TESTVECTOR_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Benchmark.h>
#include <celero/Export.h>
#include <celero/Pimpl.h>
#include <functional>
namespace celero
{
///
/// \class TestVector
///
/// \author John Farrier
///
class CELERO_EXPORT TestVector
{
public:
~TestVector();
static TestVector& Instance();
void push_back(std::shared_ptr<Benchmark> x);
void clear();
size_t size() const;
std::shared_ptr<Benchmark> operator[](size_t x);
std::shared_ptr<Benchmark> operator[](const std::string& x);
private:
///
/// Default Constructor
///
TestVector();
///
/// \brief Pimpl Idiom
///
class Impl;
///
/// \brief Pimpl Idiom
///
Pimpl<Impl> pimpl;
};
}
#endif

36
lib/celero/ThreadLocal.h Normal file
View File

@ -0,0 +1,36 @@
#ifndef H_CELERO_THREADLOCAL_H
#define H_CELERO_THREADLOCAL_H
///
/// \author Ivan Shynkarenka
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#ifndef thread_local
#if __STDC_VERSION__ >= 201112 && !defined __STDC_NO_THREADS__
#define thread_local _Thread_local
#elif defined _WIN32 && (defined _MSC_VER || defined __ICL || defined __DMC__ || defined __BORLANDC__)
#define thread_local __declspec(thread)
/* note that ICC (linux) and Clang are covered by __GNUC__ */
#elif defined __GNUC__ || defined __SUNPRO_C || defined __xlC__
#define thread_local __thread
#else
#error "Cannot define thread_local"
#endif
#endif
#endif

124
lib/celero/ThreadTestFixture.cpp Normal file
View File

@ -0,0 +1,124 @@
///
/// \author Ivan Shynkarenka
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/ThreadTestFixture.h>
#include <assert.h>
#include <celero/PimplImpl.h>
#include <algorithm>
#include <future>
#include <iostream>
using namespace celero;
class ThreadTestFixture::Impl
{
public:
static thread_local uint64_t currentCallId;
static thread_local uint64_t currentThreadId;
std::vector<std::future<void>> futures;
};
thread_local uint64_t ThreadTestFixture::Impl::currentCallId = 0;
thread_local uint64_t ThreadTestFixture::Impl::currentThreadId = 0;
ThreadTestFixture::ThreadTestFixture() : TestFixture()
{
}
ThreadTestFixture::~ThreadTestFixture()
{
}
void ThreadTestFixture::startThreads(uint64_t threads, uint64_t iterations)
{
const uint64_t iterationsPerThread = iterations / threads;
for(uint64_t i = 0; i < threads; ++i)
{
try
{
this->pimpl->futures.push_back(
// std::async(std::launch::deferred,
std::async(std::launch::async, [this, i, iterationsPerThread]() {
this->pimpl->currentThreadId = i + 1;
for(auto threadIterationCounter = size_t(0); threadIterationCounter < iterationsPerThread;)
{
this->pimpl->currentCallId = ++threadIterationCounter;
this->UserBenchmark();
}
}));
}
catch(std::system_error& e)
{
std::cerr << "Exception. Error Code: " << e.code() << ", " << e.what() << std::endl;
}
}
}
void ThreadTestFixture::stopThreads()
{
// This part will be more effective after
// wait_for_all() will be avaliable for futures!
for(auto& f : this->pimpl->futures)
{
if(f.valid() == true)
{
try
{
f.wait();
}
catch(std::system_error& e)
{
std::cerr << "Exception. Error Code: " << e.code() << ", " << e.what() << std::endl;
}
}
};
}
uint64_t ThreadTestFixture::run(uint64_t threads, uint64_t calls, const celero::TestFixture::ExperimentValue& experimentValue)
{
if(this->HardCodedMeasurement() == 0)
{
// Set up the testing fixture.
this->setUp(experimentValue);
// Get the starting time.
const auto startTime = celero::timer::GetSystemTime();
this->onExperimentStart(experimentValue);
// Start working threads.
this->startThreads(threads, calls);
// Stop working threads.
this->stopThreads();
this->onExperimentEnd();
const auto endTime = celero::timer::GetSystemTime();
// Tear down the testing fixture.
this->tearDown();
// Return the duration in microseconds for the given problem size.
return (endTime - startTime);
}
return this->HardCodedMeasurement();
}

75
lib/celero/ThreadTestFixture.h Normal file
View File

@ -0,0 +1,75 @@
#ifndef H_CELERO_THREADTESTFIXTURE_H
#define H_CELERO_THREADTESTFIXTURE_H
///
/// \author Ivan Shynkarenka
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Pimpl.h>
#include <celero/TestFixture.h>
namespace celero
{
class Benchmark;
///
/// \class ThreadTestFixture
///
/// \author Ivan Shynkarenka
///
class CELERO_EXPORT ThreadTestFixture : public TestFixture
{
public:
///
/// Default Constructor.
///
ThreadTestFixture();
///
/// Virtual destructor for inheritance.
///
virtual ~ThreadTestFixture();
///
/// Start threads before benchmark execution.
///
/// \param threads Count of working threads to start.
/// \param calls The total number of times to loop over the UserBenchmark function.
///
virtual void startThreads(uint64_t threads, uint64_t calls);
///
/// Called after test completion to stop threads.
///
virtual void stopThreads();
///
/// \param threads The number of working threads.
/// \param calls The total number of times to loop over the UserBenchmark function.
/// \param experimentValue The experiment value to pass in setUp function.
///
/// \return Returns the number of microseconds the run took.
///
uint64_t run(uint64_t threads, uint64_t calls, const celero::TestFixture::ExperimentValue& experimentValue) override;
private:
class Impl;
Pimpl<Impl> pimpl;
};
} // namespace celero
#endif

74
lib/celero/Timer.cpp Normal file
View File

@ -0,0 +1,74 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Print.h>
#include <celero/Timer.h>
#include <iostream>
#ifdef WIN32
#include <Windows.h>
LARGE_INTEGER QPCFrequency;
#else
#include <chrono>
#endif
uint64_t celero::timer::GetSystemTime()
{
#ifdef WIN32
LARGE_INTEGER timeStorage;
QueryPerformanceCounter(&timeStorage);
if(QPCFrequency.QuadPart != 0)
{
return static_cast<uint64_t>(timeStorage.QuadPart * 1000000) / static_cast<uint64_t>(QPCFrequency.QuadPart);
}
return 0;
#else
auto timePoint = std::chrono::high_resolution_clock::now();
return std::chrono::duration_cast<std::chrono::microseconds>(timePoint.time_since_epoch()).count();
#endif
}
double celero::timer::CachePerformanceFrequency(bool quiet)
{
#ifdef WIN32
QueryPerformanceFrequency(&QPCFrequency);
if(QPCFrequency.QuadPart == 0)
{
return 0;
}
auto precision = ((1.0 / static_cast<double>(QPCFrequency.QuadPart)) * 1000000.0);
#else
if(static_cast<double>(std::chrono::high_resolution_clock::period::den) == 0)
{
return 0;
}
auto precision =
(static_cast<double>(std::chrono::high_resolution_clock::period::num) / static_cast<double>(std::chrono::high_resolution_clock::period::den))
* 1000000.0;
#endif
if(quiet == false)
{
std::cout << "Timer resolution: " << std::to_string(precision) << " us" << std::endl;
}
return precision;
}

70
lib/celero/Timer.h Normal file
View File

@ -0,0 +1,70 @@
#ifndef H_CELERO_TIMER_H
#define H_CELERO_TIMER_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Utilities.h>
#include <stdint.h>
namespace celero
{
///
/// \namespace timer
///
/// \author John Farrier
///
/// \brief Provide basic cross-platform timing functions to measure code performance speed.
///
namespace timer
{
///
/// \brief Retrieves the current time.
///
/// \author John Farrier
///
/// \return The time, in ticks.
///
CELERO_EXPORT uint64_t GetSystemTime();
///
/// \brief Converts the gathered system time into seconds.
///
/// This assumes "x" is a delta and relatively small (easily fits inside of a double).
///
/// \author John Farrier
///
/// \param x The time, in ticks.
///
/// \return The time, in seconds.
///
constexpr double ConvertSystemTime(const uint64_t x)
{
return x * celero::UsToSec;
}
///
/// On Windows, this caches the frequency of the high performance clock.
///
/// \return The number of microseconds of precision that we have.
///
CELERO_EXPORT double CachePerformanceFrequency(bool quiet);
} // namespace timer
} // namespace celero
#endif

76
lib/celero/UserDefinedMeasurement.h Normal file
View File

@ -0,0 +1,76 @@
#ifndef H_CELERO_USERDEFINEDMEASUREMENT_H
#define H_CELERO_USERDEFINEDMEASUREMENT_H
///
/// \author Lukas Barth
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Export.h>
#include <functional>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
namespace celero
{
class UserDefinedMeasurement;
///
/// \brief Describes, which aggregations should be computed on a user-defined measurement.
///
/// The string names the aggregation, the UDMAggregateFunction is the function that will be called on the collected vector of user-defined
/// measurements.
///
using UDMAggregationTable = std::vector<std::pair<std::string, std::function<double(void)>>>;
///
/// \class UserDefinedMeasurement
///
/// Base class that the user must derive user-defined measurements from.
///
/// \author Lukas Barth
///
class CELERO_EXPORT UserDefinedMeasurement
{
public:
///
/// \brief Must be implemented by the user. Must return a specification which aggregations the user wants to be computed.
///
virtual UDMAggregationTable getAggregationInfo() const = 0;
///
/// \brief Must be implemented by the user. Must return the name of this user-defined measurement.
///
virtual std::string getName() const = 0;
///
/// \brief Combine the results of two user defined measurements.
///
/// As TestFixture classes are created and destroyed, this provides a mechanisim to preserve data. Internally, this function is used so that
/// each unique set of (group, experiment, problem space) has its own combined set of user defined measurements.
///
virtual void merge(const UserDefinedMeasurement* const x) = 0;
protected:
// Class may never be directly instantiated
UserDefinedMeasurement() = default;
};
} // namespace celero
#endif

94
lib/celero/UserDefinedMeasurementCollector.cpp Normal file
View File

@ -0,0 +1,94 @@
///
/// \author Lukas Barth
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <assert.h>
#include <celero/TestFixture.h>
#include <celero/UserDefinedMeasurementCollector.h>
using namespace celero;
UserDefinedMeasurementCollector::UserDefinedMeasurementCollector(std::shared_ptr<TestFixture> fixture)
{
const auto udm = fixture->getUserDefinedMeasurementNames();
if(udm.empty() == false)
{
for(auto name : fixture->getUserDefinedMeasurementNames())
{
this->collected[name] = nullptr;
}
}
}
void UserDefinedMeasurementCollector::collect(std::shared_ptr<TestFixture> fixture)
{
const auto udms = fixture->getUserDefinedMeasurements();
if(udms.empty() == false)
{
for(auto udm : udms)
{
if(this->collected[udm->getName()] == nullptr)
{
this->collected[udm->getName()] = udm;
}
else
{
this->collected[udm->getName()]->merge(&*udm);
}
}
}
}
std::vector<std::string> UserDefinedMeasurementCollector::getFields(std::shared_ptr<TestFixture> fixture) const
{
std::vector<std::string> fields;
const auto udms = fixture->getUserDefinedMeasurements();
if(udms.empty() == false)
{
for(auto udm : udms)
{
for(const auto& aggDesc : udm->getAggregationInfo())
{
fields.emplace_back(std::string(udm->getName()) + std::string(" ") + std::string(aggDesc.first));
}
}
}
return fields;
}
std::vector<std::pair<std::string, double>> UserDefinedMeasurementCollector::getAggregateValues() const
{
std::vector<std::pair<std::string, double>> aggregates;
for(const auto& collectedEntry : this->collected)
{
const auto name = collectedEntry.first;
const auto collectedUDMs = collectedEntry.second;
for(const auto& aggDesc : collectedUDMs->getAggregationInfo())
{
const auto fieldName = name + std::string(" ") + aggDesc.first;
aggregates.emplace_back(fieldName, (aggDesc.second)());
}
}
return aggregates;
}

47
lib/celero/UserDefinedMeasurementCollector.h Normal file
View File

@ -0,0 +1,47 @@
#ifndef H_CELERO_USERDEFINEDMEASUREMENTCOLLECTOR_H
#define H_CELERO_USERDEFINEDMEASUREMENTCOLLECTOR_H
///
/// \author Lukas Barth
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/UserDefinedMeasurement.h>
#include <celero/TestFixture.h>
#include <unordered_map>
namespace celero
{
///
/// \class UserDefinedMeasurementCollector
///
/// \author Lukas Barth
///
class CELERO_EXPORT UserDefinedMeasurementCollector
{
public:
UserDefinedMeasurementCollector(std::shared_ptr<TestFixture> fixture);
void collect(std::shared_ptr<TestFixture> fixture);
std::vector<std::string> getFields(std::shared_ptr<TestFixture> fixture) const;
std::vector<std::pair<std::string, double>> getAggregateValues() const;
private:
std::unordered_map<std::string, std::shared_ptr<celero::UserDefinedMeasurement>> collected;
};
} // namespace celero
#endif

182
lib/celero/UserDefinedMeasurementTemplate.h Normal file
View File

@ -0,0 +1,182 @@
#ifndef H_CELERO_USERDEFINEDMEASUREMENTTEMPLATE_H
#define H_CELERO_USERDEFINEDMEASUREMENTTEMPLATE_H
///
/// \author Lukas Barth, John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Statistics.h>
#include <celero/UserDefinedMeasurement.h>
#include <numeric>
#include <type_traits>
namespace celero
{
///
/// \class UserDefinedMeasurementTemplate
///
/// Base class that the user must derive user-defined measurements from.
///
/// \author Lukas Barth, John Farrier
///
template <typename T>
class UserDefinedMeasurementTemplate : public UserDefinedMeasurement
{
static_assert(std::is_arithmetic<T>::value, "UserDefinedMeasurementTemplate requres an arithmetic type.");
public:
///
/// Default constructor
///
UserDefinedMeasurementTemplate() = default;
///
/// Default destructor
///
virtual ~UserDefinedMeasurementTemplate() = default;
///
/// \brief Must be implemented by the user. Must return a specification which aggregations the user wants to be computed.
///
virtual UDMAggregationTable getAggregationInfo() const override
{
UDMAggregationTable table;
if(this->reportSize() == true)
{
table.push_back({"# Samp", [this]() { return static_cast<double>(this->getStatistics().getSize()); }});
}
if(this->reportMean() == true)
{
table.push_back({"Mean", [this]() { return this->getStatistics().getMean(); }});
}
if(this->reportVariance() == true)
{
table.push_back({"Var", [this]() { return this->getStatistics().getVariance(); }});
}
if(this->reportStandardDeviation() == true)
{
table.push_back({"StdDev", [this]() { return this->getStatistics().getStandardDeviation(); }});
}
if(this->reportSkewness() == true)
{
table.push_back({"Skew", [this]() { return this->getStatistics().getSkewness(); }});
}
if(this->reportKurtosis() == true)
{
table.push_back({"Kurtosis", [this]() { return this->getStatistics().getKurtosis(); }});
}
if(this->reportZScore() == true)
{
table.push_back({"ZScore", [this]() { return this->getStatistics().getZScore(); }});
}
if(this->reportMin() == true)
{
table.push_back({"Min", [this]() { return static_cast<double>(this->getStatistics().getMin()); }});
}
if(this->reportMax() == true)
{
table.push_back({"Max", [this]() { return static_cast<double>(this->getStatistics().getMax()); }});
}
return table;
}
///
/// \brief You must call this method from your fixture to add a measurement
///
void addValue(T x)
{
this->stats.addSample(x);
}
///
/// Preserve measurements within a group/experiment/problem space set.
///
virtual void merge(const UserDefinedMeasurement* const x) override
{
const auto toMerge = dynamic_cast<const UserDefinedMeasurementTemplate<T>* const>(x);
this->stats += toMerge->stats;
}
protected:
virtual bool reportSize() const
{
return true;
}
virtual bool reportMean() const
{
return true;
}
virtual bool reportVariance() const
{
return true;
}
virtual bool reportStandardDeviation() const
{
return true;
}
virtual bool reportSkewness() const
{
return true;
}
virtual bool reportKurtosis() const
{
return true;
}
virtual bool reportZScore() const
{
return true;
}
virtual bool reportMin() const
{
return true;
}
virtual bool reportMax() const
{
return true;
}
const Statistics<T>& getStatistics() const
{
return this->stats;
}
private:
/// Continuously gathers statistics without having to retain data history.
Statistics<T> stats;
};
} // namespace celero
#endif

67
lib/celero/Utilities.cpp Normal file
View File

@ -0,0 +1,67 @@
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#include <celero/Print.h>
#include <celero/Utilities.h>
#ifdef WIN32
#include <Windows.h>
#include <PowrProf.h>
#endif
#ifdef max
#undef max
#endif
#include <limits>
#include <random>
template <>
void celero::DoNotOptimizeAway(std::function<void(void)>&& x)
{
x();
// We must always do this test, but it will never pass.
static auto ttid = std::this_thread::get_id();
if(ttid == std::thread::id())
{
// This forces the value to never be optimized away
// by taking a reference then using it.
const auto* p = &x;
putchar(*reinterpret_cast<const char*>(p));
// If we do get here, kick out because something has gone wrong.
std::abort();
}
}
int celero::Random()
{
// http://en.cppreference.com/w/cpp/numeric/random/uniform_int_distribution
// Will be used to obtain a seed for the random number engine
static std::random_device rd;
// Standard mersenne_twister_engine seeded with rd()
static std::mt19937 gen(rd());
static std::uniform_int_distribution<> dis(std::numeric_limits<int>::lowest(), std::numeric_limits<int>::max());
return dis(gen);
}

156
lib/celero/Utilities.h Normal file
View File

@ -0,0 +1,156 @@
#ifndef H_CELERO_UTILITIES_H
#define H_CELERO_UTILITIES_H
///
/// \author John Farrier
///
/// \copyright Copyright 2015, 2016, 2017, 2018. 2019 John Farrier
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
/// http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
#ifndef WIN32
#include <unistd.h>
#endif
#ifdef __FreeBSD__
#include <stdio.h>
#endif
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <functional>
#include <thread>
#include <celero/Export.h>
namespace celero
{
///
/// \func DoNotOptimizeAway
///
/// Used to prevent compiler optimization of a variable
/// that performs no real purpose other than to participate
/// in a benchmark
///
/// Consider the following trivial benchmark:
///
/// \code
/// BASELINE(...)
/// {
/// int x = 0;
///
/// for(int i = 0; i < 64; i++)
/// {
/// x += i;
/// }
/// }
/// \endcode
///
/// Using Ubuntu clang v3.0, the resultant assembly is highly optimized
/// as one might expect, but not terribly useful for baselining:
///
/// \verbatim
/// movl $2016, %eax # imm = 0x7E0
/// ret
/// \endverbatim
///
/// Now, replace the inner loop with a call to DoNotOptimizeAway:
///
/// \code
/// DoNotOptimizeAway(x += i);
/// \endcode
///
/// The result is now a loop which is meaningful for establishing a
/// baseline.
///
/// \verbatim
/// xorl %ecx, %ecx
/// xorl %eax, %eax
/// .LBB0_1: # =>This Inner Loop Header: Depth=1
/// addl %ecx, %eax
/// incl %ecx
/// cmpl $64, %ecx
/// jne .LBB0_1
/// ret
/// \endverbatim
///
/// GCC 4.8 gives similar results.
///
/// gcc.godbolt.org permalink: http://goo.gl/lsngwX
///
/// Folly uses a simple bit of inline assembly:
/// > template <class T>
/// > void doNotOptimizeAway(T&& datum) {
/// > asm volatile("" : "+r" (datum));
/// >}
///
/// It would be great if that were portable with respect to both compilers and 32/64-bit targets.
///
template <class T>
void DoNotOptimizeAway(T&& x)
{
static auto ttid = std::this_thread::get_id();
if(ttid == std::thread::id())
{
// This forces the value to never be optimized away
// by taking a reference then using it.
const auto* p = &x;
putchar(*reinterpret_cast<const char*>(p));
// If we do get here, kick out because something has gone wrong.
std::abort();
}
}
/// Specialization for std::function objects which return a value.
template <class T>
void DoNotOptimizeAway(std::function<T(void)>&& x)
{
volatile auto foo = x();
static auto ttid = std::this_thread::get_id();
if(ttid == std::thread::id())
{
// This forces the value to never be optimized away
// by taking a reference then using it.
const auto* p = &foo + &x;
putchar(*reinterpret_cast<const char*>(p));
// If we do get here, kick out because something has gone wrong.
std::abort();
}
}
/// Specialization for std::function objects which return void.
template <>
CELERO_EXPORT void DoNotOptimizeAway(std::function<void(void)>&& x);
///
/// Quick definition of the number of microseconds per second.
///
constexpr uint64_t UsPerSec(1000000);
///
/// Conversion from Microseconds to Seconds.
///
constexpr double UsToSec{1.0e-6};
///
/// Drop-in replacement for std::rand();
///
CELERO_EXPORT int Random();
} // namespace celero
#endif