cpp : more robust code and added more documentation
This commit is contained in:
saundersp
187
cpp/projet.cpp
187
cpp/projet.cpp
@ -1,50 +1,61 @@
|
||||
#include <filesystem>
|
||||
namespace fs = std::filesystem;
|
||||
#include "data.hpp"
|
||||
#include "toolbox.hpp"
|
||||
#include "config.hpp"
|
||||
#include "gpu_unit_test.hpp"
|
||||
#include "toolbox_unit_test.hpp"
|
||||
#include "ViolaJones.hpp"
|
||||
#include "ViolaJonesGPU.hpp"
|
||||
#include "ViolaJonesCPU.hpp"
|
||||
|
||||
#if GPU_BOOSTED
|
||||
#include "ViolaJonesGPU.hpp"
|
||||
#include "gpu_unit_test.hpp"
|
||||
#define LABEL "GPU"
|
||||
#define apply_features apply_features_gpu
|
||||
#define set_integral_image set_integral_image_gpu
|
||||
#define argsort_2d argsort_2d_gpu
|
||||
#else
|
||||
#include "ViolaJonesCPU.hpp"
|
||||
#define LABEL "CPU"
|
||||
#define apply_features apply_features_cpu
|
||||
#define set_integral_image set_integral_image_cpu
|
||||
#define argsort_2d argsort_2d_cpu
|
||||
#endif
|
||||
|
||||
std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Array<int32_t>, np::Array<uint8_t>> preprocessing() {
|
||||
/**
|
||||
* @brief Execute the preprocessing phase
|
||||
*
|
||||
* The preprocessing phase consist of the following steps :
|
||||
* - Load the dataset
|
||||
* - Calculate features
|
||||
* - Calculate integral images
|
||||
* - Apply features to images
|
||||
* - Calculate argsort of the featured images.
|
||||
*
|
||||
* @return std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Array<int32_t>, np::Array<uint8_t>> Tuple containing in order : training features, training features sorted indexes, training labels, testing features, testing labels
|
||||
*/
|
||||
std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Array<int32_t>, np::Array<uint8_t>> preprocessing(void) {
|
||||
// Creating state saver folders if they don't exist already
|
||||
if (SAVE_STATE)
|
||||
for (const char* const folder_name : { "models", "out" })
|
||||
fs::create_directory(folder_name);
|
||||
std::filesystem::create_directory(folder_name);
|
||||
|
||||
const std::chrono::system_clock::time_point preproc_timestamp = perf_counter_ns();
|
||||
const std::array<int32_t, 3> preproc_gaps = { 49, -18, 29 };
|
||||
header({ "Preprocessing", "Time spent (ns)", "Formatted time spent" }, preproc_gaps);
|
||||
|
||||
const auto [ X_train, y_train, X_test, y_test ] = state_saver<uint8_t, 4>("Loading sets", preproc_gaps[0], {"X_train", "y_train", "X_test", "y_test"},
|
||||
const auto [ X_train, y_train, X_test, y_test ] = state_saver<uint8_t, 4>("Loading sets", preproc_gaps[0], { "X_train", "y_train", "X_test", "y_test" },
|
||||
FORCE_REDO, SAVE_STATE, OUT_DIR, load_datasets);
|
||||
|
||||
#if __DEBUG
|
||||
print("X_train");
|
||||
printf("X_train\n");
|
||||
print(X_train.shape);
|
||||
print(X_train, { IDX_INSPECT });
|
||||
print("X_test");
|
||||
printf("X_test\n");
|
||||
print(X_test.shape);
|
||||
print(X_test, { IDX_INSPECT });
|
||||
print("y_train");
|
||||
printf("y_train\n");
|
||||
print(y_train.shape);
|
||||
print(y_train, { IDX_INSPECT, IDX_INSPECT + IDX_INSPECT_OFFSET });
|
||||
print("y_test");
|
||||
printf("y_test\n");
|
||||
print(y_test.shape);
|
||||
print(y_test, { IDX_INSPECT, IDX_INSPECT + IDX_INSPECT_OFFSET });
|
||||
#endif
|
||||
@ -53,7 +64,7 @@ std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Arra
|
||||
FORCE_REDO, SAVE_STATE, OUT_DIR, build_features, X_train.shape[1], X_train.shape[2]);
|
||||
|
||||
#if __DEBUG
|
||||
print("feats");
|
||||
printf("feats\n");
|
||||
print(feats.shape);
|
||||
print_feat(feats, { IDX_INSPECT });
|
||||
#endif
|
||||
@ -64,10 +75,10 @@ std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Arra
|
||||
FORCE_REDO, SAVE_STATE, OUT_DIR, set_integral_image, X_test);
|
||||
|
||||
#if __DEBUG
|
||||
print("X_train_ii");
|
||||
printf("X_train_ii\n");
|
||||
print(X_train_ii.shape);
|
||||
print(X_train_ii, { IDX_INSPECT });
|
||||
print("X_test_ii");
|
||||
printf("X_test_ii\n");
|
||||
print(X_test_ii.shape);
|
||||
print(X_test_ii, { IDX_INSPECT });
|
||||
#endif
|
||||
@ -78,16 +89,16 @@ std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Arra
|
||||
FORCE_REDO, SAVE_STATE, OUT_DIR, apply_features, feats, X_test_ii);
|
||||
|
||||
#if __DEBUG
|
||||
print("X_train_feat");
|
||||
printf("X_train_feat\n");
|
||||
print(X_train_feat.shape);
|
||||
print(X_train_feat, { IDX_INSPECT, IDX_INSPECT + IDX_INSPECT_OFFSET });
|
||||
print("X_test_feat");
|
||||
printf("X_test_feat\n");
|
||||
print(X_test_feat.shape);
|
||||
print(X_test_feat, { IDX_INSPECT, IDX_INSPECT + IDX_INSPECT_OFFSET });
|
||||
#endif
|
||||
|
||||
// const Array<int> indices = measure_time_save<Array<int>>("Selecting best features", "indices", select_percentile, X_train_feat, d.y_train);
|
||||
// const Array<int> indices = measure_time<Array<int>>("Selecting best features", select_percentile, X_train_feat, d.y_train);
|
||||
// const np::Array<int32_t> indices = measure_time_save<Array<int>>("Selecting best features", preproc_gaps[0], "indices", select_percentile, X_train_feat, d.y_train);
|
||||
// const np::Array<int32_t> indices = measure_time<Array<int>>("Selecting best features", preproc_gaps[0], select_percentile, X_train_feat, d.y_train);
|
||||
|
||||
#if __DEBUG
|
||||
// print_feature(indices);
|
||||
@ -97,7 +108,7 @@ std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Arra
|
||||
FORCE_REDO, SAVE_STATE, OUT_DIR, argsort_2d, X_train_feat);
|
||||
|
||||
#if __DEBUG
|
||||
print("X_train_feat_argsort");
|
||||
printf("X_train_feat_argsort\n");
|
||||
print(X_train_feat_argsort.shape);
|
||||
print(X_train_feat_argsort, { IDX_INSPECT, IDX_INSPECT + IDX_INSPECT_OFFSET });
|
||||
#endif
|
||||
@ -117,6 +128,14 @@ std::tuple<np::Array<int32_t>, np::Array<uint16_t>, np::Array<uint8_t>, np::Arra
|
||||
return { X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test };
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Train the weak classifiers.
|
||||
*
|
||||
* @param X_train_feat Training images.
|
||||
* @param X_train_feat_argsort Sorted indexes of the training images features.
|
||||
* @param y_train Training labels.
|
||||
* @return Trained models
|
||||
*/
|
||||
std::array<std::array<np::Array<float64_t>, 2>, TS.size()> train(const np::Array<int32_t>& X_train_feat, const np::Array<uint16_t>& X_train_feat_argsort, const np::Array<uint8_t>& y_train) noexcept {
|
||||
const std::chrono::system_clock::time_point training_timestamp = perf_counter_ns();
|
||||
const std::array<int32_t, 3> training_gaps = { 26, -18, 29 };
|
||||
@ -136,9 +155,9 @@ std::array<std::array<np::Array<float64_t>, 2>, TS.size()> train(const np::Array
|
||||
const auto [ alphas, final_classifiers ] = state_saver<float64_t, 2>(title, training_gaps[0], { alphas_title, final_classifiers_title },
|
||||
FORCE_REDO, SAVE_STATE, MODEL_DIR, train_viola_jones, T, X_train_feat, X_train_feat_argsort, y_train);
|
||||
#if __DEBUG
|
||||
print("alphas");
|
||||
printf("alphas\n");
|
||||
print(alphas);
|
||||
print("final_classifiers");
|
||||
printf("final_classifiers\n");
|
||||
print(final_classifiers);
|
||||
#endif
|
||||
models[i++] = { alphas, final_classifiers };
|
||||
@ -151,6 +170,14 @@ std::array<std::array<np::Array<float64_t>, 2>, TS.size()> train(const np::Array
|
||||
return models;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Benchmark the trained classifiers on the training and testing sets.
|
||||
*
|
||||
* @param X_train_feat Training features.
|
||||
* @param y_train Training labels.
|
||||
* @param X_test_feat Testing features.
|
||||
* @param y_test Testing labels.
|
||||
*/
|
||||
void testing_and_evaluating(const std::array<std::array<np::Array<float64_t>, 2>, TS.size()>& models, const np::Array<int32_t>& X_train_feat, const np::Array<uint8_t>& y_train, const np::Array<int32_t>& X_test_feat, const np::Array<uint8_t>& y_test) {
|
||||
const std::array<int32_t, 5> testing_gaps = { 26, -19, 24, -19, 24 };
|
||||
header({ "Testing", "Time spent (ns) (E)", "Formatted time spent (E)", "Time spent (ns) (T)", "Formatted time spent (T)" }, testing_gaps);
|
||||
@ -201,6 +228,13 @@ void testing_and_evaluating(const std::array<std::array<np::Array<float64_t>, 2>
|
||||
footer(evaluating_gaps);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Test if the each result is equals to other devices.
|
||||
*
|
||||
* Given ViolaJones is a fully deterministic algorithm.
|
||||
* The results, regardless the device, should be the same.
|
||||
* This function check this assertion.
|
||||
*/
|
||||
void unit_test(void) {
|
||||
const std::chrono::system_clock::time_point unit_timestamp = perf_counter_ns();
|
||||
const std::array<int32_t, 4> unit_gaps = { 37, -10, -18, 29};
|
||||
@ -224,76 +258,99 @@ void unit_test(void) {
|
||||
formatted_row(unit_gaps, { title, "Failed", thousand_sep(time_spent).c_str(), format_time_ns(time_spent).c_str() });
|
||||
};
|
||||
|
||||
for (const char* label : { "train", "test" }) {
|
||||
sprintf(file_cpu, OUT_DIR "/X_%s_ii_CPU.bin", label);
|
||||
sprintf(file_gpu, OUT_DIR "/X_%s_ii_GPU.bin", label);
|
||||
if (fs::exists(file_cpu) && fs::exists(file_gpu)) {
|
||||
const np::Array<uint32_t> X_train_ii_cpu = load<uint32_t>(file_cpu);
|
||||
const np::Array<uint32_t> X_train_ii_gpu = load<uint32_t>(file_gpu);
|
||||
sprintf(tmp_title, "X_%s_ii", label);
|
||||
sprintf(title, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&X_train_ii_cpu, &X_train_ii_gpu]{ return unit_test_cpu_vs_gpu<uint32_t>(X_train_ii_cpu, X_train_ii_gpu); });
|
||||
for (const char* const label : { "train", "test" }) {
|
||||
snprintf(file_cpu, BUFFER_SIZE, OUT_DIR "/X_%s_ii_CPU.bin", label);
|
||||
snprintf(file_gpu, BUFFER_SIZE, OUT_DIR "/X_%s_ii_GPU.bin", label);
|
||||
if (std::filesystem::exists(file_cpu) && std::filesystem::exists(file_gpu)) {
|
||||
snprintf(tmp_title, BUFFER_SIZE / 2, "X_%s_ii", label);
|
||||
snprintf(title, BUFFER_SIZE, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&file_cpu, &file_gpu]{
|
||||
const np::Array<uint32_t> X_train_ii_cpu = load<uint32_t>(file_cpu);
|
||||
const np::Array<uint32_t> X_train_ii_gpu = load<uint32_t>(file_gpu);
|
||||
return unit_test_cpu_vs_gpu<uint32_t>(X_train_ii_cpu, X_train_ii_gpu);
|
||||
});
|
||||
}
|
||||
snprintf(file_cpu, BUFFER_SIZE, OUT_DIR "/X_%s_feat_CPU.bin", label);
|
||||
snprintf(file_gpu, BUFFER_SIZE, OUT_DIR "/X_%s_feat_GPU.bin", label);
|
||||
uint8_t feat = 0;
|
||||
char file_feat[BUFFER_SIZE] = { 0 };
|
||||
sprintf(file_feat, OUT_DIR "/X_%s_feat_CPU.bin", label);
|
||||
if (fs::exists(file_feat)) {
|
||||
if (std::filesystem::exists(file_cpu)) {
|
||||
strncpy(file_feat, file_cpu, BUFFER_SIZE);
|
||||
feat = 1;
|
||||
} else if (std::filesystem::exists(file_gpu)) {
|
||||
strncpy(file_feat, file_gpu, BUFFER_SIZE);
|
||||
feat = 2;
|
||||
}
|
||||
if (feat != 0) {
|
||||
const np::Array<int32_t> X_feat = load<int32_t>(file_feat);
|
||||
sprintf(file_gpu, OUT_DIR "/X_%s_feat_GPU.bin", label);
|
||||
if (fs::exists(file_gpu)) {
|
||||
const np::Array<int32_t> X_feat_gpu = load<int32_t>(file_gpu);
|
||||
sprintf(tmp_title, "X_%s_feat", label);
|
||||
sprintf(title, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&X_feat, &X_feat_gpu]{ return unit_test_cpu_vs_gpu<int32_t>(X_feat, X_feat_gpu); });
|
||||
snprintf(file_gpu, BUFFER_SIZE, feat == 1 ? OUT_DIR "/X_%s_feat_GPU.bin" : OUT_DIR "/X_%s_feat_CPU.bin", label);
|
||||
if (std::filesystem::exists(file_gpu)) {
|
||||
snprintf(tmp_title, BUFFER_SIZE / 2, "X_%s_feat", label);
|
||||
snprintf(title, BUFFER_SIZE, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&X_feat, &file_gpu]{
|
||||
const np::Array<int32_t> X_feat_aux = load<int32_t>(file_gpu);
|
||||
return unit_test_cpu_vs_gpu<int32_t>(X_feat, X_feat_aux);
|
||||
});
|
||||
}
|
||||
sprintf(file_cpu, OUT_DIR "/X_%s_feat_argsort_CPU.bin", label);
|
||||
snprintf(file_cpu, BUFFER_SIZE, OUT_DIR "/X_%s_feat_argsort_CPU.bin", label);
|
||||
np::Array<uint16_t> X_feat_argsort_cpu;
|
||||
uint8_t loaded = 0;
|
||||
if (fs::exists(file_cpu)) {
|
||||
X_feat_argsort_cpu = std::move(load<uint16_t>(file_cpu));
|
||||
if (std::filesystem::exists(file_cpu)) {
|
||||
++loaded;
|
||||
sprintf(tmp_title, "X_%s_feat_argsort", label);
|
||||
sprintf(title, "%-22s - CPU argsort", tmp_title);
|
||||
test_fnc(title, [&X_feat, &X_feat_argsort_cpu]{ return unit_test_argsort_2d<int32_t>(X_feat, X_feat_argsort_cpu); });
|
||||
snprintf(tmp_title, BUFFER_SIZE / 2, "X_%s_feat_argsort", label);
|
||||
snprintf(title, BUFFER_SIZE, "%-22s - CPU argsort", tmp_title);
|
||||
test_fnc(title, [&X_feat, &X_feat_argsort_cpu, &file_cpu]{
|
||||
X_feat_argsort_cpu = std::move(load<uint16_t>(file_cpu));
|
||||
return unit_test_argsort_2d<int32_t>(X_feat, X_feat_argsort_cpu);
|
||||
});
|
||||
}
|
||||
sprintf(file_gpu, OUT_DIR "/X_%s_feat_argsort_GPU.bin", label);
|
||||
snprintf(file_gpu, BUFFER_SIZE, OUT_DIR "/X_%s_feat_argsort_GPU.bin", label);
|
||||
np::Array<uint16_t> X_feat_argsort_gpu;
|
||||
if (fs::exists(file_gpu)) {
|
||||
X_feat_argsort_gpu = std::move(load<uint16_t>(file_gpu));
|
||||
if (std::filesystem::exists(file_gpu)) {
|
||||
++loaded;
|
||||
sprintf(tmp_title, "X_%s_feat_argsort", label);
|
||||
sprintf(title, "%-22s - GPU argsort", tmp_title);
|
||||
test_fnc(title, [&X_feat, &X_feat_argsort_gpu]{ return unit_test_argsort_2d<int32_t>(X_feat, X_feat_argsort_gpu); });
|
||||
snprintf(tmp_title, BUFFER_SIZE / 2, "X_%s_feat_argsort", label);
|
||||
snprintf(title, BUFFER_SIZE, "%-22s - GPU argsort", tmp_title);
|
||||
test_fnc(title, [&X_feat, &X_feat_argsort_gpu, &file_gpu]{
|
||||
X_feat_argsort_gpu = std::move(load<uint16_t>(file_gpu));
|
||||
return unit_test_argsort_2d<int32_t>(X_feat, X_feat_argsort_gpu);
|
||||
});
|
||||
}
|
||||
if (loaded == 2){
|
||||
sprintf(tmp_title, "X_%s_feat_argsort", label);
|
||||
sprintf(title, "%-22s - CPU vs GPU", tmp_title);
|
||||
snprintf(tmp_title, BUFFER_SIZE / 2, "X_%s_feat_argsort", label);
|
||||
snprintf(title, BUFFER_SIZE, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&X_feat_argsort_cpu, &X_feat_argsort_gpu]{ return unit_test_cpu_vs_gpu<uint16_t>(X_feat_argsort_cpu, X_feat_argsort_gpu); });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const size_t T : TS)
|
||||
for (const char* label : { "alphas", "final_classifiers" }) {
|
||||
sprintf(file_cpu, MODEL_DIR "/%s_%lu_CPU.bin", label, T);
|
||||
sprintf(file_gpu, MODEL_DIR "/%s_%lu_GPU.bin", label, T);
|
||||
if (fs::exists(file_cpu) && fs::exists(file_gpu)){
|
||||
const np::Array<float64_t> cpu = load<float64_t>(file_cpu);
|
||||
const np::Array<float64_t> gpu = load<float64_t>(file_gpu);
|
||||
sprintf(tmp_title, "%s_%ld", label, T);
|
||||
sprintf(title, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&cpu, &gpu]{ return unit_test_cpu_vs_gpu<float64_t>(cpu, gpu); });
|
||||
for (const char* const label : { "alphas", "final_classifiers" }) {
|
||||
snprintf(file_cpu, BUFFER_SIZE, MODEL_DIR "/%s_%lu_CPU.bin", label, T);
|
||||
snprintf(file_gpu, BUFFER_SIZE, MODEL_DIR "/%s_%lu_GPU.bin", label, T);
|
||||
if (std::filesystem::exists(file_cpu) && std::filesystem::exists(file_gpu)){
|
||||
snprintf(tmp_title, BUFFER_SIZE / 2, "%s_%ld", label, T);
|
||||
snprintf(title, BUFFER_SIZE, "%-22s - CPU vs GPU", tmp_title);
|
||||
test_fnc(title, [&file_cpu, &file_gpu]{
|
||||
const np::Array<float64_t> cpu = load<float64_t>(file_cpu);
|
||||
const np::Array<float64_t> gpu = load<float64_t>(file_gpu);
|
||||
return unit_test_cpu_vs_gpu<float64_t>(cpu, gpu);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const long long time_spent = duration_ns(perf_counter_ns() - unit_timestamp);
|
||||
snprintf(title, BUFFER_SIZE, "%ld/%ld", n_success, n_total);
|
||||
|
||||
formatted_line(unit_gaps, "├", "┼", "─", "┤");
|
||||
formatted_row(unit_gaps, { "Unit testing summary", title, thousand_sep(time_spent).c_str(), format_time_ns(time_spent).c_str() });
|
||||
if (n_total == 0)
|
||||
formatted_row(unit_gaps, { "Unit testing summary", "No files", thousand_sep(time_spent).c_str(), format_time_ns(time_spent).c_str() });
|
||||
else {
|
||||
snprintf(title, BUFFER_SIZE, "%ld/%ld", n_success, n_total);
|
||||
formatted_line(unit_gaps, "├", "┼", "─", "┤");
|
||||
formatted_row(unit_gaps, { "Unit testing summary", title, thousand_sep(time_spent).c_str(), format_time_ns(time_spent).c_str() });
|
||||
}
|
||||
footer(unit_gaps);
|
||||
}
|
||||
|
||||
int main(){
|
||||
int32_t main(void){
|
||||
setlocale(LC_NUMERIC, ""); // Allow proper number display
|
||||
|
||||
const std::chrono::system_clock::time_point unit_timestamp = perf_counter_ns();
|
||||
|
||||
Reference in New Issue
Block a user