From 718724b63b5f5fc9eaa51f223d276aadfccdffa2 Mon Sep 17 00:00:00 2001 From: saundersp Date: Sun, 28 Apr 2024 00:25:13 +0200 Subject: [PATCH] python : Updated code with better display, documentation and format_time --- python/ViolaJones.py | 80 +++++++-------- python/ViolaJonesCPU.py | 161 +++++++++++++++++------------- python/ViolaJonesGPU.py | 179 ++++++++++++++++++++-------------- python/activate.sh | 11 +-- python/common.py | 92 ++++++++++-------- python/convert_dataset.py | 7 +- python/decorators.py | 19 +++- python/projet.py | 106 +++++++++++--------- python/test.py | 189 ------------------------------------ python/toolbox.py | 130 ++++++++++++++++++------- python/toolbox_unit_test.py | 183 +++++++++++++++++++++++----------- 11 files changed, 591 insertions(+), 566 deletions(-) delete mode 100644 python/test.py diff --git a/python/ViolaJones.py b/python/ViolaJones.py index 1c75e4b..b7bbb64 100644 --- a/python/ViolaJones.py +++ b/python/ViolaJones.py @@ -21,10 +21,10 @@ def build_features(width: int, height: int) -> np.ndarray: """Initialize the features base on the input shape. Args: - shape (Tuple[int, int]): Shape of the image (Width, Height). + shape (Tuple[int, int]): Shape of the image (Width, Height) Returns: - np.ndarray: The initialized features. + np.ndarray: The initialized features """ feats = [] empty = (0, 0, 0, 0) @@ -63,10 +63,10 @@ def init_weights(y_train: np.ndarray) -> np.ndarray: """Initialize the weights of the weak classifiers based on the training labels. Args: - y_train (np.ndarray): Training labels. + y_train (np.ndarray): Training labels Returns: - np.ndarray: The initialized weights. + np.ndarray: The initialized weights """ weights = np.empty_like(y_train, dtype = np.float64) t = y_train.sum() @@ -79,12 +79,12 @@ def classify_weak_clf(x_feat_i: np.ndarray, threshold: int, polarity: int) -> np """Classify the integrated features based on polarity and threshold. Args: - x_feat_i (np.ndarray): Integrated features. - threshold (int): Trained threshold. - polarity (int): Trained polarity. + x_feat_i (np.ndarray): Integrated features + threshold (int): Trained threshold + polarity (int): Trained polarity Returns: - np.ndarray: Classified features. + np.ndarray: Classified features """ res = np.zeros_like(x_feat_i, dtype = np.int8) res[polarity * x_feat_i < polarity * threshold] = 1 @@ -95,10 +95,10 @@ def select_best(classifiers: np.ndarray, weights: np.ndarray, X_feat: np.ndarray """Select the best classifier given theirs predictions. Args: - classifiers (np.ndarray): The weak classifiers. - weights (np.ndarray): Trained weights of each classifiers. - X_feat (np.ndarray): Integrated features. - y (np.ndarray): Features labels. + classifiers (np.ndarray): The weak classifiers + weights (np.ndarray): Trained weights of each classifiers + X_feat (np.ndarray): Integrated features + y (np.ndarray): Features labels Returns: Tuple[int, float, np.ndarray]: Index of the best classifier, the best error and the best accuracy @@ -116,13 +116,13 @@ def train_viola_jones(T: int, X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: """Train the weak classifiers. Args: - T (int): Number of weak classifiers. - X_feat (np.ndarray): Integrated features. - X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. - y (np.ndarray): Features labels. + T (int): Number of weak classifiers + X_feat (np.ndarray): Integrated features + X_feat_argsort (np.ndarray): Sorted indexes of the integrated features + y (np.ndarray): Features labels Returns: - Tuple[np.ndarray, np.ndarray]: List of trained alphas and the list of the final classifiers. + Tuple[np.ndarray, np.ndarray]: List of trained alphas and the list of the final classifiers """ weights = init_weights(y) alphas, final_classifier = np.empty(T, dtype = np.float64), np.empty((T, 3), dtype = np.int32) @@ -144,12 +144,12 @@ def classify_viola_jones(alphas: np.ndarray, classifiers: np.ndarray, X_feat: np """Classify the trained classifiers on the given features. Args: - alphas (np.ndarray): Trained alphas. - classifiers (np.ndarray): Trained classifiers. - X_feat (np.ndarray): Integrated features. + alphas (np.ndarray): Trained alphas + classifiers (np.ndarray): Trained classifiers + X_feat (np.ndarray): Integrated features Returns: - np.ndarray: Classification results. + np.ndarray: Classification results """ total = np.zeros(X_feat.shape[1], dtype = np.float64) @@ -161,22 +161,22 @@ def classify_viola_jones(alphas: np.ndarray, classifiers: np.ndarray, X_feat: np y_pred[total >= 0.5 * np.sum(alphas)] = 1 return y_pred -@njit -def get_best_anova_features(X: np.ndarray, y: np.ndarray) -> np.ndarray: - #SelectPercentile(f_classif, percentile = 10).fit(X, y).get_support(indices = True) - classes = [X.T[y == 0].astype(np.float64), X.T[y == 1].astype(np.float64)] - n_samples_per_class = np.asarray([classes[0].shape[0], classes[1].shape[0]]) - n_samples = classes[0].shape[0] + classes[1].shape[0] - ss_alldata = (classes[0] ** 2).sum(axis = 0) + (classes[1] ** 2).sum(axis = 0) - sums_classes = [np.asarray(classes[0].sum(axis = 0)), np.asarray(classes[1].sum(axis = 0))] - sq_of_sums_alldata = (sums_classes[0] + sums_classes[1]) ** 2 - sq_of_sums_args = [sums_classes[0] ** 2, sums_classes[1] ** 2] - ss_tot = ss_alldata - sq_of_sums_alldata / n_samples - - sqd_sum_bw_n = sq_of_sums_args[0] / n_samples_per_class[0] + \ - sq_of_sums_args[1] / n_samples_per_class[1] - sq_of_sums_alldata / n_samples - ss_wn = ss_tot - sqd_sum_bw_n - df_wn = n_samples - 2 - msw = ss_wn / df_wn - f_values = sqd_sum_bw_n / msw - return np.sort(np.argsort(f_values)[::-1][: int(np.ceil(X.shape[0] / 10.0))]) +#@njit +#def get_best_anova_features(X: np.ndarray, y: np.ndarray) -> np.ndarray: +# #SelectPercentile(f_classif, percentile = 10).fit(X, y).get_support(indices = True) +# classes = [X.T[y == 0].astype(np.float64), X.T[y == 1].astype(np.float64)] +# n_samples_per_class = np.asarray([classes[0].shape[0], classes[1].shape[0]]) +# n_samples = classes[0].shape[0] + classes[1].shape[0] +# ss_all_data = (classes[0] ** 2).sum(axis = 0) + (classes[1] ** 2).sum(axis = 0) +# sums_classes = [np.asarray(classes[0].sum(axis = 0)), np.asarray(classes[1].sum(axis = 0))] +# sq_of_sums_all_data = (sums_classes[0] + sums_classes[1]) ** 2 +# sq_of_sums_args = [sums_classes[0] ** 2, sums_classes[1] ** 2] +# ss_tot = ss_all_data - sq_of_sums_all_data / n_samples +# +# sqd_sum_bw_n = sq_of_sums_args[0] / n_samples_per_class[0] + \ +# sq_of_sums_args[1] / n_samples_per_class[1] - sq_of_sums_all_data / n_samples +# ss_wn = ss_tot - sqd_sum_bw_n +# df_wn = n_samples - 2 +# msw = ss_wn / df_wn +# f_values = sqd_sum_bw_n / msw +# return np.sort(np.argsort(f_values)[::-1][: int(np.ceil(X.shape[0] / 10.0))]) diff --git a/python/ViolaJonesCPU.py b/python/ViolaJonesCPU.py index 21966bf..83be32a 100644 --- a/python/ViolaJonesCPU.py +++ b/python/ViolaJonesCPU.py @@ -18,10 +18,10 @@ def set_integral_image(X: np.ndarray) -> np.ndarray: """Transform the input images in integrated images (CPU version). Args: - X (np.ndarray): Dataset of images. + X (np.ndarray): Dataset of images Returns: - np.ndarray: Dataset of integrated images. + np.ndarray: Dataset of integrated images """ X_ii = np.empty_like(X, dtype = np.uint32) for i, Xi in enumerate(tqdm_iter(X, "Applying integral image")): @@ -34,59 +34,18 @@ def set_integral_image(X: np.ndarray) -> np.ndarray: X_ii[i] = ii return X_ii -@njit('uint32(uint32[:, :], int16, int16, int16, int16)') -def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int: - """Compute a feature on an integrated image at a specific coordinate (CPU version). - - Args: - ii (np.ndarray): Integrated image. - x (int): X coordinate. - y (int): Y coordinate. - w (int): width of the feature. - h (int): height of the feature. - - Returns: - int: Computed feature. - """ - return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w] - -@njit('int32[:, :](uint8[:, :, :, :], uint32[:, :, :])') -def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray: - """Apply the features on a integrated image dataset (CPU version). - - Args: - feats (np.ndarray): Features to apply. - X_ii (np.ndarray): Integrated image dataset. - - Returns: - np.ndarray: Applied features. - """ - X_feat = np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32) - - for i, (p, n) in enumerate(tqdm_iter(feats, "Applying features")): - for j, x_i in enumerate(X_ii): - p_x, p_y, p_w, p_h = p[0] - p1_x, p1_y, p1_w, p1_h = p[1] - n_x, n_y, n_w, n_h = n[0] - n1_x, n1_y, n1_w, n1_h = n[1] - p1 = __compute_feature__(x_i, p_x, p_y, p_w, p_h) + __compute_feature__(x_i, p1_x, p1_y, p1_w, p1_h) - n1 = __compute_feature__(x_i, n_x, n_y, n_w, n_h) + __compute_feature__(x_i, n1_x, n1_y, n1_w, n1_h) - X_feat[i, j] = int32(p1) - int32(n1) - - return X_feat - @njit('int32[:, :](int32[:, :], uint16[:, :], uint8[:], float64[:])') def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray, weights: np.ndarray) -> np.ndarray: """Train the weak classifiers on a given dataset (CPU version). Args: - X_feat (np.ndarray): Feature images dataset. - X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. - y (np.ndarray): Labels of the features. - weights (np.ndarray): Weights of the features. + X_feat (np.ndarray): Feature images dataset + X_feat_argsort (np.ndarray): Sorted indexes of the integrated features + y (np.ndarray): Labels of the features + weights (np.ndarray): Weights of the features Returns: - np.ndarray: Trained weak classifiers. + np.ndarray: Trained weak classifiers """ total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum() @@ -112,29 +71,85 @@ def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray classifiers[i] = (best_threshold, best_polarity) return classifiers +@njit('uint32(uint32[:, :], int16, int16, int16, int16)') +def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int: + """Compute a feature on an integrated image at a specific coordinate (CPU version). + + Args: + ii (np.ndarray): Integrated image + x (int): X coordinate + y (int): Y coordinate + w (int): width of the feature + h (int): height of the feature + + Returns: + int: Computed feature + """ + return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w] + +@njit('int32[:, :](uint8[:, :, :, :], uint32[:, :, :])') +def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray: + """Apply the features on a integrated image dataset (CPU version). + + Args: + feats (np.ndarray): Features to apply + X_ii (np.ndarray): Integrated image dataset + + Returns: + np.ndarray: Applied features + """ + X_feat = np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32) + + for i, (p, n) in enumerate(tqdm_iter(feats, "Applying features")): + for j, x_i in enumerate(X_ii): + p_x, p_y, p_w, p_h = p[0] + p1_x, p1_y, p1_w, p1_h = p[1] + n_x, n_y, n_w, n_h = n[0] + n1_x, n1_y, n1_w, n1_h = n[1] + p1 = __compute_feature__(x_i, p_x, p_y, p_w, p_h) + __compute_feature__(x_i, p1_x, p1_y, p1_w, p1_h) + n1 = __compute_feature__(x_i, n_x, n_y, n_w, n_h) + __compute_feature__(x_i, n1_x, n1_y, n1_w, n1_h) + X_feat[i, j] = int32(p1) - int32(n1) + + return X_feat + @njit('int32(int32[:], uint16[:], int32, int32)') -def as_partition(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> int: - i = l - 1 - j = l - for j in range(l, h + 1): - if a[indices[j]] < a[indices[h]]: +def _as_partition_(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> int: + """Partition of the argsort algorithm. + + Args: + d_a (np.ndarray): Array on device to sort + d_indices (np.ndarray): Array of indices on device to write to + low (int): lower bound to sort + high (int): higher bound to sort + + Returns: + int: Last index sorted + """ + i, j = low - 1, low + for j in range(low, high + 1): + if d_a[d_indices[j]] < d_a[d_indices[high]]: i += 1 - indices[i], indices[j] = indices[j], indices[i] + d_indices[i], d_indices[j] = d_indices[j], d_indices[i] i += 1 - indices[i], indices[j] = indices[j], indices[i] + d_indices[i], d_indices[j] = d_indices[j], d_indices[i] return i @njit('void(int32[:], uint16[:], int32, int32)') -def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int): - total = h - l + 1; - stack = np.empty((total,), dtype = np.int32) - stack[0] = l - stack[1] = h - top = 1; +def argsort_bounded(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> None: + """Perform an indirect sort of a given array within a given bound. - low = l - high = h + Args: + d_a (np.ndarray): Array to sort + d_indices (np.ndarray): Array of indices to write to + low (int): lower bound to sort + high (int): higher bound to sort + """ + total = high - low + 1 + stack = np.empty((total,), dtype = np.int32) + stack[0] = low + stack[1] = high + top = 1 while top >= 0: high = stack[top] @@ -143,24 +158,32 @@ def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int): top -= 1 if low >= high: - break; + break - p = as_partition(a, indices, low, high); + p = _as_partition_(d_a, d_indices, low, high) if p - 1 > low: top += 1 - stack[top] = low; + stack[top] = low top += 1 - stack[top] = p - 1; + stack[top] = p - 1 if p + 1 < high: top += 1 - stack[top] = p + 1; + stack[top] = p + 1 top += 1 - stack[top] = high; + stack[top] = high @njit('uint16[:, :](int32[:, :])') def argsort(X_feat: np.ndarray) -> np.ndarray: + """Perform an indirect sort of a given array. + + Args: + X_feat (np.ndarray): Array to sort + + Returns: + np.ndarray: Array of indices that sort the array + """ indices = np.empty_like(X_feat, dtype = np.uint16) indices[:, :] = np.arange(indices.shape[1]) for i in tqdm_iter(range(X_feat.shape[0]), "argsort"): diff --git a/python/ViolaJonesGPU.py b/python/ViolaJonesGPU.py index ff4973c..1bc7388 100644 --- a/python/ViolaJonesGPU.py +++ b/python/ViolaJonesGPU.py @@ -12,10 +12,10 @@ def __scanCPU_3d__(X: np.ndarray) -> np.ndarray: """Prefix Sum (scan) of a given dataset. Args: - X (np.ndarray): Dataset of images to apply sum. + X (np.ndarray): Dataset of images to apply sum Returns: - np.ndarray: Scanned dataset of images. + np.ndarray: Scanned dataset of images """ for x in range(X.shape[0]): for y in range(X.shape[1]): @@ -30,10 +30,10 @@ def __kernel_scan_3d__(n: int, j: int, d_inter: np.ndarray, d_a: np.ndarray) -> """GPU kernel used to do a parallel prefix sum (scan). Args: - n (int): - j (int): [description] - d_inter (np.ndarray): [description] - d_a (np.ndarray): [description] + n (int): Number of width blocks + j (int): Temporary sum index + d_inter (np.ndarray): Temporary sums in device to add + d_a (np.ndarray): Dataset of images in device to apply sum """ x_coor, y_coor = cuda.grid(2) @@ -76,10 +76,10 @@ def __add_3d__(d_X: np.ndarray, d_s: np.ndarray, n: int, m: int) -> None: """GPU kernel for parallel sum. Args: - d_X (np.ndarray): Dataset of images. - d_s (np.ndarray): Temporary sums to add. - n (int): Number of width blocks. - m (int): Height of a block. + d_X (np.ndarray): Dataset of images in device + d_s (np.ndarray): Temporary sums in device to add + n (int): Number of width blocks + m (int): Height of a block """ x_coor, y_coor = cuda.grid(2) if x_coor < n and y_coor < m: @@ -91,10 +91,10 @@ def __scanGPU_3d__(X: np.ndarray) -> np.ndarray: Read more: https://developer.nvidia.com/gpugems/gpugems3/part-vi-gpu-computing/chapter-39-parallel-prefix-sum-scan-cuda Args: - X (np.ndarray): Dataset of images. + X (np.ndarray): Dataset of images Returns: - np.ndarray: Scanned dataset of images. + np.ndarray: Scanned dataset of images """ k, height, n = X.shape n_block_x, n_block_y = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64) @@ -131,10 +131,10 @@ def __transpose_kernel__(d_X: np.ndarray, d_Xt: np.ndarray) -> None: """GPU kernel of the function __transpose_3d__. Args: - d_X (np.ndarray): Dataset of images. - d_Xt(np.ndarray): Transposed dataset of images. - width (int): Width of each images in the dataset. - height (int): Height of each images in the dataset. + d_X (np.ndarray): Dataset of images in device + d_Xt(np.ndarray): Transposed dataset of images + width (int): Width of each images in the dataset + height (int): Height of each images in the dataset """ temp = cuda.shared.array(NB_THREADS_2D, dtype = uint32) @@ -152,10 +152,10 @@ def __transpose_3d__(X: np.ndarray) -> np.ndarray: """Transpose every images in the given dataset. Args: - X (np.ndarray): Dataset of images. + X (np.ndarray): Dataset of images Returns: - np.ndarray: Transposed dataset of images. + np.ndarray: Transposed dataset of images """ n_block_x, n_block_z = np.ceil(np.divide(X.shape[1:], NB_THREADS_2D)).astype(np.uint64) d_X = cuda.to_device(X) @@ -167,10 +167,10 @@ def set_integral_image(X: np.ndarray) -> np.ndarray: """Transform the input images in integrated images (GPU version). Args: - X (np.ndarray): Dataset of images. + X (np.ndarray): Dataset of images Returns: - np.ndarray: Dataset of integrated images. + np.ndarray: Dataset of integrated images """ X = X.astype(np.uint32) X = __scanGPU_3d__(X) @@ -184,13 +184,13 @@ def __train_weak_clf_kernel__(d_classifiers: np.ndarray, d_y: np.ndarray, d_X_fe """GPU kernel of the function train_weak_clf. Args: - d_classifiers (np.ndarray): Weak classifiers to train. - d_y (np.ndarray): Labels of the features. - d_X_feat (np.ndarray): Feature images dataset. - d_X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. - d_weights (np.ndarray): Weights of the features. - total_pos (float): Total of positive labels in the dataset. - total_neg (float): Total of negative labels in the dataset. + d_classifiers (np.ndarray): Weak classifiers to train + d_y (np.ndarray): Labels of the features + d_X_feat (np.ndarray): Feature images dataset + d_X_feat_argsort (np.ndarray): Sorted indexes of the integrated features + d_weights (np.ndarray): Weights of the features + total_pos (float): Total of positive labels in the dataset + total_neg (float): Total of negative labels in the dataset """ i = cuda.blockIdx.x * cuda.blockDim.x * cuda.blockDim.y * cuda.blockDim.z i += cuda.threadIdx.x * cuda.blockDim.y * cuda.blockDim.z @@ -224,13 +224,13 @@ def train_weak_clf(X_feat: np.ndarray, X_feat_argsort: np.ndarray, y: np.ndarray """Train the weak classifiers on a given dataset (GPU version). Args: - X_feat (np.ndarray): Feature images dataset. - X_feat_argsort (np.ndarray): Sorted indexes of the integrated features. - y (np.ndarray): Labels of the features. - weights (np.ndarray): Weights of the features. + X_feat (np.ndarray): Feature images dataset + X_feat_argsort (np.ndarray): Sorted indexes of the integrated features + y (np.ndarray): Labels of the features + weights (np.ndarray): Weights of the features Returns: - np.ndarray: Trained weak classifiers. + np.ndarray: Trained weak classifiers """ total_pos, total_neg = weights[y == 1].sum(), weights[y == 0].sum() d_classifiers = cuda.to_device(np.empty((X_feat.shape[0], 2), dtype = np.int32)) @@ -247,14 +247,14 @@ def __compute_feature__(ii: np.ndarray, x: int, y: int, w: int, h: int) -> int: """Compute a feature on an integrated image at a specific coordinate (GPU version). Args: - ii (np.ndarray): Integrated image. - x (int): X coordinate. - y (int): Y coordinate. - w (int): width of the feature. - h (int): height of the feature. + ii (np.ndarray): Integrated image + x (int): X coordinate + y (int): Y coordinate + w (int): width of the feature + h (int): height of the feature Returns: - int: Computed feature. + int: Computed feature """ return ii[y + h, x + w] + ii[y, x] - ii[y + h, x] - ii[y, x + w] @@ -263,11 +263,11 @@ def __apply_feature_kernel__(X_feat: np.ndarray, feats: np.ndarray, X_ii: np.nda """GPU kernel of the function apply_features. Args: - X_feat (np.ndarray): Feature images dataset. - feats (np.ndarray): Features to apply. - X_ii (np.ndarray): Integrated image dataset. - n (int): Number of features. - m (int): Number of images of the dataset. + X_feat (np.ndarray): Feature images dataset on device + feats (np.ndarray): Features on device to apply + X_ii (np.ndarray): Integrated image dataset on device + n (int): Number of features + m (int): Number of images of the dataset """ x, y = cuda.grid(2) if x >= feats.shape[0] or y >= X_ii.shape[0]: @@ -288,11 +288,11 @@ def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray: """Apply the features on a integrated image dataset (GPU version). Args: - feats (np.ndarray): Features to apply. - X_ii (np.ndarray): Integrated image dataset. + feats (np.ndarray): Features to apply + X_ii (np.ndarray): Integrated image dataset Returns: - np.ndarray: Applied features. + np.ndarray: Applied features """ d_X_feat = cuda.to_device(np.empty((feats.shape[0], X_ii.shape[0]), dtype = np.int32)) d_feats = cuda.to_device(feats) @@ -303,28 +303,44 @@ def apply_features(feats: np.ndarray, X_ii: np.ndarray) -> np.ndarray: return d_X_feat.copy_to_host() @cuda.jit('int32(int32[:], uint16[:], int32, int32)', device = True) -def as_partition(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> int: +def _as_partition_(d_a: np.ndarray, d_indices: np.ndarray, l: int, h: int) -> int: + """Partition of the argsort algorithm. + + Args: + d_a (np.ndarray): Array on device to sort + d_indices (np.ndarray): Array of indices on device to write to + low (int): lower bound to sort + high (int): higher bound to sort + + Returns: + int: Last index sorted + """ i = l - 1 j = l for j in range(l, h + 1): - if a[indices[j]] < a[indices[h]]: + if d_a[d_indices[j]] < d_a[d_indices[h]]: i += 1 - indices[i], indices[j] = indices[j], indices[i] + d_indices[i], d_indices[j] = d_indices[j], d_indices[i] i += 1 - indices[i], indices[j] = indices[j], indices[i] + d_indices[i], d_indices[j] = d_indices[j], d_indices[i] return i @cuda.jit('void(int32[:], uint16[:], int32, int32)', device = True) -def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> None: - #total = h - l + 1; - stack = cuda.local.array(6977, int32) - stack[0] = l - stack[1] = h - top = 1; +def argsort_bounded(d_a: np.ndarray, d_indices: np.ndarray, low: int, high: int) -> None: + """Perform an indirect sort of a given array within a given bound. - low = l - high = h + Args: + d_a (np.ndarray): Array on device to sort + d_indices (np.ndarray): Array of indices on device to write to + low (int): lower bound to sort + high (int): higher bound to sort + """ + #total = high - low + 1; + stack = cuda.local.array(6977, int32) + stack[0] = low + stack[1] = high + top = 1 while top >= 0: high = stack[top] @@ -333,34 +349,49 @@ def argsort_bounded(a: np.ndarray, indices: np.ndarray, l: int, h: int) -> None: top -= 1 if low >= high: - break; + break - p = as_partition(a, indices, low, high); + p = _as_partition_(d_a, d_indices, low, high) if p - 1 > low: top += 1 - stack[top] = low; + stack[top] = low top += 1 - stack[top] = p - 1; + stack[top] = p - 1 if p + 1 < high: top += 1 - stack[top] = p + 1; + stack[top] = p + 1 top += 1 - stack[top] = high; + stack[top] = high @cuda.jit('void(int32[:, :], uint16[:, :])') -def argsort_flatter(X_feat: np.ndarray, indices: np.ndarray) -> None: - i = cuda.blockIdx.x * cuda.blockDim.x + cuda.threadIdx.x - if i < X_feat.shape[0]: - for j in range(indices.shape[1]): - indices[i, j] = j - argsort_bounded(X_feat[i], indices[i], 0, X_feat.shape[1] - 1) +def argsort_flatter(d_a: np.ndarray, d_indices: np.ndarray) -> None: + # TODO Finish doxygen + """Cuda kernel where argsort is applied to every columns of a given 2D array. -def argsort(X_feat: np.ndarray) -> np.ndarray: - indices = np.empty_like(X_feat, dtype = np.uint16) - n_blocks = int(np.ceil(np.divide(X_feat.shape[0], NB_THREADS))) - d_X_feat = cuda.to_device(X_feat) + Args: + d_a (np.ndarray): Array in device to sort + d_indices (np.ndarray): Array of indices on device to write to + """ + i = cuda.blockIdx.x * cuda.blockDim.x + cuda.threadIdx.x + if i < d_a.shape[0]: + for j in range(d_indices.shape[1]): + d_indices[i, j] = j + argsort_bounded(d_a[i], d_indices[i], 0, d_a.shape[1] - 1) + +def argsort(a: np.ndarray) -> np.ndarray: + """Perform an indirect sort of a given array + + Args: + a (np.ndarray): Array to sort + + Returns: + np.ndarray: Array of indices that sort the array + """ + indices = np.empty_like(a, dtype = np.uint16) + n_blocks = int(np.ceil(np.divide(a.shape[0], NB_THREADS))) + d_X_feat = cuda.to_device(a) d_indices = cuda.to_device(indices) argsort_flatter[n_blocks, NB_THREADS](d_X_feat, d_indices) cuda.synchronize() diff --git a/python/activate.sh b/python/activate.sh index 965bb8e..b8433b1 100755 --- a/python/activate.sh +++ b/python/activate.sh @@ -3,9 +3,8 @@ # Exit if any of the command doesn't exit with code 0 set -e -EXEC_DIR=$1 -test -z "$EXEC_DIR" && EXEC_DIR=.. -VENV_PATH=$EXEC_DIR/python/venv +test -z "$EXEC_DIR" && EXEC_DIR=. +test -z "$VENV_PATH" && VENV_PATH="$EXEC_DIR/venv" activate(){ if [ ! -d "$VENV_PATH" ]; then @@ -16,9 +15,9 @@ activate(){ echo 'Updating base pip packages' python -m pip install -U setuptools pip echo 'Installing requirements' - pip install -r "$EXEC_DIR"/python/requirements.txt - elif [ -f "$VENV_PATH"/Scripts/activate ]; then source "$VENV_PATH"/Scripts/activate - elif [ -f "$VENV_PATH"/bin/activate ]; then source "$VENV_PATH"/bin/activate + pip install -r requirements.txt + elif [ -f "$VENV_PATH"/Scripts/activate ]; then . "$VENV_PATH"/Scripts/activate + elif [ -f "$VENV_PATH"/bin/activate ]; then . "$VENV_PATH"/bin/activate else echo 'Python virtual environnement not detected' exit 1 diff --git a/python/common.py b/python/common.py index bf25ba9..7acfffe 100644 --- a/python/common.py +++ b/python/common.py @@ -1,29 +1,29 @@ -from toolbox import picke_multi_loader, format_time_ns, unit_test_argsort_2d +from toolbox import pickle_multi_loader, format_time_ns, unit_test_argsort_2d, header, footer, formatted_line, formatted_row from typing import List, Tuple from time import perf_counter_ns +from sys import stderr import numpy as np from config import OUT_DIR, DATA_DIR, __DEBUG -def unit_test(TS: List[int], labels: List[str] = ["CPU", "GPU", "PY", "PGPU"], tol: float = 1e-8) -> None: +def unit_test(TS: List[int], labels: List[str] = ['CPU', 'GPU', 'PY', 'PGPU'], tol: float = 1e-8) -> None: """Test if the each result is equals to other devices. - Given ViolaJones is a deterministic algorithm, the results no matter the device should be the same + Given ViolaJones is a fully deterministic algorithm. The results, regardless the device, should be the same (given the floating point fluctuations), this function check this assertion. Args: - TS (List[int]): Number of trained weak classifiers. - labels (List[str], optional): List of the trained device names. Defaults to ["CPU", "GPU", "PY", "PGPU"] (see config.py for more info). - tol (float, optional): Float difference tolerance. Defaults to 1e-8. + TS (List[int]): Number of trained weak classifiers + labels (List[str], optional): List of the trained device names. Defaults to ['CPU', 'GPU', 'PY', 'PGPU'] (see config.py for more info) + tol (float, optional): Float difference tolerance. Defaults to 1e-8 """ if len(labels) < 2: - return print("Not enough devices to test") + return print('Not enough devices to test') - print(f"\n| {'Unit testing':<37} | {'Test state':<10} | {'Time spent (ns)':<18} | {'Formatted time spent':<29} |") - print(f"|{'-'*39}|{'-'*12}|{'-'*20}|{'-'*31}|") + unit_gaps = [37, -10, -18, 29] + header(unit_gaps, ['Unit testing', 'Test state', 'Time spent (ns)', 'Formatted time spent']) - fnc_s = perf_counter_ns() - n_total = 0 - n_success = 0 + unit_timestamp = perf_counter_ns() + n_total, n_success = 0, 0 def test_fnc(title, fnc): nonlocal n_total, n_success @@ -32,96 +32,102 @@ def unit_test(TS: List[int], labels: List[str] = ["CPU", "GPU", "PY", "PGPU"], t state = fnc() e = perf_counter_ns() - s if state: - print(f"| {title:<37} | {'Passed':>10} | {e:>18,} | {format_time_ns(e):<29} |") + formatted_row(unit_gaps, [title, 'Passed', f'{e:,}', format_time_ns(e)]) n_success += 1 else: - print(f"| {title:<37} | {'Failed':>10} | {e:>18,} | {format_time_ns(e):<29} |") + formatted_row(unit_gaps, [title, 'Failed', f'{e:,}', format_time_ns(e)]) - for set_name in ["train", "test"]: - for filename in ["ii", "feat"]: - title = f"X_{set_name}_{filename}" - print(f"{filename}...", end = "\r") - bs = picke_multi_loader([f"{title}_{label}" for label in labels], OUT_DIR) + for set_name in ['train', 'test']: + for filename in ['ii', 'feat']: + title = f'X_{set_name}_{filename}' + print(f'{filename}...', file = stderr, end = '\r') + bs = pickle_multi_loader([f'{title}_{label}' for label in labels], OUT_DIR) for i, (b1, l1) in enumerate(zip(bs, labels)): if b1 is None: if __DEBUG: - print(f"| {title:<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") + formatted_row(unit_gaps, [f'{title:<22} - {l1:<12}', 'Skipped', 'None', 'None']) continue for j, (b2, l2) in enumerate(zip(bs, labels)): if i >= j: continue if b2 is None: if __DEBUG: - print(f"| {title:<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") + formatted_row(unit_gaps, [f'{title:<22} - {l1:<4} vs {l2:<4}', 'Skipped', 'None', 'None']) continue - test_fnc(f"{title:<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol) + test_fnc(f'{title:<22} - {l1:<4} vs {l2:<4}', lambda: np.abs(b1 - b2).mean() < tol) - title = f"X_{set_name}_feat_argsort" - print(f"Loading {title}...", end = "\r") + title = f'X_{set_name}_feat_argsort' + print(f'Loading {title}...', file = stderr, end = '\r') feat = None bs = [] for label in labels: if feat is None: - feat_tmp = picke_multi_loader([f"X_{set_name}_feat_{label}"], OUT_DIR)[0] + feat_tmp = pickle_multi_loader([f'X_{set_name}_feat_{label}'], OUT_DIR)[0] if feat_tmp is not None: feat = feat_tmp - bs.append(picke_multi_loader([f"{title}_{label}"], OUT_DIR)[0]) + bs.append(pickle_multi_loader([f'{title}_{label}'], OUT_DIR)[0]) for i, (b1, l1) in enumerate(zip(bs, labels)): if b1 is None: if __DEBUG: - print(f"| {title:<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") + formatted_row(unit_gaps, [f'{title:<22} - {l1:<12}', 'Skipped', 'None', 'None']) continue if feat is not None: - test_fnc(f"{title:<22} - {l1:<4} argsort", lambda: unit_test_argsort_2d(feat, b1)) + test_fnc(f'{title:<22} - {l1:<4} argsort', lambda: unit_test_argsort_2d(feat, b1)) for j, (b2, l2) in enumerate(zip(bs, labels)): if i >= j: continue if b2 is None: if __DEBUG: - print(f"| {title:<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") + formatted_row(unit_gaps, [f'{title:<22} - {l1:<4} vs {l2:<4}', 'Skipped', 'None', 'None']) continue - test_fnc(f"{title:<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol) + test_fnc(f'{title:<22} - {l1:<4} vs {l2:<4}', lambda: np.abs(b1 - b2).mean() < tol) for T in TS: - for filename in ["alphas", "final_classifiers"]: - print(f"{filename}_{T}...", end = "\r") - bs = picke_multi_loader([f"{filename}_{T}_{label}" for label in labels]) + for filename in ['alphas', 'final_classifiers']: + print(f'{filename}_{T}...', file = stderr, end = '\r') + bs = pickle_multi_loader([f'{filename}_{T}_{label}' for label in labels]) for i, (b1, l1) in enumerate(zip(bs, labels)): if b1 is None: if __DEBUG: - print(f"| {filename + '_' + str(T):<22} - {l1:<12} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") + formatted_row(unit_gaps, [f"{filename + '_' + str(T):<22} - {l1:<12}", 'Skipped', 'None', 'None']) continue for j, (b2, l2) in enumerate(zip(bs, labels)): if i >= j: continue if b2 is None: if __DEBUG: - print(f"| {filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4} | {'Skipped':>10} | {'None':>18} | {'None':<29} |") + formatted_row(unit_gaps, [f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", 'Skipped', 'None', 'None']) continue test_fnc(f"{filename + '_' + str(T):<22} - {l1:<4} vs {l2:<4}", lambda: np.abs(b1 - b2).mean() < tol) - print(f"|{'-'*39}|{'-'*12}|{'-'*20}|{'-'*31}|") - e = perf_counter_ns() - fnc_s - print(f"| {'Unit testing summary':<37} | {str(n_success) + '/' + str(n_total):>10} | {e:>18,} | {format_time_ns(e):<29} |") + time_spent = perf_counter_ns() - unit_timestamp + + if n_total == 0: + formatted_row(unit_gaps, ['Unit testing summary', 'No files', f'{time_spent:,}', format_time_ns(time_spent)]) + else: + formatted_line(unit_gaps, '├', '┼', '─', '┤') + formatted_row(unit_gaps, ['Unit testing summary', f'{n_success}/{n_total}', f'{time_spent:,}', format_time_ns(time_spent)]) + + footer(unit_gaps) def load_datasets(data_dir: str = DATA_DIR) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Load the datasets. Args: - data_dir (str, optional): [description]. Defaults to DATA_DIR (see config.py). + data_dir (str, optional): [description]. Defaults to DATA_DIR (see config.py) Returns: - Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: [description] + Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train, y_train, X_test, y_test """ - bytes_to_int_list = lambda b: list(map(int, b.rstrip().split(" "))) + bytes_to_int_list = lambda b: list(map(int, b.rstrip().split(' '))) def load(set_name: str) -> np.ndarray: - with open(f"{data_dir}/{set_name}.bin", "r") as f: + with open(f'{data_dir}/{set_name}.bin', 'r') as f: shape = bytes_to_int_list(f.readline()) return np.asarray(bytes_to_int_list(f.readline()), dtype = np.uint8).reshape(shape) - return load("X_train"), load("y_train"), load("X_test"), load("y_test") + return load('X_train'), load('y_train'), load('X_test'), load('y_test') diff --git a/python/convert_dataset.py b/python/convert_dataset.py index 244d16f..f756b01 100644 --- a/python/convert_dataset.py +++ b/python/convert_dataset.py @@ -5,6 +5,9 @@ from sys import argv import numpy as np from os import path, listdir +# Induce determinism +np.random.seed(133742) + # Makes the "leave" argument default to False tqdm = partial(tqdm, leave = False) @@ -42,8 +45,8 @@ def __main__(data_path: str) -> None: y.append(y_i) X, y = np.asarray(X), np.asarray(y) - # idx = np.random.permutation(y.shape[0]) - # X, y = X[idx], y[idx] + idx = np.random.permutation(y.shape[0]) + X, y = X[idx], y[idx] for org, s in tqdm(zip("Xy", [X, y]), desc = f"Writing {set_name}"): with open(f"{data_path}/{org}_{set_name}.bin", "w") as out: diff --git a/python/decorators.py b/python/decorators.py index 76dfcd1..944bb9d 100644 --- a/python/decorators.py +++ b/python/decorators.py @@ -2,6 +2,14 @@ from typing import Callable, Iterable, Union, Any from tqdm import tqdm def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable: + """Wrapper for optional numba's njit decorator + + Args: + f (Union[Callable, str], optional): Function to wrap with numba. Defaults to None. + + Returns: + Callable: Wrapped function. + """ def decorator(func: Callable) -> Any: return func @@ -10,4 +18,13 @@ def njit(f: Union[Callable, str] = None, *args, **kwargs) -> Callable: return decorator def tqdm_iter(iter: Iterable, desc: str): - return tqdm(iter, leave = False, desc = desc) + """Wrapper for optional tqdm iterator progress bar. + + Args: + iter (Iterable): Object to iterate over. + desc (str): Description written to stdout. + + Returns: + _type_: Wrapped iterator. + """ + return tqdm(iter, leave = False, desc = desc) \ No newline at end of file diff --git a/python/projet.py b/python/projet.py index ffb07c2..fc4658f 100644 --- a/python/projet.py +++ b/python/projet.py @@ -2,14 +2,15 @@ # Author: @saundersp from ViolaJones import train_viola_jones, classify_viola_jones -from toolbox import state_saver, picke_multi_loader, format_time_ns, benchmark_function, unit_test_argsort_2d -from toolbox_unit_test import format_time_ns_test +#from toolbox import state_saver, pickle_multi_loader, format_time_ns, benchmark_function, unit_test_argsort_2d +from toolbox import state_saver, format_time_ns, benchmark_function, unit_test_argsort_2d from toolbox import header, footer, formatted_row, formatted_line +from toolbox_unit_test import format_time_test, format_time_ns_test from sklearn.metrics import accuracy_score, f1_score, confusion_matrix -from sklearn.feature_selection import SelectPercentile, f_classif +#from sklearn.feature_selection import SelectPercentile, f_classif from common import load_datasets, unit_test -from ViolaJones import build_features, get_best_anova_features -from typing import Tuple +from ViolaJones import build_features # , get_best_anova_features +from typing import Tuple, List from time import perf_counter_ns from os import makedirs import numpy as np @@ -28,35 +29,42 @@ else: from ViolaJonesCPU import apply_features, set_integral_image, argsort label = 'CPU' if COMPILE_WITH_C else 'PY' -def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - """Load the dataset, calculate features and integral images, apply features to images and calculate argsort of the featured images. +def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Execute the preprocessing phase + + The preprocessing phase consist of the following steps : + - Load the dataset + - Calculate features + - Calculate integral images + - Apply features to images + - Calculate argsort of the featured images. Returns: - Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test + Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: Tuple containing in order : training features, training features sorted indexes, training labels, testing features, testing labels """ # Creating state saver folders if they don't exist already if SAVE_STATE: - for folder_name in ["models", "out"]: + for folder_name in ['models', 'out']: makedirs(folder_name, exist_ok = True) preproc_timestamp = perf_counter_ns() preproc_gaps = [49, -18, 29] - header(['Preprocessing', 'Time spent (ns)', 'Formatted time spent'], preproc_gaps) + header(preproc_gaps, ['Preprocessing', 'Time spent (ns)', 'Formatted time spent']) X_train, y_train, X_test, y_test = state_saver('Loading sets', preproc_gaps[0], ['X_train', 'y_train', 'X_test', 'y_test'], - load_datasets, FORCE_REDO, SAVE_STATE) + load_datasets, FORCE_REDO, SAVE_STATE) if __DEBUG: - print("X_train") + print('X_train') print(X_train.shape) print(X_train[IDX_INSPECT]) - print("X_test") + print('X_test') print(X_test.shape) print(X_test[IDX_INSPECT]) - print("y_train") + print('y_train') print(y_train.shape) print(y_train[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET]) - print("y_test") + print('y_test') print(y_test.shape) print(y_test[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET]) @@ -64,7 +72,7 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: FORCE_REDO, SAVE_STATE) if __DEBUG: - print("feats") + print('feats') print(feats.shape) print(feats[IDX_INSPECT].ravel()) @@ -74,10 +82,10 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: lambda: set_integral_image(X_test), FORCE_REDO, SAVE_STATE) if __DEBUG: - print("X_train_ii") + print('X_train_ii') print(X_train_ii.shape) print(X_train_ii[IDX_INSPECT]) - print("X_test_ii") + print('X_test_ii') print(X_test_ii.shape) print(X_test_ii[IDX_INSPECT]) @@ -88,25 +96,25 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: del X_train_ii, X_test_ii, feats if __DEBUG: - print("X_train_feat") + print('X_train_feat') print(X_train_feat.shape) print(X_train_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET]) - print("X_test_feat") + print('X_test_feat') print(X_test_feat.shape) print(X_test_feat[IDX_INSPECT, : IDX_INSPECT_OFFSET]) - #indices = state_saver("Selecting best features training set", "indices", force_redo = True, save_state = SAVE_STATE, + #indices = state_saver('Selecting best features training set', 'indices', force_redo = FORCE_REDO, save_state = SAVE_STATE, # fnc = lambda: SelectPercentile(f_classif, percentile = 10).fit(X_train_feat.T, y_train).get_support(indices = True)) - #indices = state_saver("Selecting best features training set", "indices", force_redo = FORCE_REDO, save_state = SAVE_STATE, + #indices = state_saver('Selecting best features training set', 'indices', force_redo = FORCE_REDO, save_state = SAVE_STATE, # fnc = lambda: get_best_anova_features(X_train_feat, y_train)) - #indices = benchmark_function("Selecting best features (manual)", lambda: get_best_anova_features(X_train_feat, y_train)) + #indices = benchmark_function('Selecting best features (manual)', lambda: get_best_anova_features(X_train_feat, y_train)) #if __DEBUG: - # print("indices") + # print('indices') # print(indices.shape) # print(indices[IDX_INSPECT: IDX_INSPECT + IDX_INSPECT_OFFSET]) - # assert indices.shape[0] == indices_new.shape[0], f"Indices length not equal : {indices.shape} != {indices_new.shape}" - # assert (eq := indices == indices_new).all(), f"Indices not equal : {eq.sum() / indices.shape[0]}" + # assert indices.shape[0] == indices_new.shape[0], f'Indices length not equal : {indices.shape} != {indices_new.shape}' + # assert (eq := indices == indices_new).all(), f'Indices not equal : {eq.sum() / indices.shape[0]}' # X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices] @@ -114,19 +122,20 @@ def preprocessing() -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: lambda: argsort(X_train_feat), FORCE_REDO, SAVE_STATE) if __DEBUG: - print("X_train_feat_argsort") + print('X_train_feat_argsort') print(X_train_feat_argsort.shape) print(X_train_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET]) benchmark_function('Arg unit test', preproc_gaps[0], lambda: unit_test_argsort_2d(X_train_feat, X_train_feat_argsort)) - X_test_feat_argsort = state_saver(f"Precalculating testing set argsort ({label})", f"X_test_feat_argsort_{label}", + X_test_feat_argsort = state_saver(f'Precalculating testing set argsort ({label})', preproc_gaps[0], f'X_test_feat_argsort_{label}', lambda: argsort(X_test_feat), FORCE_REDO, SAVE_STATE) if __DEBUG: - print("X_test_feat_argsort") + print('X_test_feat_argsort') print(X_test_feat_argsort.shape) print(X_test_feat_argsort[IDX_INSPECT, : IDX_INSPECT_OFFSET]) - benchmark_function("Arg unit test", lambda: unit_test_argsort_2d(X_test_feat, X_test_feat_argsort)) + benchmark_function('Arg unit test', lambda: unit_test_argsort_2d(X_test_feat, X_test_feat_argsort)) + time_spent = perf_counter_ns() - preproc_timestamp formatted_line(preproc_gaps, '├', '┼', '─', '┤') formatted_row(preproc_gaps, ['Preprocessing summary', f'{time_spent:,}', format_time_ns(time_spent)]) @@ -138,16 +147,17 @@ def train(X_train_feat: np.ndarray, X_train_feat_argsort: np.ndarray, y_train: n """Train the weak classifiers. Args: - X_train (np.ndarray): Training images. - X_train_feat_argsort (np.ndarray): Sorted indexes of the training images features. - y_train (np.ndarray): Training labels. + X_train (np.ndarray): Training images + X_train_feat_argsort (np.ndarray): Sorted indexes of the training images features + y_train (np.ndarray): Training labels - Returns: List of trained models + Returns: + List[np.ndarray]: List of trained models """ training_timestamp = perf_counter_ns() training_gaps = [26, -18, 29] - header(['Training', 'Time spent (ns)', 'Formatted time spent'], training_gaps) + header(training_gaps, ['Training', 'Time spent (ns)', 'Formatted time spent']) models = [] for T in TS: @@ -157,9 +167,9 @@ def train(X_train_feat: np.ndarray, X_train_feat_argsort: np.ndarray, y_train: n models.append([alphas, final_classifiers]) if __DEBUG: - print("alphas") + print('alphas') print(alphas) - print("final_classifiers") + print('final_classifiers') print(final_classifiers) time_spent = perf_counter_ns() - training_timestamp @@ -173,15 +183,15 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y """Benchmark the trained classifiers on the training and testing sets. Args: - models (List[np.ndarray]): List of trained models. - X_train_feat (np.ndarray): Training features. - y_train (np.ndarray): Training labels. - X_test_feat (np.ndarray): Testing features. - y_test (np.ndarray): Testing labels. + models (List[np.ndarray]): List of trained models + X_train_feat (np.ndarray): Training features + y_train (np.ndarray): Training labels + X_test_feat (np.ndarray): Testing features + y_test (np.ndarray): Testing labels """ testing_gaps = [26, -19, 24, -19, 24] - header(['Testing', 'Time spent (ns) (E)', 'Formatted time spent (E)', 'Time spent (ns) (T)', 'Formatted time spent (T)'], testing_gaps) + header(testing_gaps, ['Testing', 'Time spent (ns) (E)', 'Formatted time spent (E)', 'Time spent (ns) (T)', 'Formatted time spent (T)']) performances = [] total_train_timestamp = 0 @@ -213,7 +223,7 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y footer(testing_gaps) evaluating_gaps = [19, 7, 6, 6, 6, 7, 6, 6, 6] - header(['Evaluating', 'ACC (E)', 'F1 (E)', 'FN (E)', 'FP (E)', 'ACC (T)', 'F1 (T)', 'FN (T)', 'FP (T)'], evaluating_gaps) + header(evaluating_gaps, ['Evaluating', 'ACC (E)', 'F1 (E)', 'FN (E)', 'FP (E)', 'ACC (T)', 'F1 (T)', 'FN (T)', 'FP (T)']) for T, (e_acc, e_f1, e_FN, e_FP, t_acc, t_f1, t_FN, t_FP) in zip(TS, performances): print(f'│ ViolaJones T = {T:<4} │ {e_acc:>7.2%} │ {e_f1:>6.2f} │ {e_FN:>6,} │ {e_FP:>6,}', end = ' │ ') @@ -224,7 +234,7 @@ def testing_and_evaluating(models: List[np.ndarray], X_train_feat: np.ndarray, y def main() -> None: unit_timestamp = perf_counter_ns() unit_gaps = [27, -18, 29] - header(['Unit testing', 'Time spent (ns)', 'Formatted time spent'], unit_gaps) + header(unit_gaps, ['Unit testing', 'Time spent (ns)', 'Formatted time spent']) benchmark_function('testing format_time', unit_gaps[0], format_time_test) benchmark_function('testing format_time_ns', unit_gaps[0], format_time_ns_test) time_spent = perf_counter_ns() - unit_timestamp @@ -235,12 +245,12 @@ def main() -> None: X_train_feat, X_train_feat_argsort, y_train, X_test_feat, y_test = preprocessing() models = train(X_train_feat, X_train_feat_argsort, y_train) - # X_train_feat, X_test_feat = picke_multi_loader([f"X_train_feat_{label}", f"X_test_feat_{label}"], OUT_DIR) - # indices = picke_multi_loader(["indices"], OUT_DIR)[0] + # X_train_feat, X_test_feat = pickle_multi_loader([f'X_train_feat_{label}', f'X_test_feat_{label}'], OUT_DIR) + # indices = pickle_multi_loader(['indices'], OUT_DIR)[0] # X_train_feat, X_test_feat = X_train_feat[indices], X_test_feat[indices] testing_and_evaluating(models, X_train_feat, y_train, X_test_feat, y_test) unit_test(TS) -if __name__ == "__main__": +if __name__ == '__main__': main() diff --git a/python/test.py b/python/test.py deleted file mode 100644 index a187c4f..0000000 --- a/python/test.py +++ /dev/null @@ -1,189 +0,0 @@ -import numpy as np -from numba import cuda, config, njit -config.CUDA_LOW_OCCUPANCY_WARNINGS = 0 -#import matplotlib.pyplot as plt -from tqdm import tqdm -from time import perf_counter_ns -from toolbox import format_time_ns -from pickle import load, dump -from sys import argv - -def get(a): - with open(f"{a}.pkl", 'rb') as f: - return load(f) - -def save(a, name) -> None: - with open(name, 'wb') as f: - dump(a, f) - -def diff(folder, a, label1, label2): - af, bf = get(f"{folder}/{a}_{label1}"), get(f"{folder}/{a}_{label2}") - #print(af) - #print(bf) - print((af - bf).mean()) - -if __name__ == "__main__": - if len(argv) == 5: - diff(argv[1], argv[4], argv[2], argv[3]) - -def py_mean(a, b): - s = 0.0 - for a_i, b_i in zip(a, b): - s += a_i * b_i - return s / a.shape[0] - -def np_mean(a, b): - return np.mean(a * b) - -@njit('float64(float64[:], float64[:])', fastmath = True, nogil = True) -def nb_mean(a, b): - return np.mean(a * b) - -@njit('float64(float64[:], float64[:])', fastmath = True, nogil = True) -def nb_mean_loop(a, b): - s = 0.0 - for a_i, b_i in zip(a, b): - s += a_i * b_i - return s / a.shape[0] - -@cuda.jit('void(float64[:], float64[:], float64[:])', fastmath = True) -def cuda_mean_kernel(r, a, b): - s = 0.0 - for a_i, b_i in zip(a, b): - s += a_i * b_i - r[0] = s / a.shape[0] - -def cuda_mean(a, b): - r = cuda.to_device(np.empty(1, dtype = np.float64)) - d_a = cuda.to_device(a) - d_b = cuda.to_device(b) - cuda_mean_kernel[1, 1](r, d_a, d_b) - return r.copy_to_host()[0] - -def test_and_compare(labels, fncs, a, b): - m = [] - for fnc in tqdm(fncs, leave = False, desc = "Calculating..."): - s = perf_counter_ns() - m.append([fnc(a, b), perf_counter_ns() - s]) - print("Results:") - [print(f"\t{label:<10} {m_i:<20} {format_time_ns(time_i)}") for ((m_i, time_i), label) in zip(m, labels)] - print("Comparaison:") - for i, (m_i, label_i) in enumerate(zip(m, labels)): - for j, (m_j, label_j) in enumerate(zip(m, labels)): - if i >= j: - continue - print(f"\t{label_i:<10} vs {label_j:<10} - {abs(m_i[0] - m_j[0])}") - -if __name__ == "__main__": - np.set_printoptions(linewidth = 10000, threshold = 1000) - - N = int(2**20) - labels = ["Python", "Numpy", "Numba", "Numba loop", "CUDA"] - fncs = [py_mean, np_mean, nb_mean, nb_mean_loop, cuda_mean] - - print(f"RANDOM for N={N}") - total_size = (2 * 8 * N) - print(f"Size = {total_size} B") - print(f"Size = {total_size // 1024} kB") - print(f"Size = {total_size // 1024 // 1024} MB") - print(f"Size = {total_size // 1024 // 1024 // 1024} GB") - a, b = np.random.rand(N).astype(np.float64), np.random.rand(N).astype(np.float64) - test_and_compare(labels, fncs, a, b) - del a, b - - print(f"\nDETERMINSTIC for N={N}") - total_size = (2 * 8 * N) + (8 * N) - print(f"Size = {total_size} B") - print(f"Size = {total_size // 1024} kB") - print(f"Size = {total_size // 1024 // 1024} MB") - print(f"Size = {total_size // 1024 // 1024 // 1024} GB") - mask = np.arange(N, dtype = np.uint64) - a = np.ones(N, dtype = np.float64) - a[mask < N//2] = 0.1 - del mask - b = np.ones(N, dtype = np.float64) - test_and_compare(labels, fncs, a, b) - del a, b - - #from ViolaJonesGPU import argsort as argsort_GPU - #from ViolaJonesCPU import argsort as argsort_CPU - #from toolbox import unit_test_argsort_2d, benchmark_function - - #labels = ["Numpy", "Numba", "CUDA"] - #a = np.random.randint(2**12, size = (2**20, 2**8), dtype = np.int32) - #m = [benchmark_function(f"Argsort {label}", lambda: f(np.copy(a))) for (label, f) in zip(labels, [ - # lambda a: np.argsort(a).astype(np.uint16), argsort_CPU, argsort_GPU - #])] - #for i, (m_i, label_i) in enumerate(zip(m, labels)): - # #for j, (m_j, label_j) in enumerate(zip(m, labels)): - # # if i >= j: - # # continue - # # print(f"\t{label_i:<10} vs {label_j:<10} - {(m_i == m_j).mean()}") - # benchmark_function(f"Unit test {label_i}", lambda: unit_test_argsort_2d(a, m_i)) - - #for i in tqdm(range(X.shape[0]), leave = False, desc = "Extract image"): - # x = X[i] - # y = Y[i] - # fig = plt.figure() - # plt.imshow(x, cmap = 'gray') - # plt.savefig(f"imgs/{y}/{i}.png") - # plt.close(fig) - - #def extract_FD(Xy): - # X_c, Y_c = [], [] - # for x,y in Xy: - # X_c.append(x) - # Y_c.append(y) - # X_c = np.asarray(X_c) - # Y_c = np.asarray(Y_c) - # return X_c, Y_c - - #X_train, y_train = get('out/X_train'), get('out/y_train') - #X_test, y_test = get('out/X_test'), get('out/y_test') - - #X_train, y_train = extract_FD(get('/home/_aspil0w/git/FaceDetection/training')) - #X_test, y_test = extract_FD(get('/home/_aspil0w/git/FaceDetection/test')) - #save(X_train, 'out/X_train'), save(y_train, 'out/y_train') - #save(X_test, 'out/X_test'), save(y_test, 'out/y_test') - - #print(X_train.shape, X_train_org.shape, X_train.shape == X_train_org.shape) - #print((X_train == X_train_org).mean()) - #print(y_train.shape, y_train_org.shape, y_train.shape == y_train_org.shape) - #print((y_train == y_train_org).mean()) - - #print(X_test.shape, X_test_org.shape, X_test.shape == X_test_org.shape) - #print((X_test == X_test_org).mean()) - #print(y_test.shape, y_test_org.shape, y_test.shape == y_test_org.shape) - #print((y_test == y_test_org).mean()) - - #@njit('uint16[:](uint8[:, :, :], uint8[:, :, :])') - #def arg_find(X, X_org): - # arg = np.empty(X.shape[0], dtype = np.uint16) - # for i, x in enumerate(X_org): - # found = False - # for j, x_org in enumerate(X): - # if np.all(x == x_org): - # arg[i] = j - # found = True - # break - # assert found, "Image not found" - # return arg - - #print("Arg find results train") - #arg_train = arg_find(X_train, X_train_org) - #print((X_train[arg_train] == X_train_org).mean()) - #print((y_train[arg_train] == y_train_org).mean()) - - #print("Arg find results test") - #arg_test = arg_find(X_test, X_test_org) - #print((X_test[arg_test] == X_test_org).mean()) - #print((y_test[arg_test] == y_test_org).mean()) - - #for i in tqdm(range(X_c.shape[0]), leave = False, desc = "Extract image"): - # x = X_c[i] - # y = Y_c[i] - # fig = plt.figure() - # plt.imshow(x, cmap = 'gray') - # plt.savefig(f"imgs2/{y}/{i}.png") - # plt.close(fig) - diff --git a/python/toolbox.py b/python/toolbox.py index 79c2b9e..1111569 100644 --- a/python/toolbox.py +++ b/python/toolbox.py @@ -1,6 +1,5 @@ from typing import Any, Callable, List, Union, Final from time import perf_counter_ns -from numba import njit import numpy as np from sys import stderr import pickle @@ -8,70 +7,122 @@ import os from config import MODEL_DIR, OUT_DIR from decorators import njit -time_formats: Final = ["ns", "µs", "ms", "s", "m", "h", "j", "w", "M", "y", "c"] def formatted_row(gaps: list[int], titles: list[str], separator: str = '│') -> None: + """Print a formatted row of titles with of gaps seperated by a separator. + + Args: + gaps: List of size gaps + titles: List of titles + separator: Separator character between each gap + """ for gap, title in zip(gaps, titles): print(f"{separator} {title:{'>' if gap < 0 else '<'}{abs(gap)}} ", end = '') print(separator) -def formatted_line(gaps: list[int], right: str, middle: str, separator: str, left: str) -> None: - print(right, end = '') +def formatted_line(gaps: list[int], left: str, middle: str, separator: str, right: str) -> None: + """Print a formatted line of repeated characters. + + Args: + gaps: List of size gaps + left: Character on the left + middle: Character between each separator + separator: Separator character between each gap + right: Character on the right + """ + print(left, end = '') last_gap = len(gaps) - 1 for i, gap in enumerate(gaps): print(f'{separator * (abs(gap) + 2)}', end = '') if i != last_gap: print(middle, end = '') - print(left) + print(right) -def header(titles: list[str], gaps: list[int]) -> None: +def header(gaps: list[int], titles: list[str]) -> None: + """Print a formatted header with the given titles and sizes. + + Args: + gaps: List of size gaps + titles: List of titles + """ formatted_line(gaps, '┌', '┬', '─', '┐') formatted_row(gaps, titles) formatted_line(gaps, '├', '┼', '─', '┤') def footer(gaps: list[int]) -> None: + """Print a formatted fooder with the given sizes + + Args: + gaps: List of size gaps + """ formatted_line(gaps, '└', '┴', '─', '┘') +time_formats: Final = ['ns', 'µs', 'ms', 's', 'm', 'h', 'j', 'w', 'M', 'y', 'c'] time_numbers: Final = np.array([1, 1e3, 1e6, 1e9, 6e10, 36e11, 864e11, 6048e11, 26784e11, 31536e12, 31536e14], dtype = np.uint64) @njit('str(uint64)') def format_time_ns(time: int) -> str: """Format the time in nanoseconds in human readable format. Args: - time (int): Time in nanoseconds. + time (int): Time in nanoseconds Returns: - str: The formatted human readable string. + str: The formatted human readable string """ - assert time >= 0, "Incorrect time stamp" + assert time >= 0, 'Incorrect time stamp' if time == 0: - return "0ns" + return '0ns' - s = "" + s = '' for i in range(time_numbers.shape[0])[::-1]: if time >= time_numbers[i]: res = int(time // time_numbers[i]) time = time % time_numbers[i] - s += f"{res}{time_formats[i]} " + s += f'{res}{time_formats[i]} ' - assert time == 0, "Leftover in formatting time !" + assert time == 0, 'Leftover in formatting time !' return s.rstrip() -def picke_multi_loader(filenames: List[str], save_dir: str = MODEL_DIR) -> List[Any]: +@njit('str(uint64)') +def format_time(time: int) -> str: + """Format the time in seconds in human readable format. + + Args: + time (int): Time in seconds + + Returns: + str: The formatted human readable string + """ + assert time >= 0, 'Incorrect time stamp' + if time == 0: + return '0s' + + s = '' + for i in range(3, time_numbers.shape[0])[::-1]: + time_number = time_numbers[i] / int(1e9) + if time >= time_number: + res = int(time // time_number) + time = time % time_number + s += f'{res}{time_formats[i]} ' + + assert time == 0, 'Leftover in formatting time !' + return s.rstrip() + +def pickle_multi_loader(filenames: List[str], save_dir: str = MODEL_DIR) -> List[Any]: """Load multiple pickle data files. Args: - filenames (List[str]): List of all the filename to load. - save_dir (str, optional): Path of the files to load. Defaults to MODELS_DIR (see config.py). + filenames (List[str]): List of all the filename to load + save_dir (str, optional): Path of the files to load. Defaults to MODELS_DIR (see config.py) Returns: - List[Any]. List of loaded pickle data files. + List[Any]. List of loaded pickle data files """ b = [] for f in filenames: - filepath = f"{save_dir}/{f}.pkl" + filepath = f'{save_dir}/{f}.pkl' if os.path.exists(filepath): - with open(filepath, "rb") as filebyte: - b.append(pickle.load(filebyte)) + with open(filepath, 'rb') as file_bytes: + b.append(pickle.load(file_bytes)) else: b.append(None) return b @@ -80,11 +131,11 @@ def benchmark_function(step_name: str, column_width: int, fnc: Callable) -> Any: """Benchmark a function and display the result of stdout. Args: - step_name (str): Name of the function to call. - fnc (Callable): Function to call. + step_name (str): Name of the function to call + fnc (Callable): Function to call Returns: - Any: Result of the function. + Any: Result of the function """ print(f'{step_name}...', file = stderr, end = '\r') s = perf_counter_ns() @@ -98,34 +149,34 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str """Either execute a function then saves the result or load the already existing result. Args: - step_name (str): Name of the function to call. - filename (Union[str, List[str]]): Name or list of names of the filenames where the result(s) are saved. - fnc ([type]): Function to call. - force_redo (bool, optional): Recall the function even if the result(s) is already saved. Defaults to False. - save_dir (str, optional): Path of the directory to save the result(s). Defaults to OUT_DIR (see config.py). + step_name (str): Name of the function to call + filename (Union[str, List[str]]): Name or list of names of the filenames where the result(s) are saved + fnc ([type]): Function to call + force_redo (bool, optional): Recall the function even if the result(s) is already saved. Defaults to False + save_dir (str, optional): Path of the directory to save the result(s). Defaults to OUT_DIR (see config.py) Returns: Any: The result(s) of the called function """ if isinstance(filename, str): - if not os.path.exists(f"{save_dir}/{filename}.pkl") or force_redo: + if not os.path.exists(f'{save_dir}/{filename}.pkl') or force_redo: b = benchmark_function(step_name, column_width, fnc) if save_state: - with open(f"{save_dir}/{filename}.pkl", 'wb') as f: print(f'Saving results of {step_name}', file = stderr, end = '\r') + with open(f'{save_dir}/{filename}.pkl', 'wb') as f: pickle.dump(b, f) print(' ' * 100, file = stderr, end = '\r') return b else: - with open(f"{save_dir}/{filename}.pkl", "rb") as f: print(f'Loading results of {step_name}', file = stderr, end = '\r') + with open(f'{save_dir}/{filename}.pkl', 'rb') as f: res = pickle.load(f) print(f"│ {step_name:<{column_width}} │ {'None':>18} │ {'loaded saved state':<29} │") return res elif isinstance(filename, list): abs = False for fn in filename: - if not os.path.exists(f"{save_dir}/{fn}.pkl"): + if not os.path.exists(f'{save_dir}/{fn}.pkl'): abs = True break if abs or force_redo: @@ -133,7 +184,7 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str if save_state: print(f'Saving results of {step_name}', file = stderr, end = '\r') for bi, fnI in zip(b, filename): - with open(f"{save_dir}/{fnI}.pkl", 'wb') as f: + with open(f'{save_dir}/{fnI}.pkl', 'wb') as f: pickle.dump(bi, f) print(' ' * 100, file = stderr, end = '\r') return b @@ -142,15 +193,24 @@ def state_saver(step_name: str, column_width: int, filename: Union[str, List[str b = [] print(f'Loading results of {step_name}', file = stderr, end = '\r') for fn in filename: - with open(f"{save_dir}/{fn}.pkl", "rb") as f: + with open(f'{save_dir}/{fn}.pkl', 'rb') as f: b.append(pickle.load(f)) print(' ' * 100, file = stderr, end = '\r') return b else: - assert False, f"Incompatible filename type = {type(filename)}" + assert False, f'Incompatible filename type = {type(filename)}' @njit('boolean(int32[:, :], uint16[:, :])') def unit_test_argsort_2d(arr: np.ndarray, indices: np.ndarray) -> bool: + """Test if a given array of indices sort a given array. + + Args: + arr (np.ndarray): Array of data + indices (np.ndarray): Indices that sort arr + + Returns: + bool: Success of the test + """ n = indices.shape[0] total = indices.shape[0] * indices.shape[1] for i, sub_indices in enumerate(indices): diff --git a/python/toolbox_unit_test.py b/python/toolbox_unit_test.py index a3bc883..208c303 100644 --- a/python/toolbox_unit_test.py +++ b/python/toolbox_unit_test.py @@ -1,67 +1,132 @@ from typing import Any -from toolbox import format_time_ns +from toolbox import format_time, format_time_ns def Assert(name: str, expected: Any, result: Any): + """Test if a given result is equal of the expected one and log result + + Args: + name (str): name of the unit test + expected (Any): expected result of the function call + result (Any): result of the function + """ if expected != result: print(f"For test name {name} Expected '{expected}' but got '{result}' instead") assert False -def format_time_ns_test() -> None: - # https://en.wikipedia.org/wiki/Unit_of_time - Assert("format_time_ns null", "0ns", format_time_ns(0)); - Assert("format_time_ns nanosecond", "1ns", format_time_ns(1)); - Assert("format_time_ns shake", "10ns", format_time_ns(10)); - Assert("format_time_ns microsecond", "1µs", format_time_ns(int(1e3))); - Assert("format_time_ns millisecond", "1ms", format_time_ns(int(1e6))); - Assert("format_time_ns centisecond", "10ms", format_time_ns(int(1e7))); - Assert("format_time_ns decisecond", "100ms", format_time_ns(int(1e8))); - Assert("format_time_ns second", "1s", format_time_ns(int(1e9))); - Assert("format_time_ns decasecond", "10s", format_time_ns(int(1e10))); - Assert("format_time_ns minute", "1m", format_time_ns(int(6e10))); - Assert("format_time_ns milliday", "1m 26s 400ms", format_time_ns(int(864e8))); - Assert("format_time_ns hectosecond", "1m 40s", format_time_ns(int(1e11))); - Assert("format_time_ns kilosecond", "16m 40s", format_time_ns(int(1e12))); - Assert("format_time_ns hour", "1h", format_time_ns(int(36e11))); - Assert("format_time_ns day", "1j", format_time_ns(int(864e11))); - Assert("format_time_ns week/sennight", "1w", format_time_ns(int(6048e11))); - Assert("format_time_ns megasecond", "1w 4j 13h 46m 40s", format_time_ns(int(1e15))); - Assert("format_time_ns fortnight", "2w", format_time_ns(int(12096e11))); - Assert("format_time_ns lunar month (draconitic)", "3w 6j 5h 5m 35s 800ms", format_time_ns(int(23511358e8))); - Assert("format_time_ns lunar month (tropical)", "3w 6j 7h 43m 4s 700ms", format_time_ns(int(23605847e8))); - Assert("format_time_ns lunar month (sidereal)", "3w 6j 7h 43m 11s 600ms", format_time_ns(int(23605916e8))); - Assert("format_time_ns lunar month (anomalistic)", "3w 6j 13h 18m 33s 200ms", format_time_ns(int(23807132e8))); - Assert("format_time_ns lunar month (synodic)", "4w 1j 12h 44m 2s 900ms", format_time_ns(int(25514429e8))); - Assert("format_time_ns month", "1M", format_time_ns(int(26784e11))); - Assert("format_time_ns quarantine", "1M 1w 2j", format_time_ns(int(3456e12))); - Assert("format_time_ns semester", "4M 2j", format_time_ns(int(108864e11))); - Assert("format_time_ns lunar year", "11M 1w 6j 8h 52m 48s", format_time_ns(int(30617568e9))); - Assert("format_time_ns year", "1y", format_time_ns(int(31536e12))); - Assert("format_time_ns tropical year", "1y 5h 48m 45s 216ms", format_time_ns(int(31556925216e6))); - Assert("format_time_ns gregorian year", "1y 5h 49m 12s", format_time_ns(int(31556952e9))); - Assert("format_time_ns sidereal year", "1y 6h 9m 9s 763ms 545µs 600ns", format_time_ns(int(315581497635456e2))); - Assert("format_time_ns leap year", "1y 1j", format_time_ns(int(316224e11))); - Assert("format_time_ns olympiad", "4y", format_time_ns(int(126144e12))); - Assert("format_time_ns lusturm", "5y", format_time_ns(int(15768e13))); - Assert("format_time_ns decade", "10y", format_time_ns(int(31536e13))); - Assert("format_time_ns indiction", "15y", format_time_ns(int(47304e13))); - Assert("format_time_ns score", "20y", format_time_ns(int(63072e13))); - Assert("format_time_ns gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time_ns(int(1e18))); - Assert("format_time_ns jubilee", "50y", format_time_ns(int(15768e14))); - Assert("format_time_ns century", "1c", format_time_ns(int(31536e14))); - Assert("format_time_ns millennium", "10c", format_time_ns(int(31536e15))); - Assert("format_time_ns age", "257c 72y", format_time_ns(int(812745792e12))); - Assert("format_time_ns terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time_ns(int(1e22))); - Assert("format_time_ns megaannum", "10000c", format_time_ns(int(31536e18))); - # Cannot use number bigger than currently supported ISO Python - #Assert("format_time_ns petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time_ns(int(1e24))); - #Assert("format_time_ns galactic year", "2300000c", format_time_ns(int(725328e19))); - #Assert("format_time_ns eon", "10000000c", format_time_ns(int(31536e21))); - #Assert("format_time_ns kalpa", "43200000c", format_time_ns(int(13623552e19))); - #Assert("format_time_ns exasecond", "317097919c 83y 9M 1h 46m 40s", format_time_ns(int(1e27))); - #Assert("format_time_ns zettasecond", "", format_time_ns(int(1e30))); - #Assert("format_time_ns yottasecond", "", format_time_ns(int(1e33))); - #Assert("format_time_ns ronnasecond", "", format_time_ns(int(1e36))); - #Assert("format_time_ns quettasecond", "", format_time_ns(int(1e39))); - # uint64_t_MAX == 2**64 == 18446744073709551615I64u == -1 - Assert("format_time_ns max", "5c 84y 11M 2j 23h 34m 33s 709ms 551µs 615ns", format_time_ns(2**64 - 1)) +def format_time_test() -> None: + """Test suite for the format_time output + See https://en.wikipedia.org/wiki/Unit_of_time for details + """ + Assert("format_time null", "0s", format_time(0)) + Assert("format_time second", "1s", format_time(1)) + Assert("format_time decasecond", "10s", format_time(10)) + Assert("format_time minute", "1m", format_time(60)) + Assert("format_time milliday", "1m 26s", format_time(86)) # missing 0.4s due to precision + Assert("format_time hectosecond", "1m 40s", format_time(100)) + Assert("format_time kilosecond", "16m 40s", format_time(int(1e3))) + Assert("format_time hour", "1h", format_time(3600)) + Assert("format_time day", "1j", format_time(86400)) + Assert("format_time week/sennight", "1w", format_time(604800)) + Assert("format_time megasecond", "1w 4j 13h 46m 40s", format_time(int(1e6))) + Assert("format_time fortnight", "2w", format_time(1209600)) + Assert("format_time lunar month (draconitic)", "3w 6j 5h 5m 35s", format_time(2351135)) # missing 0.8 due to precision + Assert("format_time lunar month (tropical)", "3w 6j 7h 43m 4s", format_time(2360584)) # missing 0.7 due to precision + Assert("format_time lunar month (sidereal)", "3w 6j 7h 43m 11s", format_time(2360591)) # missing 0.6 to precision + Assert("format_time lunar month (anomalistic)", "3w 6j 13h 18m 33s", format_time(2380713)) # missing 0.2 due to precision + Assert("format_time lunar month (synodic)", "4w 1j 12h 44m 2s", format_time(2551442)) # missing 0.9 due to precision + Assert("format_time month", "1M", format_time(2678400)) + Assert("format_time quarantine", "1M 1w 2j", format_time(int(3456e3))) + Assert("format_time semester", "4M 2j", format_time(10886400)) + Assert("format_time lunar year", "11M 1w 6j 8h 52m 48s", format_time(30617568)) + Assert("format_time year", "1y", format_time(int(31536e3))) + Assert("format_time tropical year", "1y 5h 48m 45s", format_time(31556925)) # missing 0.216 due to precision + Assert("format_time gregorian year", "1y 5h 49m 12s", format_time(31556952)) + Assert("format_time sidereal year", "1y 6h 9m 9s", format_time(31558149)) # missing 0.7635456 due to precision + Assert("format_time leap year", "1y 1j", format_time(31622400)) + Assert("format_time olympiad", "4y", format_time(int(126144e3))) + Assert("format_time lusturm", "5y", format_time(int(15768e4))) + Assert("format_time decade", "10y", format_time(int(31536e4))) + Assert("format_time indiction", "15y", format_time(int(47304e4))) + Assert("format_time score", "20y", format_time(int(63072e4))) + Assert("format_time gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time(int(1e9))) + Assert("format_time jubilee", "50y", format_time(int(15768e5))) + Assert("format_time century", "1c", format_time(int(31536e5))) + Assert("format_time millennium", "10c", format_time(int(31536e6))) + Assert("format_time age", "257c 72y", format_time(int(812745792e3))) + Assert("format_time terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time(int(1e13))) + Assert("format_time megaannum", "10000c", format_time(int(31536e9))) + Assert("format_time petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time(int(1e15))) + Assert("format_time galactic year", "2300000c", format_time(int(725328e10))) + Assert("format_time eon", "10000000c", format_time(int(31536e12))) + Assert("format_time kalpa", "43200000c", format_time(int(13623552e10))) + Assert("format_time exasecond", "317097919c 83y 9M 1h 46m 40s", format_time(int(1e18))) + # Cannot use number bigger than currently supported ISO Python + #Assert("format_time zettasecond", "", format_time(1e21)) + #Assert("format_time yottasecond", "", format_time(1e24)) + #Assert("format_time ronnasecond", "", format_time(1e27)) + #Assert("format_time quettasecond", "", format_time(1e30)) + # uint64_t_MAX == 2**64 == 18446744073709551615 == -1 + Assert("format_time max", "5849424173c 55y 3w 5j 7h 16s", format_time(int(2**64 - 1))) + +def format_time_ns_test() -> None: + """Test suite for the format_time_ns output + + See https://en.wikipedia.org/wiki/Unit_of_time for details + """ + Assert("format_time_ns null", "0ns", format_time_ns(0)) + Assert("format_time_ns nanosecond", "1ns", format_time_ns(1)) + Assert("format_time_ns shake", "10ns", format_time_ns(10)) + Assert("format_time_ns microsecond", "1µs", format_time_ns(int(1e3))) + Assert("format_time_ns millisecond", "1ms", format_time_ns(int(1e6))) + Assert("format_time_ns centisecond", "10ms", format_time_ns(int(1e7))) + Assert("format_time_ns decisecond", "100ms", format_time_ns(int(1e8))) + Assert("format_time_ns second", "1s", format_time_ns(int(1e9))) + Assert("format_time_ns decasecond", "10s", format_time_ns(int(1e10))) + Assert("format_time_ns minute", "1m", format_time_ns(int(6e10))) + Assert("format_time_ns milliday", "1m 26s 400ms", format_time_ns(int(864e8))) + Assert("format_time_ns hectosecond", "1m 40s", format_time_ns(int(1e11))) + Assert("format_time_ns kilosecond", "16m 40s", format_time_ns(int(1e12))) + Assert("format_time_ns hour", "1h", format_time_ns(int(36e11))) + Assert("format_time_ns day", "1j", format_time_ns(int(864e11))) + Assert("format_time_ns week/sennight", "1w", format_time_ns(int(6048e11))) + Assert("format_time_ns megasecond", "1w 4j 13h 46m 40s", format_time_ns(int(1e15))) + Assert("format_time_ns fortnight", "2w", format_time_ns(int(12096e11))) + Assert("format_time_ns lunar month (draconitic)", "3w 6j 5h 5m 35s 800ms", format_time_ns(int(23511358e8))) + Assert("format_time_ns lunar month (tropical)", "3w 6j 7h 43m 4s 700ms", format_time_ns(int(23605847e8))) + Assert("format_time_ns lunar month (sidereal)", "3w 6j 7h 43m 11s 600ms", format_time_ns(int(23605916e8))) + Assert("format_time_ns lunar month (anomalistic)", "3w 6j 13h 18m 33s 200ms", format_time_ns(int(23807132e8))) + Assert("format_time_ns lunar month (synodic)", "4w 1j 12h 44m 2s 900ms", format_time_ns(int(25514429e8))) + Assert("format_time_ns month", "1M", format_time_ns(int(26784e11))) + Assert("format_time_ns quarantine", "1M 1w 2j", format_time_ns(int(3456e12))) + Assert("format_time_ns semester", "4M 2j", format_time_ns(int(108864e11))) + Assert("format_time_ns lunar year", "11M 1w 6j 8h 52m 48s", format_time_ns(int(30617568e9))) + Assert("format_time_ns year", "1y", format_time_ns(int(31536e12))) + Assert("format_time_ns tropical year", "1y 5h 48m 45s 216ms", format_time_ns(int(31556925216e6))) + Assert("format_time_ns gregorian year", "1y 5h 49m 12s", format_time_ns(int(31556952e9))) + Assert("format_time_ns sidereal year", "1y 6h 9m 9s 763ms 545µs 600ns", format_time_ns(int(315581497635456e2))) + Assert("format_time_ns leap year", "1y 1j", format_time_ns(int(316224e11))) + Assert("format_time_ns olympiad", "4y", format_time_ns(int(126144e12))) + Assert("format_time_ns lusturm", "5y", format_time_ns(int(15768e13))) + Assert("format_time_ns decade", "10y", format_time_ns(int(31536e13))) + Assert("format_time_ns indiction", "15y", format_time_ns(int(47304e13))) + Assert("format_time_ns score", "20y", format_time_ns(int(63072e13))) + Assert("format_time_ns gigasecond", "31y 8M 1w 4j 1h 46m 40s", format_time_ns(int(1e18))) + Assert("format_time_ns jubilee", "50y", format_time_ns(int(15768e14))) + Assert("format_time_ns century", "1c", format_time_ns(int(31536e14))) + Assert("format_time_ns millennium", "10c", format_time_ns(int(31536e15))) + Assert("format_time_ns age", "257c 72y", format_time_ns(int(812745792e12))) + Assert("format_time_ns terasecond", "3170c 97y 10M 3w 4j 17h 46m 40s", format_time_ns(int(1e22))) + Assert("format_time_ns megaannum", "10000c", format_time_ns(int(31536e18))) + # Cannot use number bigger than currently supported ISO Python + # Assert("format_time_ns petasecond", "317097c 91y 11M 2w 4j 1h 46m 40s", format_time_ns(int(1e24))) + # Assert("format_time_ns galactic year", "2300000c", format_time_ns(int(725328e19))) + # Assert("format_time_ns eon", "10000000c", format_time_ns(int(31536e21))) + # Assert("format_time_ns kalpa", "43200000c", format_time_ns(int(13623552e19))) + # Assert("format_time_ns exasecond", "317097919c 83y 9M 1h 46m 40s", format_time_ns(int(1e27))) + # Assert("format_time_ns zettasecond", "", format_time_ns(int(1e30))) + # Assert("format_time_ns yottasecond", "", format_time_ns(int(1e33))) + # Assert("format_time_ns ronnasecond", "", format_time_ns(int(1e36))) + # Assert("format_time_ns quettasecond", "", format_time_ns(int(1e39))) + # uint64_t_MAX == 2**64 == 18446744073709551615 == -1 + Assert("format_time_ns max", "5c 84y 11M 2j 23h 34m 33s 709ms 551µs 615ns", format_time_ns(2**64 - 1))