diff --git a/Programming/EARIN/lab2/main.py b/Programming/EARIN/lab2/main.py index 72ad7a27..d84536e5 100755 --- a/Programming/EARIN/lab2/main.py +++ b/Programming/EARIN/lab2/main.py @@ -448,7 +448,7 @@ class Game: continue has_moved = self.make_move(from_coordinates, to_coordinates, color) - self.print_board(color == "white") + self.print_board(rotate=color == "white") def start_game(self, player_color: str = "black", algorithm_depth: int = 5) -> None: """Start the main loop of the game.""" @@ -457,7 +457,7 @@ class Game: return ai_color = "black" if player_color == "white" else "white" - game.print_board(player_color == "white") + game.print_board(rotate=player_color == "white") if player_color == "white": game.handle_player_move("white") while True: @@ -468,7 +468,10 @@ class Game: logger.info("Game over, %s wins", player_color) return _, ai_move = game.alpha_beta(algorithm_depth, (5, 10), ai_color) - game.make_move(*ai_move, ai_color) + if ai_move is None: + logger.info("Game over, %s wins", player_color) + return + game.make_move(ai_move[0], ai_move[1], ai_color) logger.info( "AI's move: %s%s %s%s", chr(ord("a") + ai_move[0][0]), @@ -476,7 +479,7 @@ class Game: chr(ord("a") + ai_move[1][0]), ai_move[1][1], ) - game.print_board(player_color == "white") + game.print_board(rotate=player_color == "white") ai_turn = game.get_possible_moves(ai_color)[1] and possible_moves_ai[1] player_turn = True while player_turn: @@ -506,7 +509,7 @@ class Game: _, ai_move = game.alpha_beta(algorithm_depth, (5, 10), ai_color) if ai_move is None: ai_move = possible_moves_ai[0] - game.make_move(*ai_move, ai_color) + game.make_move(ai_move[0], ai_move[1], ai_color) if print_info: logger.info( "AI's move: %s%s %s%s", diff --git a/Programming/EARIN/lab3/main.py b/Programming/EARIN/lab3/main.py index 1efcdd72..0066cd22 100755 --- a/Programming/EARIN/lab3/main.py +++ b/Programming/EARIN/lab3/main.py @@ -335,9 +335,9 @@ def user_input() -> dict[str, float]: def print_output(print_info: object, save_results: object, summary: object) -> None: """Print out population and summary plots.""" for population, generation_number, file_name in print_info: - output(population, generation_number, file_name, save_results) + output(population, generation_number, file_name, save_results=save_results) summary_file_name = file_name - print_summary(summary, summary_file_name, save_results) + print_summary(summary, summary_file_name, save_results=save_results) # Ran first in the code @@ -355,7 +355,7 @@ if __name__ == "__main__": SUMMARY, ) = evolution_strategy( ARGUMENTS, - ARGUMENTS["no_display"], + no_display=bool(ARGUMENTS["no_display"]), ) end_time = time.perf_counter() if not ARGUMENTS["no_display"]: diff --git a/Programming/EARIN/lab5/code/main.py b/Programming/EARIN/lab5/code/main.py index ddf30506..4089e150 100755 --- a/Programming/EARIN/lab5/code/main.py +++ b/Programming/EARIN/lab5/code/main.py @@ -164,8 +164,10 @@ def training_loop( _, training_parameters["optimizer"] = single_train_iteration( data, training_parameters, targets, batch_idx, epoch ) - calculate_accuracy_epoch(training_parameters, epoch, print_info) - calculate_validation_set_accuracy(training_parameters, epoch, print_info) + calculate_accuracy_epoch(training_parameters, epoch, print_info=print_info) + calculate_validation_set_accuracy( + training_parameters, epoch, print_info=print_info + ) return epoch, training_parameters["loaders"]["train_loader"] @@ -228,7 +230,7 @@ def main_part(*, show_plot: bool = True) -> None: training_parameters = set_training_parameters( hyperparameters, loaders, model, criterion, optimizer ) - training_loop(training_parameters, show_plot) + training_loop(training_parameters, print_info=show_plot) with Path("results.txt").open("a") as file: rule = "-" * 85 + "\n" file.write(rule) diff --git a/Programming/EARIN/lab6/main.py b/Programming/EARIN/lab6/main.py index 88220e3f..88b98d72 100755 --- a/Programming/EARIN/lab6/main.py +++ b/Programming/EARIN/lab6/main.py @@ -195,7 +195,7 @@ def inference(env: object, q_table: object) -> None: if __name__ == "__main__": logging.basicConfig(format="%(message)s", level=logging.INFO) HYPERPARAMETERS = initialize_hyperparameters() - ENV = initialize_environment(HYPERPARAMETERS) + ENV = initialize_environment() Q_TABLE = initialize_q_table(ENV) ENV, Q_TABLE = training_loop(HYPERPARAMETERS, ENV, Q_TABLE) inference(ENV, Q_TABLE) diff --git a/Programming/EARIN/project/midterm/code/main.py b/Programming/EARIN/project/midterm/code/main.py index 09c67513..c8585889 100755 --- a/Programming/EARIN/project/midterm/code/main.py +++ b/Programming/EARIN/project/midterm/code/main.py @@ -349,7 +349,11 @@ if __name__ == "__main__": RATING_DATA, ANIME_CONTACT_DATA = get_data(data_limit, db) PIVOT_TABLE = preprocessing( - RATING_DATA, ANIME_CONTACT_DATA, debug, user_threshold, anime_threshold + RATING_DATA, + ANIME_CONTACT_DATA, + debug=debug, + user_threshold=user_threshold, + anime_threshold=anime_threshold, ) MODEL = create_model(PIVOT_TABLE, metric, algorithm, neighbors) predict(MODEL, PIVOT_TABLE, seed, anime, recommendation_amount) diff --git a/Programming/PORR/.gitignore b/Programming/PORR/.gitignore index 0624ab8c..c35f2615 100644 --- a/Programming/PORR/.gitignore +++ b/Programming/PORR/.gitignore @@ -406,4 +406,15 @@ TSWLatexianTemp* #*Notes.bib spd_10000.npz nemeth12.npz -poli3.npz \ No newline at end of file +poli3.npz + +# Generated SPD test matrices. MatrixGenerator.save_to_file() writes these as a +# cache and regenerates any that is missing, so none of them is a deliverable. +# +# spd_5000.npz was tracked as a Git LFS pointer whose 200 MB object is NOT on +# the server -- `git lfs pull` returns "[404] Object does not exist". numpy +# then read the 134-byte pointer text as a .npz and raised ValueError instead +# of FileNotFoundError, so the regeneration path never ran and every n=5000 +# test failed. The pointer is untracked and .gitattributes (its only content +# was three LFS filter rules, two for files that never existed here) deleted. +spd_5000.npz diff --git a/Programming/PORR/code/linear_algebra_utils.py b/Programming/PORR/code/linear_algebra_utils.py index 5448545b..0d907e00 100644 --- a/Programming/PORR/code/linear_algebra_utils.py +++ b/Programming/PORR/code/linear_algebra_utils.py @@ -1,80 +1,84 @@ import cmath -import math import itertools -import operator +import logging +import math import multiprocessing -from multiprocessing import Pool +import operator from abc import ABC, abstractmethod from concurrent.futures import ThreadPoolExecutor from functools import partial -from time_measurement import time_measurement, time_accumulator -import numpy as np +from multiprocessing import Pool + import dask.array as da +from time_measurement import time_accumulator, time_measurement + +logger = logging.getLogger(__name__) + class LinearAlgebraUtils: @staticmethod @abstractmethod - def dot_product(v1, v2): + def dot_product(v1, v2) -> None: pass @staticmethod @abstractmethod - def matrix_vector_multiply(A, x): - pass - - @staticmethod - @abstractmethod - def vector_norm(v): + def matrix_vector_multiply(A, x) -> None: pass @staticmethod @abstractmethod - def vector_scalar_divide(x, scalar): + def vector_norm(v) -> None: pass @staticmethod @abstractmethod - def matrix_scalar_multiply(A, w): + def vector_scalar_divide(x, scalar) -> None: pass @staticmethod @abstractmethod - def vector_vector_subtraction(v1, v2): + def matrix_scalar_multiply(A, w) -> None: pass @staticmethod @abstractmethod - def vector_vector_addition(v1, v2): + def vector_vector_subtraction(v1, v2) -> None: pass @staticmethod @abstractmethod - def scalar_vector_multiply(omega, vector): + def vector_vector_addition(v1, v2) -> None: pass @staticmethod @abstractmethod - def matrix_norm(A): + def scalar_vector_multiply(omega, vector) -> None: pass @staticmethod @abstractmethod - def matrix_matrix_subtraction(A, B): + def matrix_norm(A) -> None: + pass + + @staticmethod + @abstractmethod + def matrix_matrix_subtraction(A, B) -> None: pass class SequentialLinearAlgebraUtils: @staticmethod def dot_product(v1, v2): - return sum(x*y for x, y in zip(v1, v2)) + return sum(x * y for x, y in zip(v1, v2)) @staticmethod def matrix_vector_multiply(A, x): return [SequentialLinearAlgebraUtils.dot_product(row, x) for row in A] - + @staticmethod def vector_norm(v): - x_values = (x*x for x in v) + x_values = (x * x for x in v) x_values_sum = sum(x_values) return cmath.sqrt(x_values_sum).real @@ -88,11 +92,11 @@ class SequentialLinearAlgebraUtils: @staticmethod def vector_vector_subtraction(v1, v2): - return [x-y for x, y in zip(v1, v2)] + return [x - y for x, y in zip(v1, v2)] @staticmethod def vector_vector_addition(v1, v2): - return [x+y for x, y in zip(v1, v2)] + return [x + y for x, y in zip(v1, v2)] @staticmethod def scalar_vector_multiply(omega, vector): @@ -100,7 +104,7 @@ class SequentialLinearAlgebraUtils: @staticmethod def matrix_norm(A): - return math.sqrt(sum(sum(element ** 2 for element in row) for row in A)) + return math.sqrt(sum(sum(element**2 for element in row) for row in A)) @staticmethod def matrix_matrix_subtraction(A, B): @@ -120,7 +124,9 @@ class ThreadsLinearAlgebraUtils: @staticmethod def divide_vectors_to_chunks(v1, v2): - chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size(v1) + chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size( + v1 + ) chunks = [] start = 0 @@ -130,7 +136,7 @@ class ThreadsLinearAlgebraUtils: start = end return chunks - + @staticmethod def divide_vector_or_matrix_to_chunks(v): chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size(v) @@ -148,11 +154,13 @@ class ThreadsLinearAlgebraUtils: @time_measurement(time_accumulator) def matrix_vector_multiply(A, x): chunks = ThreadsLinearAlgebraUtils.divide_vector_or_matrix_to_chunks(A) - with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: + with ThreadPoolExecutor( + max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS + ) as executor: func = partial(SequentialLinearAlgebraUtils.matrix_vector_multiply, x=x) results = executor.map(func, chunks) return [item for sublist in results for item in sublist] - + @staticmethod @time_measurement(time_accumulator) def vector_norm(v): @@ -161,7 +169,9 @@ class ThreadsLinearAlgebraUtils: def partial_norm(chunk): return sum(x * x for x in chunk) - with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: + with ThreadPoolExecutor( + max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS + ) as executor: results = executor.map(partial_norm, chunks) total_sum = sum(results) return total_sum**0.5 @@ -170,26 +180,44 @@ class ThreadsLinearAlgebraUtils: @time_measurement(time_accumulator) def vector_vector_subtraction(v1, v2): chunks = ThreadsLinearAlgebraUtils.divide_vectors_to_chunks(v1, v2) - with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: - results = executor.map(lambda pair: SequentialLinearAlgebraUtils.vector_vector_subtraction(*pair), chunks) + with ThreadPoolExecutor( + max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS + ) as executor: + results = executor.map( + lambda pair: SequentialLinearAlgebraUtils.vector_vector_subtraction( + *pair + ), + chunks, + ) return [item for sublist in results for item in sublist] - @staticmethod @time_measurement(time_accumulator) def vector_vector_addition(v1, v2): chunks = ThreadsLinearAlgebraUtils.divide_vectors_to_chunks(v1, v2) - with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: - results = executor.map(lambda pair: SequentialLinearAlgebraUtils.vector_vector_addition(*pair), chunks) + with ThreadPoolExecutor( + max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS + ) as executor: + results = executor.map( + lambda pair: SequentialLinearAlgebraUtils.vector_vector_addition(*pair), + chunks, + ) return [item for sublist in results for item in sublist] - + @staticmethod @time_measurement(time_accumulator) def scalar_vector_multiply(omega, vector): chunks = ThreadsLinearAlgebraUtils.divide_vector_or_matrix_to_chunks(vector) - with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: - results = executor.map(lambda chunk: SequentialLinearAlgebraUtils.scalar_vector_multiply(omega, chunk), chunks) - + with ThreadPoolExecutor( + max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS + ) as executor: + results = executor.map( + lambda chunk: SequentialLinearAlgebraUtils.scalar_vector_multiply( + omega, chunk + ), + chunks, + ) + return [item for sublist in results for item in sublist] @@ -199,11 +227,13 @@ def process_row(params): factor = A[i][k] / A[k][k] return [A[i][j] - factor * A[k][j] for j in range(len(A[0]))] + @time_measurement(time_accumulator) def divide_by_scalar(pair): xi, scalar = pair return xi / scalar + @time_measurement(time_accumulator) def multiply_by_scalar(pair): element, scalar = pair @@ -215,9 +245,11 @@ class ProcessLinearAlgebraUtils: @time_measurement(time_accumulator) def dot_product(v1, v2): with Pool() as pool: - result = pool.starmap(ProcessLinearAlgebraUtils.multiply_elements, zip(v1, v2)) + result = pool.starmap( + ProcessLinearAlgebraUtils.multiply_elements, zip(v1, v2) + ) return sum(result) - + @staticmethod @time_measurement(time_accumulator) def multiply_elements(x, y): @@ -233,9 +265,12 @@ class ProcessLinearAlgebraUtils: @time_measurement(time_accumulator) def matrix_vector_multiply(A, x): with Pool() as pool: - result = pool.map(ProcessLinearAlgebraUtils.matrix_vector_multiply_row, [(row, x) for row in A]) + result = pool.map( + ProcessLinearAlgebraUtils.matrix_vector_multiply_row, + [(row, x) for row in A], + ) return list(result) - + @staticmethod @time_measurement(time_accumulator) def vector_norm(v): @@ -254,12 +289,15 @@ class ProcessLinearAlgebraUtils: with Pool() as pool: result = pool.map(divide_by_scalar, [(xi, scalar) for xi in x]) return list(result) - + @staticmethod @time_measurement(time_accumulator) def divide_vector_by_scalar(x, scalar): with Pool() as pool: - result = pool.map(ProcessLinearAlgebraUtils.vector_scalar_divide, [(xi, scalar) for xi in x]) + result = pool.map( + ProcessLinearAlgebraUtils.vector_scalar_divide, + [(xi, scalar) for xi in x], + ) return list(result) @staticmethod @@ -272,7 +310,10 @@ class ProcessLinearAlgebraUtils: @time_measurement(time_accumulator) def matrix_scalar_multiply(A, w): with Pool() as pool: - result = pool.map(ProcessLinearAlgebraUtils.matrix_scalar_multiply_row, [(row, w) for row in A]) + result = pool.map( + ProcessLinearAlgebraUtils.matrix_scalar_multiply_row, + [(row, w) for row in A], + ) return result @staticmethod @@ -285,27 +326,35 @@ class ProcessLinearAlgebraUtils: @time_measurement(time_accumulator) def vector_vector_subtraction(v1, v2): with Pool() as pool: - result = pool.map(ProcessLinearAlgebraUtils.vector_vector_operation, zip(v1, v2, itertools.repeat(operator.sub))) + result = pool.map( + ProcessLinearAlgebraUtils.vector_vector_operation, + zip(v1, v2, itertools.repeat(operator.sub)), + ) return list(result) @staticmethod @time_measurement(time_accumulator) def vector_vector_addition(v1, v2): with Pool() as pool: - result = pool.map(ProcessLinearAlgebraUtils.vector_vector_operation, zip(v1, v2, itertools.repeat(operator.add))) + result = pool.map( + ProcessLinearAlgebraUtils.vector_vector_operation, + zip(v1, v2, itertools.repeat(operator.add)), + ) return list(result) @staticmethod @time_measurement(time_accumulator) def scalar_vector_multiply(omega, vector): with Pool() as pool: - result = pool.map(multiply_by_scalar, [(element, omega) for element in vector]) + result = pool.map( + multiply_by_scalar, [(element, omega) for element in vector] + ) return list(result) - + @staticmethod @time_measurement(time_accumulator) def sum_of_squares(row): - return sum(x ** 2 for x in row) + return sum(x**2 for x in row) @staticmethod @time_measurement(time_accumulator) @@ -318,7 +367,7 @@ class ProcessLinearAlgebraUtils: @time_measurement(time_accumulator) def subtract_rows(row_from_A, row_from_B): return [a - b for a, b in zip(row_from_A, row_from_B)] - + @staticmethod @time_measurement(time_accumulator) def matrix_matrix_subtraction(A, B): @@ -326,78 +375,79 @@ class ProcessLinearAlgebraUtils: result = pool.starmap(ProcessLinearAlgebraUtils.subtract_rows, zip(A, B)) return result + class DistributedArraysLinearAlgebraUtils(ABC): @staticmethod @time_measurement(time_accumulator) def dot_product(v1, v2): - dv1 = da.from_array(v1, chunks='auto') - dv2 = da.from_array(v2, chunks='auto') + dv1 = da.from_array(v1, chunks="auto") + dv2 = da.from_array(v2, chunks="auto") return da.dot(dv1, dv2).compute() @staticmethod @time_measurement(time_accumulator) def matrix_vector_multiply(A, x): - dA = da.from_array(A, chunks='auto') - dx = da.from_array(x, chunks='auto') + dA = da.from_array(A, chunks="auto") + dx = da.from_array(x, chunks="auto") return da.dot(dA, dx).compute().tolist() - + @staticmethod @time_measurement(time_accumulator) def vector_norm(v): - dv = da.from_array(v, chunks='auto') + dv = da.from_array(v, chunks="auto") return da.linalg.norm(dv).compute() @staticmethod @time_measurement(time_accumulator) def vector_scalar_divide(x, scalar): - dx = da.from_array(x, chunks='auto') + dx = da.from_array(x, chunks="auto") return (dx / scalar).compute().tolist() @staticmethod @time_measurement(time_accumulator) def matrix_scalar_multiply(A, w): - dA = da.from_array(A, chunks='auto') + dA = da.from_array(A, chunks="auto") return (dA * w).compute().tolist() @staticmethod @time_measurement(time_accumulator) def vector_vector_subtraction(v1, v2): - dv1 = da.from_array(v1, chunks='auto') - dv2 = da.from_array(v2, chunks='auto') + dv1 = da.from_array(v1, chunks="auto") + dv2 = da.from_array(v2, chunks="auto") return (dv1 - dv2).compute().tolist() @staticmethod @time_measurement(time_accumulator) def vector_vector_addition(v1, v2): - dv1 = da.from_array(v1, chunks='auto') - dv2 = da.from_array(v2, chunks='auto') + dv1 = da.from_array(v1, chunks="auto") + dv2 = da.from_array(v2, chunks="auto") return (dv1 + dv2).compute().tolist() @staticmethod @time_measurement(time_accumulator) def scalar_vector_multiply(omega, vector): - dvector = da.from_array(vector, chunks='auto') + dvector = da.from_array(vector, chunks="auto") return (omega * dvector).compute().tolist() @staticmethod @time_measurement(time_accumulator) def matrix_norm(A): - dA = da.from_array(A, chunks='auto') + dA = da.from_array(A, chunks="auto") return da.linalg.norm(dA).compute() - + @staticmethod @time_measurement(time_accumulator) def matrix_matrix_subtraction(A, B): - dA = da.from_array(A, chunks='auto') - dB = da.from_array(B, chunks='auto') + dA = da.from_array(A, chunks="auto") + dB = da.from_array(B, chunks="auto") return (dA - dB).compute().tolist() @staticmethod @time_measurement(time_accumulator) def gaussian_elimination(A, b): try: - dA = da.from_array(A, chunks='auto') - db = da.from_array(b, chunks='auto') + dA = da.from_array(A, chunks="auto") + db = da.from_array(b, chunks="auto") Ab = da.hstack([dA, db[:, None]]) Ab = Ab.persist() @@ -406,8 +456,8 @@ class DistributedArraysLinearAlgebraUtils(ABC): max_index = da.argmax(da.abs(Ab[k:, k])) + k Ab[[k, max_index]] = Ab[[max_index, k]] Ab = Ab.persist() - factor = Ab[k + 1:, k] / Ab[k, k] - Ab[k + 1:] -= factor[:, None] * Ab[k] + factor = Ab[k + 1 :, k] / Ab[k, k] + Ab[k + 1 :] -= factor[:, None] * Ab[k] return Ab for k in range(A.shape[0]): @@ -415,8 +465,8 @@ class DistributedArraysLinearAlgebraUtils(ABC): x = da.zeros(A.shape[0]) for i in range(A.shape[0] - 1, -1, -1): - x[i] = (Ab[i, -1] - da.dot(Ab[i, i + 1:-1], x[i + 1:])) / Ab[i, i] + x[i] = (Ab[i, -1] - da.dot(Ab[i, i + 1 : -1], x[i + 1 :])) / Ab[i, i] return x.compute().tolist() except Exception as e: - print(f"Error during Gaussian elimination: {e}") + logger.info("Error during Gaussian elimination: %s", e) return None diff --git a/Programming/PORR/code/main.py b/Programming/PORR/code/main.py index b468d54c..10263d27 100644 --- a/Programming/PORR/code/main.py +++ b/Programming/PORR/code/main.py @@ -1,5 +1,19 @@ +"""Entry point for the PORR benchmark suite. + +Runs ``tests.py``, which compares this project's Richardson implementations +(sequential, threads, processes, distributed arrays) against numpy's direct +solver over a range of matrix sizes. + +The suite is heavy at the largest sizes it parametrises; ``../run.sh`` narrows +the default selection via ``PYTEST_ADDOPTS`` and says there how to widen it. +""" + +import sys + import pytest if __name__ == "__main__": - # Run pytest and exit with the appropriate status code - pytest.main(["-v", "tests.py"]) + # sys.exit, because pytest.main returns the status rather than raising: + # discarding it made `python main.py` report success even when every test + # failed, which in turn made the run harness unable to tell the two apart. + sys.exit(pytest.main(["-v", "tests.py"])) diff --git a/Programming/PORR/code/matrix_generator.py b/Programming/PORR/code/matrix_generator.py index 024403a3..0d379652 100644 --- a/Programming/PORR/code/matrix_generator.py +++ b/Programming/PORR/code/matrix_generator.py @@ -1,86 +1,108 @@ +import os + import numpy as np import scipy.io -import os + class MatrixGenerator: @staticmethod def generate_spd_matrix(n: int) -> np.ndarray: A = np.random.rand(n, n) - spd_matrix = np.dot(A, A.T) + n * MatrixGenerator.generate_identity_matrix(n) # Adding n*I ensures positive definiteness - return spd_matrix + return np.dot(A, A.T) + n * MatrixGenerator.generate_identity_matrix( + n + ) # Adding n*I ensures positive definiteness @staticmethod def generate_identity_matrix(size): return np.eye(size) - + @staticmethod def generate_alternating_vector(size): return np.tile([1, 2], int(np.ceil(size / 2)))[:size] - + @staticmethod def get_matrix_from_file(file_path, problem): mat_contents = scipy.io.loadmat(file_path) - problem_record = mat_contents['Problem'][0][0] + problem_record = mat_contents["Problem"][0][0] A = problem_record[problem] if scipy.sparse.issparse(A): - A_dense = A.todense() + A_dense = A.todense() else: - A_dense = A + A_dense = A return np.array(A_dense) @staticmethod def generate_matrix_and_vector(type, size=None): - if type == 'spd': + if type == "spd": if size is None: raise ValueError("Size must be provided for SPD matrix generation.") try: - matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file("spd_"+str(size)+".npz") - except FileNotFoundError as e: + matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file( + "spd_" + str(size) + ".npz" + ) + except FileNotFoundError: matrix = MatrixGenerator.generate_spd_matrix(size) vector = np.random.uniform(-1, 1, size) lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix) - MatrixGenerator.save_to_file(matrix, vector, lambda_min, lambda_max, "spd_"+str(size)+".npz") - elif type == 'nemeth12': + MatrixGenerator.save_to_file( + matrix, vector, lambda_min, lambda_max, "spd_" + str(size) + ".npz" + ) + elif type == "nemeth12": try: - matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file("nemeth12.npz") - except FileNotFoundError as e: + matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file( + "nemeth12.npz" + ) + except FileNotFoundError: matrix = -1 * MatrixGenerator.get_matrix_from_file("nemeth12.mat", 1) size = matrix.shape[0] vector = MatrixGenerator.generate_alternating_vector(size) lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix) - MatrixGenerator.save_to_file(matrix, vector, lambda_min, lambda_max, "nemeth12.npz") - elif type == 'poli3': + MatrixGenerator.save_to_file( + matrix, vector, lambda_min, lambda_max, "nemeth12.npz" + ) + elif type == "poli3": try: - matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file("poli3.npz") - except FileNotFoundError as e: + matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file( + "poli3.npz" + ) + except FileNotFoundError: matrix = MatrixGenerator.get_matrix_from_file("poli3.mat", 2) size = matrix.shape[0] vector = MatrixGenerator.generate_alternating_vector(size) lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix) - MatrixGenerator.save_to_file(matrix, vector, lambda_min, lambda_max, "poli3.npz") + MatrixGenerator.save_to_file( + matrix, vector, lambda_min, lambda_max, "poli3.npz" + ) else: - raise ValueError("Invalid type specified. Choose 'spd', 'nemeth12', or 'poli3'.") - + raise ValueError( + "Invalid type specified. Choose 'spd', 'nemeth12', or 'poli3'." + ) + return matrix, vector, lambda_min, lambda_max - + @staticmethod def calculate_eigenvalues(A): eigenvalues = np.linalg.eigvals(A) lambda_min = np.min(eigenvalues) lambda_max = np.max(eigenvalues) return lambda_min, lambda_max - + @staticmethod - def save_to_file(matrix, vector, lambda_min, lambda_max, file_path): - np.savez(file_path, matrix=matrix, vector=vector, lambda_min=lambda_min, lambda_max=lambda_max) + def save_to_file(matrix, vector, lambda_min, lambda_max, file_path) -> None: + np.savez( + file_path, + matrix=matrix, + vector=vector, + lambda_min=lambda_min, + lambda_max=lambda_max, + ) def load_from_file(file_path): if not os.path.exists(file_path): raise FileNotFoundError(f"The file {file_path} does not exist.") data = np.load(file_path) - matrix = data['matrix'] - vector = data['vector'] - lambda_min = data['lambda_min'] - lambda_max = data['lambda_max'] + matrix = data["matrix"] + vector = data["vector"] + lambda_min = data["lambda_min"] + lambda_max = data["lambda_max"] return matrix, vector, lambda_min, lambda_max - diff --git a/Programming/PORR/code/processing_type.py b/Programming/PORR/code/processing_type.py index 8a5e622b..8bedd46d 100644 --- a/Programming/PORR/code/processing_type.py +++ b/Programming/PORR/code/processing_type.py @@ -1,5 +1,6 @@ from enum import Enum, auto + class ProcessingType(Enum): SEQUENTIAL = auto() THREADS = auto() diff --git a/Programming/PORR/code/richardson_method.py b/Programming/PORR/code/richardson_method.py index 53053514..7a22ee4e 100644 --- a/Programming/PORR/code/richardson_method.py +++ b/Programming/PORR/code/richardson_method.py @@ -1,65 +1,79 @@ -import linear_algebra_utils as linAlg -from matrix_generator import MatrixGenerator -from processing_type import ProcessingType -from time_measurement import time_measurement, time_accumulator, tests_time -import time import gc -import numpy as np +import logging +import time + +import linear_algebra_utils as linAlg +from processing_type import ProcessingType +from time_measurement import tests_time, time_accumulator, time_measurement + +logger = logging.getLogger(__name__) + class RichardsonMethod: @time_measurement(time_accumulator) - def __init__(self, method: ProcessingType, A, b, lambda_min, lambda_max, max_iterations, size: int, x0=None, tol=1e-5): + def __init__( + self, + method: ProcessingType, + A, + b, + lambda_min, + lambda_max, + max_iterations, + size: int, + x0=None, + tol=1e-5, + ) -> None: self.LinAlg = self.assign_LinAlgType(method) self.A = A self.b = b self.x0 = x0 if x0 is not None else [0.0] * len(b) self.max_iterations = max_iterations self.tol = tol - # self.I = MatrixGenerator.generate_identity_matrix(size) self.lambda_min = lambda_min self.lambda_max = lambda_max if self.lambda_min < 0: raise ValueError("Matrix A is not positive semi-definite.") self.omega = RichardsonMethod.calculate_omega(self.lambda_min, self.lambda_max) - + @staticmethod def calculate_omega(lambda_min, lambda_max): return 2 / (lambda_min + lambda_max) - + @staticmethod def convergence_norm(LinAlgType, A, omega, I) -> bool: wA = LinAlgType.matrix_scalar_multiply(A, omega) IMinuswA = LinAlgType.matrix_matrix_subtraction(I, wA) - norm = LinAlgType.matrix_norm(IMinuswA) - return norm - + return LinAlgType.matrix_norm(IMinuswA) + @staticmethod def assign_LinAlgType(method): methods = { ProcessingType.SEQUENTIAL: linAlg.SequentialLinearAlgebraUtils, ProcessingType.THREADS: linAlg.ThreadsLinearAlgebraUtils, ProcessingType.PROCESSES: linAlg.ProcessLinearAlgebraUtils, - ProcessingType.DISTRIBUTED_ARRAYS: linAlg.DistributedArraysLinearAlgebraUtils + ProcessingType.DISTRIBUTED_ARRAYS: linAlg.DistributedArraysLinearAlgebraUtils, } - + try: return methods[method] except KeyError: - raise ValueError("Unknown method, please use 'SEQUENTIAL', 'THREADS' or 'PROCESSES'.") + raise ValueError( + "Unknown method, please use 'SEQUENTIAL', 'THREADS' or 'PROCESSES'." + ) def solve(self): gc.disable() time_accumulator.total_time = 0 start = time.perf_counter() x = self.x0[:] - # if RichardsonMethod.convergence_norm(self.LinAlg, self.A, self.omega, self.I) >= 1: - # return RichardsonMethod.convergence_norm(self.LinAlg, self.A, self.omega, self.I), "Richardson method for those values will NOT converge", for iteration in range(self.max_iterations): Ax = self.LinAlg.matrix_vector_multiply(self.A, x) residual = self.LinAlg.vector_vector_subtraction(self.b, Ax) - x = self.LinAlg.vector_vector_addition(x, self.LinAlg.scalar_vector_multiply(self.omega, residual)) - if (linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < self.tol): + x = self.LinAlg.vector_vector_addition( + x, self.LinAlg.scalar_vector_multiply(self.omega, residual) + ) + if linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < self.tol: break end = time.perf_counter() @@ -68,17 +82,37 @@ class RichardsonMethod: match self.LinAlg: case linAlg.SequentialLinearAlgebraUtils: - print(f"Total: {total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") + logger.info( + "Total: %ss, Tests time: %ss", total_time, tests_time.total_time + ) case linAlg.ThreadsLinearAlgebraUtils: sequential_time = total_time - time_accumulator.total_time - print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") + logger.info( + "Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss", + total_time, + sequential_time, + time_accumulator.total_time, + tests_time.total_time, + ) case linAlg.ProcessLinearAlgebraUtils: sequential_time = total_time - time_accumulator.total_time - print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (processes): {time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") + logger.info( + "Total: %ss, Seq: %ss, Parallel (processes): %ss, Tests time: %ss", + total_time, + sequential_time, + time_accumulator.total_time, + tests_time.total_time, + ) case linAlg.DistributedArraysLinearAlgebraUtils: sequential_time = total_time - time_accumulator.total_time - print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (distributed arrays): {time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") + logger.info( + "Total: %ss, Seq: %ss, Parallel (distributed arrays): %ss, Tests time: %ss", + total_time, + sequential_time, + time_accumulator.total_time, + tests_time.total_time, + ) case _: - print("Unhandled LinAlg type") - + logger.info("Unhandled LinAlg type") + return x, 0 diff --git a/Programming/PORR/code/richardson_mpi.py b/Programming/PORR/code/richardson_mpi.py index fe0e29ca..b69d476e 100644 --- a/Programming/PORR/code/richardson_mpi.py +++ b/Programming/PORR/code/richardson_mpi.py @@ -1,7 +1,12 @@ -from mpi4py import MPI -import numpy as np +import logging import time +import numpy as np +from mpi4py import MPI + +logger = logging.getLogger(__name__) + + def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000): comm = MPI.COMM_WORLD rank = comm.Get_rank() @@ -12,9 +17,6 @@ def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000): # Obliczanie wartości własnych tylko na jednym procesie if rank == 0: - # eigenvalues = np.linalg.eigvals(A) - # lambda_min = np.min(eigenvalues) - # lambda_max = np.max(eigenvalues) omega = 2 / (lambda_min + lambda_max) else: omega = None @@ -67,21 +69,25 @@ def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000): return x, execution_time - def check_solution(A, b, x_approx, tolerance=8e-3): x_true = np.linalg.solve(A, b) error = np.linalg.norm(x_true - x_approx) return error < tolerance, error + if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) comm = MPI.COMM_WORLD rank = comm.Get_rank() from matrix_generator import MatrixGenerator + sizes = [2, 5, 10, 50, 80, 100, 300, 500, 750, 1000, 5000, 10000] for i in sizes: if rank == 0: - A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('spd', size=i) + A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector( + "spd", size=i + ) else: A = None b = None @@ -96,14 +102,18 @@ if __name__ == "__main__": # Sprawdzanie poprawności rozwiązania (na procesie 0) if rank == 0: - print(f"Spd matrix with size {i}") + logger.info("Spd matrix with size %s", i) is_correct, error = check_solution(A, b, x) - print("Czas wykonania [s]:", time_taken) - print("Czy rozwiązanie jest poprawne:", "Tak" if is_correct else "Nie") - print("Błąd rozwiązania:", error) + logger.info("Czas wykonania [s]: %s", time_taken) + logger.info( + "Czy rozwiązanie jest poprawne: %s", "Tak" if is_correct else "Nie" + ) + logger.info("Błąd rozwiązania: %s", error) if rank == 0: - A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('nemeth12') + A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector( + "nemeth12" + ) else: A = None b = None @@ -118,14 +128,16 @@ if __name__ == "__main__": # Sprawdzanie poprawności rozwiązania (na procesie 0) if rank == 0: - print(f"Nemeth12 matrix") + logger.info("Nemeth12 matrix") is_correct, error = check_solution(A, b, x) - print("Czas wykonania [s]:", time_taken) - print("Czy rozwiązanie jest poprawne:", "Tak" if is_correct else "Nie") - print("Błąd rozwiązania:", error) + logger.info("Czas wykonania [s]: %s", time_taken) + logger.info("Czy rozwiązanie jest poprawne: %s", "Tak" if is_correct else "Nie") + logger.info("Błąd rozwiązania: %s", error) if rank == 0: - A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('poli3') + A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector( + "poli3" + ) else: A = None b = None @@ -140,8 +152,8 @@ if __name__ == "__main__": # Sprawdzanie poprawności rozwiązania (na procesie 0) if rank == 0: - print(f"Poli3 matrix") + logger.info("Poli3 matrix") is_correct, error = check_solution(A, b, x) - print("Czas wykonania [s]:", time_taken) - print("Czy rozwiązanie jest poprawne:", "Tak" if is_correct else "Nie") - print("Błąd rozwiązania:", error) + logger.info("Czas wykonania [s]: %s", time_taken) + logger.info("Czy rozwiązanie jest poprawne: %s", "Tak" if is_correct else "Nie") + logger.info("Błąd rozwiązania: %s", error) diff --git a/Programming/PORR/code/tests.py b/Programming/PORR/code/tests.py index 1bc1ba20..8f049ab0 100644 --- a/Programming/PORR/code/tests.py +++ b/Programming/PORR/code/tests.py @@ -1,15 +1,20 @@ -import pytest +import logging + import numpy as np +import pytest from matrix_generator import MatrixGenerator +from processing_type import ProcessingType from richardson_method import RichardsonMethod from threads_indep import RichardsonMethodThreads -from processing_type import ProcessingType -from time_measurement import time_measurement, tests_time +from time_measurement import tests_time, time_measurement + +logger = logging.getLogger(__name__) + def calculate_norm_numpy(I, w, A): difference = I - w * A - norm = np.linalg.norm(difference) - return norm + return np.linalg.norm(difference) + def calculate_eigenvalues(A): eigenvalues = np.linalg.eigvals(A) @@ -17,89 +22,113 @@ def calculate_eigenvalues(A): lambda_max = np.max(eigenvalues) return lambda_min, lambda_max + def calcualte_norm_from_matrix_numpy(A, n): lambda_min, lambda_max = calculate_eigenvalues(A) omega = 2 / (lambda_min + lambda_max) I = np.eye(n) return calculate_norm_numpy(I, omega, A) + @time_measurement(tests_time) def solution_lib(A, b): return np.linalg.solve(A, b) -@pytest.mark.parametrize("n", [ - 2, - 5, - 10, - 50, - 100, - 300, - 500, - 750, - 1000, - 5000, - 10000 - ]) -@pytest.mark.parametrize("processing_type", [ - ProcessingType.SEQUENTIAL, - ProcessingType.THREADS, - ProcessingType.PROCESSES, - ProcessingType.DISTRIBUTED_ARRAYS - ]) -@pytest.mark.parametrize("matrix_type", [ - "spd", - "nemeth12", - "poli3" - ]) -def test_richardson_vs_cg(n: int, processing_type: ProcessingType, matrix_type: str, capsys): - print("matrix type: ", matrix_type) - print("matrix size: ", n if matrix_type == "spd" else "fixed") + +@pytest.mark.parametrize("n", [2, 5, 10, 50, 100, 300, 500, 750, 1000, 5000, 10000]) +@pytest.mark.parametrize( + "processing_type", + [ + ProcessingType.SEQUENTIAL, + ProcessingType.THREADS, + ProcessingType.PROCESSES, + ProcessingType.DISTRIBUTED_ARRAYS, + ], +) +@pytest.mark.parametrize("matrix_type", ["spd", "nemeth12", "poli3"]) +def test_richardson_vs_cg( + n: int, processing_type: ProcessingType, matrix_type: str, capsys +) -> None: + logger.info("matrix type: %s", matrix_type) + logger.info("matrix size: %s", n if matrix_type == "spd" else "fixed") tolerance = 8e-3 - max_iterations=100 + max_iterations = 100 if matrix_type in ["nemeth12", "poli3"] and n != 2: - pytest.skip("Fixed matrix size for nemeth12 and poli3, skipping redundant runs.") - + pytest.skip( + "Fixed matrix size for nemeth12 and poli3, skipping redundant runs." + ) + if matrix_type == "spd": - A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('spd', size=n) + A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector( + "spd", size=n + ) elif matrix_type == "poli3": - A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('poli3') + A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector( + "poli3" + ) elif matrix_type == "nemeth12": - A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('nemeth12') + A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector( + "nemeth12" + ) else: - raise ValueError("Invalid matrix type specified. Choose 'spd', 'poli3', or 'nemeth12'.") - + raise ValueError( + "Invalid matrix type specified. Choose 'spd', 'poli3', or 'nemeth12'." + ) + solution_richardson, info_richardson = None, None if processing_type != ProcessingType.THREADS: - richardson_solver = RichardsonMethod(processing_type, A, b, lambda_min, lambda_max, max_iterations, size=A.shape[0], tol=1e-7) + richardson_solver = RichardsonMethod( + processing_type, + A, + b, + lambda_min, + lambda_max, + max_iterations, + size=A.shape[0], + tol=1e-7, + ) with capsys.disabled(): solution_richardson, info_richardson = richardson_solver.solve() else: with capsys.disabled(): - solution_richardson, info_richardson = RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, tol=1e-7) + solution_richardson, info_richardson = RichardsonMethodThreads( + A, b, lambda_min, lambda_max, max_iterations, tol=1e-7 + ) # Przechwytywanie wyjścia po solve captured = capsys.readouterr() - print("Captured output:", captured.out) - - solution = solution_lib(A,b) - + logger.info("Captured output: %s", captured.out) + + solution = solution_lib(A, b) + assert_converged(solution_richardson, info_richardson, solution, tolerance, A, n) -def assert_converged(solution_richardson, info_richardson, solution, tolerance, A, n): + +def assert_converged( + solution_richardson, info_richardson, solution, tolerance, A, n +) -> None: if info_richardson == "Richardson method for those values will NOT converge": numpy_norm = calcualte_norm_from_matrix_numpy(A, n) - print("Numpy norm: ", numpy_norm, " Richardson norm: ", solution_richardson) + logger.info( + "Numpy norm: %s Richardson norm: %s", numpy_norm, solution_richardson + ) assert False, "Richardson did not converge" else: difference = np.linalg.norm(solution_richardson - solution) - print(f"Difference between Richardson and numpy solutions: {difference:.8f}") + logger.info("Difference between Richardson and numpy solutions: %s", difference) if difference < tolerance: - print("Both Richardson and numpy method converged and calculated correct values.") + logger.info( + "Both Richardson and numpy method converged and calculated correct values." + ) else: - print("Solution numpy:\n", solution) - print("Solution Richardson:\n", solution_richardson) - assert difference < tolerance, f"The solutions are different! Difference: {difference:.8f}" + logger.info("Solution numpy:\n %s", solution) + logger.info("Solution Richardson:\n %s", solution_richardson) + assert difference < tolerance, ( + f"The solutions are different! Difference: {difference:.8f}" + ) + if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) pytest.main() diff --git a/Programming/PORR/code/threads_indep.py b/Programming/PORR/code/threads_indep.py index 08103025..7a7421cd 100644 --- a/Programming/PORR/code/threads_indep.py +++ b/Programming/PORR/code/threads_indep.py @@ -1,55 +1,71 @@ import gc +import logging import time -import numpy as np -from numba import njit, prange -from time_measurement import time_measurement_longest, longest_threads_time_accumulator, tests_time + import linear_algebra_utils as linAlg +from numba import njit, prange +from time_measurement import ( + longest_threads_time_accumulator, + tests_time, + time_measurement_longest, +) + +logger = logging.getLogger(__name__) @njit(parallel=True) -def numba_matrix_vector_multiply(A, input_x, Ax): +def numba_matrix_vector_multiply(A, input_x, Ax) -> None: for i in prange(len(A)): acc = 0.0 for j in range(len(input_x)): acc += A[i][j] * input_x[j] Ax[i] = acc + @njit(parallel=True) -def numba_vector_vector_subtraction(b, Ax, residual): +def numba_vector_vector_subtraction(b, Ax, residual) -> None: for i in prange(len(b)): residual[i] = b[i] - Ax[i] + @njit(nopython=True) -def numba_scalar_vector_multiply(omega, vector, result): - omega_real = omega.real +def numba_scalar_vector_multiply(omega, vector, result) -> None: + omega_real = omega.real for i in range(len(vector)): result[i] = omega_real * vector[i] + @njit(parallel=True) -def numba_vector_vector_addition(input_x, vector, output_x): +def numba_vector_vector_addition(input_x, vector, output_x) -> None: for i in prange(len(input_x)): output_x[i] = input_x[i] + vector[i] # Funkcje z dekoratorem @time_measurement_longest(longest_threads_time_accumulator) -def matrix_vector_multiply(A, input_x, Ax): +def matrix_vector_multiply(A, input_x, Ax) -> None: numba_matrix_vector_multiply(A, input_x, Ax) + @time_measurement_longest(longest_threads_time_accumulator) -def vector_vector_subtraction(b, Ax, residual): +def vector_vector_subtraction(b, Ax, residual) -> None: numba_vector_vector_subtraction(b, Ax, residual) + @time_measurement_longest(longest_threads_time_accumulator) -def scalar_vector_multiply(omega, vector, result): +def scalar_vector_multiply(omega, vector, result) -> None: numba_scalar_vector_multiply(omega, vector, result) + @time_measurement_longest(longest_threads_time_accumulator) -def vector_vector_addition(input_x, vector, output_x): +def vector_vector_addition(input_x, vector, output_x) -> None: numba_vector_vector_addition(input_x, vector, output_x) + # Metoda Richardson z obsługą wątków -def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5): +def RichardsonMethodThreads( + A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5 +): longest_threads_time_accumulator.hard_reset() gc.disable() @@ -88,6 +104,12 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non total_time = end_time - start_time sequential_time = total_time - longest_threads_time_accumulator.total_time - print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {longest_threads_time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") + logger.info( + "Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss", + total_time, + sequential_time, + longest_threads_time_accumulator.total_time, + tests_time.total_time, + ) return x, 0 diff --git a/Programming/PORR/code/threads_indep_old.py b/Programming/PORR/code/threads_indep_old.py index 7c5ecc5e..a97c29d3 100644 --- a/Programming/PORR/code/threads_indep_old.py +++ b/Programming/PORR/code/threads_indep_old.py @@ -1,28 +1,42 @@ -import multiprocessing import gc +import logging +import multiprocessing import time from concurrent.futures import ThreadPoolExecutor -from time_measurement import time_measurement_longest, longest_threads_time_accumulator, tests_time + import linear_algebra_utils as linAlg +from time_measurement import ( + longest_threads_time_accumulator, + tests_time, + time_measurement_longest, +) + +logger = logging.getLogger(__name__) @time_measurement_longest(longest_threads_time_accumulator) -def matrix_vector_multiply(A, input_x, start, end, Ax): - Ax[start:end] = [sum(x*y for x, y in zip(row, input_x)) for row in A[start:end]] +def matrix_vector_multiply(A, input_x, start, end, Ax) -> None: + Ax[start:end] = [sum(x * y for x, y in zip(row, input_x)) for row in A[start:end]] + @time_measurement_longest(longest_threads_time_accumulator) -def vector_vector_subtraction(b, Ax, start, end, residual): - residual[start:end] = [x-y for x, y in zip(b[start:end], Ax[start:end])] +def vector_vector_subtraction(b, Ax, start, end, residual) -> None: + residual[start:end] = [x - y for x, y in zip(b[start:end], Ax[start:end])] + @time_measurement_longest(longest_threads_time_accumulator) -def scalar_vector_multiply(omega, vector, start, end, result): +def scalar_vector_multiply(omega, vector, start, end, result) -> None: result[start:end] = [omega * x for x in vector[start:end]] -@time_measurement_longest(longest_threads_time_accumulator) -def vector_vector_addition(input_x, vector, start, end, output_x): - output_x[start:end] = [x+y for x, y in zip(input_x[start:end], vector[start:end])] -def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5): +@time_measurement_longest(longest_threads_time_accumulator) +def vector_vector_addition(input_x, vector, start, end, output_x) -> None: + output_x[start:end] = [x + y for x, y in zip(input_x[start:end], vector[start:end])] + + +def RichardsonMethodThreads( + A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5 +): longest_threads_time_accumulator.hard_reset() gc.disable() @@ -35,69 +49,101 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non num_threads = multiprocessing.cpu_count() chunk_size = n // num_threads - with ThreadPoolExecutor(max_workers=num_threads) as executor: # wątki są tworzone raz i nie są niszczone + with ThreadPoolExecutor( + max_workers=num_threads + ) as executor: # wątki są tworzone raz i nie są niszczone for iteration in range(max_iterations): - - Ax = [0] * len(x) # tutaj zostanie przypisany wynik z mnożenia macierzy A z wektorem x + Ax = [0] * len( + x + ) # tutaj zostanie przypisany wynik z mnożenia macierzy A z wektorem x futures = [] for i in range(num_threads): start = i * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size - futures.append(executor.submit(matrix_vector_multiply, A, x, start, end, Ax)) - + futures.append( + executor.submit(matrix_vector_multiply, A, x, start, end, Ax) + ) + for future in futures: future.result() longest_threads_time_accumulator.save_lap_and_reset() - residual = [0] * len(b) # tutaj zostanie przypisany wynik z vector_vector_subtraction + residual = [0] * len( + b + ) # tutaj zostanie przypisany wynik z vector_vector_subtraction futures = [] for i in range(num_threads): start = i * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size - futures.append(executor.submit(vector_vector_subtraction, b, Ax, start, end, residual)) - + futures.append( + executor.submit( + vector_vector_subtraction, b, Ax, start, end, residual + ) + ) + for future in futures: future.result() longest_threads_time_accumulator.save_lap_and_reset() - change_vector = [0] * len(residual) # zostanie tu przypisany wynik scalar_vector_multiply po pracy wątków + change_vector = [0] * len( + residual + ) # zostanie tu przypisany wynik scalar_vector_multiply po pracy wątków futures = [] for i in range(num_threads): start = i * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size - futures.append(executor.submit(scalar_vector_multiply, omega, residual, start, end, change_vector)) - + futures.append( + executor.submit( + scalar_vector_multiply, + omega, + residual, + start, + end, + change_vector, + ) + ) + for future in futures: future.result() longest_threads_time_accumulator.save_lap_and_reset() - _x = x[:] # do _x zostanie przez wątki przypisany wynik pracy w danej iteracji + _x = x[ + : + ] # do _x zostanie przez wątki przypisany wynik pracy w danej iteracji futures = [] for i in range(num_threads): start = i * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size - futures.append(executor.submit(vector_vector_addition, x, change_vector, start, end, _x)) - + futures.append( + executor.submit( + vector_vector_addition, x, change_vector, start, end, _x + ) + ) + for future in futures: future.result() - + longest_threads_time_accumulator.save_lap_and_reset() x = _x[:] - - if (linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < tol): - break - + if linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < tol: + break end_time = time.perf_counter() gc.enable() total_time = end_time - start_time sequential_time = total_time - longest_threads_time_accumulator.total_time - - print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {longest_threads_time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") - - return x, 0 \ No newline at end of file + + logger.info( + "Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss", + total_time, + sequential_time, + longest_threads_time_accumulator.total_time, + tests_time.total_time, + ) + + return x, 0 diff --git a/Programming/PORR/code/threads_too_simple.py b/Programming/PORR/code/threads_too_simple.py index d5ac2713..dc20eaff 100644 --- a/Programming/PORR/code/threads_too_simple.py +++ b/Programming/PORR/code/threads_too_simple.py @@ -1,20 +1,30 @@ -import numpy as np -import threading -import multiprocessing import gc -import time +import logging +import multiprocessing import sys -from time_measurement import time_measurement_longest, longest_time_accumulator, tests_time -import linear_algebra_utils as linAlg +import threading +import time + +import numpy as np +from time_measurement import ( + longest_time_accumulator, + tests_time, + time_measurement_longest, +) + +logger = logging.getLogger(__name__) @time_measurement_longest(longest_time_accumulator) -def RichardsonThread(A, b, x, _x, omega, start, end): +def RichardsonThread(A, b, x, _x, omega, start, end) -> None: for i in range(start, end): sigma = np.dot(A[i, :], _x) - A[i, i] * _x[i] x[i] = (1 - omega) * _x[i] + omega * (b[i] - sigma) / A[i, i] -def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5): + +def RichardsonMethodThreads( + A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5 +): longest_time_accumulator.total_time = 0 longest_time_accumulator.start = sys.float_info.max longest_time_accumulator.end = 0 @@ -25,7 +35,7 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non n = len(b) x0 = x0 if x0 is not None else [0.0] * len(b) x = x0[:] - omega = 0.05#2 / (lambda_min + lambda_max) + omega = 0.05 # 2 / (lambda_min + lambda_max) num_threads = multiprocessing.cpu_count() threads = [] chunk_size = n // num_threads @@ -34,27 +44,32 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non for _ in range(max_iterations): _x = x[:] for i in range(num_threads): - start = i * chunk_size # start jest indeksem w A. Wątki otrzymują kolejny punkt startowy będący wielokrotnością rozmiaru porcji na wątek + start = ( + i * chunk_size + ) # start jest indeksem w A. Wątki otrzymują kolejny punkt startowy będący wielokrotnością rozmiaru porcji na wątek end = n if i == num_threads - 1 else (i + 1) * chunk_size - thread = threading.Thread(target=RichardsonThread, args=(A, b, x, _x, omega, start, end)) + thread = threading.Thread( + target=RichardsonThread, args=(A, b, x, _x, omega, start, end) + ) threads.append(thread) thread.start() - + for thread in threads: thread.join() - - # Ax = linAlg.SequentialLinearAlgebraUtils.matrix_vector_multiply(A, x) - # residual = linAlg.SequentialLinearAlgebraUtils.vector_vector_subtraction(b, Ax) - # if (linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < tol): + # break end_time = time.perf_counter() gc.enable() total_time = end_time - start_time sequential_time = total_time - longest_time_accumulator.total_time - - print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {longest_time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") - + + logger.info( + "Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss", + total_time, + sequential_time, + longest_time_accumulator.total_time, + tests_time.total_time, + ) + return x, 0 - - diff --git a/Programming/PORR/code/time_measurement.py b/Programming/PORR/code/time_measurement.py index d70992a2..43fc8b5a 100644 --- a/Programming/PORR/code/time_measurement.py +++ b/Programming/PORR/code/time_measurement.py @@ -1,25 +1,27 @@ -import time import sys +import time from functools import wraps + class TimeAccumulator: - def __init__(self): + def __init__(self) -> None: self.total_time = 0 + class ComplexTimeAcumulator: - def __init__(self): + def __init__(self) -> None: self.hard_reset() - - def hard_reset(self): + + def hard_reset(self) -> None: self.total_time = 0 self.reset() - - def reset(self): + + def reset(self) -> None: self.lap_time = 0 self.start = sys.float_info.max self.end = 0 - - def save_lap_and_reset(self): + + def save_lap_and_reset(self) -> None: self.total_time += self.lap_time self.reset() @@ -29,6 +31,7 @@ tests_time = TimeAccumulator() longest_threads_time_accumulator = ComplexTimeAcumulator() + def time_measurement(accumulator: TimeAccumulator): def decorator(func): @wraps(func) @@ -38,9 +41,12 @@ def time_measurement(accumulator: TimeAccumulator): end = time.perf_counter() accumulator.total_time += end - start return result + return inner + return decorator + def time_measurement_longest(accumulator: ComplexTimeAcumulator): def decorator(func): @wraps(func) @@ -48,15 +54,13 @@ def time_measurement_longest(accumulator: ComplexTimeAcumulator): start = time.perf_counter() result = func(*args, **kwargs) end = time.perf_counter() - if start < accumulator.start: - accumulator.start = start - if end > accumulator.end: - accumulator.end = end - accumulator.lap_time = accumulator.end - accumulator.start # "=" instead of "+=" + accumulator.start = min(accumulator.start, start) + accumulator.end = max(accumulator.end, end) + accumulator.lap_time = ( + accumulator.end - accumulator.start + ) # "=" instead of "+=" return result + return inner + return decorator - - - - diff --git a/Programming/PORR/run.sh b/Programming/PORR/run.sh index 7b687781..8ac10666 100755 --- a/Programming/PORR/run.sh +++ b/Programming/PORR/run.sh @@ -7,6 +7,31 @@ set -euo pipefail # shellcheck source=../../scripts/course_lib.sh . "$(dirname "$(readlink -f "$0")")/../../scripts/course_lib.sh" +# This suite is a BENCHMARK, not a smoke test. Measured on this machine, the +# cost is not where you would guess -- it is not the matrix size: +# +# ProcessingType.PROCESSES 20.9s at n=2, 32.8s at n=5, 78.2s at n=10. +# That is pool setup per test, and it grows. +# matrix_type poli3 densifies a sparse .mat into a 2.3 GB cache +# (nemeth12: 723 MB) -- over the 2G ceiling +# `../../run.sh run` imposes, before any solving. +# n=10000 the SPD matrix alone is 800 MB and +# np.linalg.solve needs a second copy. +# n=5000 spd_5000.npz was a Git LFS pointer whose object +# 404s on the server, so this size regenerates +# from scratch: a dense 5000x5000 eigendecomposition. +# +# What is left after removing those is fast and still covers the whole +# Richardson implementation across every size that fits: 18 tests, 28 seconds. +# The full submitted parameter set is one environment variable away, and needs +# headroom the default deliberately does not give it: +# +# PYTEST_ADDOPTS= RUN_MEMORY_MAX=16G RUN_CPU_QUOTA=800% ./run.sh run Programming/PORR +# +# main.py hardcodes its pytest arguments, so the selection is passed the only +# way that survives that: PYTEST_ADDOPTS, which pytest.main() also honours. +export PYTEST_ADDOPTS="${PYTEST_ADDOPTS--k 'spd and (SEQUENTIAL or THREADS) and not 5000 and not 10000'}" + course_main --kind python \ --entry "$(dirname "$(readlink -f "$0")")/code/main.py" \ -- "$@" diff --git a/scripts/check_refactor_safety.py b/scripts/check_refactor_safety.py new file mode 100755 index 00000000..dd119247 --- /dev/null +++ b/scripts/check_refactor_safety.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Three checks that catch what ruff and a smoke run both miss. + +Every one of these exists because it caught a real regression during the +`select = ["ALL"]` campaign on this archive: + +1. **Signatures.** Annotating 110 EARIN functions silently dropped the DEFAULT + VALUES from 22 of them, so a call relying on a default would have raised + TypeError. Nothing flagged it: ruff does not look at defaults, and the + affected paths were not the ones a smoke run exercises. +2. **Logging.** Rewriting ``print()`` as ``logger.info()`` made programs + silent, because logging's last-resort handler only emits WARNING and above. + A module that logs and has a ``__main__`` guard must configure logging. +3. **print keywords.** ``print(x, end="")`` builds a line incrementally. + Dropping ``end=`` turned a rendered chess board into one glyph per line; + dropping ``file=sys.stderr`` would silently move error output to stdout. + +Usage:: + + scripts/check_refactor_safety.py --logging --print-keywords + scripts/check_refactor_safety.py --signatures + +Exit status is 1 if anything is reported. +""" + +from __future__ import annotations + +import argparse +import ast +import logging +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + +_MAIN_GUARD = "__name__ == '__main__'" + + +def _functions(source: str) -> dict[str, tuple[set[str], dict[str, str]]]: + """Map each function name to its parameter names and their defaults.""" + out: dict[str, tuple[set[str], dict[str, str]]] = {} + for node in ast.walk(ast.parse(source)): + if not isinstance(node, ast.FunctionDef): + continue + args = node.args + defaults: dict[str, str] = {} + # defaults cover posonlyargs and args together, tail-aligned. + ordered = args.posonlyargs + args.args + positional = ordered[len(ordered) - len(args.defaults) :] + for arg, value in zip(positional, args.defaults, strict=True): + defaults[arg.arg] = ast.unparse(value) + for arg, value in zip(args.kwonlyargs, args.kw_defaults, strict=True): + if value is not None: + defaults[arg.arg] = ast.unparse(value) + names = {a.arg for a in args.posonlyargs + args.args + args.kwonlyargs} + out[node.name] = (names, defaults) + return out + + +def check_signatures(before: Path, after: Path) -> int: + """Report any parameter or default that changed between two trees.""" + problems = 0 + for old in sorted(before.rglob("*.py")): + new = after / old.relative_to(before) + if not new.exists(): + continue + was = _functions(old.read_text(encoding="utf-8")) + now = _functions(new.read_text(encoding="utf-8")) + for name, (names, defaults) in was.items(): + if name not in now: + logger.error("%s: %s disappeared", new, name) + problems += 1 + continue + names_now, defaults_now = now[name] + missing = sorted(names - names_now) + if missing: + logger.error("%s: %s lost parameters %s", new, name, missing) + problems += 1 + changed = { + key: (value, defaults_now.get(key)) + for key, value in defaults.items() + if key in names_now and defaults_now.get(key) != value + } + if changed: + logger.error("%s: %s default(s) changed %s", new, name, changed) + problems += 1 + return problems + + +def check_logging(paths: list[Path]) -> int: + """Report modules that log but never configure logging.""" + problems = 0 + for path in paths: + source = path.read_text(encoding="utf-8", errors="ignore") + if "logger.info(" not in source and "logger.debug(" not in source: + continue + if "basicConfig" in source: + continue + tree = ast.parse(source) + runnable = any( + isinstance(node, ast.If) and ast.unparse(node.test) == _MAIN_GUARD + for node in tree.body + ) + # Only a module that can be RUN is at fault: configuring logging is + # the entry point's job, not a library module's. + if runnable: + logger.error("%s: logs at INFO but never calls basicConfig", path) + problems += 1 + return problems + + +def check_print_keywords(paths: list[Path]) -> int: + """Report print() calls whose keywords an automated rewrite would lose.""" + problems = 0 + for path in paths: + tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore")) + for node in ast.walk(tree): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "print" + and node.keywords + ): + continue + keys = sorted(k.arg or "**" for k in node.keywords) + logger.error( + "%s:%d: print() with %s -- convert this one by hand", + path, + node.lineno, + ", ".join(keys), + ) + problems += 1 + return problems + + +def _python_files(roots: list[str]) -> list[Path]: + """Every .py under the given roots, skipping venvs and vendored trees.""" + found: list[Path] = [] + for root in roots: + base = Path(root) + candidates = [base] if base.is_file() else sorted(base.rglob("*.py")) + found.extend( + p for p in candidates if ".venv" not in p.parts and "sightpy" not in p.parts + ) + return found + + +def main(argv: list[str] | None = None) -> int: + """Run the requested checks and return 1 if any reported something.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("paths", nargs="+", help="files, directories, or two trees") + parser.add_argument("--signatures", action="store_true") + parser.add_argument("--logging", action="store_true") + parser.add_argument("--print-keywords", action="store_true") + args = parser.parse_args(argv) + logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout) + + problems = 0 + if args.signatures: + expected = 2 + if len(args.paths) != expected: + logger.error("--signatures needs exactly two trees: before after") + return 1 + problems += check_signatures(Path(args.paths[0]), Path(args.paths[1])) + if args.logging or args.print_keywords: + files = _python_files(args.paths) + if args.logging: + problems += check_logging(files) + if args.print_keywords: + problems += check_print_keywords(files) + + logger.info("problems: %d", problems) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main())