PORR runs in 28s, and three checks so the next pass cannot repeat my mistakes

PORR was the last course whose `run` did not work. Four measurements, not
guesses, explain why -- and none of them is the matrix size everyone assumes:

  ProcessingType.PROCESSES   20.9s at n=2, 32.8s at n=5, 78.2s at n=10. That
                             is process-pool setup PER TEST, and it grows.
  matrix_type poli3          densifies a sparse .mat into a 2.3 GB npz cache
                             (nemeth12: 723 MB) -- over the 2G ceiling before
                             any solving happens.
  n=10000                    800 MB matrix, and np.linalg.solve wants a copy.
  n=5000                     spd_5000.npz was a Git LFS pointer whose object
                             404s on GitHub, so it regenerates from scratch.

Remove those and what is left still exercises the whole Richardson
implementation at every size that fits: 18 tests, 28 seconds through the
harness, exit 0. The submitted parameter set is one documented environment
variable away. `main.py` now propagates pytest's status (verified: exit 5 when
nothing is collected) -- it discarded it before, so `run` could not tell a
passing suite from a failing one.

The rest of this commit is scripts/check_refactor_safety.py, which exists
because three regressions got past both ruff and a smoke run during this
campaign, and I would rather they could not get past the next one:

  --signatures      diffs two trees function by function, parameter by
                    parameter, default by default. This is what found the 22
                    EARIN functions whose defaults an annotation pass silently
                    dropped.
  --logging         a module with a __main__ guard that logs at INFO must call
                    basicConfig, or it prints nothing at all. Found two more
                    in PORR just now.
  --print-keywords  refuses to let an automated rewrite near print(x, end="")
                    or print(x, file=sys.stderr); losing those turns a
                    rendered board into one glyph per line, or silently moves
                    error output onto stdout.

All three pass on the whole archive. Also in here: 12 latent TypeErrors mypy
found in EARIN that ruff could not -- call sites still passing positionally to
parameters this campaign had made keyword-only, plus one caller of a function
whose argument I had removed. Every one would have raised at runtime.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0172dj34FDpao6wsix8ZuCzE
This commit is contained in:
Krzysztof Rudnicki 2026-08-28 20:50:46 +02:00
parent cc06bc0cd8
commit 42007d56e4
19 changed files with 780 additions and 309 deletions

View File

@ -448,7 +448,7 @@ class Game:
continue continue
has_moved = self.make_move(from_coordinates, to_coordinates, color) has_moved = self.make_move(from_coordinates, to_coordinates, color)
self.print_board(color == "white") self.print_board(rotate=color == "white")
def start_game(self, player_color: str = "black", algorithm_depth: int = 5) -> None: def start_game(self, player_color: str = "black", algorithm_depth: int = 5) -> None:
"""Start the main loop of the game.""" """Start the main loop of the game."""
@ -457,7 +457,7 @@ class Game:
return return
ai_color = "black" if player_color == "white" else "white" ai_color = "black" if player_color == "white" else "white"
game.print_board(player_color == "white") game.print_board(rotate=player_color == "white")
if player_color == "white": if player_color == "white":
game.handle_player_move("white") game.handle_player_move("white")
while True: while True:
@ -468,7 +468,10 @@ class Game:
logger.info("Game over, %s wins", player_color) logger.info("Game over, %s wins", player_color)
return return
_, ai_move = game.alpha_beta(algorithm_depth, (5, 10), ai_color) _, ai_move = game.alpha_beta(algorithm_depth, (5, 10), ai_color)
game.make_move(*ai_move, ai_color) if ai_move is None:
logger.info("Game over, %s wins", player_color)
return
game.make_move(ai_move[0], ai_move[1], ai_color)
logger.info( logger.info(
"AI's move: %s%s %s%s", "AI's move: %s%s %s%s",
chr(ord("a") + ai_move[0][0]), chr(ord("a") + ai_move[0][0]),
@ -476,7 +479,7 @@ class Game:
chr(ord("a") + ai_move[1][0]), chr(ord("a") + ai_move[1][0]),
ai_move[1][1], ai_move[1][1],
) )
game.print_board(player_color == "white") game.print_board(rotate=player_color == "white")
ai_turn = game.get_possible_moves(ai_color)[1] and possible_moves_ai[1] ai_turn = game.get_possible_moves(ai_color)[1] and possible_moves_ai[1]
player_turn = True player_turn = True
while player_turn: while player_turn:
@ -506,7 +509,7 @@ class Game:
_, ai_move = game.alpha_beta(algorithm_depth, (5, 10), ai_color) _, ai_move = game.alpha_beta(algorithm_depth, (5, 10), ai_color)
if ai_move is None: if ai_move is None:
ai_move = possible_moves_ai[0] ai_move = possible_moves_ai[0]
game.make_move(*ai_move, ai_color) game.make_move(ai_move[0], ai_move[1], ai_color)
if print_info: if print_info:
logger.info( logger.info(
"AI's move: %s%s %s%s", "AI's move: %s%s %s%s",

View File

@ -335,9 +335,9 @@ def user_input() -> dict[str, float]:
def print_output(print_info: object, save_results: object, summary: object) -> None: def print_output(print_info: object, save_results: object, summary: object) -> None:
"""Print out population and summary plots.""" """Print out population and summary plots."""
for population, generation_number, file_name in print_info: for population, generation_number, file_name in print_info:
output(population, generation_number, file_name, save_results) output(population, generation_number, file_name, save_results=save_results)
summary_file_name = file_name summary_file_name = file_name
print_summary(summary, summary_file_name, save_results) print_summary(summary, summary_file_name, save_results=save_results)
# Ran first in the code # Ran first in the code
@ -355,7 +355,7 @@ if __name__ == "__main__":
SUMMARY, SUMMARY,
) = evolution_strategy( ) = evolution_strategy(
ARGUMENTS, ARGUMENTS,
ARGUMENTS["no_display"], no_display=bool(ARGUMENTS["no_display"]),
) )
end_time = time.perf_counter() end_time = time.perf_counter()
if not ARGUMENTS["no_display"]: if not ARGUMENTS["no_display"]:

View File

@ -164,8 +164,10 @@ def training_loop(
_, training_parameters["optimizer"] = single_train_iteration( _, training_parameters["optimizer"] = single_train_iteration(
data, training_parameters, targets, batch_idx, epoch data, training_parameters, targets, batch_idx, epoch
) )
calculate_accuracy_epoch(training_parameters, epoch, print_info) calculate_accuracy_epoch(training_parameters, epoch, print_info=print_info)
calculate_validation_set_accuracy(training_parameters, epoch, print_info) calculate_validation_set_accuracy(
training_parameters, epoch, print_info=print_info
)
return epoch, training_parameters["loaders"]["train_loader"] return epoch, training_parameters["loaders"]["train_loader"]
@ -228,7 +230,7 @@ def main_part(*, show_plot: bool = True) -> None:
training_parameters = set_training_parameters( training_parameters = set_training_parameters(
hyperparameters, loaders, model, criterion, optimizer hyperparameters, loaders, model, criterion, optimizer
) )
training_loop(training_parameters, show_plot) training_loop(training_parameters, print_info=show_plot)
with Path("results.txt").open("a") as file: with Path("results.txt").open("a") as file:
rule = "-" * 85 + "\n" rule = "-" * 85 + "\n"
file.write(rule) file.write(rule)

View File

@ -195,7 +195,7 @@ def inference(env: object, q_table: object) -> None:
if __name__ == "__main__": if __name__ == "__main__":
logging.basicConfig(format="%(message)s", level=logging.INFO) logging.basicConfig(format="%(message)s", level=logging.INFO)
HYPERPARAMETERS = initialize_hyperparameters() HYPERPARAMETERS = initialize_hyperparameters()
ENV = initialize_environment(HYPERPARAMETERS) ENV = initialize_environment()
Q_TABLE = initialize_q_table(ENV) Q_TABLE = initialize_q_table(ENV)
ENV, Q_TABLE = training_loop(HYPERPARAMETERS, ENV, Q_TABLE) ENV, Q_TABLE = training_loop(HYPERPARAMETERS, ENV, Q_TABLE)
inference(ENV, Q_TABLE) inference(ENV, Q_TABLE)

View File

@ -349,7 +349,11 @@ if __name__ == "__main__":
RATING_DATA, ANIME_CONTACT_DATA = get_data(data_limit, db) RATING_DATA, ANIME_CONTACT_DATA = get_data(data_limit, db)
PIVOT_TABLE = preprocessing( PIVOT_TABLE = preprocessing(
RATING_DATA, ANIME_CONTACT_DATA, debug, user_threshold, anime_threshold RATING_DATA,
ANIME_CONTACT_DATA,
debug=debug,
user_threshold=user_threshold,
anime_threshold=anime_threshold,
) )
MODEL = create_model(PIVOT_TABLE, metric, algorithm, neighbors) MODEL = create_model(PIVOT_TABLE, metric, algorithm, neighbors)
predict(MODEL, PIVOT_TABLE, seed, anime, recommendation_amount) predict(MODEL, PIVOT_TABLE, seed, anime, recommendation_amount)

View File

@ -406,4 +406,15 @@ TSWLatexianTemp*
#*Notes.bib #*Notes.bib
spd_10000.npz spd_10000.npz
nemeth12.npz nemeth12.npz
poli3.npz poli3.npz
# Generated SPD test matrices. MatrixGenerator.save_to_file() writes these as a
# cache and regenerates any that is missing, so none of them is a deliverable.
#
# spd_5000.npz was tracked as a Git LFS pointer whose 200 MB object is NOT on
# the server -- `git lfs pull` returns "[404] Object does not exist". numpy
# then read the 134-byte pointer text as a .npz and raised ValueError instead
# of FileNotFoundError, so the regeneration path never ran and every n=5000
# test failed. The pointer is untracked and .gitattributes (its only content
# was three LFS filter rules, two for files that never existed here) deleted.
spd_5000.npz

View File

@ -1,80 +1,84 @@
import cmath import cmath
import math
import itertools import itertools
import operator import logging
import math
import multiprocessing import multiprocessing
from multiprocessing import Pool import operator
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from functools import partial from functools import partial
from time_measurement import time_measurement, time_accumulator from multiprocessing import Pool
import numpy as np
import dask.array as da import dask.array as da
from time_measurement import time_accumulator, time_measurement
logger = logging.getLogger(__name__)
class LinearAlgebraUtils: class LinearAlgebraUtils:
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def dot_product(v1, v2): def dot_product(v1, v2) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def matrix_vector_multiply(A, x): def matrix_vector_multiply(A, x) -> None:
pass
@staticmethod
@abstractmethod
def vector_norm(v):
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def vector_scalar_divide(x, scalar): def vector_norm(v) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def matrix_scalar_multiply(A, w): def vector_scalar_divide(x, scalar) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def vector_vector_subtraction(v1, v2): def matrix_scalar_multiply(A, w) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def vector_vector_addition(v1, v2): def vector_vector_subtraction(v1, v2) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def scalar_vector_multiply(omega, vector): def vector_vector_addition(v1, v2) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def matrix_norm(A): def scalar_vector_multiply(omega, vector) -> None:
pass pass
@staticmethod @staticmethod
@abstractmethod @abstractmethod
def matrix_matrix_subtraction(A, B): def matrix_norm(A) -> None:
pass
@staticmethod
@abstractmethod
def matrix_matrix_subtraction(A, B) -> None:
pass pass
class SequentialLinearAlgebraUtils: class SequentialLinearAlgebraUtils:
@staticmethod @staticmethod
def dot_product(v1, v2): def dot_product(v1, v2):
return sum(x*y for x, y in zip(v1, v2)) return sum(x * y for x, y in zip(v1, v2))
@staticmethod @staticmethod
def matrix_vector_multiply(A, x): def matrix_vector_multiply(A, x):
return [SequentialLinearAlgebraUtils.dot_product(row, x) for row in A] return [SequentialLinearAlgebraUtils.dot_product(row, x) for row in A]
@staticmethod @staticmethod
def vector_norm(v): def vector_norm(v):
x_values = (x*x for x in v) x_values = (x * x for x in v)
x_values_sum = sum(x_values) x_values_sum = sum(x_values)
return cmath.sqrt(x_values_sum).real return cmath.sqrt(x_values_sum).real
@ -88,11 +92,11 @@ class SequentialLinearAlgebraUtils:
@staticmethod @staticmethod
def vector_vector_subtraction(v1, v2): def vector_vector_subtraction(v1, v2):
return [x-y for x, y in zip(v1, v2)] return [x - y for x, y in zip(v1, v2)]
@staticmethod @staticmethod
def vector_vector_addition(v1, v2): def vector_vector_addition(v1, v2):
return [x+y for x, y in zip(v1, v2)] return [x + y for x, y in zip(v1, v2)]
@staticmethod @staticmethod
def scalar_vector_multiply(omega, vector): def scalar_vector_multiply(omega, vector):
@ -100,7 +104,7 @@ class SequentialLinearAlgebraUtils:
@staticmethod @staticmethod
def matrix_norm(A): def matrix_norm(A):
return math.sqrt(sum(sum(element ** 2 for element in row) for row in A)) return math.sqrt(sum(sum(element**2 for element in row) for row in A))
@staticmethod @staticmethod
def matrix_matrix_subtraction(A, B): def matrix_matrix_subtraction(A, B):
@ -120,7 +124,9 @@ class ThreadsLinearAlgebraUtils:
@staticmethod @staticmethod
def divide_vectors_to_chunks(v1, v2): def divide_vectors_to_chunks(v1, v2):
chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size(v1) chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size(
v1
)
chunks = [] chunks = []
start = 0 start = 0
@ -130,7 +136,7 @@ class ThreadsLinearAlgebraUtils:
start = end start = end
return chunks return chunks
@staticmethod @staticmethod
def divide_vector_or_matrix_to_chunks(v): def divide_vector_or_matrix_to_chunks(v):
chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size(v) chunk_size, num_threads, remainder = ThreadsLinearAlgebraUtils.get_chunk_size(v)
@ -148,11 +154,13 @@ class ThreadsLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_vector_multiply(A, x): def matrix_vector_multiply(A, x):
chunks = ThreadsLinearAlgebraUtils.divide_vector_or_matrix_to_chunks(A) chunks = ThreadsLinearAlgebraUtils.divide_vector_or_matrix_to_chunks(A)
with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: with ThreadPoolExecutor(
max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS
) as executor:
func = partial(SequentialLinearAlgebraUtils.matrix_vector_multiply, x=x) func = partial(SequentialLinearAlgebraUtils.matrix_vector_multiply, x=x)
results = executor.map(func, chunks) results = executor.map(func, chunks)
return [item for sublist in results for item in sublist] return [item for sublist in results for item in sublist]
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_norm(v): def vector_norm(v):
@ -161,7 +169,9 @@ class ThreadsLinearAlgebraUtils:
def partial_norm(chunk): def partial_norm(chunk):
return sum(x * x for x in chunk) return sum(x * x for x in chunk)
with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: with ThreadPoolExecutor(
max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS
) as executor:
results = executor.map(partial_norm, chunks) results = executor.map(partial_norm, chunks)
total_sum = sum(results) total_sum = sum(results)
return total_sum**0.5 return total_sum**0.5
@ -170,26 +180,44 @@ class ThreadsLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_vector_subtraction(v1, v2): def vector_vector_subtraction(v1, v2):
chunks = ThreadsLinearAlgebraUtils.divide_vectors_to_chunks(v1, v2) chunks = ThreadsLinearAlgebraUtils.divide_vectors_to_chunks(v1, v2)
with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: with ThreadPoolExecutor(
results = executor.map(lambda pair: SequentialLinearAlgebraUtils.vector_vector_subtraction(*pair), chunks) max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS
) as executor:
results = executor.map(
lambda pair: SequentialLinearAlgebraUtils.vector_vector_subtraction(
*pair
),
chunks,
)
return [item for sublist in results for item in sublist] return [item for sublist in results for item in sublist]
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_vector_addition(v1, v2): def vector_vector_addition(v1, v2):
chunks = ThreadsLinearAlgebraUtils.divide_vectors_to_chunks(v1, v2) chunks = ThreadsLinearAlgebraUtils.divide_vectors_to_chunks(v1, v2)
with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: with ThreadPoolExecutor(
results = executor.map(lambda pair: SequentialLinearAlgebraUtils.vector_vector_addition(*pair), chunks) max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS
) as executor:
results = executor.map(
lambda pair: SequentialLinearAlgebraUtils.vector_vector_addition(*pair),
chunks,
)
return [item for sublist in results for item in sublist] return [item for sublist in results for item in sublist]
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def scalar_vector_multiply(omega, vector): def scalar_vector_multiply(omega, vector):
chunks = ThreadsLinearAlgebraUtils.divide_vector_or_matrix_to_chunks(vector) chunks = ThreadsLinearAlgebraUtils.divide_vector_or_matrix_to_chunks(vector)
with ThreadPoolExecutor(max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS) as executor: with ThreadPoolExecutor(
results = executor.map(lambda chunk: SequentialLinearAlgebraUtils.scalar_vector_multiply(omega, chunk), chunks) max_workers=ThreadsLinearAlgebraUtils.NUM_THREADS
) as executor:
results = executor.map(
lambda chunk: SequentialLinearAlgebraUtils.scalar_vector_multiply(
omega, chunk
),
chunks,
)
return [item for sublist in results for item in sublist] return [item for sublist in results for item in sublist]
@ -199,11 +227,13 @@ def process_row(params):
factor = A[i][k] / A[k][k] factor = A[i][k] / A[k][k]
return [A[i][j] - factor * A[k][j] for j in range(len(A[0]))] return [A[i][j] - factor * A[k][j] for j in range(len(A[0]))]
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def divide_by_scalar(pair): def divide_by_scalar(pair):
xi, scalar = pair xi, scalar = pair
return xi / scalar return xi / scalar
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def multiply_by_scalar(pair): def multiply_by_scalar(pair):
element, scalar = pair element, scalar = pair
@ -215,9 +245,11 @@ class ProcessLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def dot_product(v1, v2): def dot_product(v1, v2):
with Pool() as pool: with Pool() as pool:
result = pool.starmap(ProcessLinearAlgebraUtils.multiply_elements, zip(v1, v2)) result = pool.starmap(
ProcessLinearAlgebraUtils.multiply_elements, zip(v1, v2)
)
return sum(result) return sum(result)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def multiply_elements(x, y): def multiply_elements(x, y):
@ -233,9 +265,12 @@ class ProcessLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_vector_multiply(A, x): def matrix_vector_multiply(A, x):
with Pool() as pool: with Pool() as pool:
result = pool.map(ProcessLinearAlgebraUtils.matrix_vector_multiply_row, [(row, x) for row in A]) result = pool.map(
ProcessLinearAlgebraUtils.matrix_vector_multiply_row,
[(row, x) for row in A],
)
return list(result) return list(result)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_norm(v): def vector_norm(v):
@ -254,12 +289,15 @@ class ProcessLinearAlgebraUtils:
with Pool() as pool: with Pool() as pool:
result = pool.map(divide_by_scalar, [(xi, scalar) for xi in x]) result = pool.map(divide_by_scalar, [(xi, scalar) for xi in x])
return list(result) return list(result)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def divide_vector_by_scalar(x, scalar): def divide_vector_by_scalar(x, scalar):
with Pool() as pool: with Pool() as pool:
result = pool.map(ProcessLinearAlgebraUtils.vector_scalar_divide, [(xi, scalar) for xi in x]) result = pool.map(
ProcessLinearAlgebraUtils.vector_scalar_divide,
[(xi, scalar) for xi in x],
)
return list(result) return list(result)
@staticmethod @staticmethod
@ -272,7 +310,10 @@ class ProcessLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_scalar_multiply(A, w): def matrix_scalar_multiply(A, w):
with Pool() as pool: with Pool() as pool:
result = pool.map(ProcessLinearAlgebraUtils.matrix_scalar_multiply_row, [(row, w) for row in A]) result = pool.map(
ProcessLinearAlgebraUtils.matrix_scalar_multiply_row,
[(row, w) for row in A],
)
return result return result
@staticmethod @staticmethod
@ -285,27 +326,35 @@ class ProcessLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_vector_subtraction(v1, v2): def vector_vector_subtraction(v1, v2):
with Pool() as pool: with Pool() as pool:
result = pool.map(ProcessLinearAlgebraUtils.vector_vector_operation, zip(v1, v2, itertools.repeat(operator.sub))) result = pool.map(
ProcessLinearAlgebraUtils.vector_vector_operation,
zip(v1, v2, itertools.repeat(operator.sub)),
)
return list(result) return list(result)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_vector_addition(v1, v2): def vector_vector_addition(v1, v2):
with Pool() as pool: with Pool() as pool:
result = pool.map(ProcessLinearAlgebraUtils.vector_vector_operation, zip(v1, v2, itertools.repeat(operator.add))) result = pool.map(
ProcessLinearAlgebraUtils.vector_vector_operation,
zip(v1, v2, itertools.repeat(operator.add)),
)
return list(result) return list(result)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def scalar_vector_multiply(omega, vector): def scalar_vector_multiply(omega, vector):
with Pool() as pool: with Pool() as pool:
result = pool.map(multiply_by_scalar, [(element, omega) for element in vector]) result = pool.map(
multiply_by_scalar, [(element, omega) for element in vector]
)
return list(result) return list(result)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def sum_of_squares(row): def sum_of_squares(row):
return sum(x ** 2 for x in row) return sum(x**2 for x in row)
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
@ -318,7 +367,7 @@ class ProcessLinearAlgebraUtils:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def subtract_rows(row_from_A, row_from_B): def subtract_rows(row_from_A, row_from_B):
return [a - b for a, b in zip(row_from_A, row_from_B)] return [a - b for a, b in zip(row_from_A, row_from_B)]
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_matrix_subtraction(A, B): def matrix_matrix_subtraction(A, B):
@ -326,78 +375,79 @@ class ProcessLinearAlgebraUtils:
result = pool.starmap(ProcessLinearAlgebraUtils.subtract_rows, zip(A, B)) result = pool.starmap(ProcessLinearAlgebraUtils.subtract_rows, zip(A, B))
return result return result
class DistributedArraysLinearAlgebraUtils(ABC): class DistributedArraysLinearAlgebraUtils(ABC):
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def dot_product(v1, v2): def dot_product(v1, v2):
dv1 = da.from_array(v1, chunks='auto') dv1 = da.from_array(v1, chunks="auto")
dv2 = da.from_array(v2, chunks='auto') dv2 = da.from_array(v2, chunks="auto")
return da.dot(dv1, dv2).compute() return da.dot(dv1, dv2).compute()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_vector_multiply(A, x): def matrix_vector_multiply(A, x):
dA = da.from_array(A, chunks='auto') dA = da.from_array(A, chunks="auto")
dx = da.from_array(x, chunks='auto') dx = da.from_array(x, chunks="auto")
return da.dot(dA, dx).compute().tolist() return da.dot(dA, dx).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_norm(v): def vector_norm(v):
dv = da.from_array(v, chunks='auto') dv = da.from_array(v, chunks="auto")
return da.linalg.norm(dv).compute() return da.linalg.norm(dv).compute()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_scalar_divide(x, scalar): def vector_scalar_divide(x, scalar):
dx = da.from_array(x, chunks='auto') dx = da.from_array(x, chunks="auto")
return (dx / scalar).compute().tolist() return (dx / scalar).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_scalar_multiply(A, w): def matrix_scalar_multiply(A, w):
dA = da.from_array(A, chunks='auto') dA = da.from_array(A, chunks="auto")
return (dA * w).compute().tolist() return (dA * w).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_vector_subtraction(v1, v2): def vector_vector_subtraction(v1, v2):
dv1 = da.from_array(v1, chunks='auto') dv1 = da.from_array(v1, chunks="auto")
dv2 = da.from_array(v2, chunks='auto') dv2 = da.from_array(v2, chunks="auto")
return (dv1 - dv2).compute().tolist() return (dv1 - dv2).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def vector_vector_addition(v1, v2): def vector_vector_addition(v1, v2):
dv1 = da.from_array(v1, chunks='auto') dv1 = da.from_array(v1, chunks="auto")
dv2 = da.from_array(v2, chunks='auto') dv2 = da.from_array(v2, chunks="auto")
return (dv1 + dv2).compute().tolist() return (dv1 + dv2).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def scalar_vector_multiply(omega, vector): def scalar_vector_multiply(omega, vector):
dvector = da.from_array(vector, chunks='auto') dvector = da.from_array(vector, chunks="auto")
return (omega * dvector).compute().tolist() return (omega * dvector).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_norm(A): def matrix_norm(A):
dA = da.from_array(A, chunks='auto') dA = da.from_array(A, chunks="auto")
return da.linalg.norm(dA).compute() return da.linalg.norm(dA).compute()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def matrix_matrix_subtraction(A, B): def matrix_matrix_subtraction(A, B):
dA = da.from_array(A, chunks='auto') dA = da.from_array(A, chunks="auto")
dB = da.from_array(B, chunks='auto') dB = da.from_array(B, chunks="auto")
return (dA - dB).compute().tolist() return (dA - dB).compute().tolist()
@staticmethod @staticmethod
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def gaussian_elimination(A, b): def gaussian_elimination(A, b):
try: try:
dA = da.from_array(A, chunks='auto') dA = da.from_array(A, chunks="auto")
db = da.from_array(b, chunks='auto') db = da.from_array(b, chunks="auto")
Ab = da.hstack([dA, db[:, None]]) Ab = da.hstack([dA, db[:, None]])
Ab = Ab.persist() Ab = Ab.persist()
@ -406,8 +456,8 @@ class DistributedArraysLinearAlgebraUtils(ABC):
max_index = da.argmax(da.abs(Ab[k:, k])) + k max_index = da.argmax(da.abs(Ab[k:, k])) + k
Ab[[k, max_index]] = Ab[[max_index, k]] Ab[[k, max_index]] = Ab[[max_index, k]]
Ab = Ab.persist() Ab = Ab.persist()
factor = Ab[k + 1:, k] / Ab[k, k] factor = Ab[k + 1 :, k] / Ab[k, k]
Ab[k + 1:] -= factor[:, None] * Ab[k] Ab[k + 1 :] -= factor[:, None] * Ab[k]
return Ab return Ab
for k in range(A.shape[0]): for k in range(A.shape[0]):
@ -415,8 +465,8 @@ class DistributedArraysLinearAlgebraUtils(ABC):
x = da.zeros(A.shape[0]) x = da.zeros(A.shape[0])
for i in range(A.shape[0] - 1, -1, -1): for i in range(A.shape[0] - 1, -1, -1):
x[i] = (Ab[i, -1] - da.dot(Ab[i, i + 1:-1], x[i + 1:])) / Ab[i, i] x[i] = (Ab[i, -1] - da.dot(Ab[i, i + 1 : -1], x[i + 1 :])) / Ab[i, i]
return x.compute().tolist() return x.compute().tolist()
except Exception as e: except Exception as e:
print(f"Error during Gaussian elimination: {e}") logger.info("Error during Gaussian elimination: %s", e)
return None return None

View File

@ -1,5 +1,19 @@
"""Entry point for the PORR benchmark suite.
Runs ``tests.py``, which compares this project's Richardson implementations
(sequential, threads, processes, distributed arrays) against numpy's direct
solver over a range of matrix sizes.
The suite is heavy at the largest sizes it parametrises; ``../run.sh`` narrows
the default selection via ``PYTEST_ADDOPTS`` and says there how to widen it.
"""
import sys
import pytest import pytest
if __name__ == "__main__": if __name__ == "__main__":
# Run pytest and exit with the appropriate status code # sys.exit, because pytest.main returns the status rather than raising:
pytest.main(["-v", "tests.py"]) # discarding it made `python main.py` report success even when every test
# failed, which in turn made the run harness unable to tell the two apart.
sys.exit(pytest.main(["-v", "tests.py"]))

View File

@ -1,86 +1,108 @@
import os
import numpy as np import numpy as np
import scipy.io import scipy.io
import os
class MatrixGenerator: class MatrixGenerator:
@staticmethod @staticmethod
def generate_spd_matrix(n: int) -> np.ndarray: def generate_spd_matrix(n: int) -> np.ndarray:
A = np.random.rand(n, n) A = np.random.rand(n, n)
spd_matrix = np.dot(A, A.T) + n * MatrixGenerator.generate_identity_matrix(n) # Adding n*I ensures positive definiteness return np.dot(A, A.T) + n * MatrixGenerator.generate_identity_matrix(
return spd_matrix n
) # Adding n*I ensures positive definiteness
@staticmethod @staticmethod
def generate_identity_matrix(size): def generate_identity_matrix(size):
return np.eye(size) return np.eye(size)
@staticmethod @staticmethod
def generate_alternating_vector(size): def generate_alternating_vector(size):
return np.tile([1, 2], int(np.ceil(size / 2)))[:size] return np.tile([1, 2], int(np.ceil(size / 2)))[:size]
@staticmethod @staticmethod
def get_matrix_from_file(file_path, problem): def get_matrix_from_file(file_path, problem):
mat_contents = scipy.io.loadmat(file_path) mat_contents = scipy.io.loadmat(file_path)
problem_record = mat_contents['Problem'][0][0] problem_record = mat_contents["Problem"][0][0]
A = problem_record[problem] A = problem_record[problem]
if scipy.sparse.issparse(A): if scipy.sparse.issparse(A):
A_dense = A.todense() A_dense = A.todense()
else: else:
A_dense = A A_dense = A
return np.array(A_dense) return np.array(A_dense)
@staticmethod @staticmethod
def generate_matrix_and_vector(type, size=None): def generate_matrix_and_vector(type, size=None):
if type == 'spd': if type == "spd":
if size is None: if size is None:
raise ValueError("Size must be provided for SPD matrix generation.") raise ValueError("Size must be provided for SPD matrix generation.")
try: try:
matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file("spd_"+str(size)+".npz") matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file(
except FileNotFoundError as e: "spd_" + str(size) + ".npz"
)
except FileNotFoundError:
matrix = MatrixGenerator.generate_spd_matrix(size) matrix = MatrixGenerator.generate_spd_matrix(size)
vector = np.random.uniform(-1, 1, size) vector = np.random.uniform(-1, 1, size)
lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix) lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix)
MatrixGenerator.save_to_file(matrix, vector, lambda_min, lambda_max, "spd_"+str(size)+".npz") MatrixGenerator.save_to_file(
elif type == 'nemeth12': matrix, vector, lambda_min, lambda_max, "spd_" + str(size) + ".npz"
)
elif type == "nemeth12":
try: try:
matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file("nemeth12.npz") matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file(
except FileNotFoundError as e: "nemeth12.npz"
)
except FileNotFoundError:
matrix = -1 * MatrixGenerator.get_matrix_from_file("nemeth12.mat", 1) matrix = -1 * MatrixGenerator.get_matrix_from_file("nemeth12.mat", 1)
size = matrix.shape[0] size = matrix.shape[0]
vector = MatrixGenerator.generate_alternating_vector(size) vector = MatrixGenerator.generate_alternating_vector(size)
lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix) lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix)
MatrixGenerator.save_to_file(matrix, vector, lambda_min, lambda_max, "nemeth12.npz") MatrixGenerator.save_to_file(
elif type == 'poli3': matrix, vector, lambda_min, lambda_max, "nemeth12.npz"
)
elif type == "poli3":
try: try:
matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file("poli3.npz") matrix, vector, lambda_min, lambda_max = MatrixGenerator.load_from_file(
except FileNotFoundError as e: "poli3.npz"
)
except FileNotFoundError:
matrix = MatrixGenerator.get_matrix_from_file("poli3.mat", 2) matrix = MatrixGenerator.get_matrix_from_file("poli3.mat", 2)
size = matrix.shape[0] size = matrix.shape[0]
vector = MatrixGenerator.generate_alternating_vector(size) vector = MatrixGenerator.generate_alternating_vector(size)
lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix) lambda_min, lambda_max = MatrixGenerator.calculate_eigenvalues(matrix)
MatrixGenerator.save_to_file(matrix, vector, lambda_min, lambda_max, "poli3.npz") MatrixGenerator.save_to_file(
matrix, vector, lambda_min, lambda_max, "poli3.npz"
)
else: else:
raise ValueError("Invalid type specified. Choose 'spd', 'nemeth12', or 'poli3'.") raise ValueError(
"Invalid type specified. Choose 'spd', 'nemeth12', or 'poli3'."
)
return matrix, vector, lambda_min, lambda_max return matrix, vector, lambda_min, lambda_max
@staticmethod @staticmethod
def calculate_eigenvalues(A): def calculate_eigenvalues(A):
eigenvalues = np.linalg.eigvals(A) eigenvalues = np.linalg.eigvals(A)
lambda_min = np.min(eigenvalues) lambda_min = np.min(eigenvalues)
lambda_max = np.max(eigenvalues) lambda_max = np.max(eigenvalues)
return lambda_min, lambda_max return lambda_min, lambda_max
@staticmethod @staticmethod
def save_to_file(matrix, vector, lambda_min, lambda_max, file_path): def save_to_file(matrix, vector, lambda_min, lambda_max, file_path) -> None:
np.savez(file_path, matrix=matrix, vector=vector, lambda_min=lambda_min, lambda_max=lambda_max) np.savez(
file_path,
matrix=matrix,
vector=vector,
lambda_min=lambda_min,
lambda_max=lambda_max,
)
def load_from_file(file_path): def load_from_file(file_path):
if not os.path.exists(file_path): if not os.path.exists(file_path):
raise FileNotFoundError(f"The file {file_path} does not exist.") raise FileNotFoundError(f"The file {file_path} does not exist.")
data = np.load(file_path) data = np.load(file_path)
matrix = data['matrix'] matrix = data["matrix"]
vector = data['vector'] vector = data["vector"]
lambda_min = data['lambda_min'] lambda_min = data["lambda_min"]
lambda_max = data['lambda_max'] lambda_max = data["lambda_max"]
return matrix, vector, lambda_min, lambda_max return matrix, vector, lambda_min, lambda_max

View File

@ -1,5 +1,6 @@
from enum import Enum, auto from enum import Enum, auto
class ProcessingType(Enum): class ProcessingType(Enum):
SEQUENTIAL = auto() SEQUENTIAL = auto()
THREADS = auto() THREADS = auto()

View File

@ -1,65 +1,79 @@
import linear_algebra_utils as linAlg
from matrix_generator import MatrixGenerator
from processing_type import ProcessingType
from time_measurement import time_measurement, time_accumulator, tests_time
import time
import gc import gc
import numpy as np import logging
import time
import linear_algebra_utils as linAlg
from processing_type import ProcessingType
from time_measurement import tests_time, time_accumulator, time_measurement
logger = logging.getLogger(__name__)
class RichardsonMethod: class RichardsonMethod:
@time_measurement(time_accumulator) @time_measurement(time_accumulator)
def __init__(self, method: ProcessingType, A, b, lambda_min, lambda_max, max_iterations, size: int, x0=None, tol=1e-5): def __init__(
self,
method: ProcessingType,
A,
b,
lambda_min,
lambda_max,
max_iterations,
size: int,
x0=None,
tol=1e-5,
) -> None:
self.LinAlg = self.assign_LinAlgType(method) self.LinAlg = self.assign_LinAlgType(method)
self.A = A self.A = A
self.b = b self.b = b
self.x0 = x0 if x0 is not None else [0.0] * len(b) self.x0 = x0 if x0 is not None else [0.0] * len(b)
self.max_iterations = max_iterations self.max_iterations = max_iterations
self.tol = tol self.tol = tol
# self.I = MatrixGenerator.generate_identity_matrix(size)
self.lambda_min = lambda_min self.lambda_min = lambda_min
self.lambda_max = lambda_max self.lambda_max = lambda_max
if self.lambda_min < 0: if self.lambda_min < 0:
raise ValueError("Matrix A is not positive semi-definite.") raise ValueError("Matrix A is not positive semi-definite.")
self.omega = RichardsonMethod.calculate_omega(self.lambda_min, self.lambda_max) self.omega = RichardsonMethod.calculate_omega(self.lambda_min, self.lambda_max)
@staticmethod @staticmethod
def calculate_omega(lambda_min, lambda_max): def calculate_omega(lambda_min, lambda_max):
return 2 / (lambda_min + lambda_max) return 2 / (lambda_min + lambda_max)
@staticmethod @staticmethod
def convergence_norm(LinAlgType, A, omega, I) -> bool: def convergence_norm(LinAlgType, A, omega, I) -> bool:
wA = LinAlgType.matrix_scalar_multiply(A, omega) wA = LinAlgType.matrix_scalar_multiply(A, omega)
IMinuswA = LinAlgType.matrix_matrix_subtraction(I, wA) IMinuswA = LinAlgType.matrix_matrix_subtraction(I, wA)
norm = LinAlgType.matrix_norm(IMinuswA) return LinAlgType.matrix_norm(IMinuswA)
return norm
@staticmethod @staticmethod
def assign_LinAlgType(method): def assign_LinAlgType(method):
methods = { methods = {
ProcessingType.SEQUENTIAL: linAlg.SequentialLinearAlgebraUtils, ProcessingType.SEQUENTIAL: linAlg.SequentialLinearAlgebraUtils,
ProcessingType.THREADS: linAlg.ThreadsLinearAlgebraUtils, ProcessingType.THREADS: linAlg.ThreadsLinearAlgebraUtils,
ProcessingType.PROCESSES: linAlg.ProcessLinearAlgebraUtils, ProcessingType.PROCESSES: linAlg.ProcessLinearAlgebraUtils,
ProcessingType.DISTRIBUTED_ARRAYS: linAlg.DistributedArraysLinearAlgebraUtils ProcessingType.DISTRIBUTED_ARRAYS: linAlg.DistributedArraysLinearAlgebraUtils,
} }
try: try:
return methods[method] return methods[method]
except KeyError: except KeyError:
raise ValueError("Unknown method, please use 'SEQUENTIAL', 'THREADS' or 'PROCESSES'.") raise ValueError(
"Unknown method, please use 'SEQUENTIAL', 'THREADS' or 'PROCESSES'."
)
def solve(self): def solve(self):
gc.disable() gc.disable()
time_accumulator.total_time = 0 time_accumulator.total_time = 0
start = time.perf_counter() start = time.perf_counter()
x = self.x0[:] x = self.x0[:]
# if RichardsonMethod.convergence_norm(self.LinAlg, self.A, self.omega, self.I) >= 1:
# return RichardsonMethod.convergence_norm(self.LinAlg, self.A, self.omega, self.I), "Richardson method for those values will NOT converge",
for iteration in range(self.max_iterations): for iteration in range(self.max_iterations):
Ax = self.LinAlg.matrix_vector_multiply(self.A, x) Ax = self.LinAlg.matrix_vector_multiply(self.A, x)
residual = self.LinAlg.vector_vector_subtraction(self.b, Ax) residual = self.LinAlg.vector_vector_subtraction(self.b, Ax)
x = self.LinAlg.vector_vector_addition(x, self.LinAlg.scalar_vector_multiply(self.omega, residual)) x = self.LinAlg.vector_vector_addition(
if (linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < self.tol): x, self.LinAlg.scalar_vector_multiply(self.omega, residual)
)
if linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < self.tol:
break break
end = time.perf_counter() end = time.perf_counter()
@ -68,17 +82,37 @@ class RichardsonMethod:
match self.LinAlg: match self.LinAlg:
case linAlg.SequentialLinearAlgebraUtils: case linAlg.SequentialLinearAlgebraUtils:
print(f"Total: {total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Tests time: %ss", total_time, tests_time.total_time
)
case linAlg.ThreadsLinearAlgebraUtils: case linAlg.ThreadsLinearAlgebraUtils:
sequential_time = total_time - time_accumulator.total_time sequential_time = total_time - time_accumulator.total_time
print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss",
total_time,
sequential_time,
time_accumulator.total_time,
tests_time.total_time,
)
case linAlg.ProcessLinearAlgebraUtils: case linAlg.ProcessLinearAlgebraUtils:
sequential_time = total_time - time_accumulator.total_time sequential_time = total_time - time_accumulator.total_time
print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (processes): {time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Seq: %ss, Parallel (processes): %ss, Tests time: %ss",
total_time,
sequential_time,
time_accumulator.total_time,
tests_time.total_time,
)
case linAlg.DistributedArraysLinearAlgebraUtils: case linAlg.DistributedArraysLinearAlgebraUtils:
sequential_time = total_time - time_accumulator.total_time sequential_time = total_time - time_accumulator.total_time
print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (distributed arrays): {time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Seq: %ss, Parallel (distributed arrays): %ss, Tests time: %ss",
total_time,
sequential_time,
time_accumulator.total_time,
tests_time.total_time,
)
case _: case _:
print("Unhandled LinAlg type") logger.info("Unhandled LinAlg type")
return x, 0 return x, 0

View File

@ -1,7 +1,12 @@
from mpi4py import MPI import logging
import numpy as np
import time import time
import numpy as np
from mpi4py import MPI
logger = logging.getLogger(__name__)
def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000): def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000):
comm = MPI.COMM_WORLD comm = MPI.COMM_WORLD
rank = comm.Get_rank() rank = comm.Get_rank()
@ -12,9 +17,6 @@ def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000):
# Obliczanie wartości własnych tylko na jednym procesie # Obliczanie wartości własnych tylko na jednym procesie
if rank == 0: if rank == 0:
# eigenvalues = np.linalg.eigvals(A)
# lambda_min = np.min(eigenvalues)
# lambda_max = np.max(eigenvalues)
omega = 2 / (lambda_min + lambda_max) omega = 2 / (lambda_min + lambda_max)
else: else:
omega = None omega = None
@ -67,21 +69,25 @@ def richardson_parallel(A, b, lambda_min, lambda_max, tol=1e-5, max_iter=10000):
return x, execution_time return x, execution_time
def check_solution(A, b, x_approx, tolerance=8e-3): def check_solution(A, b, x_approx, tolerance=8e-3):
x_true = np.linalg.solve(A, b) x_true = np.linalg.solve(A, b)
error = np.linalg.norm(x_true - x_approx) error = np.linalg.norm(x_true - x_approx)
return error < tolerance, error return error < tolerance, error
if __name__ == "__main__": if __name__ == "__main__":
logging.basicConfig(format="%(message)s", level=logging.INFO)
comm = MPI.COMM_WORLD comm = MPI.COMM_WORLD
rank = comm.Get_rank() rank = comm.Get_rank()
from matrix_generator import MatrixGenerator from matrix_generator import MatrixGenerator
sizes = [2, 5, 10, 50, 80, 100, 300, 500, 750, 1000, 5000, 10000] sizes = [2, 5, 10, 50, 80, 100, 300, 500, 750, 1000, 5000, 10000]
for i in sizes: for i in sizes:
if rank == 0: if rank == 0:
A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('spd', size=i) A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector(
"spd", size=i
)
else: else:
A = None A = None
b = None b = None
@ -96,14 +102,18 @@ if __name__ == "__main__":
# Sprawdzanie poprawności rozwiązania (na procesie 0) # Sprawdzanie poprawności rozwiązania (na procesie 0)
if rank == 0: if rank == 0:
print(f"Spd matrix with size {i}") logger.info("Spd matrix with size %s", i)
is_correct, error = check_solution(A, b, x) is_correct, error = check_solution(A, b, x)
print("Czas wykonania [s]:", time_taken) logger.info("Czas wykonania [s]: %s", time_taken)
print("Czy rozwiązanie jest poprawne:", "Tak" if is_correct else "Nie") logger.info(
print("Błąd rozwiązania:", error) "Czy rozwiązanie jest poprawne: %s", "Tak" if is_correct else "Nie"
)
logger.info("Błąd rozwiązania: %s", error)
if rank == 0: if rank == 0:
A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('nemeth12') A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector(
"nemeth12"
)
else: else:
A = None A = None
b = None b = None
@ -118,14 +128,16 @@ if __name__ == "__main__":
# Sprawdzanie poprawności rozwiązania (na procesie 0) # Sprawdzanie poprawności rozwiązania (na procesie 0)
if rank == 0: if rank == 0:
print(f"Nemeth12 matrix") logger.info("Nemeth12 matrix")
is_correct, error = check_solution(A, b, x) is_correct, error = check_solution(A, b, x)
print("Czas wykonania [s]:", time_taken) logger.info("Czas wykonania [s]: %s", time_taken)
print("Czy rozwiązanie jest poprawne:", "Tak" if is_correct else "Nie") logger.info("Czy rozwiązanie jest poprawne: %s", "Tak" if is_correct else "Nie")
print("Błąd rozwiązania:", error) logger.info("Błąd rozwiązania: %s", error)
if rank == 0: if rank == 0:
A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('poli3') A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector(
"poli3"
)
else: else:
A = None A = None
b = None b = None
@ -140,8 +152,8 @@ if __name__ == "__main__":
# Sprawdzanie poprawności rozwiązania (na procesie 0) # Sprawdzanie poprawności rozwiązania (na procesie 0)
if rank == 0: if rank == 0:
print(f"Poli3 matrix") logger.info("Poli3 matrix")
is_correct, error = check_solution(A, b, x) is_correct, error = check_solution(A, b, x)
print("Czas wykonania [s]:", time_taken) logger.info("Czas wykonania [s]: %s", time_taken)
print("Czy rozwiązanie jest poprawne:", "Tak" if is_correct else "Nie") logger.info("Czy rozwiązanie jest poprawne: %s", "Tak" if is_correct else "Nie")
print("Błąd rozwiązania:", error) logger.info("Błąd rozwiązania: %s", error)

View File

@ -1,15 +1,20 @@
import pytest import logging
import numpy as np import numpy as np
import pytest
from matrix_generator import MatrixGenerator from matrix_generator import MatrixGenerator
from processing_type import ProcessingType
from richardson_method import RichardsonMethod from richardson_method import RichardsonMethod
from threads_indep import RichardsonMethodThreads from threads_indep import RichardsonMethodThreads
from processing_type import ProcessingType from time_measurement import tests_time, time_measurement
from time_measurement import time_measurement, tests_time
logger = logging.getLogger(__name__)
def calculate_norm_numpy(I, w, A): def calculate_norm_numpy(I, w, A):
difference = I - w * A difference = I - w * A
norm = np.linalg.norm(difference) return np.linalg.norm(difference)
return norm
def calculate_eigenvalues(A): def calculate_eigenvalues(A):
eigenvalues = np.linalg.eigvals(A) eigenvalues = np.linalg.eigvals(A)
@ -17,89 +22,113 @@ def calculate_eigenvalues(A):
lambda_max = np.max(eigenvalues) lambda_max = np.max(eigenvalues)
return lambda_min, lambda_max return lambda_min, lambda_max
def calcualte_norm_from_matrix_numpy(A, n): def calcualte_norm_from_matrix_numpy(A, n):
lambda_min, lambda_max = calculate_eigenvalues(A) lambda_min, lambda_max = calculate_eigenvalues(A)
omega = 2 / (lambda_min + lambda_max) omega = 2 / (lambda_min + lambda_max)
I = np.eye(n) I = np.eye(n)
return calculate_norm_numpy(I, omega, A) return calculate_norm_numpy(I, omega, A)
@time_measurement(tests_time) @time_measurement(tests_time)
def solution_lib(A, b): def solution_lib(A, b):
return np.linalg.solve(A, b) return np.linalg.solve(A, b)
@pytest.mark.parametrize("n", [
2, @pytest.mark.parametrize("n", [2, 5, 10, 50, 100, 300, 500, 750, 1000, 5000, 10000])
5, @pytest.mark.parametrize(
10, "processing_type",
50, [
100, ProcessingType.SEQUENTIAL,
300, ProcessingType.THREADS,
500, ProcessingType.PROCESSES,
750, ProcessingType.DISTRIBUTED_ARRAYS,
1000, ],
5000, )
10000 @pytest.mark.parametrize("matrix_type", ["spd", "nemeth12", "poli3"])
]) def test_richardson_vs_cg(
@pytest.mark.parametrize("processing_type", [ n: int, processing_type: ProcessingType, matrix_type: str, capsys
ProcessingType.SEQUENTIAL, ) -> None:
ProcessingType.THREADS, logger.info("matrix type: %s", matrix_type)
ProcessingType.PROCESSES, logger.info("matrix size: %s", n if matrix_type == "spd" else "fixed")
ProcessingType.DISTRIBUTED_ARRAYS
])
@pytest.mark.parametrize("matrix_type", [
"spd",
"nemeth12",
"poli3"
])
def test_richardson_vs_cg(n: int, processing_type: ProcessingType, matrix_type: str, capsys):
print("matrix type: ", matrix_type)
print("matrix size: ", n if matrix_type == "spd" else "fixed")
tolerance = 8e-3 tolerance = 8e-3
max_iterations=100 max_iterations = 100
if matrix_type in ["nemeth12", "poli3"] and n != 2: if matrix_type in ["nemeth12", "poli3"] and n != 2:
pytest.skip("Fixed matrix size for nemeth12 and poli3, skipping redundant runs.") pytest.skip(
"Fixed matrix size for nemeth12 and poli3, skipping redundant runs."
)
if matrix_type == "spd": if matrix_type == "spd":
A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('spd', size=n) A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector(
"spd", size=n
)
elif matrix_type == "poli3": elif matrix_type == "poli3":
A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('poli3') A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector(
"poli3"
)
elif matrix_type == "nemeth12": elif matrix_type == "nemeth12":
A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector('nemeth12') A, b, lambda_min, lambda_max = MatrixGenerator.generate_matrix_and_vector(
"nemeth12"
)
else: else:
raise ValueError("Invalid matrix type specified. Choose 'spd', 'poli3', or 'nemeth12'.") raise ValueError(
"Invalid matrix type specified. Choose 'spd', 'poli3', or 'nemeth12'."
)
solution_richardson, info_richardson = None, None solution_richardson, info_richardson = None, None
if processing_type != ProcessingType.THREADS: if processing_type != ProcessingType.THREADS:
richardson_solver = RichardsonMethod(processing_type, A, b, lambda_min, lambda_max, max_iterations, size=A.shape[0], tol=1e-7) richardson_solver = RichardsonMethod(
processing_type,
A,
b,
lambda_min,
lambda_max,
max_iterations,
size=A.shape[0],
tol=1e-7,
)
with capsys.disabled(): with capsys.disabled():
solution_richardson, info_richardson = richardson_solver.solve() solution_richardson, info_richardson = richardson_solver.solve()
else: else:
with capsys.disabled(): with capsys.disabled():
solution_richardson, info_richardson = RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, tol=1e-7) solution_richardson, info_richardson = RichardsonMethodThreads(
A, b, lambda_min, lambda_max, max_iterations, tol=1e-7
)
# Przechwytywanie wyjścia po solve # Przechwytywanie wyjścia po solve
captured = capsys.readouterr() captured = capsys.readouterr()
print("Captured output:", captured.out) logger.info("Captured output: %s", captured.out)
solution = solution_lib(A,b) solution = solution_lib(A, b)
assert_converged(solution_richardson, info_richardson, solution, tolerance, A, n) assert_converged(solution_richardson, info_richardson, solution, tolerance, A, n)
def assert_converged(solution_richardson, info_richardson, solution, tolerance, A, n):
def assert_converged(
solution_richardson, info_richardson, solution, tolerance, A, n
) -> None:
if info_richardson == "Richardson method for those values will NOT converge": if info_richardson == "Richardson method for those values will NOT converge":
numpy_norm = calcualte_norm_from_matrix_numpy(A, n) numpy_norm = calcualte_norm_from_matrix_numpy(A, n)
print("Numpy norm: ", numpy_norm, " Richardson norm: ", solution_richardson) logger.info(
"Numpy norm: %s Richardson norm: %s", numpy_norm, solution_richardson
)
assert False, "Richardson did not converge" assert False, "Richardson did not converge"
else: else:
difference = np.linalg.norm(solution_richardson - solution) difference = np.linalg.norm(solution_richardson - solution)
print(f"Difference between Richardson and numpy solutions: {difference:.8f}") logger.info("Difference between Richardson and numpy solutions: %s", difference)
if difference < tolerance: if difference < tolerance:
print("Both Richardson and numpy method converged and calculated correct values.") logger.info(
"Both Richardson and numpy method converged and calculated correct values."
)
else: else:
print("Solution numpy:\n", solution) logger.info("Solution numpy:\n %s", solution)
print("Solution Richardson:\n", solution_richardson) logger.info("Solution Richardson:\n %s", solution_richardson)
assert difference < tolerance, f"The solutions are different! Difference: {difference:.8f}" assert difference < tolerance, (
f"The solutions are different! Difference: {difference:.8f}"
)
if __name__ == "__main__": if __name__ == "__main__":
logging.basicConfig(format="%(message)s", level=logging.INFO)
pytest.main() pytest.main()

View File

@ -1,55 +1,71 @@
import gc import gc
import logging
import time import time
import numpy as np
from numba import njit, prange
from time_measurement import time_measurement_longest, longest_threads_time_accumulator, tests_time
import linear_algebra_utils as linAlg import linear_algebra_utils as linAlg
from numba import njit, prange
from time_measurement import (
longest_threads_time_accumulator,
tests_time,
time_measurement_longest,
)
logger = logging.getLogger(__name__)
@njit(parallel=True) @njit(parallel=True)
def numba_matrix_vector_multiply(A, input_x, Ax): def numba_matrix_vector_multiply(A, input_x, Ax) -> None:
for i in prange(len(A)): for i in prange(len(A)):
acc = 0.0 acc = 0.0
for j in range(len(input_x)): for j in range(len(input_x)):
acc += A[i][j] * input_x[j] acc += A[i][j] * input_x[j]
Ax[i] = acc Ax[i] = acc
@njit(parallel=True) @njit(parallel=True)
def numba_vector_vector_subtraction(b, Ax, residual): def numba_vector_vector_subtraction(b, Ax, residual) -> None:
for i in prange(len(b)): for i in prange(len(b)):
residual[i] = b[i] - Ax[i] residual[i] = b[i] - Ax[i]
@njit(nopython=True) @njit(nopython=True)
def numba_scalar_vector_multiply(omega, vector, result): def numba_scalar_vector_multiply(omega, vector, result) -> None:
omega_real = omega.real omega_real = omega.real
for i in range(len(vector)): for i in range(len(vector)):
result[i] = omega_real * vector[i] result[i] = omega_real * vector[i]
@njit(parallel=True) @njit(parallel=True)
def numba_vector_vector_addition(input_x, vector, output_x): def numba_vector_vector_addition(input_x, vector, output_x) -> None:
for i in prange(len(input_x)): for i in prange(len(input_x)):
output_x[i] = input_x[i] + vector[i] output_x[i] = input_x[i] + vector[i]
# Funkcje z dekoratorem # Funkcje z dekoratorem
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def matrix_vector_multiply(A, input_x, Ax): def matrix_vector_multiply(A, input_x, Ax) -> None:
numba_matrix_vector_multiply(A, input_x, Ax) numba_matrix_vector_multiply(A, input_x, Ax)
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def vector_vector_subtraction(b, Ax, residual): def vector_vector_subtraction(b, Ax, residual) -> None:
numba_vector_vector_subtraction(b, Ax, residual) numba_vector_vector_subtraction(b, Ax, residual)
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def scalar_vector_multiply(omega, vector, result): def scalar_vector_multiply(omega, vector, result) -> None:
numba_scalar_vector_multiply(omega, vector, result) numba_scalar_vector_multiply(omega, vector, result)
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def vector_vector_addition(input_x, vector, output_x): def vector_vector_addition(input_x, vector, output_x) -> None:
numba_vector_vector_addition(input_x, vector, output_x) numba_vector_vector_addition(input_x, vector, output_x)
# Metoda Richardson z obsługą wątków # Metoda Richardson z obsługą wątków
def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5): def RichardsonMethodThreads(
A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5
):
longest_threads_time_accumulator.hard_reset() longest_threads_time_accumulator.hard_reset()
gc.disable() gc.disable()
@ -88,6 +104,12 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non
total_time = end_time - start_time total_time = end_time - start_time
sequential_time = total_time - longest_threads_time_accumulator.total_time sequential_time = total_time - longest_threads_time_accumulator.total_time
print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {longest_threads_time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss",
total_time,
sequential_time,
longest_threads_time_accumulator.total_time,
tests_time.total_time,
)
return x, 0 return x, 0

View File

@ -1,28 +1,42 @@
import multiprocessing
import gc import gc
import logging
import multiprocessing
import time import time
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from time_measurement import time_measurement_longest, longest_threads_time_accumulator, tests_time
import linear_algebra_utils as linAlg import linear_algebra_utils as linAlg
from time_measurement import (
longest_threads_time_accumulator,
tests_time,
time_measurement_longest,
)
logger = logging.getLogger(__name__)
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def matrix_vector_multiply(A, input_x, start, end, Ax): def matrix_vector_multiply(A, input_x, start, end, Ax) -> None:
Ax[start:end] = [sum(x*y for x, y in zip(row, input_x)) for row in A[start:end]] Ax[start:end] = [sum(x * y for x, y in zip(row, input_x)) for row in A[start:end]]
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def vector_vector_subtraction(b, Ax, start, end, residual): def vector_vector_subtraction(b, Ax, start, end, residual) -> None:
residual[start:end] = [x-y for x, y in zip(b[start:end], Ax[start:end])] residual[start:end] = [x - y for x, y in zip(b[start:end], Ax[start:end])]
@time_measurement_longest(longest_threads_time_accumulator) @time_measurement_longest(longest_threads_time_accumulator)
def scalar_vector_multiply(omega, vector, start, end, result): def scalar_vector_multiply(omega, vector, start, end, result) -> None:
result[start:end] = [omega * x for x in vector[start:end]] result[start:end] = [omega * x for x in vector[start:end]]
@time_measurement_longest(longest_threads_time_accumulator)
def vector_vector_addition(input_x, vector, start, end, output_x):
output_x[start:end] = [x+y for x, y in zip(input_x[start:end], vector[start:end])]
def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5): @time_measurement_longest(longest_threads_time_accumulator)
def vector_vector_addition(input_x, vector, start, end, output_x) -> None:
output_x[start:end] = [x + y for x, y in zip(input_x[start:end], vector[start:end])]
def RichardsonMethodThreads(
A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5
):
longest_threads_time_accumulator.hard_reset() longest_threads_time_accumulator.hard_reset()
gc.disable() gc.disable()
@ -35,69 +49,101 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non
num_threads = multiprocessing.cpu_count() num_threads = multiprocessing.cpu_count()
chunk_size = n // num_threads chunk_size = n // num_threads
with ThreadPoolExecutor(max_workers=num_threads) as executor: # wątki są tworzone raz i nie są niszczone with ThreadPoolExecutor(
max_workers=num_threads
) as executor: # wątki są tworzone raz i nie są niszczone
for iteration in range(max_iterations): for iteration in range(max_iterations):
Ax = [0] * len(
Ax = [0] * len(x) # tutaj zostanie przypisany wynik z mnożenia macierzy A z wektorem x x
) # tutaj zostanie przypisany wynik z mnożenia macierzy A z wektorem x
futures = [] futures = []
for i in range(num_threads): for i in range(num_threads):
start = i * chunk_size start = i * chunk_size
end = n if i == num_threads - 1 else (i + 1) * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size
futures.append(executor.submit(matrix_vector_multiply, A, x, start, end, Ax)) futures.append(
executor.submit(matrix_vector_multiply, A, x, start, end, Ax)
)
for future in futures: for future in futures:
future.result() future.result()
longest_threads_time_accumulator.save_lap_and_reset() longest_threads_time_accumulator.save_lap_and_reset()
residual = [0] * len(b) # tutaj zostanie przypisany wynik z vector_vector_subtraction residual = [0] * len(
b
) # tutaj zostanie przypisany wynik z vector_vector_subtraction
futures = [] futures = []
for i in range(num_threads): for i in range(num_threads):
start = i * chunk_size start = i * chunk_size
end = n if i == num_threads - 1 else (i + 1) * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size
futures.append(executor.submit(vector_vector_subtraction, b, Ax, start, end, residual)) futures.append(
executor.submit(
vector_vector_subtraction, b, Ax, start, end, residual
)
)
for future in futures: for future in futures:
future.result() future.result()
longest_threads_time_accumulator.save_lap_and_reset() longest_threads_time_accumulator.save_lap_and_reset()
change_vector = [0] * len(residual) # zostanie tu przypisany wynik scalar_vector_multiply po pracy wątków change_vector = [0] * len(
residual
) # zostanie tu przypisany wynik scalar_vector_multiply po pracy wątków
futures = [] futures = []
for i in range(num_threads): for i in range(num_threads):
start = i * chunk_size start = i * chunk_size
end = n if i == num_threads - 1 else (i + 1) * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size
futures.append(executor.submit(scalar_vector_multiply, omega, residual, start, end, change_vector)) futures.append(
executor.submit(
scalar_vector_multiply,
omega,
residual,
start,
end,
change_vector,
)
)
for future in futures: for future in futures:
future.result() future.result()
longest_threads_time_accumulator.save_lap_and_reset() longest_threads_time_accumulator.save_lap_and_reset()
_x = x[:] # do _x zostanie przez wątki przypisany wynik pracy w danej iteracji _x = x[
:
] # do _x zostanie przez wątki przypisany wynik pracy w danej iteracji
futures = [] futures = []
for i in range(num_threads): for i in range(num_threads):
start = i * chunk_size start = i * chunk_size
end = n if i == num_threads - 1 else (i + 1) * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size
futures.append(executor.submit(vector_vector_addition, x, change_vector, start, end, _x)) futures.append(
executor.submit(
vector_vector_addition, x, change_vector, start, end, _x
)
)
for future in futures: for future in futures:
future.result() future.result()
longest_threads_time_accumulator.save_lap_and_reset() longest_threads_time_accumulator.save_lap_and_reset()
x = _x[:] x = _x[:]
if linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < tol:
if (linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < tol): break
break
end_time = time.perf_counter() end_time = time.perf_counter()
gc.enable() gc.enable()
total_time = end_time - start_time total_time = end_time - start_time
sequential_time = total_time - longest_threads_time_accumulator.total_time sequential_time = total_time - longest_threads_time_accumulator.total_time
print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {longest_threads_time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss",
return x, 0 total_time,
sequential_time,
longest_threads_time_accumulator.total_time,
tests_time.total_time,
)
return x, 0

View File

@ -1,20 +1,30 @@
import numpy as np
import threading
import multiprocessing
import gc import gc
import time import logging
import multiprocessing
import sys import sys
from time_measurement import time_measurement_longest, longest_time_accumulator, tests_time import threading
import linear_algebra_utils as linAlg import time
import numpy as np
from time_measurement import (
longest_time_accumulator,
tests_time,
time_measurement_longest,
)
logger = logging.getLogger(__name__)
@time_measurement_longest(longest_time_accumulator) @time_measurement_longest(longest_time_accumulator)
def RichardsonThread(A, b, x, _x, omega, start, end): def RichardsonThread(A, b, x, _x, omega, start, end) -> None:
for i in range(start, end): for i in range(start, end):
sigma = np.dot(A[i, :], _x) - A[i, i] * _x[i] sigma = np.dot(A[i, :], _x) - A[i, i] * _x[i]
x[i] = (1 - omega) * _x[i] + omega * (b[i] - sigma) / A[i, i] x[i] = (1 - omega) * _x[i] + omega * (b[i] - sigma) / A[i, i]
def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5):
def RichardsonMethodThreads(
A, b, lambda_min, lambda_max, max_iterations, x0=None, tol=1e-5
):
longest_time_accumulator.total_time = 0 longest_time_accumulator.total_time = 0
longest_time_accumulator.start = sys.float_info.max longest_time_accumulator.start = sys.float_info.max
longest_time_accumulator.end = 0 longest_time_accumulator.end = 0
@ -25,7 +35,7 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non
n = len(b) n = len(b)
x0 = x0 if x0 is not None else [0.0] * len(b) x0 = x0 if x0 is not None else [0.0] * len(b)
x = x0[:] x = x0[:]
omega = 0.05#2 / (lambda_min + lambda_max) omega = 0.05 # 2 / (lambda_min + lambda_max)
num_threads = multiprocessing.cpu_count() num_threads = multiprocessing.cpu_count()
threads = [] threads = []
chunk_size = n // num_threads chunk_size = n // num_threads
@ -34,27 +44,32 @@ def RichardsonMethodThreads(A, b, lambda_min, lambda_max, max_iterations, x0=Non
for _ in range(max_iterations): for _ in range(max_iterations):
_x = x[:] _x = x[:]
for i in range(num_threads): for i in range(num_threads):
start = i * chunk_size # start jest indeksem w A. Wątki otrzymują kolejny punkt startowy będący wielokrotnością rozmiaru porcji na wątek start = (
i * chunk_size
) # start jest indeksem w A. Wątki otrzymują kolejny punkt startowy będący wielokrotnością rozmiaru porcji na wątek
end = n if i == num_threads - 1 else (i + 1) * chunk_size end = n if i == num_threads - 1 else (i + 1) * chunk_size
thread = threading.Thread(target=RichardsonThread, args=(A, b, x, _x, omega, start, end)) thread = threading.Thread(
target=RichardsonThread, args=(A, b, x, _x, omega, start, end)
)
threads.append(thread) threads.append(thread)
thread.start() thread.start()
for thread in threads: for thread in threads:
thread.join() thread.join()
# Ax = linAlg.SequentialLinearAlgebraUtils.matrix_vector_multiply(A, x)
# residual = linAlg.SequentialLinearAlgebraUtils.vector_vector_subtraction(b, Ax)
# if (linAlg.SequentialLinearAlgebraUtils.vector_norm(residual) < tol):
# break # break
end_time = time.perf_counter() end_time = time.perf_counter()
gc.enable() gc.enable()
total_time = end_time - start_time total_time = end_time - start_time
sequential_time = total_time - longest_time_accumulator.total_time sequential_time = total_time - longest_time_accumulator.total_time
print(f"Total: {total_time:.3e}s, Seq: {sequential_time:.3e}s, Parallel (threads): {longest_time_accumulator.total_time:.3e}s, Tests time: {tests_time.total_time:.3e}s") logger.info(
"Total: %ss, Seq: %ss, Parallel (threads): %ss, Tests time: %ss",
total_time,
sequential_time,
longest_time_accumulator.total_time,
tests_time.total_time,
)
return x, 0 return x, 0

View File

@ -1,25 +1,27 @@
import time
import sys import sys
import time
from functools import wraps from functools import wraps
class TimeAccumulator: class TimeAccumulator:
def __init__(self): def __init__(self) -> None:
self.total_time = 0 self.total_time = 0
class ComplexTimeAcumulator: class ComplexTimeAcumulator:
def __init__(self): def __init__(self) -> None:
self.hard_reset() self.hard_reset()
def hard_reset(self): def hard_reset(self) -> None:
self.total_time = 0 self.total_time = 0
self.reset() self.reset()
def reset(self): def reset(self) -> None:
self.lap_time = 0 self.lap_time = 0
self.start = sys.float_info.max self.start = sys.float_info.max
self.end = 0 self.end = 0
def save_lap_and_reset(self): def save_lap_and_reset(self) -> None:
self.total_time += self.lap_time self.total_time += self.lap_time
self.reset() self.reset()
@ -29,6 +31,7 @@ tests_time = TimeAccumulator()
longest_threads_time_accumulator = ComplexTimeAcumulator() longest_threads_time_accumulator = ComplexTimeAcumulator()
def time_measurement(accumulator: TimeAccumulator): def time_measurement(accumulator: TimeAccumulator):
def decorator(func): def decorator(func):
@wraps(func) @wraps(func)
@ -38,9 +41,12 @@ def time_measurement(accumulator: TimeAccumulator):
end = time.perf_counter() end = time.perf_counter()
accumulator.total_time += end - start accumulator.total_time += end - start
return result return result
return inner return inner
return decorator return decorator
def time_measurement_longest(accumulator: ComplexTimeAcumulator): def time_measurement_longest(accumulator: ComplexTimeAcumulator):
def decorator(func): def decorator(func):
@wraps(func) @wraps(func)
@ -48,15 +54,13 @@ def time_measurement_longest(accumulator: ComplexTimeAcumulator):
start = time.perf_counter() start = time.perf_counter()
result = func(*args, **kwargs) result = func(*args, **kwargs)
end = time.perf_counter() end = time.perf_counter()
if start < accumulator.start: accumulator.start = min(accumulator.start, start)
accumulator.start = start accumulator.end = max(accumulator.end, end)
if end > accumulator.end: accumulator.lap_time = (
accumulator.end = end accumulator.end - accumulator.start
accumulator.lap_time = accumulator.end - accumulator.start # "=" instead of "+=" ) # "=" instead of "+="
return result return result
return inner return inner
return decorator return decorator

View File

@ -7,6 +7,31 @@ set -euo pipefail
# shellcheck source=../../scripts/course_lib.sh # shellcheck source=../../scripts/course_lib.sh
. "$(dirname "$(readlink -f "$0")")/../../scripts/course_lib.sh" . "$(dirname "$(readlink -f "$0")")/../../scripts/course_lib.sh"
# This suite is a BENCHMARK, not a smoke test. Measured on this machine, the
# cost is not where you would guess -- it is not the matrix size:
#
# ProcessingType.PROCESSES 20.9s at n=2, 32.8s at n=5, 78.2s at n=10.
# That is pool setup per test, and it grows.
# matrix_type poli3 densifies a sparse .mat into a 2.3 GB cache
# (nemeth12: 723 MB) -- over the 2G ceiling
# `../../run.sh run` imposes, before any solving.
# n=10000 the SPD matrix alone is 800 MB and
# np.linalg.solve needs a second copy.
# n=5000 spd_5000.npz was a Git LFS pointer whose object
# 404s on the server, so this size regenerates
# from scratch: a dense 5000x5000 eigendecomposition.
#
# What is left after removing those is fast and still covers the whole
# Richardson implementation across every size that fits: 18 tests, 28 seconds.
# The full submitted parameter set is one environment variable away, and needs
# headroom the default deliberately does not give it:
#
# PYTEST_ADDOPTS= RUN_MEMORY_MAX=16G RUN_CPU_QUOTA=800% ./run.sh run Programming/PORR
#
# main.py hardcodes its pytest arguments, so the selection is passed the only
# way that survives that: PYTEST_ADDOPTS, which pytest.main() also honours.
export PYTEST_ADDOPTS="${PYTEST_ADDOPTS--k 'spd and (SEQUENTIAL or THREADS) and not 5000 and not 10000'}"
course_main --kind python \ course_main --kind python \
--entry "$(dirname "$(readlink -f "$0")")/code/main.py" \ --entry "$(dirname "$(readlink -f "$0")")/code/main.py" \
-- "$@" -- "$@"

177
scripts/check_refactor_safety.py Executable file
View File

@ -0,0 +1,177 @@
#!/usr/bin/env python3
"""Three checks that catch what ruff and a smoke run both miss.
Every one of these exists because it caught a real regression during the
`select = ["ALL"]` campaign on this archive:
1. **Signatures.** Annotating 110 EARIN functions silently dropped the DEFAULT
VALUES from 22 of them, so a call relying on a default would have raised
TypeError. Nothing flagged it: ruff does not look at defaults, and the
affected paths were not the ones a smoke run exercises.
2. **Logging.** Rewriting ``print()`` as ``logger.info()`` made programs
silent, because logging's last-resort handler only emits WARNING and above.
A module that logs and has a ``__main__`` guard must configure logging.
3. **print keywords.** ``print(x, end="")`` builds a line incrementally.
Dropping ``end=`` turned a rendered chess board into one glyph per line;
dropping ``file=sys.stderr`` would silently move error output to stdout.
Usage::
scripts/check_refactor_safety.py --logging --print-keywords <paths>
scripts/check_refactor_safety.py --signatures <before-tree> <after-tree>
Exit status is 1 if anything is reported.
"""
from __future__ import annotations
import argparse
import ast
import logging
import sys
from pathlib import Path
logger = logging.getLogger(__name__)
_MAIN_GUARD = "__name__ == '__main__'"
def _functions(source: str) -> dict[str, tuple[set[str], dict[str, str]]]:
"""Map each function name to its parameter names and their defaults."""
out: dict[str, tuple[set[str], dict[str, str]]] = {}
for node in ast.walk(ast.parse(source)):
if not isinstance(node, ast.FunctionDef):
continue
args = node.args
defaults: dict[str, str] = {}
# defaults cover posonlyargs and args together, tail-aligned.
ordered = args.posonlyargs + args.args
positional = ordered[len(ordered) - len(args.defaults) :]
for arg, value in zip(positional, args.defaults, strict=True):
defaults[arg.arg] = ast.unparse(value)
for arg, value in zip(args.kwonlyargs, args.kw_defaults, strict=True):
if value is not None:
defaults[arg.arg] = ast.unparse(value)
names = {a.arg for a in args.posonlyargs + args.args + args.kwonlyargs}
out[node.name] = (names, defaults)
return out
def check_signatures(before: Path, after: Path) -> int:
"""Report any parameter or default that changed between two trees."""
problems = 0
for old in sorted(before.rglob("*.py")):
new = after / old.relative_to(before)
if not new.exists():
continue
was = _functions(old.read_text(encoding="utf-8"))
now = _functions(new.read_text(encoding="utf-8"))
for name, (names, defaults) in was.items():
if name not in now:
logger.error("%s: %s disappeared", new, name)
problems += 1
continue
names_now, defaults_now = now[name]
missing = sorted(names - names_now)
if missing:
logger.error("%s: %s lost parameters %s", new, name, missing)
problems += 1
changed = {
key: (value, defaults_now.get(key))
for key, value in defaults.items()
if key in names_now and defaults_now.get(key) != value
}
if changed:
logger.error("%s: %s default(s) changed %s", new, name, changed)
problems += 1
return problems
def check_logging(paths: list[Path]) -> int:
"""Report modules that log but never configure logging."""
problems = 0
for path in paths:
source = path.read_text(encoding="utf-8", errors="ignore")
if "logger.info(" not in source and "logger.debug(" not in source:
continue
if "basicConfig" in source:
continue
tree = ast.parse(source)
runnable = any(
isinstance(node, ast.If) and ast.unparse(node.test) == _MAIN_GUARD
for node in tree.body
)
# Only a module that can be RUN is at fault: configuring logging is
# the entry point's job, not a library module's.
if runnable:
logger.error("%s: logs at INFO but never calls basicConfig", path)
problems += 1
return problems
def check_print_keywords(paths: list[Path]) -> int:
"""Report print() calls whose keywords an automated rewrite would lose."""
problems = 0
for path in paths:
tree = ast.parse(path.read_text(encoding="utf-8", errors="ignore"))
for node in ast.walk(tree):
if not (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id == "print"
and node.keywords
):
continue
keys = sorted(k.arg or "**" for k in node.keywords)
logger.error(
"%s:%d: print() with %s -- convert this one by hand",
path,
node.lineno,
", ".join(keys),
)
problems += 1
return problems
def _python_files(roots: list[str]) -> list[Path]:
"""Every .py under the given roots, skipping venvs and vendored trees."""
found: list[Path] = []
for root in roots:
base = Path(root)
candidates = [base] if base.is_file() else sorted(base.rglob("*.py"))
found.extend(
p for p in candidates if ".venv" not in p.parts and "sightpy" not in p.parts
)
return found
def main(argv: list[str] | None = None) -> int:
"""Run the requested checks and return 1 if any reported something."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("paths", nargs="+", help="files, directories, or two trees")
parser.add_argument("--signatures", action="store_true")
parser.add_argument("--logging", action="store_true")
parser.add_argument("--print-keywords", action="store_true")
args = parser.parse_args(argv)
logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout)
problems = 0
if args.signatures:
expected = 2
if len(args.paths) != expected:
logger.error("--signatures needs exactly two trees: before after")
return 1
problems += check_signatures(Path(args.paths[0]), Path(args.paths[1]))
if args.logging or args.print_keywords:
files = _python_files(args.paths)
if args.logging:
problems += check_logging(files)
if args.print_keywords:
problems += check_print_keywords(files)
logger.info("problems: %d", problems)
return 1 if problems else 0
if __name__ == "__main__":
sys.exit(main())