diff --git a/CLAUDE.md b/CLAUDE.md index 8eb4d7ba..3b531e8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,9 +71,18 @@ check. shared gate in `~/utils`. There is deliberately no baseline and no allowlist: split the file. For a LaTeX report that means `\input{}` per section; for a markdown report, sibling `DOCS-*.md` files behind an index. -- Do not "fix" coursework to be better than it was submitted. Correct what is - broken, keep what is merely old-fashioned. The archive should still show - what was actually handed in. +- **Coursework is held to the same bar as the tooling.** It gets the full + `select = ["ALL"]` treatment -- annotations, docstrings, naming, the lot -- + and there are no per-file-ignores carving the two course trees out. The + version that was handed in is preserved by git history, which is a better + archive than a frozen working tree; this repo's job is code that still + builds, runs and reads well. (This reverses an earlier rule that said not to + improve coursework beyond what was submitted. kuhy overruled it on + 2026-08-28: "yes fix it if necessary we can always go back to worse version + by going back in git repo.") +- The one thing that is still off limits is *what was delivered*: do not add + content to a submitted `.pdf`/report that was never in it, and do not merge + the near-duplicate submission directories listed above. ### Vendored content @@ -87,8 +96,12 @@ is mirrored into `~/utils/file_length/_tables.py`, `[tool.ruff] exclude` and - `ruff` runs with `select = ["ALL"]` and mypy with `strict = true`. - No `# noqa`, no `# type: ignore`. A pre-commit hook rejects both outright. -- Notebooks: outputs are stripped on commit, lint via `nbqa`. Real logic - belongs in a sibling importable module — a notebook cannot be covered. +- Notebooks: `ruff` reads and formats `.ipynb` natively (cell-aware, so an + import used two cells later is not reported unused), which is why there is + no `nbqa` layer. It rewrites `source` only — outputs are left byte-identical. + Nothing strips outputs on commit yet; the four notebooks here were committed + with theirs. Real logic belongs in a sibling importable module — a notebook + cannot be covered. ### LaTeX diff --git a/DOCS-runnability.md b/DOCS-runnability.md index 0ebbda11..8daa0d51 100644 --- a/DOCS-runnability.md +++ b/DOCS-runnability.md @@ -51,7 +51,7 @@ run cannot clobber the local column. | `Programming/PORR` | runnable | - | python3 /home/kuhy/WUT_Computer_Science/Programming/PORR/code/main.py | | `Programming/PSD` | runnable | - | starts the Kafka/Flink stack (installs packages with sudo) | | `Programming/psd_project` | runnable | - | mvn package | -| `Programming/TRAK` | runnable | - | python3 the entry script | +| `Programming/TRAK` | runnable | - | python3 /home/kuhy/WUT_Computer_Science/Programming/TRAK/main.py | | `Programming/twm_4` | blocked | - | jupyter is on PATH but broken (bad interpreter) | | `Programming/USD` | no-code | - | README describes a MetaDrive RL assignment; no implementation was submitted | | `Programming/WDWR` | runnable | - | python3 the entry script | diff --git a/NotProgramming/MOM/report_three/model.py b/NotProgramming/MOM/report_three/model.py old mode 100644 new mode 100755 index 441ab27f..39c62806 --- a/NotProgramming/MOM/report_three/model.py +++ b/NotProgramming/MOM/report_three/model.py @@ -1,45 +1,112 @@ -import pulp +#!/usr/bin/env python3 +"""Bi-criteria distribution model for MOM project 3 (data set 1.6). + +A PuLP transport problem over two factories, four warehouses and six +customers, minimising distribution cost while maximising customer +satisfaction. It is solved three ways: once on cost alone, then swept over +alpha in 0.0..1.0 to trace the efficient frontier, then at the five +weightings the report tabulates. +""" + +import logging import sys +import pulp + +logger = logging.getLogger(__name__) +logging.basicConfig(format="%(message)s", level=logging.INFO) + +# alpha runs over 0.0, 0.1 ... 1.0, so eleven steps. +ALPHA_STEPS = 11 + # Create a problem variable: model = pulp.LpProblem("Optimal_Distribution", pulp.LpMinimize) # Zdefiniowanie dostawcy (fabryki and magazyny) -fabryki = ['F1', 'F2'] -magazyny = ['M1', 'M2', 'M3', 'M4'] +fabryki = ["F1", "F2"] +magazyny = ["M1", "M2", "M3", "M4"] dostawcy = fabryki + magazyny # Combining both lists # Zdefiniowanie klientów -klienci = ['K1', 'K2', 'K3', 'K4', 'K5', 'K6'] +klienci = ["K1", "K2", "K3", "K4", "K5", "K6"] koszt = { - 'F1': {'M1': 0.3, 'M2': 0.5, 'M3': 1.2, 'M4': 0.8, 'K1': 1.2, 'K2': 999, - 'K3': 1.2, 'K4': 2.0, 'K5': 999, 'K6': 1.1}, - 'F2': {'M1': 999, 'M2': 0.4, 'M3': 0.5, 'M4': 0.3, 'K1': 1.8, 'K2': 999, - 'K3': 999, 'K4': 999, 'K5': 999, 'K6': 999}, - 'M1': {'K1': 999, 'K2': 1.2, 'K3': 0.2, 'K4': 1.7, 'K5': 999, 'K6': 2.0}, - 'M2': {'K1': 1.4, 'K2': 0.3, 'K3': 1.8, 'K4': 1.3, 'K5': 0.5, 'K6': 999}, - 'M3': {'K1': 999, 'K2': 1.3, 'K3': 2.0, 'K4': 999, 'K5': 0.3, 'K6': 1.4}, - 'M4': {'K1': 999, 'K2': 999, 'K3': 0.4, 'K4': 2.0, 'K5': 0.5, 'K6': 1.6} + "F1": { + "M1": 0.3, + "M2": 0.5, + "M3": 1.2, + "M4": 0.8, + "K1": 1.2, + "K2": 999, + "K3": 1.2, + "K4": 2.0, + "K5": 999, + "K6": 1.1, + }, + "F2": { + "M1": 999, + "M2": 0.4, + "M3": 0.5, + "M4": 0.3, + "K1": 1.8, + "K2": 999, + "K3": 999, + "K4": 999, + "K5": 999, + "K6": 999, + }, + "M1": {"K1": 999, "K2": 1.2, "K3": 0.2, "K4": 1.7, "K5": 999, "K6": 2.0}, + "M2": {"K1": 1.4, "K2": 0.3, "K3": 1.8, "K4": 1.3, "K5": 0.5, "K6": 999}, + "M3": {"K1": 999, "K2": 1.3, "K3": 2.0, "K4": 999, "K5": 0.3, "K6": 1.4}, + "M4": {"K1": 999, "K2": 999, "K3": 0.4, "K4": 2.0, "K5": 0.5, "K6": 1.6}, } -P = pulp.LpVariable.dicts("P", [(i, j) for i in dostawcy for j in klienci], cat='Binary') +P = pulp.LpVariable.dicts( + "P", [(i, j) for i in dostawcy for j in klienci], cat="Binary" +) maksymalne_zamowienie = 60 -poziom_satysfakcji = {'K1': 50 / maksymalne_zamowienie, 'K2': 10 / maksymalne_zamowienie, 'K3': 40 / maksymalne_zamowienie, 'K4': 35 / maksymalne_zamowienie, 'K5': 60 / maksymalne_zamowienie, 'K6': 20 / maksymalne_zamowienie} -preferencja_klienta = {'K1': ['F2'], 'K2': ['M1'], 'K3': ['M2', 'M3'], 'K4': ['F1'], 'K5': [], 'K6': ['M3', 'M4']} -suma_satysfakcji = pulp.lpSum([poziom_satysfakcji[j] * P[(i, j)] for i in dostawcy for j in klienci]) +poziom_satysfakcji = { + "K1": 50 / maksymalne_zamowienie, + "K2": 10 / maksymalne_zamowienie, + "K3": 40 / maksymalne_zamowienie, + "K4": 35 / maksymalne_zamowienie, + "K5": 60 / maksymalne_zamowienie, + "K6": 20 / maksymalne_zamowienie, +} +preferencja_klienta = { + "K1": ["F2"], + "K2": ["M1"], + "K3": ["M2", "M3"], + "K4": ["F1"], + "K5": [], + "K6": ["M3", "M4"], +} +suma_satysfakcji = pulp.lpSum( + [poziom_satysfakcji[j] * P[(i, j)] for i in dostawcy for j in klienci] +) # Zmienne decyzyjne -x = pulp.LpVariable.dicts("x", [(i, j) for i in dostawcy for j in klienci], lowBound=0, cat='Integer') -y = pulp.LpVariable.dicts("y", [(i, k) for i in fabryki for k in magazyny], lowBound=0, cat='Integer') +x = pulp.LpVariable.dicts( + "x", [(i, j) for i in dostawcy for j in klienci], lowBound=0, cat="Integer" +) +y = pulp.LpVariable.dicts( + "y", [(i, k) for i in fabryki for k in magazyny], lowBound=0, cat="Integer" +) # Funkcje Celu -koszt_dystrybucji = pulp.lpSum([koszt[i][j] * x[(i, j)] for i in dostawcy for j in klienci]) -koszt_magazynowania = pulp.lpSum([koszt[i][k] * y[(i, k)] for i in fabryki for k in magazyny]) +koszt_dystrybucji = pulp.lpSum( + [koszt[i][j] * x[(i, j)] for i in dostawcy for j in klienci] +) +koszt_magazynowania = pulp.lpSum( + [koszt[i][k] * y[(i, k)] for i in fabryki for k in magazyny] +) alpha = 0.5 beta = 0.5 model += alpha * (koszt_dystrybucji + koszt_magazynowania) - beta * suma_satysfakcji -mozliwosci_fabryki = {'F1': 150, 'F2': 200} -pojemnosc_magazynu = {'M1': 70, 'M2': 50 , 'M3': 100, 'M4': 40 } -zamowienia_klientow = {'K1': 50, 'K2': 10, 'K3': 40, 'K4': 35, 'K5': 60, 'K6': 20} +mozliwosci_fabryki = {"F1": 150, "F2": 200} +pojemnosc_magazynu = {"M1": 70, "M2": 50, "M3": 100, "M4": 40} +zamowienia_klientow = {"K1": 50, "K2": 10, "K3": 40, "K4": 35, "K5": 60, "K6": 20} for i in fabryki: - model += pulp.lpSum([x[(i, j)] for j in klienci] + [y[(i, k)] for k in magazyny]) <= mozliwosci_fabryki[i] + model += ( + pulp.lpSum([x[(i, j)] for j in klienci] + [y[(i, k)] for k in magazyny]) + <= mozliwosci_fabryki[i] + ) for k in magazyny: model += pulp.lpSum([x[(k, j)] for j in klienci]) <= pojemnosc_magazynu[k] @@ -49,22 +116,21 @@ for j in klienci: model.solve() for v in model.variables(): - print(v.name, "=", v.varValue) + logger.info("%s = %s", v.name, v.varValue) koszt_wyniki = [] zadowolenie_wyniki = [] wyniki_funkcji = [] -maksymalny_wynik = 0; -for alpha in range(0, 11): - beta = 10 - alpha + sys.float_info.epsilon - alpha /= 10.0 - beta /= 10.0 +maksymalny_wynik = 0 +for step in range(ALPHA_STEPS): + alpha = step / 10.0 + beta = (ALPHA_STEPS - 1 - step + sys.float_info.epsilon) / 10.0 # Update objective function model.objective = alpha * koszt_dystrybucji - beta * suma_satysfakcji # Solve the model model.solve() - print(alpha) + logger.info("%s", alpha) # Record the wyniki calkowity_koszt = pulp.value(koszt_dystrybucji) @@ -72,14 +138,19 @@ for alpha in range(0, 11): wynik_funkcji = pulp.value(alpha * koszt_dystrybucji - beta * suma_satysfakcji) koszt_wyniki.append(calkowity_koszt) zadowolenie_wyniki.append(calkowite_zadowolenie) - if wynik_funkcji > maksymalny_wynik: - maksymalny_wynik = wynik_funkcji + maksymalny_wynik = max(maksymalny_wynik, wynik_funkcji) wyniki_funkcji.append(wynik_funkcji) - -print("maksymalny_wynik", maksymalny_wynik, wyniki_funkcji) + +logger.info("maksymalny_wynik %s %s", maksymalny_wynik, wyniki_funkcji) # Pseudo-code, assuming model setup as previously discussed -scemariusze = [(1.0, sys.float_info.epsilon), (0.8, 0.2), (0.5, 0.5), (0.2, 0.8), (sys.float_info.epsilon, 1.0)] +scemariusze = [ + (1.0, sys.float_info.epsilon), + (0.8, 0.2), + (0.5, 0.5), + (0.2, 0.8), + (sys.float_info.epsilon, 1.0), +] wyniki = [] for alpha, beta in scemariusze: @@ -92,7 +163,8 @@ for alpha, beta in scemariusze: wyniki.append((alpha, beta, calkowity_koszt, calkowite_zadowolenie)) for idx, (alpha, beta, koszt, zadowolenie) in enumerate(wyniki): - print(f"Krok {idx+1}:") - print(f" koszt: {alpha}, zadowolenie: {beta}") - print(f" Calkowity koszt: {koszt}, zadowolenie klienta: {zadowolenie}\n") - + logger.info("Krok %d:", idx + 1) + logger.info(" koszt: %s, zadowolenie: %s", alpha, beta) + logger.info( + " Calkowity koszt: %s, zadowolenie klienta: %s\n", koszt, zadowolenie + ) diff --git a/NotProgramming/MOM/report_two/model.py b/NotProgramming/MOM/report_two/model.py deleted file mode 100644 index e69de29b..00000000 diff --git a/NotProgramming/SPD/lab4_spd/main.py b/NotProgramming/SPD/lab4_spd/main.py old mode 100644 new mode 100755 index b50d9926..28e568b7 --- a/NotProgramming/SPD/lab4_spd/main.py +++ b/NotProgramming/SPD/lab4_spd/main.py @@ -1,128 +1,167 @@ +#!/usr/bin/env python3 +"""Plot the SPD lab-4 masking thresholds for the 200 Hz and 400 Hz tone complexes. + +Each dataset is one measured masking curve -- probe frequency against the +level at which it became audible -- and the two figures overlay every curve +recorded at that fundamental. +""" + import matplotlib.pyplot as plt # Prepare the datasets -datasets_200Hz = [] -datasets_400Hz = [] +DATASETS_200_HZ = [] +DATASETS_400_HZ = [] # Dataset 1: Wieloton 200 Hz, 7 (1400 Hz), 3 składowe # Starting x value is 1400 Hz -datasets_200Hz.append({ - 'name': '200 Hz, 7 (1400 Hz), 3 składowe', - 'x': [1400 + 0, 1400 + 50, 1400 + 100, 1400 + 150], - 'y': [200, 200, 215, 230] -}) +DATASETS_200_HZ.append( + { + "name": "200 Hz, 7 (1400 Hz), 3 składowe", + "x": [1400 + 0, 1400 + 50, 1400 + 100, 1400 + 150], + "y": [200, 200, 215, 230], + } +) # Dataset 2: Wieloton 200 Hz, 7 (1400 Hz), 7 składowych -datasets_200Hz.append({ - 'name': '200 Hz, 7 (1400 Hz), 7 składowych', - 'x': [1400 + 0, 1400 + 50, 1400 + 100, 1400 + 150], - 'y': [190, 190, 200, 195] -}) +DATASETS_200_HZ.append( + { + "name": "200 Hz, 7 (1400 Hz), 7 składowych", + "x": [1400 + 0, 1400 + 50, 1400 + 100, 1400 + 150], + "y": [190, 190, 200, 195], + } +) # Dataset 3: Wieloton 200 Hz, 10 (2000 Hz), 3 składowe # Starting x value is 2000 Hz -datasets_200Hz.append({ - 'name': '200 Hz, 10 (2000 Hz), 3 składowe', - 'x': [2000 + 0, 2000 + 50, 2000 + 100, 2000 + 150], - 'y': [225, 215, 220, 265] -}) +DATASETS_200_HZ.append( + { + "name": "200 Hz, 10 (2000 Hz), 3 składowe", + "x": [2000 + 0, 2000 + 50, 2000 + 100, 2000 + 150], + "y": [225, 215, 220, 265], + } +) # Dataset 4: Wieloton 200 Hz, 10 (2000 Hz), 7 składowych -datasets_200Hz.append({ - 'name': '200 Hz, 10 (2000 Hz), 7 składowych', - 'x': [2000 + 0, 2000 + 50, 2000 + 100, 2000 + 150], - 'y': [190, 205, 205, 210] -}) +DATASETS_200_HZ.append( + { + "name": "200 Hz, 10 (2000 Hz), 7 składowych", + "x": [2000 + 0, 2000 + 50, 2000 + 100, 2000 + 150], + "y": [190, 205, 205, 210], + } +) # Dataset 4: Wieloton 200 Hz, 10 (2000 Hz), 7 składowych -datasets_200Hz.append({ - 'name': 'Shouten - 7', - 'x': [1400 + 0, 1400 + 50, 1400 + 100, 1400 + 150], - 'y': [200, 207.5, 214, 222] -}) +DATASETS_200_HZ.append( + { + "name": "Shouten - 7", + "x": [1400 + 0, 1400 + 50, 1400 + 100, 1400 + 150], + "y": [200, 207.5, 214, 222], + } +) # Dataset 4: Wieloton 200 Hz, 10 (2000 Hz), 7 składowych -datasets_200Hz.append({ - 'name': 'Shouten - 10', - 'x': [2000 + 0, 2000 + 50, 2000 + 100, 2000 + 150], - 'y': [200, 205, 210, 215] -}) - +DATASETS_200_HZ.append( + { + "name": "Shouten - 10", + "x": [2000 + 0, 2000 + 50, 2000 + 100, 2000 + 150], + "y": [200, 205, 210, 215], + } +) # Dataset 5: Wieloton 400 Hz, 7 (2800 Hz), 3 składowe # Starting x value is 2800 Hz -datasets_400Hz.append({ - 'name': '400 Hz, 7 (2800 Hz), 3 składowe', - 'x': [2800 + 0, 2800 + 100, 2800 + 200, 2800 + 300], - 'y': [405, 410, 410, 405] -}) +DATASETS_400_HZ.append( + { + "name": "400 Hz, 7 (2800 Hz), 3 składowe", + "x": [2800 + 0, 2800 + 100, 2800 + 200, 2800 + 300], + "y": [405, 410, 410, 405], + } +) # Dataset 6: Wieloton 400 Hz, 7 (2800 Hz), 7 składowych -datasets_400Hz.append({ - 'name': '400 Hz, 7 (2800 Hz), 7 składowych', - 'x': [2800 + 0, 2800 + 100, 2800 + 200, 2800 + 300], - 'y': [410, 420, 420, 425] -}) +DATASETS_400_HZ.append( + { + "name": "400 Hz, 7 (2800 Hz), 7 składowych", + "x": [2800 + 0, 2800 + 100, 2800 + 200, 2800 + 300], + "y": [410, 420, 420, 425], + } +) # Dataset 7: Wieloton 400 Hz, 10 (4000 Hz), 3 składowe # Starting x value is 4000 Hz -datasets_400Hz.append({ - 'name': '400 Hz, 10 (4000 Hz), 3 składowe', - 'x': [4000 + 0, 4000 + 100, 4000 + 200, 4000 + 300], - 'y': [407, 419, 425, 425] -}) +DATASETS_400_HZ.append( + { + "name": "400 Hz, 10 (4000 Hz), 3 składowe", + "x": [4000 + 0, 4000 + 100, 4000 + 200, 4000 + 300], + "y": [407, 419, 425, 425], + } +) # Dataset 8: Wieloton 400 Hz, 10 (4000 Hz), 7 składowych # Note: The last two 'Wynik [Hz]' values seem inconsistent. # They are much lower than expected based on prior values. # Assuming there might be typos, and they should be 2200 and 2400 # instead of 220 and 240. -datasets_400Hz.append({ - 'name': '400 Hz, 10 (4000 Hz), 7 składowych', - 'x': [4000 + 0, 4000 + 100, 4000 + 200, 4000 + 300], - 'y': [405, 410, 415, 415] # Original data -}) +DATASETS_400_HZ.append( + { + "name": "400 Hz, 10 (4000 Hz), 7 składowych", + "x": [4000 + 0, 4000 + 100, 4000 + 200, 4000 + 300], + "y": [405, 410, 415, 415], # Original data + } +) -datasets_400Hz.append({ - 'name': 'Shouten - 7', - 'x': [2800 + 0, 2800 + 100, 2800 + 200, 2800 + 300], - 'y': [400, 414, 428, 443] -}) +DATASETS_400_HZ.append( + { + "name": "Shouten - 7", + "x": [2800 + 0, 2800 + 100, 2800 + 200, 2800 + 300], + "y": [400, 414, 428, 443], + } +) -datasets_400Hz.append({ - 'name': 'Shouten - 10', - 'x': [4000 + 0, 4000 + 100, 4000 + 200, 4000 + 300], - 'y': [400, 410, 420, 430] # Original data -}) +DATASETS_400_HZ.append( + { + "name": "Shouten - 10", + "x": [4000 + 0, 4000 + 100, 4000 + 200, 4000 + 300], + "y": [400, 410, 420, 430], # Original data + } +) # Plot the data for 200 Hz plt.figure(figsize=(12, 8)) -for dataset in datasets_200Hz: - if 'Shouten' in dataset['name']: - plt.plot(dataset['x'], dataset['y'], marker='s', color='red', label=dataset['name']) # Shouten datasets +for dataset in DATASETS_200_HZ: + if "Shouten" in dataset["name"]: + plt.plot( + dataset["x"], dataset["y"], marker="s", color="red", label=dataset["name"] + ) # Shouten datasets else: - plt.plot(dataset['x'], dataset['y'], marker='o', label=dataset['name']) # Other datasets -plt.xlabel('Frequency [Hz]') -plt.ylabel('Result [Hz]') -plt.title('Combined Graph of Results for 200 Hz') + plt.plot( + dataset["x"], dataset["y"], marker="o", label=dataset["name"] + ) # Other datasets +plt.xlabel("Frequency [Hz]") +plt.ylabel("Result [Hz]") +plt.title("Combined Graph of Results for 200 Hz") plt.legend() -plt.grid(True) -plt.savefig('results_200Hz.png') # Save the plot +plt.grid(visible=True) +plt.savefig("results_200Hz.png") # Save the plot plt.show() # Plot the data for 400 Hz plt.figure(figsize=(12, 8)) -for dataset in datasets_400Hz: - if 'Shouten' in dataset['name']: - plt.plot(dataset['x'], dataset['y'], marker='s', color='red', label=dataset['name']) # Shouten datasets +for dataset in DATASETS_400_HZ: + if "Shouten" in dataset["name"]: + plt.plot( + dataset["x"], dataset["y"], marker="s", color="red", label=dataset["name"] + ) # Shouten datasets else: - plt.plot(dataset['x'], dataset['y'], marker='o', label=dataset['name']) # Other datasets -plt.xlabel('Frequency [Hz]') -plt.ylabel('Result [Hz]') -plt.title('Combined Graph of Results for 400 Hz') + plt.plot( + dataset["x"], dataset["y"], marker="o", label=dataset["name"] + ) # Other datasets +plt.xlabel("Frequency [Hz]") +plt.ylabel("Result [Hz]") +plt.title("Combined Graph of Results for 400 Hz") plt.legend() -plt.grid(True) -plt.savefig('results_400Hz.png') # Save the plot -plt.show() \ No newline at end of file +plt.grid(visible=True) +plt.savefig("results_400Hz.png") # Save the plot +plt.show() diff --git a/NotProgramming/SPD/spd3/main.py b/NotProgramming/SPD/spd3/main.py old mode 100644 new mode 100755 index fab8d2fa..2dfeb630 --- a/NotProgramming/SPD/spd3/main.py +++ b/NotProgramming/SPD/spd3/main.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +"""Plot the SPD lab-3 measurement series.""" + import matplotlib.pyplot as plt # Values 1 @@ -16,20 +19,22 @@ y3 = [22.06, 15.48, 18.74, 17.71] plt.figure(figsize=(10, 6)) # Plot each line with different styles -plt.plot(x1, y1, marker='o', linestyle='-', color='blue', label='symetrycznie 500 Hz') -plt.plot(x2, y2, marker='s', linestyle='--', color='green', label='asymetrycznie 550 Hz') -plt.plot(x3, y3, marker='^', linestyle='-.', color='red', label='asymetrycznie 450 Hz') +plt.plot(x1, y1, marker="o", linestyle="-", color="blue", label="symetrycznie 500 Hz") +plt.plot( + x2, y2, marker="s", linestyle="--", color="green", label="asymetrycznie 550 Hz" +) +plt.plot(x3, y3, marker="^", linestyle="-.", color="red", label="asymetrycznie 450 Hz") # Add labels and title -plt.xlabel('parametr g', fontsize=40) -plt.ylabel('Prod detekcji syngalu [dB]', fontsize=40) -plt.title('500 Hz', fontsize=48) +plt.xlabel("parametr g", fontsize=40) +plt.ylabel("Prod detekcji syngalu [dB]", fontsize=40) +plt.title("500 Hz", fontsize=48) # Add a legend plt.legend(fontsize=32) # Optional: Add gridlines -plt.grid(True) +plt.grid(visible=True) plt.xticks(fontsize=32) plt.yticks(fontsize=32) @@ -52,20 +57,22 @@ y3 = [13.0, 13.06, 7.8, 1.7] plt.figure(figsize=(10, 6)) # Plot each line with different styles -plt.plot(x1, y1, marker='o', linestyle='-', color='blue', label='symetrycznie 1500 Hz') -plt.plot(x2, y2, marker='s', linestyle='--', color='green', label='asymetrycznie 1650 Hz') -plt.plot(x3, y3, marker='^', linestyle='-.', color='red', label='asymetrycznie 1350 Hz') +plt.plot(x1, y1, marker="o", linestyle="-", color="blue", label="symetrycznie 1500 Hz") +plt.plot( + x2, y2, marker="s", linestyle="--", color="green", label="asymetrycznie 1650 Hz" +) +plt.plot(x3, y3, marker="^", linestyle="-.", color="red", label="asymetrycznie 1350 Hz") # Add labels and title -plt.xlabel('parametr g', fontsize=40) -plt.ylabel('Prod detekcji syngalu [dB]', fontsize=40) -plt.title('1500 Hz', fontsize=48) +plt.xlabel("parametr g", fontsize=40) +plt.ylabel("Prod detekcji syngalu [dB]", fontsize=40) +plt.title("1500 Hz", fontsize=48) # Add a legend plt.legend(fontsize=32) # Optional: Add gridlines -plt.grid(True) +plt.grid(visible=True) plt.xticks(fontsize=32) plt.yticks(fontsize=32) @@ -84,19 +91,21 @@ y2 = [33.06, 35.28, 23.33, 17.14, 14.33, 10.01, 4.43] plt.figure(figsize=(10, 6)) # Plot each line with different styles -plt.plot(x1, y1, marker='o', linestyle='-', color='blue', label='symetrycznie 500 Hz') -plt.plot(x2, y2, marker='s', linestyle='--', color='green', label='symetrycznie 1500 Hz') +plt.plot(x1, y1, marker="o", linestyle="-", color="blue", label="symetrycznie 500 Hz") +plt.plot( + x2, y2, marker="s", linestyle="--", color="green", label="symetrycznie 1500 Hz" +) # Add labels and title -plt.xlabel('parametr g', fontsize=40) -plt.ylabel('Prod detekcji syngalu [dB]', fontsize=40) -plt.title('500 Hz vs 1500 Hz', fontsize=48) +plt.xlabel("parametr g", fontsize=40) +plt.ylabel("Prod detekcji syngalu [dB]", fontsize=40) +plt.title("500 Hz vs 1500 Hz", fontsize=48) # Add a legend plt.legend(fontsize=32) # Optional: Add gridlines -plt.grid(True) +plt.grid(visible=True) plt.xticks(fontsize=32) plt.yticks(fontsize=32) @@ -115,22 +124,23 @@ y2 = [81.59, 60.08, 68.48, 58.77, 53.6, 55.46] plt.figure(figsize=(10, 6)) # Plot each line with different styles -plt.plot(x1, y1, marker='o', linestyle='-', color='blue', label='pasmowy 500 Hz') -plt.plot(x2, y2, marker='s', linestyle='--', color='green', label='pasmowo-zaporowy 500 Hz') +plt.plot(x1, y1, marker="o", linestyle="-", color="blue", label="pasmowy 500 Hz") +plt.plot( + x2, y2, marker="s", linestyle="--", color="green", label="pasmowo-zaporowy 500 Hz" +) # Add labels and title -plt.xlabel('parametr g', fontsize=40) -plt.ylabel('Prod detekcji syngalu [dB]', fontsize=40) -plt.title('Porownanie szum pasmowy vs szum pasmowo zaporowy dla 500 Hz', fontsize=48) +plt.xlabel("parametr g", fontsize=40) +plt.ylabel("Prod detekcji syngalu [dB]", fontsize=40) +plt.title("Porownanie szum pasmowy vs szum pasmowo zaporowy dla 500 Hz", fontsize=48) # Add a legend plt.legend(fontsize=32) # Optional: Add gridlines -plt.grid(True) +plt.grid(visible=True) plt.xticks(fontsize=32) plt.yticks(fontsize=32) # Display the plot plt.show() - diff --git a/NotProgramming/SPD/spd_5/main.py b/NotProgramming/SPD/spd_5/main.py old mode 100644 new mode 100755 index d3d6a5e8..0c814ed5 --- a/NotProgramming/SPD/spd_5/main.py +++ b/NotProgramming/SPD/spd_5/main.py @@ -1,3 +1,6 @@ +#!/usr/bin/env python3 +"""Plot the SPD lab-5 measurement series.""" + import matplotlib.pyplot as plt import numpy as np @@ -23,20 +26,22 @@ y_pure_tone_line = 9 * log_delta_t_line - 19 plt.figure(figsize=(10, 6)) # Plot the experimental data -plt.semilogx(delta_t, ton_60_db_minus, 'o', label='Ton eksperymentalny 60 dB') -plt.semilogx(delta_t, szum_20_db_per_hz_minus, 's', label='Szum eksperymentalny 20 dB/Hz') +plt.semilogx(delta_t, ton_60_db_minus, "o", label="Ton eksperymentalny 60 dB") +plt.semilogx( + delta_t, szum_20_db_per_hz_minus, "s", label="Szum eksperymentalny 20 dB/Hz" +) # Plot the theoretical lines -plt.semilogx(delta_t_line, y_pure_tone_line, label='Ton - krzywa teoretyczna') -plt.semilogx(delta_t_line, y_broadband_line, label='Szum - krzywa teoereyczna') +plt.semilogx(delta_t_line, y_pure_tone_line, label="Ton - krzywa teoretyczna") +plt.semilogx(delta_t_line, y_broadband_line, label="Szum - krzywa teoereyczna") -plt.xlabel('Czas trwania sygnału [ms]', fontsize=40) -plt.ylabel('Poziom względny głośności [dB]', fontsize=40) -plt.title('Głośnośc vs czas', fontsize=48) +plt.xlabel("Czas trwania sygnału [ms]", fontsize=40) +plt.ylabel("Poziom względny głośności [dB]", fontsize=40) +plt.title("Głośnośc vs czas", fontsize=48) plt.legend(fontsize=32) -plt.grid(True, which="both", ls="--") +plt.grid(visible=True, which="both", ls="--") plt.xticks(fontsize=32) plt.yticks(fontsize=32) -plt.show() \ No newline at end of file +plt.show() diff --git a/NotProgramming/SPD/spd_5/main2.py b/NotProgramming/SPD/spd_5/main2.py old mode 100644 new mode 100755 index a6c95e5d..48122ac3 --- a/NotProgramming/SPD/spd_5/main2.py +++ b/NotProgramming/SPD/spd_5/main2.py @@ -1,24 +1,27 @@ +#!/usr/bin/env python3 +"""Plot the SPD lab-5 detection thresholds at 600, 2200 and 4400 Hz.""" + import matplotlib.pyplot as plt import numpy as np # Data (decreasing values) bandwidth = np.array([40, 100, 160, 300, 600, 1000]) -detection_threshold_600Hz = np.array([50, 30, 20, 12, 10, 8]) -detection_threshold_2200Hz = np.array([40, 25, 18, 10, 8, 7]) -detection_threshold_4400Hz = np.array([35, 22, 15, 9, 7, 6]) +DETECTION_THRESHOLD_600_HZ = np.array([50, 30, 20, 12, 10, 8]) +DETECTION_THRESHOLD_2200_HZ = np.array([40, 25, 18, 10, 8, 7]) +DETECTION_THRESHOLD_4400_HZ = np.array([35, 22, 15, 9, 7, 6]) # Plot plt.figure(figsize=(10, 6)) -plt.xscale('log') +plt.xscale("log") -plt.scatter(bandwidth, detection_threshold_600Hz, label='600 Hz', marker='o') -plt.scatter(bandwidth, detection_threshold_2200Hz, label='2200 Hz', marker='^') -plt.scatter(bandwidth, detection_threshold_4400Hz, label='4400 Hz', marker='s') +plt.scatter(bandwidth, DETECTION_THRESHOLD_600_HZ, label="600 Hz", marker="o") +plt.scatter(bandwidth, DETECTION_THRESHOLD_2200_HZ, label="2200 Hz", marker="^") +plt.scatter(bandwidth, DETECTION_THRESHOLD_4400_HZ, label="4400 Hz", marker="s") -plt.xlabel('Szerokość pasma [Hz]') -plt.ylabel('Próg detekcji interwału ciszy [ms]') -plt.title('Górna częstotliwość odcięcia (GCO)') -plt.legend(title='GCO') +plt.xlabel("Szerokość pasma [Hz]") +plt.ylabel("Próg detekcji interwału ciszy [ms]") +plt.title("Górna częstotliwość odcięcia (GCO)") +plt.legend(title="GCO") -plt.grid(True) -plt.show() \ No newline at end of file +plt.grid(visible=True) +plt.show() diff --git a/Programming/EARIN/lab1/main.py b/Programming/EARIN/lab1/main.py index ce1d4b8d..5a0c17d9 100644 --- a/Programming/EARIN/lab1/main.py +++ b/Programming/EARIN/lab1/main.py @@ -1,5 +1,4 @@ -""" -Write horizontal_line program that solves horizontal_line maze +"""Write horizontal_line program that solves horizontal_line maze using greedy best-first search algorithm. The maze is horizontal_line 2D grid with empty space, walls, horizontal_line start, and an end position. @@ -24,14 +23,14 @@ Does not work if no path (Should print out NO PATH FOUND) """ import heapq +import os import sys import time -import os -from random import shuffle, randrange, random +from random import random, randrange, shuffle class MazeSolver: - """Maze Solver""" + """Maze Solver.""" # self corresponds to "this" in js, it refers to object of MazeSolver class @@ -45,7 +44,7 @@ class MazeSolver: # Start/End character def find_start_and_end(self): - """Finds start and end points in the maze""" + """Finds start and end points in the maze.""" start = end = None for row_i, row in enumerate(self.maze): @@ -66,7 +65,7 @@ class MazeSolver: # If it is not horizontal_line "wall" (#) add its position to list of neighbors def get_neighbors(self, position): - """Finds point'maze_data neighbors""" + """Finds point'maze_data neighbors.""" row, col = position neighbors = [] if row > 0 and self.maze[row - 1][col] != "#": @@ -82,7 +81,7 @@ class MazeSolver: # find path through maze def solve_loop(self, queue, visited): - """ Goes through maze and finds the path """ + """Goes through maze and finds the path.""" heuristic_total_time = 0 heuristics_called = 0 while queue: @@ -100,20 +99,17 @@ class MazeSolver: for neighbor in self.get_neighbors(current): if neighbor not in visited: new_path = path + [neighbor] - heuristic, heuristic_time = self.heuristic_euclidean( - neighbor) + heuristic, heuristic_time = self.heuristic_euclidean(neighbor) heuristic_total_time += heuristic_time heuristics_called += 1 - heapq.heappush( - queue, (heuristic, neighbor, new_path) - ) + heapq.heappush(queue, (heuristic, neighbor, new_path)) if not self.test: print_maze(self.maze, path, visited) print() return path, visited, heuristic_total_time, heuristics_called def solve(self): - """Solves the maze""" + """Solves the maze.""" queue = [] # set means that values inside can not repeat visited = set() @@ -122,9 +118,7 @@ class MazeSolver: # we use heapq so the element with lowest heuristic value will always # be at the top of heap heuristic = self.heuristic_euclidean(self.start) - heapq.heappush( - queue, (heuristic, self.start, [self.start]) - ) + heapq.heappush(queue, (heuristic, self.start, [self.start])) # Go through queue until it'maze_data empty # Find neighbor (which is not wall) closest to the @@ -137,10 +131,9 @@ class MazeSolver: # This heuristic returns the Manhattan distance between the given position # and the maze'maze_data end def heuristic_manhattan(self, position): - """Heuristic function that uses Manhattan distance""" + """Heuristic function that uses Manhattan distance.""" start_time = time.perf_counter() - heuristic = abs(position[0] - self.end[0]) + \ - abs(position[1] - self.end[1]) + heuristic = abs(position[0] - self.end[0]) + abs(position[1] - self.end[1]) end_time = time.perf_counter() heuristic_time = end_time - start_time return heuristic, heuristic_time @@ -148,18 +141,17 @@ class MazeSolver: # This heuristic returns the Euclidean distance between the given position # and the maze'maze_data end def heuristic_euclidean(self, position): - """Heuristic function that uses Euclidean distance""" + """Heuristic function that uses Euclidean distance.""" start_time = time.perf_counter() heuristic = ( - abs(position[0] - self.end[0]) ** 2 + - abs(position[1] - self.end[1]) ** 2 + abs(position[0] - self.end[0]) ** 2 + abs(position[1] - self.end[1]) ** 2 ) ** 0.5 end_time = time.perf_counter() heuristic_time = end_time - start_time return heuristic, heuristic_time def heuristic_random(self, position): - """Heuristic function that just returns random value between 0 and 1""" + """Heuristic function that just returns random value between 0 and 1.""" start_time = time.perf_counter() heuristic = random() end_time = time.perf_counter() @@ -169,9 +161,9 @@ class MazeSolver: # Open and load text file to array def load_maze(maze_file_name): - """Loads horizontal_line maze from the specified file""" + """Loads horizontal_line maze from the specified file.""" # Open for reading only and save to fileContents - with open(maze_file_name, "r", encoding="utf8") as file_contents: + with open(maze_file_name, encoding="utf8") as file_contents: # strip() removes extra white spaces from the beginning and the end of # horizontal_line string # list() changes string to array of chars @@ -184,7 +176,7 @@ def load_maze(maze_file_name): def print_maze(maze, path=None, visited=None): - """Prints the maze""" + """Prints the maze.""" if path is None: path = [] if visited is None: @@ -201,7 +193,7 @@ def print_maze(maze, path=None, visited=None): def create_maze_folder(solved): - """ Creates folder for generated or solved mazes""" + """Creates folder for generated or solved mazes.""" if solved: folder_name = "solvedMazes" else: @@ -211,14 +203,18 @@ def create_maze_folder(solved): return folder_name -def save_maze(maze, solved=True, path=None, visited=None, saved_file="Maze", iteration=0): - """Saves maze from array to txt file""" +def save_maze( + maze, solved=True, path=None, visited=None, saved_file="Maze", iteration=0 +): + """Saves maze from array to txt file.""" folder_name = create_maze_folder(solved) if path is None: path = [] if visited is None: visited = [] - with open(f"{folder_name}/{iteration}{os.path.basename(saved_file)}", "w", encoding="utf8") as maze_file: + with open( + f"{folder_name}/{iteration}{os.path.basename(saved_file)}", "w", encoding="utf8" + ) as maze_file: for row_i, row in enumerate(maze): for col_i, cell in enumerate(row): if (row_i, col_i) in path and cell == " ": @@ -234,7 +230,7 @@ def save_maze(maze, solved=True, path=None, visited=None, saved_file="Maze", ite def fill_generated_maze(hor, ver, width): - """ Fills generated maze array from horizontal and vertical lines """ + """Fills generated maze array from horizontal and vertical lines.""" maze_data = "" for horizontal_line, vertical_line in zip(hor, ver): maze_data += "".join(horizontal_line + ["\n"] + vertical_line + ["\n"]) @@ -246,7 +242,7 @@ def fill_generated_maze(hor, ver, width): def make_maze(width=16, height=8): - """ generate maze with given width and height """ + """Generate maze with given width and height.""" vis = [[0] * width + [1] for _ in range(height)] + [[1] * (width + 1)] ver = [["# "] * width + ["#"] for _ in range(height)] + [[]] hor = [["###"] * width + ["#"] for _ in range(height + 1)] @@ -254,20 +250,20 @@ def make_maze(width=16, height=8): def walk(x_coordinate, y_coordinate): vis[y_coordinate][x_coordinate] = 1 - neighbors = [(x_coordinate - 1, y_coordinate), - (x_coordinate, y_coordinate + 1), - (x_coordinate + 1, y_coordinate), - (x_coordinate, y_coordinate - 1)] + neighbors = [ + (x_coordinate - 1, y_coordinate), + (x_coordinate, y_coordinate + 1), + (x_coordinate + 1, y_coordinate), + (x_coordinate, y_coordinate - 1), + ] shuffle(neighbors) for x_coordinate_neighbor, y_coordinate_neighbor in neighbors: if vis[y_coordinate_neighbor][x_coordinate_neighbor]: continue if x_coordinate_neighbor == x_coordinate: - hor[max(y_coordinate, y_coordinate_neighbor) - ][x_coordinate] = "# " + hor[max(y_coordinate, y_coordinate_neighbor)][x_coordinate] = "# " if y_coordinate_neighbor == y_coordinate: - ver[y_coordinate][max( - x_coordinate, x_coordinate_neighbor)] = " " + ver[y_coordinate][max(x_coordinate, x_coordinate_neighbor)] = " " walk(x_coordinate_neighbor, y_coordinate_neighbor) walk(randrange(width), randrange(height)) @@ -276,7 +272,7 @@ def make_maze(width=16, height=8): def print_help(): - """prints help""" + """Prints help.""" print( """python main.py - run the script against default maze file (any file named maze.txt in the code directory) @@ -296,7 +292,7 @@ Number parameter and puts it in the generatedMazes folder""" def test_mode(): - """ Loads and solves multiple mazes in order to compare heuristics """ + """Loads and solves multiple mazes in order to compare heuristics.""" create_maze_folder(False) sum_of_paths = 0 files_amount = 0 @@ -311,7 +307,9 @@ def test_mode(): solver_test = MazeSolver(loaded_maze, TEST_MODE) # Find path using MazeSolver solve method start_time = time.perf_counter() - solved_path, visited, heuristic_total_time, heuristics_called = solver_test.solve() + solved_path, visited, heuristic_total_time, heuristics_called = ( + solver_test.solve() + ) heuristic_total_total_time += heuristic_total_time all_heuristic_called += heuristics_called end_time = time.perf_counter() @@ -335,7 +333,7 @@ def test_mode(): def default(): - """ Runs default operation - reads, solves and prints single maze from file """ + """Runs default operation - reads, solves and prints single maze from file.""" # Open and load text file to array loaded_maze = load_maze(FILE_NAME) # Initialize MazeSolver object with maze as parameter @@ -365,13 +363,12 @@ if __name__ == "__main__": FOLDER_NAME = sys.argv[2] test_mode() sys.exit() - if sys.argv[1] == '-g' or sys.argv[1] == '--generate': + if sys.argv[1] == "-g" or sys.argv[1] == "--generate": if len(sys.argv) > 2: GENERATE_AMOUNT = int(sys.argv[2]) for n in range(GENERATE_AMOUNT): GENERATED_MAZE = make_maze() - save_maze(GENERATED_MAZE, False, None, - None, f'generated{n}.txt') + save_maze(GENERATED_MAZE, False, None, None, f"generated{n}.txt") sys.exit() FILE_NAME = sys.argv[1] default() diff --git a/Programming/EARIN/lab2/main.py b/Programming/EARIN/lab2/main.py index ef384b0f..3fcf703d 100644 --- a/Programming/EARIN/lab2/main.py +++ b/Programming/EARIN/lab2/main.py @@ -1,14 +1,13 @@ -""" -Program that plays draughts (checkers) with user on 8x8 board using min-max with alpha-beta pruning -""" -import re +"""Program that plays draughts (checkers) with user on 8x8 board using min-max with alpha-beta pruning.""" + import copy import math +import re import sys class Game: - """Game""" + """Game.""" def __init__(self, size): self.board_size = size @@ -16,20 +15,19 @@ class Game: self.black_positions = self.initialize_black() def initialize_white(self): - """Initialize white pieces""" + """Initialize white pieces.""" white_positions = [] for y_coordinate in range(math.floor((self.board_size - 2) / 2)): for x_coordinate in range(self.board_size): if y_coordinate % 2 == 0: if x_coordinate % 2 == 1: white_positions.append((x_coordinate, y_coordinate, False)) - else: - if x_coordinate % 2 == 0: - white_positions.append((x_coordinate, y_coordinate, False)) + elif x_coordinate % 2 == 0: + white_positions.append((x_coordinate, y_coordinate, False)) return white_positions def initialize_black(self): - """Initialize black pieces""" + """Initialize black pieces.""" black_positions = [] for y_coordinate in range( self.board_size - math.floor((self.board_size - 2) / 2), self.board_size @@ -38,13 +36,12 @@ class Game: if y_coordinate % 2 == 0: if x_coordinate % 2 == 1: black_positions.append((x_coordinate, y_coordinate, False)) - else: - if x_coordinate % 2 == 0: - black_positions.append((x_coordinate, y_coordinate, False)) + elif x_coordinate % 2 == 0: + black_positions.append((x_coordinate, y_coordinate, False)) return black_positions def check_move_out_of_bounds(self, to_): - """Check if the move destination is out of the bounds of the board""" + """Check if the move destination is out of the bounds of the board.""" if to_[0] < 0 or to_[0] > self.board_size - 1: # print(f"Illegal move! Final x coordinate must be between 0 and {self.board_size-1}!") return True @@ -54,24 +51,24 @@ class Game: return False def check_piece_exists(self, coords, color): - """Check if a piece of given color exists at a given spot""" + """Check if a piece of given color exists at a given spot.""" if color == "white": if any( piece in self.white_positions for piece in ((*coords, False), (*coords, True)) ): return True - else: - if any( - piece in self.black_positions - for piece in ((*coords, False), (*coords, True)) - ): - return True + elif any( + piece in self.black_positions + for piece in ((*coords, False), (*coords, True)) + ): + return True return False def check_piece_king(self, coords, color): """Check if a piece of in a given spot and of a given color is a king. - Return false if no piece is found""" + Return false if no piece is found. + """ if color == "white": return (*coords, True) in self.white_positions if color == "black": @@ -80,7 +77,7 @@ class Game: # https://stackoverflow.com/a/2191707 def check_move_piece_capable(self, from_, to_, color): - """Check if the move is exactly one square diagonally""" + """Check if the move is exactly one square diagonally.""" if abs(to_[0] - from_[0]) == 1: if self.check_piece_king(from_, color): return True @@ -92,7 +89,8 @@ class Game: def check_capture(self, from_, to_, color): """Check if a piece was captured for a given move. - Return captured piece coordinates or None""" + Return captured piece coordinates or None. + """ # captures can only happen if the player moved twice-diagonally if abs(to_[0] - from_[0]) != 2 or abs(to_[1] - from_[1]) != 2: @@ -107,7 +105,7 @@ class Game: return None def check_move_legal(self, from_, to_, color, give_feedback=False): - """Check if a move is legal. Return a boolean or coordinates of captured piece""" + """Check if a move is legal. Return a boolean or coordinates of captured piece.""" if self.check_move_out_of_bounds(to_): if give_feedback: print("Illegal move! Final position is out of the bounds of the board") @@ -135,7 +133,7 @@ class Game: return capture def make_move(self, from_, to_, color): - """Move a piece on the board and remove any captured pieces""" + """Move a piece on the board and remove any captured pieces.""" move_legal = self.check_move_legal(from_, to_, color) if move_legal is False: return False @@ -165,19 +163,19 @@ class Game: return True def print_board(self, rotate=False): - """Print the board in the console""" + """Print the board in the console.""" def print_letters(): - """Print the letters above or under the board""" + """Print the letters above or under the board.""" print(" ", end="") for col in range(self.board_size): if rotate: - print(f" {chr(ord('a')+self.board_size-1-col)}", end=" ") + print(f" {chr(ord('a') + self.board_size - 1 - col)}", end=" ") else: - print(f" {chr(ord('a')+col)}", end=" ") + print(f" {chr(ord('a') + col)}", end=" ") def get_square_code(pos, background): - """Return the code of a given square on the board""" + """Return the code of a given square on the board.""" if (*pos, False) in self.white_positions: return "w" if (*pos, True) in self.white_positions: @@ -210,7 +208,7 @@ class Game: if col == 0: if line % 4 == 2: - print(f"{row//4:3d}", end="") + print(f"{row // 4:3d}", end="") else: print(" ", end="") @@ -218,13 +216,13 @@ class Game: case 0: print("+---", end="") case 1 | 3: - print(f"|{3*background}", end="") + print(f"|{3 * background}", end="") case 2: print(f"|{background}{checker}{background}", end="") if line % 4 == 0: print("+") else: - print(f"|{row//4}" if line % 4 == 2 else "|") + print(f"|{row // 4}" if line % 4 == 2 else "|") line += 1 print(" ", end="") for col in range(self.board_size): @@ -235,7 +233,7 @@ class Game: # Ran first in the code def get_possible_moves_capture(self, from_, color): - """Return all possible captures for a piece""" + """Return all possible captures for a piece.""" # all capturing moves: legal_moves = [] move_down_left_two = (from_[0] + 2, from_[1] - 2) @@ -253,7 +251,7 @@ class Game: return legal_moves def get_possible_moves_non_capture(self, from_, color): - """Return all possible moves that are not captures for a piece""" + """Return all possible moves that are not captures for a piece.""" # all non-capturing moves legal_moves = [] move_down_left_one = (from_[0] + 1, from_[1] - 1) @@ -271,7 +269,7 @@ class Game: return legal_moves def get_possible_moves(self, color): - """Return all possible moves for a given piece color""" + """Return all possible moves for a given piece color.""" legal_moves = [] captures = [] if color == "white": @@ -298,7 +296,8 @@ class Game: def alpha_beta(self, depth, alpha_beta, color, current_color=None): """Do alpha beta pruning for given parameters - and return the best move and its evaluated points""" + and return the best move and its evaluated points. + """ if current_color is None: current_color = color @@ -353,7 +352,7 @@ class Game: return None def evaluate(self, color): - """Evaluates the state of the board for a given color""" + """Evaluates the state of the board for a given color.""" white_score = 0 black_score = 0 @@ -374,13 +373,13 @@ class Game: ) def input_to_coordinates(self, user_input): - """Change input from a1 form to tuple form""" + """Change input from a1 form to tuple form.""" pos_x = ord(user_input[0]) - ord("a") pos_y = int(user_input[1::]) return pos_x, pos_y def handle_player_move(self, color): - """Prompt player to move, validate their input and make move""" + """Prompt player to move, validate their input and make move.""" has_moved = False possible_moves = self.get_possible_moves(color)[0] while not has_moved: @@ -396,7 +395,7 @@ class Game: from_coordinates = self.input_to_coordinates(move_from) to_coordinates = self.input_to_coordinates(move_to) - if not (from_coordinates, to_coordinates) in possible_moves: + if (from_coordinates, to_coordinates) not in possible_moves: legal_no_captures = self.check_move_legal( from_coordinates, to_coordinates, color, True ) @@ -408,7 +407,7 @@ class Game: self.print_board(color == "white") def start_game(self, player_color="black", algorithm_depth=5): - """Start the main loop of the game""" + """Start the main loop of the game.""" if player_color not in ("black", "white"): print("Invalid color! Color can be black or white") return @@ -428,8 +427,8 @@ class Game: game.make_move(*ai_move, ai_color) print( "AI's move: " - f"{chr(ord('a')+ai_move[0][0])}{ai_move[0][1]} " - f"{chr(ord('a')+ai_move[1][0])}{ai_move[1][1]}" + f"{chr(ord('a') + ai_move[0][0])}{ai_move[0][1]} " + f"{chr(ord('a') + ai_move[1][0])}{ai_move[1][1]}" ) game.print_board(player_color == "white") ai_turn = game.get_possible_moves(ai_color)[1] and possible_moves_ai[1] @@ -446,7 +445,7 @@ class Game: ) def ai_turn(self, ai_color, algorithm_depth, possible_moves_ai, print_info): - """ Calculates ai move and makes it """ + """Calculates ai move and makes it.""" if len(possible_moves_ai) == 0: if print_info: print(f"Game over, {ai_color} loses") @@ -458,14 +457,14 @@ class Game: if print_info: print( "AI's move: " - f"{chr(ord('a')+ai_move[0][0])}{ai_move[0][1]} " - f"{chr(ord('a')+ai_move[1][0])}{ai_move[1][1]}" + f"{chr(ord('a') + ai_move[0][0])}{ai_move[0][1]} " + f"{chr(ord('a') + ai_move[1][0])}{ai_move[1][1]}" ) game.print_board(True) return False def auto_game(self, white_depth, black_depth): - """Auto game mode between two bots""" + """Auto game mode between two bots.""" game_turn = 0 max_turns = 250 while game_turn < max_turns: @@ -491,8 +490,9 @@ class Game: return "" return "" + def auto_simulation(white_depth, black_depth, iterations): - """Runs iterations amount of simulations""" + """Runs iterations amount of simulations.""" print( f"""Running {iterations} simulations with white depth = {white_depth},black depth = {black_depth}""" @@ -522,7 +522,7 @@ def auto_simulation(white_depth, black_depth, iterations): def print_help(): - """prints help""" + """Prints help.""" print( """python main.py [algorithm_depth] - play the game against the bot as black, if no algorithm depth is specified the default (5) will be set @@ -541,7 +541,7 @@ if no algorithm depth is specified the default (5) will be set def default(color="black", algorithm_depth=5): - """default program function -> allows to play a game against bot (by default as black)""" + """Default program function -> allows to play a game against bot (by default as black).""" game.start_game(color, algorithm_depth) diff --git a/Programming/EARIN/lab3/example.py b/Programming/EARIN/lab3/example.py index 5e79a028..796ee2a1 100644 --- a/Programming/EARIN/lab3/example.py +++ b/Programming/EARIN/lab3/example.py @@ -1,9 +1,9 @@ -import tempfile import os +import tempfile import cv2 -import numpy as np import matplotlib.pyplot as plt +import numpy as np def main(): @@ -32,7 +32,7 @@ def main(): image = cv2.imread(f.name) # show the image, provide window name first - cv2.imshow('visualization', image) + cv2.imshow("visualization", image) # add wait key. window waits until user presses a key and quits if the key is 'q' if cv2.waitKey(0) == 113: @@ -49,5 +49,5 @@ def main(): cv2.destroyAllWindows() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/Programming/EARIN/lab3/main.py b/Programming/EARIN/lab3/main.py index 7dc5fc85..cba4a3c1 100644 --- a/Programming/EARIN/lab3/main.py +++ b/Programming/EARIN/lab3/main.py @@ -1,19 +1,20 @@ -""" -Program that optimizes Rastrigin function: file_ (x_point_value, y_point_value) = +"""Program that optimizes Rastrigin function: file_ (x_point_value, y_point_value) = 20 + (x_point_value^2 - 10cos(2πx)) + (y_point_value^2 - 10 cos(2πy)). Using Evolutionary Strategy (μ, λ). """ -import sys + import os -import time +import sys import tempfile +import time + import cv2 import matplotlib.pyplot as plt import numpy as np def rastrigin(x_argument, y_argument): - """Define the Rastrigin function""" + """Define the Rastrigin function.""" return ( 20 + x_argument**2 @@ -23,11 +24,8 @@ def rastrigin(x_argument, y_argument): ) -def generate( - population, - arguments -): - """Run single generation""" +def generate(population, arguments): + """Run single generation.""" # Evaluate the fitness of each individual fitness = np.array( [ @@ -37,20 +35,24 @@ def generate( ) # Select the top arguments["number_of_parents"] individuals - parents = population[np.argsort(fitness)[:arguments["number_of_parents"]]] + parents = population[np.argsort(fitness)[: arguments["number_of_parents"]]] # Generate the next generation of lambda individuals by recombination children = np.concatenate( [ np.random.permutation(parents) - for i in range((arguments["size_of_population"] // arguments["number_of_parents"]) + 1) + for i in range( + (arguments["size_of_population"] // arguments["number_of_parents"]) + 1 + ) ] ) - children = children[:arguments["size_of_population"]] + children = children[: arguments["size_of_population"]] # Add mutation to the children mutation = np.random.normal( - loc=0, scale=arguments["mutation_strength"], size=(arguments["size_of_population"], 2) + loc=0, + scale=arguments["mutation_strength"], + size=(arguments["size_of_population"], 2), ) population = children + mutation return fitness, population @@ -60,45 +62,52 @@ def evolution_strategy( arguments, no_display=False, ): - """Define the Evolutionary Strategy (μ, λ) algorithm""" + """Define the Evolutionary Strategy (μ, λ) algorithm.""" # Initialize the population print_info = [] population = np.random.uniform( - low=arguments["min"], high=arguments["max"], size=(arguments["size_of_population"], 2) + low=arguments["min"], + high=arguments["max"], + size=(arguments["size_of_population"], 2), ) summary = [] if not no_display: print_info.append( - (population, - 0, - f"""0:nop-{arguments["number_of_parents"]}:sop-{arguments["size_of_population"]}: + ( + population, + 0, + f"""0:nop-{arguments["number_of_parents"]}:sop-{arguments["size_of_population"]}: ms-{arguments["mutation_strength"]}:nog-{arguments["number_of_generations"]}: min-max-{arguments["min"], arguments["max"]}:noo-{arguments["number_of_outputs"]}""", - )) + ) + ) arguments["number_of_outputs"] = min( - [arguments["number_of_outputs"] - 1, arguments["number_of_generations"]]) + [arguments["number_of_outputs"] - 1, arguments["number_of_generations"]] + ) # Iterate until we reach max number of generate and terminate for generation_number in range(1, arguments["number_of_generations"] + 1): - fitness, population = generate( - population, arguments) + fitness, population = generate(population, arguments) step = ( arguments["number_of_generations"] // arguments["number_of_outputs"] if arguments["number_of_generations"] % arguments["number_of_outputs"] == 0 - else arguments["number_of_generations"] // (arguments["number_of_outputs"] - 1) + else arguments["number_of_generations"] + // (arguments["number_of_outputs"] - 1) ) offset = arguments["number_of_generations"] % step if (generation_number - offset) % step == 0 and not no_display: print_info.append( - (population, - generation_number, - f"""{generation_number}:nop_{arguments["number_of_parents"]}: + ( + population, + generation_number, + f"""{generation_number}:nop_{arguments["number_of_parents"]}: sop_{arguments["size_of_population"]}:ms_{arguments["mutation_strength"]}: nog_{arguments["number_of_generations"]}: min_max_{arguments["min"], arguments["max"]}: noo_{arguments["number_of_outputs"]}""", - )) + ) + ) summary.append(population) # Evaluate the fitness of the final population @@ -121,7 +130,7 @@ def evolution_strategy( def print_help(): - """Print program functionality and how to access it""" + """Print program functionality and how to access it.""" print( """ python main.py - Default functionality optimizing @@ -158,7 +167,7 @@ def print_help(): def get_output_bounds(x_data, y_data): - """Get x and y output limits for pyplot""" + """Get x and y output limits for pyplot.""" # min_size = 0.2 min_output_size = ARGUMENTS["mutation_strength"] * 10 @@ -185,13 +194,8 @@ def get_output_bounds(x_data, y_data): return x_bounds, y_bounds -def output( - population_output, - generation_number, - file_name="temp", - save_results=False): - """Draw result of our function""" - +def output(population_output, generation_number, file_name="temp", save_results=False): + """Draw result of our function.""" # define the visualization params colors = np.random.rand(len(population_output)) @@ -233,8 +237,7 @@ def output( def print_summary(populations, file_name="temp_summary", save_results=False): - """Draw result of our function for chosen generations""" - + """Draw result of our function for chosen generations.""" with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as file_: # iterate over the optimization steps # generate random 2D data - replace it with the results from your @@ -254,12 +257,7 @@ def print_summary(populations, file_name="temp_summary", save_results=False): # plot the data transparency = ind / (len(populations) - 1) color = [[transparency, 0, 0]] * len(pop) - plt.scatter( - x_data, - y_data, - c=color, - alpha=transparency, - label=f"{ind}") + plt.scatter(x_data, y_data, c=color, alpha=transparency, label=f"{ind}") plt.xlim(bounds[0]) plt.ylim(bounds[1]) plt.savefig(file_.name) @@ -285,7 +283,7 @@ def print_summary(populations, file_name="temp_summary", save_results=False): def user_input(): - """Handle user terminal arguments""" + """Handle user terminal arguments.""" arguments = { "number_of_parents": 5, "size_of_population": 20, @@ -324,7 +322,7 @@ def user_input(): def print_output(print_info, save_results, summary): - """ Prints out population and summary plots """ + """Prints out population and summary plots.""" for population, generation_number, file_name in print_info: output(population, generation_number, file_name, save_results) summary_file_name = file_name diff --git a/Programming/EARIN/lab4/main.py b/Programming/EARIN/lab4/main.py index 5dd6ff14..fcf5ef68 100644 --- a/Programming/EARIN/lab4/main.py +++ b/Programming/EARIN/lab4/main.py @@ -1,26 +1,23 @@ -""" -Program that predicts wine quality based on variant2.csv data -""" +"""Program that predicts wine quality based on variant2.csv data.""" + +import matplotlib.pyplot as plt +import numpy as np import pandas as pd import seaborn as sns -import numpy as np -import matplotlib.pyplot as plt -from sklearn.preprocessing import StandardScaler -from sklearn.model_selection import train_test_split -from sklearn.metrics import mean_squared_error, accuracy_score, f1_score from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score, f1_score, mean_squared_error +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler class LinearRegression: - """Implements Linear regression method""" + """Implements Linear regression method.""" def __init__(self): self.theta = None def fit(self, x_values, y_values): - """ - Fit linear regression model to our training data - """ + """Fit linear regression model to our training data.""" # Add a column of ones to X for the intercept term x_values = np.concatenate((np.ones((x_values.shape[0], 1)), y_values), axis=1) @@ -30,9 +27,7 @@ class LinearRegression: ) def predict(self, x_values): - """ - Predict target values for our input data using the trained linear regression model. - """ + """Predict target values for our input data using the trained linear regression model.""" # Add a column of ones to X for the intercept term x_values = np.concatenate((np.ones((x_values.shape[0], 1)), x_values), axis=1) @@ -42,9 +37,7 @@ class LinearRegression: return y_predicted def score(self, x_values, y_values): - """ - Compute the R-squared score of the linear regression model on our test data. - """ + """Compute the R-squared score of the linear regression model on our test data.""" y_predicted = self.predict(x_values) ss_res = np.sum((y_values - y_predicted) ** 2) ss_tot = np.sum((y_values - np.mean(y_values)) ** 2) diff --git a/Programming/EARIN/lab5/code/example.py b/Programming/EARIN/lab5/code/example.py index 299bcc73..491d3668 100644 --- a/Programming/EARIN/lab5/code/example.py +++ b/Programming/EARIN/lab5/code/example.py @@ -1,25 +1,26 @@ # numpy for loading dataset import numpy as np + # pytorch for deep learning models import torch + # nn like neural network -import torch.nn as nn -import torch.optim as optim +from torch import nn, optim # Pima indians describes patient medical data and whether they had diabetes for last 5 years # It is binary classification (they could either have diabetes 1 or not 0) # load the file as a matrix of numbers, -dataset = np.loadtxt('pima-indians-diabetes.csv', delimiter=',') +dataset = np.loadtxt("pima-indians-diabetes.csv", delimiter=",") input_columns = 8 # split into input (X) -> in this case everything beside info whether patient had diabetes or not is input # We are splitting data into two subsets by using NumPy slice operator : and choose first 8 columns using 0:8 slice -X = dataset[:,0:input_columns] +X = dataset[:, 0:input_columns] # and output (y) variables -> in this case we are only interested whether patient had diabetes or not as an output # you can simplify that y = f(X) # We are splitting the data by using slice operator : and choosing last column -y = dataset[:,input_columns] +y = dataset[:, input_columns] -# we need to convert this data to pytorch tensors +# we need to convert this data to pytorch tensors # Pytorch usually operates on 32-bit floating point and NumPy by default uses 64 bit floating point X = torch.tensor(X, dtype=torch.float32) # We can also correct the shape to fit what PyTorch would expect (here we are converting n vectors to n x 1 matrix) @@ -27,6 +28,7 @@ X = torch.tensor(X, dtype=torch.float32) # reshape is converting the output variable y from a 1-dimensional NumPy array to a 2-dimensional PyTorch tensor with a shape of (n, 1), where n is the number of samples in the dataset. y = torch.tensor(y, dtype=torch.float32).reshape(-1, 1) + # define the model # this class is a subclass of nn.Module -> base class provided by PyTorch for building neural network models. class PimaClassifier(nn.Module): @@ -38,7 +40,7 @@ class PimaClassifier(nn.Module): first_output_neurons = 12 self.hidden1 = nn.Linear(input_columns, first_output_neurons) # This creates ReLU (rectified linear unit) activation function applied after first hidden layer - self.act1 = nn.ReLU() + self.act1 = nn.ReLU() # This maps the output of first layer (which was 12 neurons) to new hidden layer of 8 neurons second_output_neurons = 8 @@ -46,13 +48,13 @@ class PimaClassifier(nn.Module): # ReLU activation function applied after second hidden layer self.act2 = nn.ReLU() - # We map output of second layer to a single output neuron -> which will represent the predicted + # We map output of second layer to a single output neuron -> which will represent the predicted # probability of a sample having diabetes self.output = nn.Linear(second_output_neurons, 1) # sigmoid function forces output to be either 0 or 1 self.act_output = nn.Sigmoid() - # forward pass is computation of output based on input 'x' + # forward pass is computation of output based on input 'x' def forward(self, x): # Applies first hidden layer (and then ReLU activation) to input x x = self.act1(self.hidden1(x)) @@ -63,21 +65,22 @@ class PimaClassifier(nn.Module): # returns final output (0 or 1) return x + # Create object from model class model = PimaClassifier() print(model) # train the model -# first we need to specify what is the goal of training +# first we need to specify what is the goal of training # we have input X and output y and we want the model to be as close to y as possible -# Since this is binary classification problem we will use "binary cross entropy" to measure the distance between +# Since this is binary classification problem we will use "binary cross entropy" to measure the distance between # our prediction and y -loss_fn = nn.BCELoss() # binary cross entropy -# Optimizer adjust model weights to produce better output +loss_fn = nn.BCELoss() # binary cross entropy +# Optimizer adjust model weights to produce better output # Its described as being able to tune itself to a lot of problems # inputs are: # parameters which it will optimize (from the model) -# and lr (learning rate) which is step size of each iteration +# and lr (learning rate) which is step size of each iteration optimizer = optim.Adam(model.parameters(), lr=0.001) @@ -95,11 +98,11 @@ for epoch in range(n_epochs): # and one for batches for i in range(0, len(X), batch_size): # Split X data into a batch with the size from batch_size - Xbatch = X[i:i+batch_size] + Xbatch = X[i : i + batch_size] # run the model on the batch and return "batched" output y_pred = model(Xbatch) # Split y data into a batch with the size from batch_size - ybatch = y[i:i+batch_size] + ybatch = y[i : i + batch_size] # Compare loss loss = loss_fn(y_pred, ybatch) # optimize model @@ -117,4 +120,4 @@ print(f"Accuracy {accuracy}") # make class predictions with the model predictions = (model(X) > 0.5).int() for i in range(5): - print('%s => %d (expected %d)' % (X[i].tolist(), predictions[i], y[i])) \ No newline at end of file + print("%s => %d (expected %d)" % (X[i].tolist(), predictions[i], y[i])) diff --git a/Programming/EARIN/lab5/code/main.py b/Programming/EARIN/lab5/code/main.py index a612d5e0..1c8a2b24 100644 --- a/Programming/EARIN/lab5/code/main.py +++ b/Programming/EARIN/lab5/code/main.py @@ -1,14 +1,15 @@ -""" Implementation of a network analyzing MNIST dataset """ -import torch -from torch import nn -from torch import optim -from torchvision import datasets, transforms -import matplotlib.pyplot as plt +"""Implementation of a network analyzing MNIST dataset.""" + import time +import matplotlib.pyplot as plt +import torch +from torch import nn, optim +from torchvision import datasets, transforms + def set_hyperparameters(): - """ sets hyperparameters used throughout the network """ + """Sets hyperparameters used throughout the network.""" return { "num_epochs": 5, "init_input_size": 28 * 28, # MNIST images are 28x28 pixels @@ -17,7 +18,7 @@ def set_hyperparameters(): def load_datasets(): - """ Loads train and test dataset from MNIST """ + """Loads train and test dataset from MNIST.""" train_dataset = datasets.MNIST( root="./data", train=True, transform=transforms.ToTensor(), download=True ) @@ -28,7 +29,7 @@ def load_datasets(): def create_data_loaders(train_dataset, test_dataset): - """ Create train and test data loaders """ + """Create train and test data loaders.""" train_loader = torch.utils.data.DataLoader( dataset=train_dataset, batch_size=BATCH_SIZE, shuffle=True ) @@ -45,42 +46,38 @@ val_acc_values = [] def define_model(hyperparameters): - """ Define the multilayer perceptron training_parameters['model'] """ + """Define the multilayer perceptron training_parameters['model'].""" # Define the multilayer perceptron model model = nn.Sequential() - model.add_module('flatten', nn.Flatten()) - input_size = hyperparameters['init_input_size'] + model.add_module("flatten", nn.Flatten()) + input_size = hyperparameters["init_input_size"] for i in range(NUM_HIDDEN_LAYERS): - model.add_module(f'linear{i}', nn.Linear(input_size, WIDTH)) - model.add_module(f'relu{i}', nn.ReLU()) + model.add_module(f"linear{i}", nn.Linear(input_size, WIDTH)) + model.add_module(f"relu{i}", nn.ReLU()) input_size = WIDTH - model.add_module('output', nn.Linear( - input_size, hyperparameters['num_classes'])) + model.add_module("output", nn.Linear(input_size, hyperparameters["num_classes"])) return model def get_optimizer(model): - """ Return optimizer function """ - if OPTIMIZER_TYPE == 'SGD': + """Return optimizer function.""" + if OPTIMIZER_TYPE == "SGD": return optim.SGD(model.parameters(), lr=LEARNING_RATE) - if OPTIMIZER_TYPE == 'SGD_Momentum': + if OPTIMIZER_TYPE == "SGD_Momentum": return optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=0.9) - if OPTIMIZER_TYPE == 'Adam': + if OPTIMIZER_TYPE == "Adam": return optim.Adam(model.parameters(), lr=LEARNING_RATE) raise ValueError("Unsupported optimizer type!") def initial_configuration(): - """ - Perform all operations needed for training network - """ + """Perform all operations needed for training network.""" # Set random seed for reproducibility torch.manual_seed(42) hyperparameters = set_hyperparameters() # Load MNIST dataset and apply transformations train_dataset, test_dataset = load_datasets() - train_loader, test_loader = create_data_loaders( - train_dataset, test_dataset) + train_loader, test_loader = create_data_loaders(train_dataset, test_dataset) model = define_model(hyperparameters) # Loss function criterion = nn.CrossEntropyLoss() @@ -89,103 +86,89 @@ def initial_configuration(): return hyperparameters, train_loader, test_loader, model, criterion, optimizer -def single_train_iteration( - data, training_parameters, targets, batch_idx, epoch -): - """ - Train network for single batch - """ +def single_train_iteration(data, training_parameters, targets, batch_idx, epoch): + """Train network for single batch.""" # Reshape the input data data = data.view(data.size(0), -1) # Forward pass - outputs = training_parameters['model'](data) - loss = training_parameters['criterion'](outputs, targets) + outputs = training_parameters["model"](data) + loss = training_parameters["criterion"](outputs, targets) # Backward pass and optimization - training_parameters['optimizer'].zero_grad() + training_parameters["optimizer"].zero_grad() loss.backward() - training_parameters['optimizer'].step() + training_parameters["optimizer"].step() # Print loss value for every learning step if (batch_idx + 1) % 100 == 0: print( - f''' - Epoch [{epoch+1}/{training_parameters['hyperparameters']["num_epochs"]}], - Step [{batch_idx+1}/{len(training_parameters['loaders']['train_loader'])}], + f""" + Epoch [{epoch + 1}/{training_parameters["hyperparameters"]["num_epochs"]}], + Step [{batch_idx + 1}/{len(training_parameters["loaders"]["train_loader"])}], Loss: {loss.item():.4f} - ''' + """ ) # Append loss value for every learning step loss_values.append(loss.item()) - return data, training_parameters['optimizer'] + return data, training_parameters["optimizer"] def set_loaders(train_loader, test_loader): - """ - Put train and test loaders into one object - """ - return { - 'train_loader': train_loader, - 'test_loader': test_loader - } + """Put train and test loaders into one object.""" + return {"train_loader": train_loader, "test_loader": test_loader} def set_training_parameters(hyperparameters, loaders, model, criterion, optimizer): - """ - Put all training parameters into one object - """ + """Put all training parameters into one object.""" return { - 'hyperparameters': hyperparameters, - 'loaders': { - 'train_loader': loaders['train_loader'], - 'test_loader': loaders['test_loader'] + "hyperparameters": hyperparameters, + "loaders": { + "train_loader": loaders["train_loader"], + "test_loader": loaders["test_loader"], }, - 'model': model, - 'criterion': criterion, - 'optimizer': optimizer, + "model": model, + "criterion": criterion, + "optimizer": optimizer, } def training_loop(training_parameters, print_info=True): - """ - Train network for all epochs - """ + """Train network for all epochs.""" epochs_num = training_parameters["hyperparameters"]["num_epochs"] # Training loop for epoch in range(epochs_num): - for batch_idx, (data, targets) in enumerate(training_parameters['loaders']['train_loader']): - data, training_parameters['optimizer'] = single_train_iteration( + for batch_idx, (data, targets) in enumerate( + training_parameters["loaders"]["train_loader"] + ): + data, training_parameters["optimizer"] = single_train_iteration( data, training_parameters, targets, batch_idx, epoch ) - calculate_accuracy_epoch( - training_parameters, epoch, print_info) - calculate_validation_set_accuracy( - training_parameters, epoch, print_info) - return epoch, training_parameters['loaders']['train_loader'] + calculate_accuracy_epoch(training_parameters, epoch, print_info) + calculate_validation_set_accuracy(training_parameters, epoch, print_info) + return epoch, training_parameters["loaders"]["train_loader"] def calculate_accuracy_epoch(training_parameters, epoch, print_info=True): - """ Calculate accuracy on train set after each epoch """ + """Calculate accuracy on train set after each epoch.""" correct = 0 total = 0 - for data, targets in training_parameters['loaders']['train_loader']: + for data, targets in training_parameters["loaders"]["train_loader"]: data = data.view(data.size(0), -1) - outputs = training_parameters['model'](data) + outputs = training_parameters["model"](data) _, predicted = torch.max(outputs.data, 1) total += targets.size(0) correct += (predicted == targets).sum().item() train_accuracy = 100 * correct / total if print_info: - print( - f"Accuracy on Train Set after Epoch {epoch+1}: {train_accuracy:.2f}%") + print(f"Accuracy on Train Set after Epoch {epoch + 1}: {train_accuracy:.2f}%") train_acc_values.append(train_accuracy) def calculate_validation_set_accuracy(training_parameters, epoch, print_info=True): - """ Calculate accuracy on validation set after each epoch """ + """Calculate accuracy on validation set after each epoch.""" correct = 0 total = 0 - for data, targets in training_parameters['loaders']['test_loader']: + for data, targets in training_parameters["loaders"]["test_loader"]: data = data.view(data.size(0), -1) - outputs = training_parameters['model'](data) + outputs = training_parameters["model"](data) _, predicted = torch.max(outputs.data, 1) total += targets.size(0) correct += (predicted == targets).sum().item() @@ -193,7 +176,7 @@ def calculate_validation_set_accuracy(training_parameters, epoch, print_info=Tru validation_accuracy = 100 * correct / total if print_info: print( - f"Accuracy on Validation Set after Epoch {epoch+1}: {validation_accuracy:.2f}%" + f"Accuracy on Validation Set after Epoch {epoch + 1}: {validation_accuracy:.2f}%" ) print("---") val_acc_values.append(validation_accuracy) @@ -209,52 +192,55 @@ def main_part(show_plot=True): OPTIMIZER, ) = initial_configuration() start_time = time.time() - LOADERS = set_loaders( - TRAIN_LOADER, TEST_LOADER) + LOADERS = set_loaders(TRAIN_LOADER, TEST_LOADER) TRAINING_PARAMETERS = set_training_parameters( - HYPERPARAMETERS, LOADERS, MODEL, CRITERION, OPTIMIZER) + HYPERPARAMETERS, LOADERS, MODEL, CRITERION, OPTIMIZER + ) training_loop(TRAINING_PARAMETERS, show_plot) file = open("results.txt", "a") file.write( - "-------------------------------------------------------------------------------------" + "\n") + "-------------------------------------------------------------------------------------" + "\n" + ) file.write( - f"loss-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}" + "\n") + f"loss-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}" + "\n" + ) file.write(f"Execution time: {(time.time() - start_time)}" + "\n") file.write( - "-------------------------------------------------------------------------------------" + "\n") + "-------------------------------------------------------------------------------------" + "\n" + ) # Plot the loss value for every learning step - learning_step_title = f'loss-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}.png' + learning_step_title = f"loss-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}.png" plt.plot(loss_values) - plt.xlabel('Learning Step') - plt.ylabel('Loss') + plt.xlabel("Learning Step") + plt.ylabel("Loss") plt.title(learning_step_title) - plt.savefig(learning_step_title - ) + plt.savefig(learning_step_title) if show_plot: plt.show() plt.close() # Plot the accuracy on train set after each epoch - train_accuracy_title = f'trainAccuracy-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}.png' + train_accuracy_title = f"trainAccuracy-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}.png" plt.plot(train_acc_values) - plt.xlabel('Epoch') - plt.ylabel('Train Accuracy') + plt.xlabel("Epoch") + plt.ylabel("Train Accuracy") plt.title(train_accuracy_title) - plt.savefig( - train_accuracy_title) + plt.savefig(train_accuracy_title) if show_plot: plt.show() plt.close() # Plot the accuracy on validation set after each epoch - validation_accuracy_title = f'validationAccuracy-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}.png' + validation_accuracy_title = f"validationAccuracy-lr{LEARNING_RATE}-bs{BATCH_SIZE}-hl{NUM_HIDDEN_LAYERS}-w{WIDTH}-{OPTIMIZER_TYPE}.png" plt.plot(val_acc_values) - plt.xlabel('Epoch') - plt.ylabel('Validation Accuracy') + plt.xlabel("Epoch") + plt.ylabel("Validation Accuracy") plt.title(validation_accuracy_title) - plt.savefig( - validation_accuracy_title) + plt.savefig(validation_accuracy_title) if show_plot: plt.show() plt.close() @@ -265,7 +251,7 @@ if __name__ == "__main__": BATCH_SIZE = 64 NUM_HIDDEN_LAYERS = 2 WIDTH = 128 - OPTIMIZER_TYPE = 'Adam' + OPTIMIZER_TYPE = "Adam" main_part(True) """ learning_rate_values = [0.1, 0.01, 0.001] diff --git a/Programming/EARIN/lab6/main.py b/Programming/EARIN/lab6/main.py index 3dde8d0a..d3ee1094 100644 --- a/Programming/EARIN/lab6/main.py +++ b/Programming/EARIN/lab6/main.py @@ -1,7 +1,7 @@ -""" -Code used to solve MountainCar-v0 gymnasium problem using Q-Learning algorithm -""" +"""Code used to solve MountainCar-v0 gymnasium problem using Q-Learning algorithm.""" + from datetime import datetime + import gymnasium as gym import numpy as np @@ -9,58 +9,54 @@ import numpy as np def discretize_state(state, env, first_time): - # print( - # f"state: {state}, state[0]: {state[0]}, env.observation_space.low: {env.observation_space.low}") + # print( + # f"state: {state}, state[0]: {state[0]}, env.observation_space.low: {env.observation_space.low}") # print(f"state[0] - env {state[0] - env.observation_space.low}") # print(f"state - env {state - env.observation_space.low}") if first_time: substract_from_state = state[0] - env.observation_space.low else: substract_from_state = state - env.observation_space.low - discretized_state = ( - substract_from_state) * np.array([10, 100]) + discretized_state = (substract_from_state) * np.array([10, 100]) discretized_state = np.round(discretized_state, 0).astype(int) return discretized_state def initialize_environment(hyperparameters): - """ - Initialize environment and video recording - """ + """Initialize environment and video recording.""" # Initialize environment - env = gym.make('MountainCar-v0', render_mode='rgb_array') + env = gym.make("MountainCar-v0", render_mode="rgb_array") # Save video now = datetime.now() time_string = now.strftime("%H:%M:%S") env = gym.wrappers.RecordVideo( env, - video_folder='vid', + video_folder="vid", disable_logger=True, - name_prefix=time_string, episode_trigger=lambda x: x > 600 and x % 2 == 0) + name_prefix=time_string, + episode_trigger=lambda x: x > 600 and x % 2 == 0, + ) return env def initialize_q_table(env): - """ - Initialize "empty" Q-table - """ + """Initialize "empty" Q-table.""" # Initialize Q-table # n_actions = env.action_space.n # Number of possible actions, should be 3 # 0 accelerate left # 1 dont accelerate # 2 accelerate to the right # q_table = np.zeros((n_actions,)) - num_states = (env.observation_space.high - - env.observation_space.low) * np.array([10, 100]) + num_states = (env.observation_space.high - env.observation_space.low) * np.array( + [10, 100] + ) num_states = np.round(num_states, 0).astype(int) + 1 q_table = np.zeros((num_states[0], num_states[1], env.action_space.n)) return q_table def initialize_hyperparameters(): - """ - Initialize hyperparameters used by algorithm - """ + """Initialize hyperparameters used by algorithm.""" hyperparameters = { "learning_rate": 0.1, "discount_factor": 0.99, @@ -70,15 +66,13 @@ def initialize_hyperparameters(): "min_max_car_position": [-1.2, 0.6], "min_max_car_velocity": [-0.07, 0.07], "goal_x": 0.5, - "truncation": 200 + "truncation": 200, } return hyperparameters def choose_action(hyperparameters, env, q_table, discretized_state): - """ - Choose one of 3 actions possible for the algorithm - """ + """Choose one of 3 actions possible for the algorithm.""" # hyperparameters["epsilon"]-greedy exploration-exploitation tradeoff if np.random.uniform(0, 1) < hyperparameters["epsilon"]: action = env.action_space.sample() # Choose a random action @@ -89,30 +83,34 @@ def choose_action(hyperparameters, env, q_table, discretized_state): def update_q_table(q_table, action, hyperparameters, reward): - """ - Update q_table with newest reward - """ + """Update q_table with newest reward.""" # Q-table update q_value = q_table[action] max_q_value = np.max(q_table) - new_q_value = (1 - hyperparameters["learning_rate"]) * q_value + \ - hyperparameters["learning_rate"] * \ - (reward + hyperparameters["discount_factor"] * max_q_value) + new_q_value = (1 - hyperparameters["learning_rate"]) * q_value + hyperparameters[ + "learning_rate" + ] * (reward + hyperparameters["discount_factor"] * max_q_value) q_table[action] = new_q_value return q_table -def movement(hyperparameters, env, q_table, discretized_state, total_reward=0, episode_number=0): - """ - Choose action and observe consequences - """ +def movement( + hyperparameters, env, q_table, discretized_state, total_reward=0, episode_number=0 +): + """Choose action and observe consequences.""" action = choose_action(hyperparameters, env, q_table, discretized_state) # Take the action and observe the next state next_state, reward, terminated, truncated, _ = env.step(action) discretized_next_state = discretize_state(next_state, env, False) # print(discretized_next_state[0], discretized_next_state[1]) - q_table[discretized_state[0], discretized_state[1], action] += hyperparameters["learning_rate"] * (reward + hyperparameters["discount_factor"] * np.max( - q_table[discretized_next_state[0], discretized_next_state[1]]) - q_table[discretized_state[0], discretized_state[1], action]) + q_table[discretized_state[0], discretized_state[1], action] += hyperparameters[ + "learning_rate" + ] * ( + reward + + hyperparameters["discount_factor"] + * np.max(q_table[discretized_next_state[0], discretized_next_state[1]]) + - q_table[discretized_state[0], discretized_state[1], action] + ) total_reward += reward discretized_state = discretized_next_state @@ -123,9 +121,7 @@ def movement(hyperparameters, env, q_table, discretized_state, total_reward=0, e def episode_step(env, hyperparameters, q_table, episode_rewards, episode_number): - """ - Actions done with every episode - """ + """Actions done with every episode.""" state = env.reset() # Reset the environment to an initial state discretized_state = discretize_state(state, env, True) done = False # Boolean to indicate episode completion @@ -133,7 +129,13 @@ def episode_step(env, hyperparameters, q_table, episode_rewards, episode_number) for step in range(hyperparameters["max_steps"]): hyperparameters, env, q_table, done, discretized_state, total_reward = movement( - hyperparameters, env, q_table, discretized_state, total_reward, episode_number) + hyperparameters, + env, + q_table, + discretized_state, + total_reward, + episode_number, + ) if done: break @@ -142,22 +144,19 @@ def episode_step(env, hyperparameters, q_table, episode_rewards, episode_number) def training_loop(hyperparameters, env, q_table): - """ - Actual training for MountainCar - """ + """Actual training for MountainCar.""" episode_rewards = [] # List to store episode rewards for episode_number in range(hyperparameters["max_episodes"]): env, hyperparameters, q_table, episode_rewards = episode_step( - env, hyperparameters, q_table, episode_rewards, episode_number) + env, hyperparameters, q_table, episode_rewards, episode_number + ) return env, q_table def inference(env, q_table): - """ - Inference using the updated Q-table - """ + """Inference using the updated Q-table.""" env.reset() done = False @@ -169,7 +168,7 @@ def inference(env, q_table): done = terminated or truncated -if __name__ == '__main__': +if __name__ == "__main__": HYPERPARAMETERS = initialize_hyperparameters() ENV = initialize_environment(HYPERPARAMETERS) Q_TABLE = initialize_q_table(ENV) diff --git a/Programming/EARIN/project/final/code/main.py b/Programming/EARIN/project/final/code/main.py index 44ede9c4..1ad34c89 100644 --- a/Programming/EARIN/project/final/code/main.py +++ b/Programming/EARIN/project/final/code/main.py @@ -1,28 +1,25 @@ +"""Code for preprocessing data and creating model that predicts and +recomends anime based on another anime entered by user. """ -Code for preprocessing data and creating model that predicts and -recomends anime based on another anime entered by user -""" -import math + import argparse -import shutil -import os import datetime -import pandas as pd +import math +import os +import shutil + import numpy as np -from sklearn.neighbors import NearestNeighbors -from sklearn.neighbors import VALID_METRICS_SPARSE +import pandas as pd from scipy.sparse import csr_matrix +from sklearn.neighbors import NearestNeighbors def get_data_cpu(limit_data=-1, data_folder_path="database"): - """ - Reads anime from csv database - """ + """Reads anime from csv database.""" if limit_data > -1: # User can limit number of data taken into consideration, # model seems to work with limit_data value as low as 500,000 - rating_data = pd.read_csv( - data_folder_path + "/animelist.csv", nrows=limit_data) + rating_data = pd.read_csv(data_folder_path + "/animelist.csv", nrows=limit_data) else: rating_data = pd.read_csv(data_folder_path + "/animelist.csv") anime_data = pd.read_csv(data_folder_path + "/anime.csv") @@ -40,15 +37,12 @@ def get_data(limit_data=-1, data_folder_path="database", gpu=False): def merge_rating_anime_data(rating_data, anime_contact_data, debug=False): - """ - Preprocesses the data used for rating - """ + """Preprocesses the data used for rating.""" rating_data = rating_data.merge( anime_contact_data, left_on="anime_id", right_on="anime_id", how="left" ) rating_data = rating_data[ - ["user_id", "Name", "anime_id", "rating", - "watching_status", "watched_episodes"] + ["user_id", "Name", "anime_id", "rating", "watching_status", "watched_episodes"] ] rating_head = rating_data.head() if debug: @@ -60,9 +54,7 @@ def merge_rating_anime_data(rating_data, anime_contact_data, debug=False): def split_data_below_thresholds(rating_data, data_name, threshold=-1, debug=False): - """ - Removes data with data_name which is below given threshold - """ + """Removes data with data_name which is below given threshold.""" if threshold != -1: count = rating_data[data_name].value_counts() rating_data = rating_data[ @@ -75,9 +67,8 @@ def split_data_below_thresholds(rating_data, data_name, threshold=-1, debug=Fals def combine_name_and_ratings(rating_data, debug=False): - """ - Create table which holds name of the anime and number of its reviews - then we merge this with rating_data + """Create table which holds name of the anime and number of its reviews + then we merge this with rating_data. """ combine_movie_rating = rating_data.dropna(axis=0, subset=["Name"]) movie_rating_count = ( @@ -95,44 +86,39 @@ def combine_name_and_ratings(rating_data, debug=False): def get_length_of_data(rating_data, data_name): - """ - We get amount of data in the database with a given column data_name - """ + """We get amount of data in the database with a given column data_name.""" # Encoding categorical data column_ids = rating_data[data_name + "_id"].unique().tolist() column_to_column = {x: i for i, x in enumerate(column_ids)} - rating_data[data_name] = rating_data[data_name + - "_id"].map(column_to_column) + rating_data[data_name] = rating_data[data_name + "_id"].map(column_to_column) users_number = len(column_to_column) return users_number def get_top_ranked(rating_data, data_name, join_table=None, top_data_taken=20): - """ - Get anime with highest ranking - """ + """Get anime with highest ranking.""" if join_table is None: join_table = rating_data - group_data_by_rating = rating_data.groupby( - data_name + "_id")["rating"].count() + group_data_by_rating = rating_data.groupby(data_name + "_id")["rating"].count() top_users = group_data_by_rating.dropna().sort_values(ascending=False)[ - :top_data_taken] - top_rated = join_table.join(top_users, rsuffix="_r", - how="inner", on=data_name + "_id") + :top_data_taken + ] + top_rated = join_table.join( + top_users, rsuffix="_r", how="inner", on=data_name + "_id" + ) return top_rated def get_data_info(rating_data, debug=False, gpu=False): - """ - Get some informations about data - """ + """Get some informations about data.""" users_number = get_length_of_data(rating_data, "user") animes_number = get_length_of_data(rating_data, "anime") top_rated = get_top_ranked(rating_data, "user") top_rated = get_top_ranked(rating_data, "anime", top_rated) - pivot = pd.crosstab(top_rated.user_id, top_rated.anime_id, - top_rated.rating, aggfunc=np.sum) + pivot = pd.crosstab( + top_rated.user_id, top_rated.anime_id, top_rated.rating, aggfunc=np.sum + ) pivot.fillna(0, inplace=True) smallest_rating = min(rating_data["rating"]) @@ -142,19 +128,22 @@ def get_data_info(rating_data, debug=False, gpu=False): if debug: print(f"Num of users: {users_number}, Num of animes: {animes_number}") print( - f"Min total rating: {smallest_rating}, Max total rating: {highest_rating}") + f"Min total rating: {smallest_rating}, Max total rating: {highest_rating}" + ) -def preprocessing(rating_data, anime_contact_data, - debug=False, user_threshold=500, anime_threshold=200, auto=False): - """ - Preprocesses data for making model more accurate and/or faster - """ +def preprocessing( + rating_data, + anime_contact_data, + debug=False, + user_threshold=500, + anime_threshold=200, + auto=False, +): + """Preprocesses data for making model more accurate and/or faster.""" rating_data = merge_rating_anime_data(rating_data, anime_contact_data) - rating_data = split_data_below_thresholds( - rating_data, "user_id", user_threshold) - rating_data = split_data_below_thresholds( - rating_data, "anime_id", anime_threshold) + rating_data = split_data_below_thresholds(rating_data, "user_id", user_threshold) + rating_data = split_data_below_thresholds(rating_data, "anime_id", anime_threshold) rating_data = combine_name_and_ratings(rating_data) rating_data = rating_data.drop(columns="rating_y") @@ -171,10 +160,16 @@ def preprocessing(rating_data, anime_contact_data, return pivot_table -def predict(prediction_model, pivot_table, seed=42, anime="RANDOM", recommendation_number=6, auto=False, debug=False): - """ - This will choose a random anime name and our prediction_model will predict similar anime. - """ +def predict( + prediction_model, + pivot_table, + seed=42, + anime="RANDOM", + recommendation_number=6, + auto=False, + debug=False, +): + """This will choose a random anime name and our prediction_model will predict similar anime.""" np.random.seed(seed) if anime == "RANDOM": chosen_anime = np.random.choice(pivot_table.shape[0]) @@ -183,11 +178,10 @@ def predict(prediction_model, pivot_table, seed=42, anime="RANDOM", recommendati else: query = pivot_table.loc[anime].values.reshape(1, -1) chosen_anime_name = anime - distance, suggestions = prediction_model.kneighbors( - query) + distance, suggestions = prediction_model.kneighbors(query) if debug: print("prediction model, distance: ", distance) - for i in range(0, 2): + for i in range(2): if i == 0: print(f"Recommendations for {chosen_anime_name}:\n") else: @@ -200,9 +194,15 @@ def predict(prediction_model, pivot_table, seed=42, anime="RANDOM", recommendati closest_anime_distance = distance.flatten()[1] average_minus_closest_distance = average_distance - closest_anime_distance print( - f"Average distance: {average_distance}, average_minus_closest_distance: {average_minus_closest_distance}") + f"Average distance: {average_distance}, average_minus_closest_distance: {average_minus_closest_distance}" + ) - return chosen_anime, suggestions.flatten()[1:recommendation_number+1], distance.flatten()[1:recommendation_number+1], f"{closest_anime_distance}_{average_distance}_{average_minus_closest_distance}" + return ( + chosen_anime, + suggestions.flatten()[1 : recommendation_number + 1], + distance.flatten()[1 : recommendation_number + 1], + f"{closest_anime_distance}_{average_distance}_{average_minus_closest_distance}", + ) # return f"{chosen_anime_name}_{closest_anime_name}_{closest_anime_distance}_{average_distance}_{average_minus_closest_distance}" @@ -212,25 +212,25 @@ def calculate_neighbors(rows_number, neighbors=5): "sqrt": math.floor(math.sqrt(rows_number)), "half": math.floor(rows_number / 2), "log": math.floor(math.log(rows_number)), - "n-1": rows_number - 1 + "n-1": rows_number - 1, } if isinstance(neighbors, str): return neighbor_value[neighbors] return neighbors -def create_model(pivot_table, rows_number, metric="cosine", algorithm="brute", neighbors=5): - """ - Creates model based on neaarest neighbor for anime prediction - """ +def create_model( + pivot_table, rows_number, metric="cosine", algorithm="brute", neighbors=5 +): + """Creates model based on neaarest neighbor for anime prediction.""" neighbors_number = calculate_neighbors(pivot_table.shape[0], neighbors) pivot_table_matrix = csr_matrix(pivot_table.values) if algorithm == "brute": - model = NearestNeighbors(n_neighbors=neighbors_number, - metric=metric, algorithm=algorithm) - else: model = NearestNeighbors( - n_neighbors=neighbors_number, algorithm=algorithm) + n_neighbors=neighbors_number, metric=metric, algorithm=algorithm + ) + else: + model = NearestNeighbors(n_neighbors=neighbors_number, algorithm=algorithm) try: model.fit(pivot_table_matrix) except: @@ -241,103 +241,180 @@ def create_model(pivot_table, rows_number, metric="cosine", algorithm="brute", n def handle_arguments(): - """ - Handles all arguments that can be used to change algorithm behaviour or program display - """ - parser = argparse.ArgumentParser(description='Example script with pyargs') - parser.add_argument('--data_limit', '-dl', - help="""Specify data limit, + """Handles all arguments that can be used to change algorithm behaviour or program display.""" + parser = argparse.ArgumentParser(description="Example script with pyargs") + parser.add_argument( + "--data_limit", + "-dl", + help="""Specify data limit, Recommended at least 500k, set to -1 for no limit""", - required=False, type=int, default=-1) - parser.add_argument('--seed', '-s', - help='Specify seed', - type=int, required=False, default=42) - parser.add_argument('--debug', '-d', - help='Use debug (more information) prints', - type=bool, required=False, default=False) - parser.add_argument('--database', '-db', - help='Specify database path', - required=False, default="database") + required=False, + type=int, + default=-1, + ) + parser.add_argument( + "--seed", "-s", help="Specify seed", type=int, required=False, default=42 + ) + parser.add_argument( + "--debug", + "-d", + help="Use debug (more information) prints", + type=bool, + required=False, + default=False, + ) + parser.add_argument( + "--database", + "-db", + help="Specify database path", + required=False, + default="database", + ) allowed_metric = ["cosine", "mahalanobis", "euclidean"] - parser.add_argument('--metric', '-m', - help='Specify metric for NearestNeighbor learner', - required=False, default="cosine", choices=allowed_metric) - allowed_algorithms = ['auto', 'brute'] - parser.add_argument('--algorithm', '-a', - help='Specify algorithm for Nearest Neighbor learner', - required=False, default="brute", choices=allowed_algorithms) - parser.add_argument('--anime', '-an', - help='Specify anime to choose', - required=False, default="RANDOM") - parser.add_argument('--neighbors', '-n', - help='Specify number of nearest neighbors', - required=False, default=5) - parser.add_argument('--user_threshold', '-ut', - help="""Specify minimal number of votes required for user to be + parser.add_argument( + "--metric", + "-m", + help="Specify metric for NearestNeighbor learner", + required=False, + default="cosine", + choices=allowed_metric, + ) + allowed_algorithms = ["auto", "brute"] + parser.add_argument( + "--algorithm", + "-a", + help="Specify algorithm for Nearest Neighbor learner", + required=False, + default="brute", + choices=allowed_algorithms, + ) + parser.add_argument( + "--anime", + "-an", + help="Specify anime to choose", + required=False, + default="RANDOM", + ) + parser.add_argument( + "--neighbors", + "-n", + help="Specify number of nearest neighbors", + required=False, + default=5, + ) + parser.add_argument( + "--user_threshold", + "-ut", + help="""Specify minimal number of votes required for user to be included in the data, set to -1 for no threshold""", - required=False, type=int, default=500) - parser.add_argument('--anime_threshold', '-at', - help="""Specify minimal number of votes required for anime + required=False, + type=int, + default=500, + ) + parser.add_argument( + "--anime_threshold", + "-at", + help="""Specify minimal number of votes required for anime to be included in the data, set to -1 for no threshold""", - required=False, type=int, default=200) - parser.add_argument('--recommendation_amount', '-ra', - help='Specify how much anime should be recommended', - required=False, type=int, default=5) - parser.add_argument('--auto', '-au', - help="""Enable auto mode, no debug, no user parameters, + required=False, + type=int, + default=200, + ) + parser.add_argument( + "--recommendation_amount", + "-ra", + help="Specify how much anime should be recommended", + required=False, + type=int, + default=5, + ) + parser.add_argument( + "--auto", + "-au", + help="""Enable auto mode, no debug, no user parameters, automatic testing and saving results""", - type=bool, required=False, default=False) + type=bool, + required=False, + default=False, + ) # Parse the command-line arguments args = parser.parse_args() args.recommendation_amount = args.recommendation_amount + 1 # Access the values of the arguments - return args.seed, args.debug, args.data_limit, args.database, args.metric, args.algorithm, args.anime, args.neighbors, args.user_threshold, args.anime_threshold, args.recommendation_amount, args.auto + return ( + args.seed, + args.debug, + args.data_limit, + args.database, + args.metric, + args.algorithm, + args.anime, + args.neighbors, + args.user_threshold, + args.anime_threshold, + args.recommendation_amount, + args.auto, + ) def auto_mode(data_limit=-1, seed=42, anime="RANDOM"): print("Started auto mode") - algorithm_spread = ['auto', 'brute'] - metric_spread = ['manhattan', 'euclidean', 'cosine'] + algorithm_spread = ["auto", "brute"] + metric_spread = ["manhattan", "euclidean", "cosine"] neighbor_spread = [5, "sqrt", "half", "log", "n-1"] # No reason to access and waste computational power every time we run the simulation starting_rating_data, starting_anime_contact_data, starting_rows_number = get_data( - limit_data=data_limit) + limit_data=data_limit + ) original_pivot_table = preprocessing( - starting_rating_data, starting_anime_contact_data) - if os.path.exists('test_results'): - shutil.rmtree('test_results') + starting_rating_data, starting_anime_contact_data + ) + if os.path.exists("test_results"): + shutil.rmtree("test_results") for algorithm in algorithm_spread: possibleMetrics = [] - if algorithm != 'auto': + if algorithm != "auto": possibleMetrics = metric_spread print("testing for algorithm: ", algorithm, possibleMetrics) if possibleMetrics == []: possibleMetrics = [""] for metric in possibleMetrics: - if metric != 'precomputed': + if metric != "precomputed": print("testing for algorithm, metric: ", algorithm, metric) for neighbor_amount in neighbor_spread: - print("testing for algorithm, metric, neighbor_amount: ", - algorithm, metric, neighbor_amount) - preprocess_model_predict(starting_rating_data, starting_anime_contact_data, - starting_rows_number, original_pivot_table, seed=seed, anime=anime, neighbors=neighbor_amount, algorithm=algorithm, metric=metric) + print( + "testing for algorithm, metric, neighbor_amount: ", + algorithm, + metric, + neighbor_amount, + ) + preprocess_model_predict( + starting_rating_data, + starting_anime_contact_data, + starting_rows_number, + original_pivot_table, + seed=seed, + anime=anime, + neighbors=neighbor_amount, + algorithm=algorithm, + metric=metric, + ) def write_test_results(title, result=""): # Create directory if it doesn't already exist - if not os.path.exists('test_results'): - os.makedirs('test_results') + if not os.path.exists("test_results"): + os.makedirs("test_results") # Generate timestamped filename - timestamp = datetime.datetime.now().strftime( - '%Y%m%d%H%M%S') # e.g., 20230611235959 + timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") # e.g., 20230611235959 filename = f"{title}_{timestamp}.txt" # Create and write to the file - with open(os.path.join('test_results', filename), 'a') as file: + with open(os.path.join("test_results", filename), "a") as file: file.write(result) @@ -347,42 +424,94 @@ def calculate_precision(predictions, threshold=8): return np.mean(precision) -def preprocess_model_predict(rating_data, anime_contact_data, rows_number, pivot_table, data_limit=-1, db="database", debug=False, user_threshold=500, anime_threshold=200, metric="cosine", algorithm="brute", neighbors=5, seed=42, anime="RANDOM", recommendation_amount=5): - MODEL = create_model(pivot_table, rows_number, - metric, algorithm, neighbors) +def preprocess_model_predict( + rating_data, + anime_contact_data, + rows_number, + pivot_table, + data_limit=-1, + db="database", + debug=False, + user_threshold=500, + anime_threshold=200, + metric="cosine", + algorithm="brute", + neighbors=5, + seed=42, + anime="RANDOM", + recommendation_amount=5, +): + MODEL = create_model(pivot_table, rows_number, metric, algorithm, neighbors) result = "" if MODEL != "Error!": - chosen_anime, suggestions, distance, distance_data = predict(MODEL, pivot_table, seed, - anime, recommendation_amount) + chosen_anime, suggestions, distance, distance_data = predict( + MODEL, pivot_table, seed, anime, recommendation_amount + ) chosen_anime_name = pivot_table.index[chosen_anime] # average_distance = np.mean(distance) # closest_anime_name = pivot_table.index[suggestions[1]] # closest_anime_distance = distance[1] # average_minus_closest_distance = closest_anime_distance - average_distance - precision = calculate_precision( - [pivot_table.iloc[s] for s in suggestions]) + precision = calculate_precision([pivot_table.iloc[s] for s in suggestions]) result = f"{chosen_anime_name}:\n" for i in range(len(suggestions)): result += f"{pivot_table.index[suggestions[i]]}; Distance: {distance[i]}\n" - result += f"Precision: {precision*100}%\n" - result += "Smallest distance, average distance, Average - Smallest distance: " + distance_data + result += f"Precision: {precision * 100}%\n" + result += ( + "Smallest distance, average distance, Average - Smallest distance: " + + distance_data + ) # result = f"{chosen_anime_name}_{closest_anime_name}_{closest_anime_distance}_{average_distance}_{average_minus_closest_distance}" write_test_results( - f"dl={rows_number}&s={seed}&m={metric}&a={algorithm}&ut={user_threshold}&at={anime_threshold}&n={neighbors}", result) + f"dl={rows_number}&s={seed}&m={metric}&a={algorithm}&ut={user_threshold}&at={anime_threshold}&n={neighbors}", + result, + ) if __name__ == "__main__": - SEED, DEBUG, DATA_LIMIT, DB, METRIC, ALGORITHM, ANIME, NEIGHBORS, USER_THRESHOLD, ANIME_THRESHOLD, RECOMMENDATION_AMOUNT, AUTO = handle_arguments() + ( + SEED, + DEBUG, + DATA_LIMIT, + DB, + METRIC, + ALGORITHM, + ANIME, + NEIGHBORS, + USER_THRESHOLD, + ANIME_THRESHOLD, + RECOMMENDATION_AMOUNT, + AUTO, + ) = handle_arguments() if not AUTO: print("Entered not auto mode") - starting_rating_data, starting_anime_contact_data, starting_rows_number = get_data( - limit_data=DATA_LIMIT, data_folder_path=DB) + starting_rating_data, starting_anime_contact_data, starting_rows_number = ( + get_data(limit_data=DATA_LIMIT, data_folder_path=DB) + ) pivot_table = preprocessing( - starting_rating_data, starting_anime_contact_data, USER_THRESHOLD, ANIME_THRESHOLD) - preprocess_model_predict(starting_rating_data, starting_anime_contact_data, starting_rows_number, - pivot_table, data_limit=DATA_LIMIT, db=DB, debug=DEBUG, user_threshold=USER_THRESHOLD, anime_threshold=ANIME_THRESHOLD, - metric=METRIC, algorithm=ALGORITHM, neighbors=NEIGHBORS, seed=SEED, anime=ANIME, recommendation_amount=RECOMMENDATION_AMOUNT) + starting_rating_data, + starting_anime_contact_data, + USER_THRESHOLD, + ANIME_THRESHOLD, + ) + preprocess_model_predict( + starting_rating_data, + starting_anime_contact_data, + starting_rows_number, + pivot_table, + data_limit=DATA_LIMIT, + db=DB, + debug=DEBUG, + user_threshold=USER_THRESHOLD, + anime_threshold=ANIME_THRESHOLD, + metric=METRIC, + algorithm=ALGORITHM, + neighbors=NEIGHBORS, + seed=SEED, + anime=ANIME, + recommendation_amount=RECOMMENDATION_AMOUNT, + ) if AUTO: auto_mode(DATA_LIMIT, SEED, ANIME) diff --git a/Programming/EARIN/project/midterm/code/main.py b/Programming/EARIN/project/midterm/code/main.py index cc9b662a..2b509332 100644 --- a/Programming/EARIN/project/midterm/code/main.py +++ b/Programming/EARIN/project/midterm/code/main.py @@ -1,26 +1,21 @@ +"""Code for preprocessing data and creating model that predicts and +recomends anime based on another anime entered by user. """ -Code for preprocessing data and creating model that predicts and -recomends anime based on another anime entered by user -""" -import pandas as pd -import numpy as np import argparse -import sklearn -from sklearn.neighbors import NearestNeighbors +import numpy as np +import pandas as pd from scipy.sparse import csr_matrix +from sklearn.neighbors import NearestNeighbors def get_data(limit_data=-1, data_folder_path="database"): - """ - Reads anime from csv database - """ + """Reads anime from csv database.""" if limit_data > -1: # User can limit number of data taken into consideration, # model seems to work with limit_data value as low as 500,000 - rating_data = pd.read_csv( - data_folder_path + "/animelist.csv", nrows=limit_data) + rating_data = pd.read_csv(data_folder_path + "/animelist.csv", nrows=limit_data) else: rating_data = pd.read_csv(data_folder_path + "/animelist.csv") anime_data = pd.read_csv(data_folder_path + "/anime.csv") @@ -32,15 +27,12 @@ def get_data(limit_data=-1, data_folder_path="database"): def merge_rating_anime_data(rating_data, anime_contact_data, debug=False): - """ - Preprocesses the data used for rating - """ + """Preprocesses the data used for rating.""" rating_data = rating_data.merge( anime_contact_data, left_on="anime_id", right_on="anime_id", how="left" ) rating_data = rating_data[ - ["user_id", "Name", "anime_id", "rating", - "watching_status", "watched_episodes"] + ["user_id", "Name", "anime_id", "rating", "watching_status", "watched_episodes"] ] rating_head = rating_data.head() if debug: @@ -52,9 +44,7 @@ def merge_rating_anime_data(rating_data, anime_contact_data, debug=False): def split_data_below_thresholds(rating_data, data_name, threshold=-1, debug=False): - """ - Removes data with data_name which is below given threshold - """ + """Removes data with data_name which is below given threshold.""" if threshold != -1: count = rating_data[data_name].value_counts() rating_data = rating_data[ @@ -67,9 +57,8 @@ def split_data_below_thresholds(rating_data, data_name, threshold=-1, debug=Fals def combine_name_and_ratings(rating_data, debug=False): - """ - Create table which holds name of the anime and number of its reviews - then we merge this with rating_data + """Create table which holds name of the anime and number of its reviews + then we merge this with rating_data. """ combine_movie_rating = rating_data.dropna(axis=0, subset=["Name"]) movie_rating_count = ( @@ -87,45 +76,40 @@ def combine_name_and_ratings(rating_data, debug=False): def get_length_of_data(rating_data, data_name): - """ - We get amount of data in the database with a given column data_name - """ + """We get amount of data in the database with a given column data_name.""" # Encoding categorical data column_ids = rating_data[data_name + "_id"].unique().tolist() column_to_column = {x: i for i, x in enumerate(column_ids)} - rating_data[data_name] = rating_data[data_name + - "_id"].map(column_to_column) + rating_data[data_name] = rating_data[data_name + "_id"].map(column_to_column) users_number = len(column_to_column) return users_number def get_top_ranked(rating_data, data_name, join_table=None, top_data_taken=20): - """ - Get anime with highest ranking - """ + """Get anime with highest ranking.""" if join_table is None: join_table = rating_data - group_data_by_rating = rating_data.groupby( - data_name + "_id")["rating"].count() + group_data_by_rating = rating_data.groupby(data_name + "_id")["rating"].count() top_users = group_data_by_rating.dropna().sort_values(ascending=False)[ - :top_data_taken] - top_rated = join_table.join(top_users, rsuffix="_r", - how="inner", on=data_name + "_id") + :top_data_taken + ] + top_rated = join_table.join( + top_users, rsuffix="_r", how="inner", on=data_name + "_id" + ) return top_rated def get_data_info(rating_data, debug=False): - """ - Get some informations about data - """ + """Get some informations about data.""" users_number = get_length_of_data(rating_data, "user") animes_number = get_length_of_data(rating_data, "anime") top_rated = get_top_ranked(rating_data, "user") top_rated = get_top_ranked(rating_data, "anime", top_rated) - pivot = pd.crosstab(top_rated.user_id, top_rated.anime_id, - top_rated.rating, aggfunc=np.sum) + pivot = pd.crosstab( + top_rated.user_id, top_rated.anime_id, top_rated.rating, aggfunc=np.sum + ) pivot.fillna(0, inplace=True) smallest_rating = min(rating_data["rating"]) @@ -135,18 +119,21 @@ def get_data_info(rating_data, debug=False): if debug: print(f"Num of users: {users_number}, Num of animes: {animes_number}") print( - f"Min total rating: {smallest_rating}, Max total rating: {highest_rating}") + f"Min total rating: {smallest_rating}, Max total rating: {highest_rating}" + ) -def preprocessing(rating_data, anime_contact_data, debug=False, user_threshold=500, anime_threshold=200): - """ - Preprocesses data for making model more accurate and/or faster - """ +def preprocessing( + rating_data, + anime_contact_data, + debug=False, + user_threshold=500, + anime_threshold=200, +): + """Preprocesses data for making model more accurate and/or faster.""" rating_data = merge_rating_anime_data(rating_data, anime_contact_data) - rating_data = split_data_below_thresholds( - rating_data, "user_id", user_threshold) - rating_data = split_data_below_thresholds( - rating_data, "anime_id", anime_threshold) + rating_data = split_data_below_thresholds(rating_data, "user_id", user_threshold) + rating_data = split_data_below_thresholds(rating_data, "anime_id", anime_threshold) rating_data = combine_name_and_ratings(rating_data) rating_data = rating_data.drop(columns="rating_x") @@ -163,10 +150,10 @@ def preprocessing(rating_data, anime_contact_data, debug=False, user_threshold=5 return pivot_table -def predict(prediction_model, pivot_table, seed=42, anime="RANDOM", recommendation_number=6): - """ - This will choose a random anime name and our prediction_model will predict similar anime. - """ +def predict( + prediction_model, pivot_table, seed=42, anime="RANDOM", recommendation_number=6 +): + """This will choose a random anime name and our prediction_model will predict similar anime.""" np.random.seed(seed) print(pivot_table) if anime == "RANDOM": @@ -178,8 +165,9 @@ def predict(prediction_model, pivot_table, seed=42, anime="RANDOM", recommendati chosen_anime_name = anime distance, suggestions = prediction_model.kneighbors( - query, n_neighbors=recommendation_number) - for i in range(0, len(distance.flatten())): + query, n_neighbors=recommendation_number + ) + for i in range(len(distance.flatten())): if i == 0: print(f"Recommendations for {chosen_anime_name}:\n") else: @@ -189,56 +177,136 @@ def predict(prediction_model, pivot_table, seed=42, anime="RANDOM", recommendati def create_model(pivot_table, metric="cosine", algorithm="brute", neighbors=5): - """ - Creates model based on neaarest neighbor for anime prediction - """ + """Creates model based on neaarest neighbor for anime prediction.""" pivot_table_matrix = csr_matrix(pivot_table.values) - model = NearestNeighbors(n_neighbors=neighbors, - metric=metric, algorithm=algorithm) + model = NearestNeighbors(n_neighbors=neighbors, metric=metric, algorithm=algorithm) model.fit(pivot_table_matrix) return model def handle_arguments(): - parser = argparse.ArgumentParser(description='Example script with pyargs') - parser.add_argument('--data_limit', '-dl', - help='Specify data limit, Recommended at least 500k, set to -1 for no limit', required=False, type=int, default=-1) - parser.add_argument('--seed', '-s', help='Specify seed', - type=int, required=False, default=42) - parser.add_argument('--debug', '-d', help='Use debug (more information) prints', - type=bool, required=False, default=False) - parser.add_argument('--database', '-db', help='Specify database path', - required=False, default="database") + parser = argparse.ArgumentParser(description="Example script with pyargs") + parser.add_argument( + "--data_limit", + "-dl", + help="Specify data limit, Recommended at least 500k, set to -1 for no limit", + required=False, + type=int, + default=-1, + ) + parser.add_argument( + "--seed", "-s", help="Specify seed", type=int, required=False, default=42 + ) + parser.add_argument( + "--debug", + "-d", + help="Use debug (more information) prints", + type=bool, + required=False, + default=False, + ) + parser.add_argument( + "--database", + "-db", + help="Specify database path", + required=False, + default="database", + ) allowed_metric = ["cosine", "mahalanobis", "euclidean"] - parser.add_argument('--metric', '-m', help='Specify metric for NearestNeighbor learner', - required=False, default="cosine", choices=allowed_metric) - allowed_algorithms = ['auto', 'ball_tree', 'kd_tree', 'brute'] - parser.add_argument('--algorithm', '-a', help='Specify algorithm for Nearest Neighbor learner', - required=False, default="brute", choices=allowed_algorithms) - parser.add_argument('--anime', '-an', help='Specify anime to choose', - required=False, default="RANDOM") - parser.add_argument('--neighbors', '-n', help='Specify number of nearest neighbors', - required=False, default=5) - parser.add_argument('--user_threshold', '-ut', help='Specify minimal number of votes required for user to be included in the data, set to -1 for no threshold', - required=False, type=int, default=500) - parser.add_argument('--anime_threshold', '-at', help='Specify minimal number of votes required for anime to be included in the data, set to -1 for no threshold', - required=False, type=int, default=200) - parser.add_argument('--recommendation_amount', '-ra', help='Specify how much anime should be recommended', - required=False, type=int, default=5) + parser.add_argument( + "--metric", + "-m", + help="Specify metric for NearestNeighbor learner", + required=False, + default="cosine", + choices=allowed_metric, + ) + allowed_algorithms = ["auto", "ball_tree", "kd_tree", "brute"] + parser.add_argument( + "--algorithm", + "-a", + help="Specify algorithm for Nearest Neighbor learner", + required=False, + default="brute", + choices=allowed_algorithms, + ) + parser.add_argument( + "--anime", + "-an", + help="Specify anime to choose", + required=False, + default="RANDOM", + ) + parser.add_argument( + "--neighbors", + "-n", + help="Specify number of nearest neighbors", + required=False, + default=5, + ) + parser.add_argument( + "--user_threshold", + "-ut", + help="Specify minimal number of votes required for user to be included in the data, set to -1 for no threshold", + required=False, + type=int, + default=500, + ) + parser.add_argument( + "--anime_threshold", + "-at", + help="Specify minimal number of votes required for anime to be included in the data, set to -1 for no threshold", + required=False, + type=int, + default=200, + ) + parser.add_argument( + "--recommendation_amount", + "-ra", + help="Specify how much anime should be recommended", + required=False, + type=int, + default=5, + ) # Parse the command-line arguments args = parser.parse_args() args.recommendation_amount = args.recommendation_amount + 1 # Access the values of the arguments - return args.seed, args.debug, args.data_limit, args.database, args.metric, args.algorithm, args.anime, args.neighbors, args.user_threshold, args.anime_threshold, args.recommendation_amount + return ( + args.seed, + args.debug, + args.data_limit, + args.database, + args.metric, + args.algorithm, + args.anime, + args.neighbors, + args.user_threshold, + args.anime_threshold, + args.recommendation_amount, + ) if __name__ == "__main__": - seed, debug, data_limit, db, metric, algorithm, anime, neighbors, user_threshold, anime_threshold, recommendation_amount = handle_arguments() + ( + seed, + debug, + data_limit, + db, + metric, + algorithm, + anime, + neighbors, + user_threshold, + anime_threshold, + recommendation_amount, + ) = handle_arguments() RATING_DATA, ANIME_CONTACT_DATA = get_data(data_limit, db) PIVOT_TABLE = preprocessing( - RATING_DATA, ANIME_CONTACT_DATA, debug, user_threshold, anime_threshold) + RATING_DATA, ANIME_CONTACT_DATA, debug, user_threshold, anime_threshold + ) MODEL = create_model(PIVOT_TABLE, metric, algorithm, neighbors) predict(MODEL, PIVOT_TABLE, seed, anime, recommendation_amount) diff --git a/Programming/ECOTE/program/translator/main.py b/Programming/ECOTE/program/translator/main.py index 9b7c045c..1f8686b2 100644 --- a/Programming/ECOTE/program/translator/main.py +++ b/Programming/ECOTE/program/translator/main.py @@ -1,23 +1,19 @@ -""" - Program for converting Latex files into html files -""" +"""Program for converting Latex files into html files.""" + import argparse def command_name_check(latex_string, command_name): - """ - Checks if command name given from latex was spelled correctly - """ - if latex_string[1: (len(command_name) + 1)] != command_name: + """Checks if command name given from latex was spelled correctly.""" + if latex_string[1 : (len(command_name) + 1)] != command_name: print(latex_string + "Error! command misspeled!") return "Error!" return "" def generic_checks(latex_string): - """ - Checks if string is given and whether latex parameters - ends with curly bracket + """Checks if string is given and whether latex parameters + ends with curly bracket. """ if latex_string == "": print(latex_string + "Error! No input given to function") @@ -29,8 +25,7 @@ def generic_checks(latex_string): def generic_checks_command(latex_string): - """ - In additon to generic checks it also checks if command starts with \ + """In additon to generic checks it also checks if command starts with \ """ if generic_checks(latex_string) == "Error!": return "Error!" @@ -41,9 +36,7 @@ def generic_checks_command(latex_string): def document_class_only_checks(latex_string): - """ - Checks only errors connected to document class - """ + """Checks only errors connected to document class.""" if len(latex_string) < len("\\documentclass{"): print(latex_string + "Error! latex string to short to be document class!") return "Error!" @@ -56,9 +49,8 @@ def document_class_only_checks(latex_string): def document_class(latex_string): - r""" - Converts LaTeX documentclass method to html - \documentclass{article} + r"""Converts LaTeX documentclass method to html + \documentclass{article}. """ if generic_checks_command(latex_string) == "Error!": return "Error!" @@ -70,9 +62,8 @@ def document_class(latex_string): def begin_document(latex_string): - r""" - Converts LaTeX begin document method to html - \begin{document} + r"""Converts LaTeX begin document method to html + \begin{document}. """ if generic_checks_command(latex_string) == "Error!": return "Error!" @@ -82,9 +73,7 @@ def begin_document(latex_string): def begin_tabular(latex_string): - r""" - Checks if LaTeX begin tabular method is correct - """ + r"""Checks if LaTeX begin tabular method is correct.""" if generic_checks_command(latex_string) == "Error!": return "Error!" if command_name_check(latex_string, "begin") == "Error!": @@ -93,9 +82,7 @@ def begin_tabular(latex_string): def tabular_parameters(latex_string): - r""" - Checks if LaTeX tabular environment has any parameters - """ + r"""Checks if LaTeX tabular environment has any parameters.""" if latex_string == "": return "" if latex_string == "[]": @@ -105,16 +92,15 @@ def tabular_parameters(latex_string): def only_pipes_and_space(latex_string): """Checks if latex string only contains | or " " - if yes returns True, if no returns false""" + if yes returns True, if no returns false. + """ return all(char in ("|", " ", "{", "}") for char in latex_string) def handle_complicated_parameter(latex_string, return_array, i): - """ - handles single complicated parameter from loop - """ + """Handles single complicated parameter from loop.""" closing_bracket = latex_string.find("}", i + 1) - columns_string = latex_string[i: closing_bracket + 1] + columns_string = latex_string[i : closing_bracket + 1] result = tabular_columns_parameters(columns_string) if result == "Error!": return return_array, i, result @@ -123,16 +109,13 @@ def handle_complicated_parameter(latex_string, return_array, i): return return_array, i, "" -def main_tabular_parameters_loop_inside(latex_string, i, - simple_parameters_dictionary, - return_array): - """ - Handles single tabular parameter character - """ +def main_tabular_parameters_loop_inside( + latex_string, i, simple_parameters_dictionary, return_array +): + """Handles single tabular parameter character.""" character = latex_string[i] if character in ["l", "c", "r", "|"]: - return_array.append( - simple_parameters_dictionary.get(latex_string[i])) + return_array.append(simple_parameters_dictionary.get(latex_string[i])) i += 1 return return_array, i, "" if character in ["p", "m", "b"]: @@ -142,24 +125,22 @@ def main_tabular_parameters_loop_inside(latex_string, i, def main_tabular_parameters_loop(latex_string, simple_parameters_dictionary): - """ - Converts simple and paragraph table parameters into html style code - """ + """Converts simple and paragraph table parameters into html style code.""" return_array = [] i = 0 latex_string_length = len(latex_string) while i < latex_string_length: return_array, i, result = main_tabular_parameters_loop_inside( - latex_string, i, simple_parameters_dictionary, return_array) + latex_string, i, simple_parameters_dictionary, return_array + ) if result == "Error!": return result return return_array def tabular_required_errors(latex_string): - """ - Runs generic checks and - checks if there are any parameters put to latex table + """Runs generic checks and + checks if there are any parameters put to latex table. """ if generic_checks(latex_string) == "Error!": return "Error!" @@ -174,9 +155,7 @@ def tabular_required_errors(latex_string): def simple_parameters_dictionary_return(): - """ - Defines html styles for simple tabular parameters - """ + """Defines html styles for simple tabular parameters.""" simple_parameters_dictionary = { "l": "align='left'", "c": "align='center'", @@ -187,9 +166,7 @@ def simple_parameters_dictionary_return(): def tabular_required_parameters(latex_string): - """ - Converts simple tabular parameters to html style commands - """ + """Converts simple tabular parameters to html style commands.""" if tabular_required_errors(latex_string) == "Error!": return "Error!" simple_parameters_dictionary = simple_parameters_dictionary_return() @@ -200,14 +177,11 @@ def tabular_required_parameters(latex_string): latex_string, ) return "Error!" - return main_tabular_parameters_loop( - latex_string, simple_parameters_dictionary) + return main_tabular_parameters_loop(latex_string, simple_parameters_dictionary) def length_conversions(latex_length): - """ - Converts latex lengths to html lengths - """ + """Converts latex lengths to html lengths.""" length_dictionary = { "pt": [1.3, "px"], "mm": [1, "mm"], @@ -220,9 +194,7 @@ def length_conversions(latex_length): def construct_return_string(vertical_align_type, final_length, conversed_unit): - """ - Construsts entire html style of parameters - """ + """Construsts entire html style of parameters.""" return_string = ( "style='" + vertical_align_type @@ -235,9 +207,7 @@ def construct_return_string(vertical_align_type, final_length, conversed_unit): def return_parameter_dictionary(): - """ - Returns html styles for each complicated latex parameter - """ + """Returns html styles for each complicated latex parameter.""" parameter_dictionary = { "p": "vertical-align: top;", "m": "vertical-align: middle;", @@ -247,13 +217,11 @@ def return_parameter_dictionary(): def tabular_columns_parameters_errors(latex_string, vertical_align_type): - """ - Checks for intermediate errors - Checks if parameter length starts with curly bracket + """Checks for intermediate errors + Checks if parameter length starts with curly bracket. """ if vertical_align_type == "Error!": - print("tabular_columns_parameters, unknown parameter: ", - latex_string[0]) + print("tabular_columns_parameters, unknown parameter: ", latex_string[0]) return "Error!" if latex_string[1] != "{": print( @@ -266,9 +234,7 @@ def tabular_columns_parameters_errors(latex_string, vertical_align_type): def string_length_no_whitespace(length_parameter): - """ - Calculates string width without whitespace - """ + """Calculates string width without whitespace.""" i = 0 length_value = "" for character in length_parameter: @@ -280,15 +246,12 @@ def string_length_no_whitespace(length_parameter): def tabular_columns_parameters(latex_string): - """ - Handles tabular paragraph parameters and translates them to html style - """ + """Handles tabular paragraph parameters and translates them to html style.""" if generic_checks(latex_string) == "Error!": return "Error!" parameter_dictionary = return_parameter_dictionary() vertical_align_type = parameter_dictionary.get(latex_string[0], "Error!") - if tabular_columns_parameters_errors( - latex_string, vertical_align_type) == "Error!": + if tabular_columns_parameters_errors(latex_string, vertical_align_type) == "Error!": return "Error!" length_parameter_with_bracket = latex_string.partition("{")[2] length_parameter = length_parameter_with_bracket.partition("}")[0] @@ -296,50 +259,44 @@ def tabular_columns_parameters(latex_string): length_unit = length_parameter[i:] conversed_unit = length_conversions(length_unit) if conversed_unit == "Error!": - print("""tabular_columns_parameters, - Unit could not be conversed!""", latex_string) + print( + """tabular_columns_parameters, + Unit could not be conversed!""", + latex_string, + ) return "Error!" final_length = round(float(length_value) * conversed_unit[0], 2) - return construct_return_string(vertical_align_type, - final_length, conversed_unit) + return construct_return_string(vertical_align_type, final_length, conversed_unit) def split_rows(latex_string): - """ - Splits table row string into separate strings and puts them in array - """ + """Splits table row string into separate strings and puts them in array.""" double_backslash = "\\" rows = latex_string.split(double_backslash) return rows def split_columns(table_row, column_count): - """ - Splits table row columns into separate strings and puts them in array - """ + """Splits table row columns into separate strings and puts them in array.""" columns = table_row.split("&") if len(columns) != column_count and columns != [""]: print( - f"Error! extracted columns length: {columns} and columns_count: {column_count} is different!") + f"Error! extracted columns length: {columns} and columns_count: {column_count} is different!" + ) return "Error!" return columns def translate_column(latex_column): - """ - Translate insides of a single latex tabular column to html - """ + """Translate insides of a single latex tabular column to html.""" hline_string_literal = "\\hline" replaced_hline = latex_column.replace(hline_string_literal, "
") replaced_newline = replaced_hline.replace("\newline", "
") return replaced_newline -def handle_line_strings(line_string, - column_style, column_number, return_string): - """ - Converts lines untill there are no more lines - """ +def handle_line_strings(line_string, column_style, column_number, return_string): + """Converts lines untill there are no more lines.""" current_style = column_style[column_number] if current_style == line_string: while current_style == line_string: @@ -349,11 +306,10 @@ def handle_line_strings(line_string, return return_string, column_number -def handle_single_column(column, column_style, - column_number, line_string, return_string): - """ - Handles single column from columns loop - """ +def handle_single_column( + column, column_style, column_number, line_string, return_string +): + """Handles single column from columns loop.""" return_string += "= len(column_style): print( @@ -362,7 +318,8 @@ def handle_single_column(column, column_style, ) return "Error!", column_number return_string, column_number = handle_line_strings( - line_string, column_style, column_number, return_string) + line_string, column_style, column_number, return_string + ) return_string += column_style[column_number] + ">" return_string += translate_column(column) @@ -371,42 +328,36 @@ def handle_single_column(column, column_style, return return_string, column_number -def handle_latex_columns(columns, column_style, - column_number, line_string, return_string): - """ - Goes through every column in a row and translates it to html - """ - +def handle_latex_columns( + columns, column_style, column_number, line_string, return_string +): + """Goes through every column in a row and translates it to html.""" if columns != "Error!": for column in columns: - return_string, column_number = handle_single_column(column, - column_style, - column_number, - line_string, - return_string) + return_string, column_number = handle_single_column( + column, column_style, column_number, line_string, return_string + ) return return_string, column_number -def go_through_latex_rows(rows, return_string, - column_amount, column_style, line_string): - """ - Goes through latex table insides row by row - """ +def go_through_latex_rows( + rows, return_string, column_amount, column_style, line_string +): + """Goes through latex table insides row by row.""" for row in rows: return_string += "" columns = split_columns(row, column_amount) column_number = 0 return_string, column_number = handle_latex_columns( - columns, column_style, column_number, line_string, return_string) + columns, column_style, column_number, line_string, return_string + ) return_string += "" return return_string def translate_inside_to_html(latex_table_inside, column_style): - """ - Translates entire table insides to html - """ + """Translates entire table insides to html.""" return_string = "" column_amount = 0 line_string = "style='border-left: 1px solid black'" @@ -415,65 +366,58 @@ def translate_inside_to_html(latex_table_inside, column_style): column_amount += 1 rows = split_rows(latex_table_inside) return_string = go_through_latex_rows( - rows, return_string, column_amount, column_style, line_string) + rows, return_string, column_amount, column_style, line_string + ) return_string += "
" return return_string def read_file(tex_filename): + """opens, reads and saves latex file + content into python string, then closes file. """ - opens, reads and saves latex file - content into python string, then closes file - """ - with open(tex_filename, "r", encoding="UTF-8") as tex_file: + with open(tex_filename, encoding="UTF-8") as tex_file: data = tex_file.read().replace("\n", "") tex_file.close() return data def read_document_class(latex_full_string): - """ - Finds documentclass in latex file and checks function correctness - """ + """Finds documentclass in latex file and checks function correctness.""" document_class_index = latex_full_string.find("\\documentclass") if document_class_index == -1: print("Main function error! documentclass not found") return "Error!" - document_class_close_bracket = latex_full_string.find( - "}", document_class_index) + document_class_close_bracket = latex_full_string.find("}", document_class_index) document_class_string = latex_full_string[ - document_class_index: document_class_close_bracket + 1 + document_class_index : document_class_close_bracket + 1 ] latex_full_string = latex_full_string[ - document_class_close_bracket + 1: len(latex_full_string) - 1 + document_class_close_bracket + 1 : len(latex_full_string) - 1 ] return document_class(document_class_string), latex_full_string def read_begin_document(latex_full_string): - """ - Finds LaTeX \begin{document} command and returns html counterpart to it - """ + """Finds LaTeX \begin{document} command and returns html counterpart to it.""" begin_document_index = latex_full_string.find(r"\begin{document}") if begin_document_index == -1: print("read_begin_document error! begin{document not found") return "Error!", latex_full_string - begin_document_close_bracket = latex_full_string.find( - "}", begin_document_index) + begin_document_close_bracket = latex_full_string.find("}", begin_document_index) begin_document_string = latex_full_string[ - begin_document_index: begin_document_close_bracket + 1 + begin_document_index : begin_document_close_bracket + 1 ] latex_full_string = latex_full_string[ - begin_document_close_bracket + 1: len(latex_full_string) - 1 + begin_document_close_bracket + 1 : len(latex_full_string) - 1 ] return_string = begin_document(begin_document_string) return return_string, latex_full_string def read_start(html_string, data): - """ - Finds, reads latex document class - and begin document and translates them to html + """Finds, reads latex document class + and begin document and translates them to html. """ document_class_result, data = read_document_class(data) if document_class_result == "Error!": @@ -488,28 +432,24 @@ def read_start(html_string, data): def handle_table_whole(html_string, data, table_start, table_end): - """ - Reads entire table, handles its parameters - and insides and translates to html + """Reads entire table, handles its parameters + and insides and translates to html. """ if table_start == -1 or table_end == -1: return html_string, data tabular_begin_string = r"\begin{tabular}" - parameters_start_index = data.find( - "{", table_start + len(tabular_begin_string)) + parameters_start_index = data.find("{", table_start + len(tabular_begin_string)) parameters_end_index = data.find("}", parameters_start_index) - parameters_string = data[parameters_start_index: parameters_end_index + 1] + parameters_string = data[parameters_start_index : parameters_end_index + 1] parameters_array = tabular_required_parameters(parameters_string) - inside_table = data[parameters_end_index + 1: table_end] + inside_table = data[parameters_end_index + 1 : table_end] html_string += translate_inside_to_html(inside_table, parameters_array) return html_string def handle_insides(html_string, data): - """ - Finds inside of table and translates them to html - """ + """Finds inside of table and translates them to html.""" tabular_index = 0 tabular_end_string = r"\end{tabular}" while tabular_index != -1: @@ -522,15 +462,14 @@ def handle_insides(html_string, data): html_string = handle_table_whole( html_string, data, tabular_index_start, tabular_index_end ) - data = data[tabular_index_end + len(tabular_end_string): len(data)] + data = data[tabular_index_end + len(tabular_end_string) : len(data)] tabular_index = tabular_index_end return html_string, data def main_function(tex_filename): - """ - Ran by default by program, - gets latex file and translates to html + """Ran by default by program, + gets latex file and translates to html. """ data = read_file(tex_filename) html_string = "" @@ -540,15 +479,19 @@ def main_function(tex_filename): when reading documentclass or begin document""") return "Error" html_string, data = handle_insides(html_string, data) - html_string = html_string.replace('}', '') + html_string = html_string.replace("}", "") return html_string def handle_arguments(): - parser = argparse.ArgumentParser( - description='Specify tex filename and path') - parser.add_argument('--filename', '-f', - help='Specify tex filename, include path if needed ', required=False, default="texfile.tex") + parser = argparse.ArgumentParser(description="Specify tex filename and path") + parser.add_argument( + "--filename", + "-f", + help="Specify tex filename, include path if needed ", + required=False, + default="texfile.tex", + ) args = parser.parse_args() return args.filename diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_begin_document.py b/Programming/ECOTE/program/unit_tests/test_code/test_begin_document.py index ae404221..a952b438 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_begin_document.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_begin_document.py @@ -1,59 +1,53 @@ -""" - Tests begin document function -""" +"""Tests begin document function.""" + from translator.main import begin_document + # Write python tests for a function translating LaTeX documentclass to html def given_empty_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert begin_document("") == "Error!" def given_not_closed_then_error(): - r""" - Given: "\\begin\{document" + r"""Given: "\\begin\{document" When: N/A - Then: Error message + Then: Error message. """ assert begin_document(r"\\begin{document") == "Error!" def given_no_opening_then_error(): - """ - Given: No opening curly bracket + """Given: No opening curly bracket When: N/A - Then: Error message + Then: Error message. """ assert begin_document(r"\\begindocument}") == "Error!" def given_misspeled_then_error(): - """ - Given: misspelled begin document + """Given: misspelled begin document When: N/A - Then: Error message + Then: Error message. """ assert begin_document(r"\\begim{document}") == "Error!" def given_no_slash_then_error(): - """ - Given: no backslash at start + """Given: no backslash at start When: N/A - Then: Error message + Then: Error message. """ assert begin_document(r"begin{document}") == "Error!" def given_correct_then_html(): - """ - Given: \\documentclass{article} + """Given: \\documentclass{article} When: N/A - Then: + Then: . """ assert begin_document(r"\begin{document}") == "" diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_begin_tabular.py b/Programming/ECOTE/program/unit_tests/test_code/test_begin_tabular.py index 91d6ae32..f798e75d 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_begin_tabular.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_begin_tabular.py @@ -1,67 +1,61 @@ -""" - Tests begin document function -""" +"""Tests begin document function.""" + from translator.main import begin_tabular + # Write python tests for a function translating LaTeX documentclass to html def given_empty_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert begin_tabular("") == "Error!" def given_not_closed_then_error(): - r""" - Given: command not closed + r"""Given: command not closed When: N/A - Then: Error message + Then: Error message. """ assert begin_tabular(r"\\begin{tabular") == "Error!" def given_no_opening_then_error(): - """ - Given: No opening curly bracket + """Given: No opening curly bracket When: N/A - Then: Error message + Then: Error message. """ assert begin_tabular(r"\\begintabular}") == "Error!" def given_misspeled_then_error(): - """ - Given: misspelled begin tabular + """Given: misspelled begin tabular When: N/A - Then: Error message + Then: Error message. """ assert begin_tabular(r"\\begim{tabular}") == "Error!" def given_no_slash_then_error(): - """ - Given: no backslash at start + """Given: no backslash at start When: N/A - Then: Error message + Then: Error message. """ assert begin_tabular(r"begin{tabular}") == "Error!" + def given_tabular_star_then_error(): - """ - Given: no backslash at start + """Given: no backslash at start When: N/A - Then: Error message + Then: Error message. """ assert begin_tabular(r"begin{tabular*}") == "Error!" def given_correct_then_html(): - """ - Given: \begin{tabular} + """Given: \begin{tabular} When: N/A - Then: + Then: . """ assert begin_tabular(r"\begin{tabular}") == "" diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_document_class.py b/Programming/ECOTE/program/unit_tests/test_code/test_document_class.py index a06d6770..bf8df2dc 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_document_class.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_document_class.py @@ -1,82 +1,70 @@ -""" - Tests document class function -""" +"""Tests document class function.""" + from translator.main import document_class # Write python tests for a function translating LaTeX documentclass to html def given_empty_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert document_class("") == "Error!" def given_not_closed_then_error(): - """ - Given: "\\documentclass{article" + """Given: "\\documentclass{article" When: N/A - Then: Error message + Then: Error message. """ assert document_class("\\documentclass{article") == "Error!" def given_parameters_then_error(): - """ - Given: class parameters + """Given: class parameters When: N/A - Then: Error message + Then: Error message. """ assert document_class("\\documentclass[12pt]{article}") == "Error!" def given_no_opening_then_error(): - """ - Given: No opening curly bracket + """Given: No opening curly bracket When: N/A - Then: Error message + Then: Error message. """ assert document_class("\\documentclassarticle}") == "Error!" def given_misspeled_then_error(): - """ - Given: misspelled document class + """Given: misspelled document class When: N/A - Then: Error message + Then: Error message. """ assert document_class("\\documentclasZ{article}") == "Error!" def given_class_not_recognized_then_error(): - """ - Given: class not recognized + """Given: class not recognized When: N/A - Then: Error message + Then: Error message. """ - assert ( - document_class("\\documentclass{idonotexist}") - == "" - ) + assert document_class("\\documentclass{idonotexist}") == "" def given_no_slash_then_error(): - """ - Given: no backslash at start + """Given: no backslash at start When: N/A - Then: Error message + Then: Error message. """ assert document_class("documentclass{article}") == "Error!" def given_correct_then_html(): - """ - Given: \\documentclass{article} + """Given: \\documentclass{article} When: N/A - Then: + Then: . """ assert document_class("\\documentclass{article}") == "" diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks.py b/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks.py index 454ebe95..eb0006a0 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks.py @@ -6,13 +6,11 @@ def test_empty_string(): def test_no_curly_bracket(): - assert generic_checks( - "latex_string_without_curly_bracket") == "Error!" + assert generic_checks("latex_string_without_curly_bracket") == "Error!" def test_with_curly_bracket_at_end(): - assert generic_checks( - "latex_string_with_curly_bracket}") == "" + assert generic_checks("latex_string_with_curly_bracket}") == "" def test_generic_checks(): diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks_command.py b/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks_command.py index cd15fba0..9cccc3de 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks_command.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_generic_checks_command.py @@ -6,18 +6,19 @@ def test_empty_string(): def test_no_curly_bracket(): - assert (generic_checks_command( - "latex_string_without_curly_bracket")) == "Error!" + assert (generic_checks_command("latex_string_without_curly_bracket")) == "Error!" def test_no_slash_at_beginning(): - assert (generic_checks_command( - "latex_string_without_slash_at_beginning}")) == "Error!" + assert ( + generic_checks_command("latex_string_without_slash_at_beginning}") + ) == "Error!" def test_with_slash_and_curly_bracket(): - assert (generic_checks_command( - "\\latex_string_with_slash_and_curly_bracket}")) == "" + assert ( + generic_checks_command("\\latex_string_with_slash_and_curly_bracket}") + ) == "" def test_generic_checks_command(): diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_length_conversions.py b/Programming/ECOTE/program/unit_tests/test_code/test_length_conversions.py index cedcb55f..7cfcd521 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_length_conversions.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_length_conversions.py @@ -1,97 +1,86 @@ -""" - Tests begin document function -""" +"""Tests begin document function.""" + from translator.main import length_conversions # Write python tests for a function translating LaTeX documentclass to html def given_empty_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("") == "Error!" def given_pt_then_px(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("pt") == [1.3, "px"] def given_mm_then_mm(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("mm") == [1, "mm"] def given_cm_then_cm(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("cm") == [1, "cm"] def given_in_then_in(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("in") == [1, "in"] def given_ex_then_ex(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("ex") == [1, "ex"] def given_em_then_em(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("em") == [1, "em"] def given_mu_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("mu") == "Error!" def given_sp_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("sp") == "Error!" def given_unknown_then_error(): - """ - Given: "" + """Given: "" When: N/A - Then: Error message + Then: Error message. """ assert length_conversions("unknown") == "Error!" diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_split_columns.py b/Programming/ECOTE/program/unit_tests/test_code/test_split_columns.py index 48dfc59a..fe734d06 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_split_columns.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_split_columns.py @@ -2,15 +2,15 @@ from translator.main import split_columns def given_empty_then_correct(): - assert split_columns("", 0) == [''] + assert split_columns("", 0) == [""] def given_no_and_then_correct(): - assert split_columns("estsegsegtswegseg", 1) == ['estsegsegtswegseg'] + assert split_columns("estsegsegtswegseg", 1) == ["estsegsegtswegseg"] def given_single_and_then_correct(): - assert split_columns("&", 2) == ['', ''] + assert split_columns("&", 2) == ["", ""] def given_too_much_columns_then_error(): diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_split_rows.py b/Programming/ECOTE/program/unit_tests/test_code/test_split_rows.py index 3424ee8c..8cf12c1f 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_split_rows.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_split_rows.py @@ -2,7 +2,7 @@ from translator.main import split_rows def given_empty_then_correct(): - assert split_rows("") == [''] + assert split_rows("") == [""] def given_no_slash_then_correct(): @@ -10,7 +10,7 @@ def given_no_slash_then_correct(): def given_double_slash_then_correct(): - assert split_rows("\\") == ['', ''] + assert split_rows("\\") == ["", ""] def given_actual_string_then_correct(): diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_tabular_columns_parameters.py b/Programming/ECOTE/program/unit_tests/test_code/test_tabular_columns_parameters.py index f289bdfd..cc01ea1d 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_tabular_columns_parameters.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_tabular_columns_parameters.py @@ -2,51 +2,52 @@ from translator.main import tabular_columns_parameters def given_empty_then_error(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ assert tabular_columns_parameters("") == "Error!" def given_unknown_then_error(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ assert tabular_columns_parameters("l{1.3cm}") == "Error!" def given_p_then_array(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ - assert tabular_columns_parameters( - "p{1.3pt}") == "style='vertical-align: top; width: 1.69px;'" + assert ( + tabular_columns_parameters("p{1.3pt}") + == "style='vertical-align: top; width: 1.69px;'" + ) def given_m_then_array(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ - assert tabular_columns_parameters( - "m{1.3pt}") == "style='vertical-align: middle; width: 1.69px;'" + assert ( + tabular_columns_parameters("m{1.3pt}") + == "style='vertical-align: middle; width: 1.69px;'" + ) def given_b_then_array(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ - assert tabular_columns_parameters( - "b{1.3pt}") == "style='vertical-align: bottom; width: 1.69px;'" + assert ( + tabular_columns_parameters("b{1.3pt}") + == "style='vertical-align: bottom; width: 1.69px;'" + ) def test_tabular_columns_parameters(): diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_tabular_parameters.py b/Programming/ECOTE/program/unit_tests/test_code/test_tabular_parameters.py index 90a47d86..68d81d04 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_tabular_parameters.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_tabular_parameters.py @@ -1,35 +1,32 @@ -""" - Tests tabular_parameters function -""" +"""Tests tabular_parameters function.""" from translator.main import tabular_parameters + def given_empty_then_empty(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ assert tabular_parameters("") == "" + def given_empty_brackets_then_empty(): - """ - Given: [] + """Given: [] When: N/A - Then: + Then: . """ assert tabular_parameters("[]") == "" + def given_non_empty_then_error(): - """ - Given: [c] + """Given: [c] When: N/A - Then: + Then: . """ assert tabular_parameters("[c]") == "Error!" - def test_tabular_parameters(): given_empty_then_empty() given_empty_brackets_then_empty() diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_tabular_required_parameters.py b/Programming/ECOTE/program/unit_tests/test_code/test_tabular_required_parameters.py index f700737c..17d717a4 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_tabular_required_parameters.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_tabular_required_parameters.py @@ -1,135 +1,120 @@ -""" - Tests tabular_parameters function -""" +"""Tests tabular_parameters function.""" from translator.main import tabular_required_parameters def given_empty_then_error(): - """ - Given: + """Given: When: N/A - Then: + Then: . """ assert tabular_required_parameters("") == "Error!" def given_empty_brackets_then_error(): - """ - Given: {} + """Given: {} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{}") == "Error!" def given_just_line_then_error(): - """ - Given: {} + """Given: {} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{|}") == "Error!" def given_just_lines_then_error(): - """ - Given: {} + """Given: {} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{|||||||}") == "Error!" def given_single_left_then_correct(): - """ - Given: {l} + """Given: {l} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{l}") == ["align='left'"] def given_single_center_then_correct(): - """ - Given: {c} + """Given: {c} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{c}") == ["align='center'"] def given_single_right_then_correct(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{r}") == ["align='right'"] def given_empty_wrap_p_then_error(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{p}") == "Error!" def given_empty_wrap_m_then_error(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{m}") == "Error!" def given_empty_wrap_b_then_error(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{b}") == "Error!" def given_empty_wrap_p_brackets_then_error(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{p{}}") == "Error!" def given_empty_wrap_m_brackets_then_error(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{m{}}") == "Error!" def given_empty_wrap_b_brackets_then_error(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{b{}}") == "Error!" def given_filled_p_brackets_then_correct(): - """ - Given: {r} + """Given: {r} When: N/A - Then: + Then: . """ assert tabular_required_parameters("{p{1.3pt}}") == [ - "style='vertical-align: top; width: 1.69px;'"] + "style='vertical-align: top; width: 1.69px;'" + ] def given_complicated_then_correct(): @@ -140,7 +125,7 @@ def given_complicated_then_correct(): "align='center'", "style='border-left: 1px solid black'", "align='right'", - "style='border-left: 1px solid black'" + "style='border-left: 1px solid black'", ] diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_translate_column.py b/Programming/ECOTE/program/unit_tests/test_code/test_translate_column.py index a7861f19..897f31c0 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_translate_column.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_translate_column.py @@ -14,12 +14,14 @@ def given_just_hline_then_correct(): def given_just_newline_then_correct(): - assert translate_column('\newline') == "
" + assert translate_column("\newline") == "
" def given_all_then_correct(): - assert translate_column( - '\\hline \newline hline newline test') == "

hline newline test" + assert ( + translate_column("\\hline \newline hline newline test") + == "

hline newline test" + ) def test_translate_column(): diff --git a/Programming/ECOTE/program/unit_tests/test_code/test_translate_inside_to_html.py b/Programming/ECOTE/program/unit_tests/test_code/test_translate_inside_to_html.py index 9ac4f2c3..7b98d8f6 100644 --- a/Programming/ECOTE/program/unit_tests/test_code/test_translate_inside_to_html.py +++ b/Programming/ECOTE/program/unit_tests/test_code/test_translate_inside_to_html.py @@ -1,14 +1,14 @@ -from translator.main import translate_inside_to_html -from translator.main import tabular_required_parameters +from translator.main import tabular_required_parameters, translate_inside_to_html def given_correct_then_correct(): latex_string = "test & 2 & test \\ 4 & 5 & 6 \\" parameters_string = "{ l | c | r }" column_styles = tabular_required_parameters(parameters_string) - assert translate_inside_to_html(latex_string, column_styles) == "
test 2 test
4 5 6
" - - + assert ( + translate_inside_to_html(latex_string, column_styles) + == "
test 2 test
4 5 6
" + ) def test_translate_inside_to_html(): diff --git a/Programming/ECRYPT/test1.py b/Programming/ECRYPT/test1.py deleted file mode 100644 index e69de29b..00000000 diff --git a/Programming/ECRYPT_PROJECT/sieve/conftest.py b/Programming/ECRYPT_PROJECT/sieve/conftest.py index b7c34a90..e6d18854 100644 --- a/Programming/ECRYPT_PROJECT/sieve/conftest.py +++ b/Programming/ECRYPT_PROJECT/sieve/conftest.py @@ -8,14 +8,14 @@ def pytest_addoption(parser): parser.addoption( "--primes", action="store", - default='100', - help="Upper bound for primes to test." + default="100", + help="Upper bound for primes to test.", ) parser.addoption( "--composites", action="store", - default='100', - help="Upper bound for composites to test." + default="100", + help="Upper bound for composites to test.", ) @@ -26,16 +26,26 @@ def pytest_configure(config): def pytest_generate_tests(metafunc): df_prime = pd.read_csv("P-100000.csv", header=None) - df_prime = df_prime[df_prime.iloc[:, 1] < pytest.primes] + df_prime = df_prime[df_prime.iloc[:, 1] < pytest.primes] primes = set(df_prime.iloc[:, 1]) df_composite = pd.read_csv("C-100000.csv", header=None, delimiter="=") # interpret first oclumns as numbers df_composite = df_composite[df_composite.iloc[:, 0] < pytest.composites] composites = set(df_composite.iloc[:, 0]) - if metafunc.function.__name__ == 'test_positives': - metafunc.parametrize('prime', primes) - metafunc.parametrize('primes_obtained', [sieve_of_eratosthenes(max(primes) + 1),]) + if metafunc.function.__name__ == "test_positives": + metafunc.parametrize("prime", primes) + metafunc.parametrize( + "primes_obtained", + [ + sieve_of_eratosthenes(max(primes) + 1), + ], + ) - if metafunc.function.__name__ == 'test_composites': - metafunc.parametrize('composite', composites) - metafunc.parametrize('primes_obtained', [sieve_of_eratosthenes(max(composites) + 1),]) + if metafunc.function.__name__ == "test_composites": + metafunc.parametrize("composite", composites) + metafunc.parametrize( + "primes_obtained", + [ + sieve_of_eratosthenes(max(composites) + 1), + ], + ) diff --git a/Programming/ECRYPT_PROJECT/sieve/project.py b/Programming/ECRYPT_PROJECT/sieve/project.py index 9acb0831..ac3ec671 100644 --- a/Programming/ECRYPT_PROJECT/sieve/project.py +++ b/Programming/ECRYPT_PROJECT/sieve/project.py @@ -1,33 +1,36 @@ -'''Python program to print all Primes Smaller than or equal to N using Sieve of Eratosthenes''' +"""Python program to print all Primes Smaller than or equal to N using Sieve of Eratosthenes.""" + import math - + + def sieve_of_eratosthenes(num: int) -> list[int]: """Finds all prime number from 1 to num. - + Args: num (int): Number up to which the prime numbers should be printed - + Returns: list[int]: List of prime numbers + """ if num <= 0: raise ValueError("Number should not be negative.") # boolean list to store if a number is prime or not - prime = [True] * (num+1) - p = 2 # starting prime number - # we will iterate till square root of num because + prime = [True] * (num + 1) + p = 2 # starting prime number + # we will iterate till square root of num because # if a number is not prime then it will have a factor less than or equal to square root of that number while p <= math.sqrt(num): # If prime[p] is not # changed, then it is a prime if prime[p]: - i = (p << 1) # i = p * 2 + i = p << 1 # i = p * 2 # Updating all multiples of p while i <= num: prime[i] = False i += p p += 1 - return [p for p in range(2, num+1) if prime[p]] + return [p for p in range(2, num + 1) if prime[p]] def print_sieve(num: int) -> None: @@ -35,13 +38,14 @@ def print_sieve(num: int) -> None: Args: num (int): Number up to which the prime numbers should be printed + """ for number in sieve_of_eratosthenes(num): print(number, end=" ") - - + + # Driver code -if __name__ == '__main__': +if __name__ == "__main__": num = int(input("Enter a number: ")) print(f"Following are the prime numbers smaller than or equal to {num}") - print_sieve(num) \ No newline at end of file + print_sieve(num) diff --git a/Programming/ECRYPT_PROJECT/sieve/test_main.py b/Programming/ECRYPT_PROJECT/sieve/test_main.py index bcc7488c..25f456b7 100644 --- a/Programming/ECRYPT_PROJECT/sieve/test_main.py +++ b/Programming/ECRYPT_PROJECT/sieve/test_main.py @@ -1,7 +1,8 @@ -import pandas as pd import pytest + from .project import sieve_of_eratosthenes + def test_positives(prime, primes_obtained): assert prime in primes_obtained @@ -12,4 +13,4 @@ def test_composites(composite, primes_obtained): def test_negatives(): with pytest.raises(ValueError): - sieve_of_eratosthenes(-1) \ No newline at end of file + sieve_of_eratosthenes(-1) diff --git a/Programming/ERSMS-project/analytics/analytics.py b/Programming/ERSMS-project/analytics/analytics.py index 56f0b3ce..9dcdd1bb 100644 --- a/Programming/ERSMS-project/analytics/analytics.py +++ b/Programming/ERSMS-project/analytics/analytics.py @@ -1,14 +1,11 @@ -from flask import Flask, request, jsonify -from flask_caching import Cache -import psycopg2 -import pandas -import json from configparser import ConfigParser -from datetime import datetime +import psycopg2 +from flask import Flask, jsonify +from flask_caching import Cache app = Flask(__name__) -cache = Cache(config={'CACHE_TYPE': 'SimpleCache'}) +cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) db_connector = None conn = None @@ -33,13 +30,16 @@ def get_movie_ratings(movie_id): rating_values = [5, 4, 3, 2, 1] for rating in rating_values: - cursor.execute(""" + cursor.execute( + """ SELECT COUNT(*) as count FROM ratings WHERE rating = %s AND movie_ID = %s; - """, (rating, movie_id)) + """, + (rating, movie_id), + ) result = cursor.fetchone() - ratings[f'{rating}_star'] = result[0] + ratings[f"{rating}_star"] = result[0] cursor.close() @@ -62,11 +62,14 @@ def get_number_of_users(): @cache.cached(timeout=50) def get_movie_rating_avg(movie_id): cursor = conn.cursor() - cursor.execute(""" + cursor.execute( + """ SELECT AVG(rating) as avg_rating FROM ratings WHERE movie_ID = %s; - """, (movie_id,)) + """, + (movie_id,), + ) res = cursor.fetchall() cursor.close() @@ -78,11 +81,14 @@ def get_movie_rating_avg(movie_id): @cache.cached(timeout=50) def get_user_ratings(user_id): cursor = conn.cursor() - cursor.execute(""" + cursor.execute( + """ SELECT movie_ID, rating FROM ratings WHERE oauth_ID = %s; - """, (user_id,)) + """, + (user_id,), + ) res = cursor.fetchall() cursor.close() @@ -101,7 +107,7 @@ if __name__ == "__main__": database=config["postgres"]["database"], user=config["postgres"]["user"], password=config["postgres"]["password"], - port=int(config["postgres"]["port"]) + port=int(config["postgres"]["port"]), ) except Exception: diff --git a/Programming/ERSMS-project/backend/app.py b/Programming/ERSMS-project/backend/app.py index cc06df10..d1d7fd5f 100644 --- a/Programming/ERSMS-project/backend/app.py +++ b/Programming/ERSMS-project/backend/app.py @@ -1,24 +1,25 @@ -from flask import Flask, request, jsonify +import os + +import firebase_admin +import requests +from dotenv import load_dotenv +from firebase_admin import auth, credentials +from flask import Flask, jsonify, request from flask_cors import CORS from flask_sqlalchemy import SQLAlchemy -import firebase_admin -from firebase_admin import credentials, auth -from dotenv import load_dotenv -import os -import requests load_dotenv() -TMDB_BEARER_TOKEN = os.getenv('TMDB_BEARER_TOKEN') +TMDB_BEARER_TOKEN = os.getenv("TMDB_BEARER_TOKEN") app = Flask(__name__) -app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///app.sqlite3' -app.config['SQLALCHEY_TRACK_MODIFICATIONS'] = False +app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///app.sqlite3" +app.config["SQLALCHEY_TRACK_MODIFICATIONS"] = False db = SQLAlchemy(app) CORS(app) -cred = credentials.Certificate('movie-recommendation-firebase-adminsdk.json') +cred = credentials.Certificate("movie-recommendation-firebase-adminsdk.json") firebase_admin.initialize_app(cred) @@ -31,7 +32,7 @@ class User(db.Model): class Rating(db.Model): id = db.Column(db.Integer, primary_key=True) - user_id = db.Column(db.Integer, db.ForeignKey('user.uid'), nullable=False) + user_id = db.Column(db.Integer, db.ForeignKey("user.uid"), nullable=False) movie_id = db.Column(db.Integer, nullable=False) value = db.Column(db.Integer, nullable=False) @@ -40,14 +41,14 @@ with app.app_context(): db.create_all() -@app.route('/login', methods=['POST']) +@app.route("/login", methods=["POST"]) def login(): - token = request.json.get('token') + token = request.json.get("token") try: decoded_token = auth.verify_id_token(token) - uid = decoded_token['uid'] + uid = decoded_token["uid"] print(uid) - email = decoded_token.get('email') + email = decoded_token.get("email") user = User.query.filter_by(uid=uid).first() if user is None: @@ -56,36 +57,39 @@ def login(): db.session.add(user) db.session.commit() - return jsonify({'message': 'Login successful!', 'email': email, 'is_admin': user.is_admin}), 200 + return jsonify( + {"message": "Login successful!", "email": email, "is_admin": user.is_admin} + ), 200 except Exception as e: print(e) - return jsonify({'message': 'Login failed'}), 401 + return jsonify({"message": "Login failed"}), 401 -@app.route('/count_user_ratings', methods=['POST']) +@app.route("/count_user_ratings", methods=["POST"]) def count_user_ratings(): - token = request.json.get('token') + token = request.json.get("token") try: decoded_token = auth.verify_id_token(token) - user_id = decoded_token['uid'] + user_id = decoded_token["uid"] user = User.query.filter_by(uid=user_id).first() if user is None: - return jsonify({'message': 'Error'}), 500 + return jsonify({"message": "Error"}), 500 rating_count = Rating.query.filter_by(user_id=user_id).count() - return jsonify({'message': 'Ratings counted!', 'rating_count': rating_count}), 200 + return jsonify( + {"message": "Ratings counted!", "rating_count": rating_count} + ), 200 except Exception as e: print(e) - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 -@app.route('/movie/', methods=['GET']) +@app.route("/movie/", methods=["GET"]) def get_tmdb_data_movie_id(movie_id): - url = f'https://api.themoviedb.org/3/movie/{movie_id}' + url = f"https://api.themoviedb.org/3/movie/{movie_id}" - headers = { - 'Authorization': f'Bearer {TMDB_BEARER_TOKEN}'} + headers = {"Authorization": f"Bearer {TMDB_BEARER_TOKEN}"} try: response = requests.get(url, headers=headers) @@ -93,20 +97,19 @@ def get_tmdb_data_movie_id(movie_id): return jsonify(response.json()), 200 except Exception as e: print(e) - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 -@app.route('/movie', methods=['GET']) +@app.route("/movie", methods=["GET"]) def get_tmdb_data_query(): - query = request.args.get('query') + query = request.args.get("query") if query: - url = f'https://api.themoviedb.org/3/search/movie?query={query}' + url = f"https://api.themoviedb.org/3/search/movie?query={query}" else: - url = 'https://api.themoviedb.org/3/trending/movie/day' + url = "https://api.themoviedb.org/3/trending/movie/day" - headers = { - 'Authorization': f'Bearer {TMDB_BEARER_TOKEN}'} + headers = {"Authorization": f"Bearer {TMDB_BEARER_TOKEN}"} try: response = requests.get(url, headers=headers) @@ -114,88 +117,90 @@ def get_tmdb_data_query(): return jsonify(response.json()), 200 except Exception as e: print(e) - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 -@app.route('/rating', methods=['POST']) +@app.route("/rating", methods=["POST"]) def add_rating(): - token = request.json.get('token') - movie = request.json.get('movie') - value = request.json.get('value') + token = request.json.get("token") + movie = request.json.get("movie") + value = request.json.get("value") try: decoded_token = auth.verify_id_token(token) - user_id = decoded_token['uid'] + user_id = decoded_token["uid"] # user = User.query.filter_by(uid=user_id).first() # if user is None: # return jsonify({'message': 'Error'}), 500 - rating = Rating.query.filter_by( - user_id=user_id, movie_id=movie).first() + rating = Rating.query.filter_by(user_id=user_id, movie_id=movie).first() if rating is None: rating = Rating(user_id=user_id, movie_id=movie, value=value) db.session.add(rating) db.session.commit() - return jsonify({'message': 'Rating added successfully!'}), 201 - else: - rating.value = value + return jsonify({"message": "Rating added successfully!"}), 201 + rating.value = value - db.session.commit() + db.session.commit() - return jsonify({'message': 'Rating updated successfully!'}), 200 + return jsonify({"message": "Rating updated successfully!"}), 200 except Exception as e: print(e) - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 -@app.route('/rating', methods=['DELETE']) +@app.route("/rating", methods=["DELETE"]) def remove_rating(): - token = request.json.get('token') - movie = request.json.get('movie') + token = request.json.get("token") + movie = request.json.get("movie") try: decoded_token = auth.verify_id_token(token) - user_id = decoded_token['uid'] + user_id = decoded_token["uid"] # user = User.query.filter_by(uid=user_id).first() # if user is None: # return jsonify({'message': 'Error'}), 500 - rating = Rating.query.filter_by( - user_id=user_id, movie_id=movie).first() + rating = Rating.query.filter_by(user_id=user_id, movie_id=movie).first() if rating is None: - return jsonify({'message': 'Error'}), 500 + return jsonify({"message": "Error"}), 500 db.session.delete(rating) db.session.commit() - return jsonify({'message': 'Rating removed successfully!'}), 200 + return jsonify({"message": "Rating removed successfully!"}), 200 except Exception as e: print(e) - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 -@app.route('/get_rating', methods=['POST']) +@app.route("/get_rating", methods=["POST"]) def get_rating(): - token = request.json.get('token') - movie = request.json.get('movie') + token = request.json.get("token") + movie = request.json.get("movie") try: decoded_token = auth.verify_id_token(token) - user_id = decoded_token['uid'] + user_id = decoded_token["uid"] # user = User.query.filter_by(uid=user_id).first() # if user is None: # return jsonify({'message': 'Error'}), 500 - rating = Rating.query.filter_by( - user_id=user_id, movie_id=movie).first() + rating = Rating.query.filter_by(user_id=user_id, movie_id=movie).first() if rating is None: - return jsonify({'message': 'Rating not found!'}), 200 - else: - return jsonify({'message': 'Rating found!', 'movie': rating.movie_id, 'value': rating.value}), 200 + return jsonify({"message": "Rating not found!"}), 200 + return jsonify( + { + "message": "Rating found!", + "movie": rating.movie_id, + "value": rating.value, + } + ), 200 except Exception as e: print(e) - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 + app.run(host="0.0.0.0", port=8084) diff --git a/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py b/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py index c4326042..656501a6 100644 --- a/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py +++ b/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py @@ -1,14 +1,15 @@ -from flask import Flask, request, jsonify -from flask_caching import Cache -import psycopg2 -import pandas import json from configparser import ConfigParser from datetime import datetime + +import pandas +import psycopg2 import requests +from flask import Flask, jsonify +from flask_caching import Cache app = Flask(__name__) -cache = Cache(config={'CACHE_TYPE': 'SimpleCache'}) +cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) db_connector = None conn = None movie_list = None @@ -19,7 +20,9 @@ def error_decorator(fun): try: fun(*args, **kwargs) except psycopg2.DatabaseError: - return jsonify({"status": "Something... unexpected has occurred :sweat_smile:"}), 500 + return jsonify( + {"status": "Something... unexpected has occurred :sweat_smile:"} + ), 500 return inner1 @@ -34,7 +37,7 @@ def hello(): @app.route("/api/v3/get/", methods=["GET"]) def access_user(username): cursor = conn.cursor() - cursor.execute("select * from users where username='{}';".format(username)) + cursor.execute(f"select * from users where username='{username}';") res = cursor.fetchall() cursor.close() @@ -47,16 +50,16 @@ def access_user(username): @app.route("/api/v3/add//", methods=["POST"]) def add_user(oauth_ID, username): cursor = conn.cursor() - cursor.execute("select * from users where username='{}';".format(username)) + cursor.execute(f"select * from users where username='{username}';") res = cursor.fetchall() if len(res): cursor.close() return jsonify({"status": "User already exists"}), 409 - cursor.execute("INSERT INTO users (username, oauth_ID) VALUES ('{}','{}');".format( - username, oauth_ID - )) + cursor.execute( + f"INSERT INTO users (username, oauth_ID) VALUES ('{username}','{oauth_ID}');" + ) conn.commit() cursor.close() @@ -71,50 +74,54 @@ def get_recommendations(oauth_ID): cursor.execute("select movie_ID from ratings where oauth_ID='{}'", oauth_ID) res = cursor.fetchall() movies = [int(i) for i in res[0]] - url = 'http://localhost:8081/api/v3/AI_recommendations' - response = requests.post(url, - json=movies, - headers={'Content-Type': 'application/json'}) + url = "http://localhost:8081/api/v3/AI_recommendations" + response = requests.post( + url, json=movies, headers={"Content-Type": "application/json"} + ) return jsonify(response.json()), 200 @app.route("/api/v3/get_movie/", methods=["GET"]) def get_movie(movie_ID): - movie_info = movie_list.loc[movie_list['movie_id'] == movie_ID] + movie_info = movie_list.loc[movie_list["movie_id"] == movie_ID] if movie_info.empty: - return jsonify({"status": "Movie with ID {} doesn't exist".format(movie_ID)} - ), 404 + return jsonify({"status": f"Movie with ID {movie_ID} doesn't exist"}), 404 cast = json.loads(movie_info["cast"][0].replace('\\"', '"')) crew = json.loads(movie_info["crew"][0].replace('\\"', '"')) - output_json = {"movie_id": movie_ID, - "title": movie_info["title"][0], - "cast": cast, - "crew": crew} + output_json = { + "movie_id": movie_ID, + "title": movie_info["title"][0], + "cast": cast, + "crew": crew, + } return jsonify(output_json), 200 -@app.route("/api/v3/rate_movie///", methods=["POST"]) +@app.route( + "/api/v3/rate_movie///", methods=["POST"] +) def rate_movie(uID, movie_ID, rating): - movie_info = movie_list.loc[movie_list['movie_id'] == int(movie_ID)] + movie_info = movie_list.loc[movie_list["movie_id"] == int(movie_ID)] if movie_info.empty: - return jsonify({"status": "Movie with ID {} doesn't exist".format(movie_ID)} - ), 404 + return jsonify({"status": f"Movie with ID {movie_ID} doesn't exist"}), 404 if rating < 1 or rating > 5: return jsonify({"status": "Incorrect rating"}), 400 cursor = conn.cursor() - cursor.execute("select * from users where oauth_ID='{}';".format(uID)) + cursor.execute(f"select * from users where oauth_ID='{uID}';") res = cursor.fetchall() if not len(res): cursor.close() return jsonify({"status": "User doesn't exists"}), 404 - cursor.execute("select * from ratings where oauth_ID='{}' AND movie_ID='{}';".format(uID, movie_ID)) + cursor.execute( + f"select * from ratings where oauth_ID='{uID}' AND movie_ID='{movie_ID}';" + ) res = cursor.fetchall() if len(res): @@ -126,9 +133,9 @@ def rate_movie(uID, movie_ID, rating): """ cursor.execute(sql.format(rating, uID, movie_ID)) else: - cursor.execute("INSERT INTO ratings (movie_ID, oauth_ID, rating) VALUES ('{}','{}',{});".format( - movie_ID, uID, rating - )) + cursor.execute( + f"INSERT INTO ratings (movie_ID, oauth_ID, rating) VALUES ('{movie_ID}','{uID}',{rating});" + ) conn.commit() cursor.close() @@ -147,7 +154,7 @@ if __name__ == "__main__": database=config["postgres"]["database"], user=config["postgres"]["user"], password=config["postgres"]["password"], - port=int(config["postgres"]["port"]) + port=int(config["postgres"]["port"]), ) except Exception: @@ -161,4 +168,3 @@ if __name__ == "__main__": app.run(host="localhost", port=8090, debug=True) conn.close() - diff --git a/Programming/ERSMS-project/movie_recommendations/movie_recommender.py b/Programming/ERSMS-project/movie_recommendations/movie_recommender.py index 4c24fa7f..bb49693c 100644 --- a/Programming/ERSMS-project/movie_recommendations/movie_recommender.py +++ b/Programming/ERSMS-project/movie_recommendations/movie_recommender.py @@ -1,32 +1,32 @@ -import pandas as pd -import numpy as np -from ast import literal_eval -from sklearn.feature_extraction.text import CountVectorizer -from sklearn.metrics.pairwise import cosine_similarity import hashlib import json +from ast import literal_eval from configparser import ConfigParser -import psycopg2 -from flask import Flask, request, jsonify -from flask_caching import Cache +import numpy as np +import pandas as pd +import psycopg2 +from flask import Flask, jsonify, request +from flask_caching import Cache +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.metrics.pairwise import cosine_similarity app = Flask(__name__) -cache = Cache(config={'CACHE_TYPE': 'SimpleCache'}) +cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) db_connector = None conn = None def get_director(x): for i in x: - if i['job'] == 'Director': - return i['name'] + if i["job"] == "Director": + return i["name"] return np.nan def get_list(x): if isinstance(x, list): - names = [i['name'] for i in x] + names = [i["name"] for i in x] if len(names) > 3: names = names[:3] return names @@ -36,15 +36,21 @@ def get_list(x): def clean_data(x): if isinstance(x, list): return [str.lower(i.replace(" ", "")) for i in x] - else: - if isinstance(x, str): - return str.lower(x.replace(" ", "")) - else: - return '' + if isinstance(x, str): + return str.lower(x.replace(" ", "")) + return "" def create_soup(x): - return ' '.join(x['keywords']) + ' ' + ' '.join(x['cast']) + ' ' + x['director'] + ' ' + ' '.join(x['genres']) + return ( + " ".join(x["keywords"]) + + " " + + " ".join(x["cast"]) + + " " + + x["director"] + + " " + + " ".join(x["genres"]) + ) class MovieRecommender: @@ -53,61 +59,58 @@ class MovieRecommender: self.cosine_sim = None def fit(self, credits_file, movies_file): - """ - Fittuje AI do przekazanych danych + """Fittuje AI do przekazanych danych :param credits_file: csv z creditsami :param movies_file: csv z filmami - :return: Nic + :return: Nic. """ df1 = pd.read_csv(credits_file) df2 = pd.read_csv(movies_file) - df1.columns = ['id', 'tittle', 'cast', 'crew'] - df2 = df2.merge(df1, on='id') - df2['overview'] = df2['overview'].fillna('') + df1.columns = ["id", "tittle", "cast", "crew"] + df2 = df2.merge(df1, on="id") + df2["overview"] = df2["overview"].fillna("") self.df = df2 - features = ['cast', 'crew', 'keywords', 'genres'] + features = ["cast", "crew", "keywords", "genres"] for feature in features: df2[feature] = df2[feature].apply(literal_eval) - df2['director'] = df2['crew'].apply(get_director) + df2["director"] = df2["crew"].apply(get_director) - features = ['cast', 'keywords', 'genres'] + features = ["cast", "keywords", "genres"] for feature in features: df2[feature] = df2[feature].apply(get_list) - features = ['cast', 'keywords', 'director', 'genres'] + features = ["cast", "keywords", "director", "genres"] for feature in features: df2[feature] = df2[feature].apply(clean_data) - df2['soup'] = df2.apply(create_soup, axis=1) + df2["soup"] = df2.apply(create_soup, axis=1) - count = CountVectorizer(stop_words='english') - count_matrix = count.fit_transform(df2['soup']) + count = CountVectorizer(stop_words="english") + count_matrix = count.fit_transform(df2["soup"]) self.cosine_sim = cosine_similarity(count_matrix, count_matrix) self.df = df2.reset_index() def _get_recommendations_one_input(self, movie_id): - """ - Tworzy rekomendacje, bazując na jednym filmie + """Tworzy rekomendacje, bazując na jednym filmie :param movie_id: id filmu, dla którego ma zrobić rekomendację - :return: Zwraca listę [movie_ids, similarity_scores] gdzie oba argumenty są np.array + :return: Zwraca listę [movie_ids, similarity_scores] gdzie oba argumenty są np.array. """ - indices = pd.Series(self.df.index, index=self.df['id']).drop_duplicates() + indices = pd.Series(self.df.index, index=self.df["id"]).drop_duplicates() idx = indices[movie_id] sim_scores = list(enumerate(self.cosine_sim[idx])) sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True) sim_scores = sim_scores[1:101] movie_indices = [i[0] for i in sim_scores] sim_scores = np.array([t[1] for t in sim_scores]) - return [self.df['id'].iloc[movie_indices].values, sim_scores] + return [self.df["id"].iloc[movie_indices].values, sim_scores] def get_recommendations(self, movie_ids: list) -> {}: - """ - Tworzy listę rekomendacji bazującą na id podanych filmów + """Tworzy listę rekomendacji bazującą na id podanych filmów :param movie_ids: id filmów, na podstawie których ma wybrać rekomendowane filmy - :return: Zwraca dicta {movie_id: similarity_scores} + :return: Zwraca dicta {movie_id: similarity_scores}. """ recommended_movies = {} for movie_id in movie_ids: @@ -117,22 +120,25 @@ class MovieRecommender: continue if recommended_movies.get(int(recommended_id)) is None: - recommended_movies[int(recommended_id)] = float(round((sim_score / len(movie_ids)), 4)) + recommended_movies[int(recommended_id)] = float( + round((sim_score / len(movie_ids)), 4) + ) else: - recommended_movies[int(recommended_id)] += float(round((sim_score / len(movie_ids)), 4)) + recommended_movies[int(recommended_id)] += float( + round((sim_score / len(movie_ids)), 4) + ) return recommended_movies recommender = MovieRecommender() -recommender.fit('datasets/tmdb_5000_credits.csv', - 'datasets/tmdb_5000_movies.csv') +recommender.fit("datasets/tmdb_5000_credits.csv", "datasets/tmdb_5000_movies.csv") def make_cache_key(): data = request.get_json() if isinstance(data, list): data = sorted(data) - key = hashlib.md5(json.dumps(data).encode('utf-8')).hexdigest() + key = hashlib.md5(json.dumps(data).encode("utf-8")).hexdigest() return key @@ -155,7 +161,7 @@ if __name__ == "__main__": database=config["postgres"]["database"], user=config["postgres"]["user"], password=config["postgres"]["password"], - port=int(config["postgres"]["port"]) + port=int(config["postgres"]["port"]), ) except Exception: diff --git a/Programming/NLP/compare_chunks.py b/Programming/NLP/compare_chunks.py index 90a87c6c..90768264 100644 --- a/Programming/NLP/compare_chunks.py +++ b/Programming/NLP/compare_chunks.py @@ -1,58 +1,65 @@ - - def reformat_sentence(sentence): - """ - Reformats a sentence by replacing slashes '/' with ' ]'. - """ - return sentence.replace('/', ' ]') + """Reformats a sentence by replacing slashes '/' with ' ]'.""" + return sentence.replace("/", " ]") + def insert_commas(sentence, reference_sentence): - """ - Inserts commas into 'sentence' at the same positions as they appear in 'reference_sentence'. - """ + """Inserts commas into 'sentence' at the same positions as they appear in 'reference_sentence'.""" # Splitting the sentences into chunks - sentence_chunks = sentence.split(' ] [ ') - reference_chunks = reference_sentence.split(' ] [ ') + sentence_chunks = sentence.split(" ] [ ") + reference_chunks = reference_sentence.split(" ] [ ") # Insert commas into the original sentence based on the reference sentence for i, chunk in enumerate(reference_chunks): - if ',' in chunk and i < len(sentence_chunks): - sentence_chunks[i] += ',' + if "," in chunk and i < len(sentence_chunks): + sentence_chunks[i] += "," # Reconstruct the sentence with inserted commas - return ' ] [ '.join(sentence_chunks) + return " ] [ ".join(sentence_chunks) def process_sentences(sentences1, sentences2): - """ - Processes two lists of sentences according to the specified rules. - """ + """Processes two lists of sentences according to the specified rules.""" processed_sentences = [] for sentence1, sentence2 in zip(sentences1, sentences2): - # Reformat the first sentence + # Reformat the first sentence reformatted_sentence1 = reformat_sentence(sentence1) reformatted_sentence1 = insert_commas(reformatted_sentence1, sentence2) # Splitting the sentences into words, ignoring the square brackets - words1 = set(' '.join(reformatted_sentence1.strip().split('] [')).replace('[', '').replace(']', '').lower().split()) - words2 = set(' '.join(sentence2.strip().split('] [')).replace('[', '').replace(']', '').lower().split()) + words1 = set( + " ".join(reformatted_sentence1.strip().split("] [")) + .replace("[", "") + .replace("]", "") + .lower() + .split() + ) + words2 = set( + " ".join(sentence2.strip().split("] [")) + .replace("[", "") + .replace("]", "") + .lower() + .split() + ) # Finding differences in words diff = words2 - words1 # Preparing the output sentence output_sentence = reformatted_sentence1.strip() - if '[ . ]' in sentence2: - output_sentence += ' [ . ]' + if "[ . ]" in sentence2: + output_sentence += " [ . ]" # Adding the differences and the output sentence to the processed_sentences list - processed_sentences.append({ - "sentence1": reformatted_sentence1.strip(), - "sentence2": sentence2.strip(), - "differences": diff, - "merged_sentence": output_sentence - }) - if diff and diff != {'.'}: + processed_sentences.append( + { + "sentence1": reformatted_sentence1.strip(), + "sentence2": sentence2.strip(), + "differences": diff, + "merged_sentence": output_sentence, + } + ) + if diff and diff != {"."}: print("Difference found!") print("Sentence 1:", reformatted_sentence1.strip()) print("Sentence 2:", sentence2.strip()) @@ -61,14 +68,15 @@ def process_sentences(sentences1, sentences2): return processed_sentences + # First, let's open and read the contents of both files to understand their structure and content. -file_path_1 = 'test_goldStandard/images/images-chunks-gpt-one.txt' -file_path_2 = 'test_goldStandard/images/STSint.testinput.images.sent1.chunk.txt' +file_path_1 = "test_goldStandard/images/images-chunks-gpt-one.txt" +file_path_2 = "test_goldStandard/images/STSint.testinput.images.sent1.chunk.txt" # Reading the files -with open(file_path_1, 'r') as file1, open(file_path_2, 'r') as file2: +with open(file_path_1) as file1, open(file_path_2) as file2: sentences_file_1 = file1.readlines() sentences_file_2 = file2.readlines() # Process the sentences -processed_sentences = process_sentences(sentences_file_1, sentences_file_2) \ No newline at end of file +processed_sentences = process_sentences(sentences_file_1, sentences_file_2) diff --git a/Programming/NLP/create_alignments.py b/Programming/NLP/create_alignments.py index aef87960..5d448f45 100644 --- a/Programming/NLP/create_alignments.py +++ b/Programming/NLP/create_alignments.py @@ -1,14 +1,14 @@ -import processing -import pandas as pd import gpt_alignment +import pandas as pd +import processing # Specify the output file path file_path = "alignments_unformatted_headlines.txt" # paths to students answers database -#chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" -#chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" -#alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" +# chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" +# chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" +# alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" # paths to headlines chunked_path1 = "test_goldStandard/headlines/STSint.testinput.headlines.sent1.chunk.txt" @@ -16,22 +16,23 @@ chunked_path2 = "test_goldStandard/headlines/STSint.testinput.headlines.sent1.ch alignment_path = "test_goldStandard/headlines/STSint.testinput.headlines.wa" # load data -#studentAnserws = processing.load_sentences(studentAnswers1_path, studentAnswers1_path) +# studentAnserws = processing.load_sentences(studentAnswers1_path, studentAnswers1_path) goldstandard_chunked = processing.load_chunked(chunked_path1, chunked_path2) goldstandard_alignment = processing.load_alignment(alignment_path) # get a nice anwser-student table -data = pd.merge(goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True) -#print(data) +data = pd.merge( + goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True +) +# print(data) data_for_chat, indexes = processing.get_chunks_as_text(data) client = gpt_alignment.createGPT() responses = [] -for i in range(0, len(data_for_chat)): +for i in range(len(data_for_chat)): responses.append(gpt_alignment.callApi(client, data_for_chat[i])) # Writing to the file with repr() to preserve "\n" characters -with open(file_path, 'w') as file: - for string in responses: - file.write(repr(string) + '\n') \ No newline at end of file +with open(file_path, "w") as file: + file.writelines(repr(string) + "\n" for string in responses) diff --git a/Programming/NLP/create_alignments_with_examples.py b/Programming/NLP/create_alignments_with_examples.py index e7dab70f..314115b2 100644 --- a/Programming/NLP/create_alignments_with_examples.py +++ b/Programming/NLP/create_alignments_with_examples.py @@ -1,14 +1,14 @@ -import processing -import pandas as pd import gpt_alignment +import pandas as pd +import processing # Specify the output file path file_path = "alignments_with_training_headlines2.wa" # paths to students andsewrs database -#chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" -#chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" -#alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" +# chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" +# chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" +# alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" # paths to headlines chunked_path1 = "test_goldStandard/headlines/STSint.testinput.headlines.sent1.chunk.txt" @@ -20,27 +20,31 @@ goldstandard_chunked = processing.load_chunked(chunked_path1, chunked_path2) goldstandard_alignment = processing.load_alignment(alignment_path) # get a nice anwser-student table -data = pd.merge(goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True) -#print(data) +data = pd.merge( + goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True +) +# print(data) train, test = processing.generate_train_test_split(data) data_for_chat, indexes = processing.get_chunks_as_text(test) _, indexes_of_training = processing.get_chunks_as_text(train) -indexes_of_training = [i+1 for i in indexes_of_training] -indexes = [i+1 for i in indexes] +indexes_of_training = [i + 1 for i in indexes_of_training] +indexes = [i + 1 for i in indexes] print(indexes_of_training) print(indexes) client = gpt_alignment.createGPT() responses = [] -for i in range(0, 10):#len(data_for_chat)): - responses.append([gpt_alignment.callApi_examples(client, train, data_for_chat[i]), indexes[i]]) +for i in range(10): # len(data_for_chat)): + responses.append( + [gpt_alignment.callApi_examples(client, train, data_for_chat[i]), indexes[i]] + ) -with open(file_path, 'w') as file: +with open(file_path, "w") as file: for i, r in enumerate(responses): - file.write("\n") + file.write('\n') file.write("\n") file.write(r[0]) file.write("\n\n") - file.write("\n\n") \ No newline at end of file + file.write("\n\n") diff --git a/Programming/NLP/format_alignments.py b/Programming/NLP/format_alignments.py index 9a3841d2..bf36d0ad 100644 --- a/Programming/NLP/format_alignments.py +++ b/Programming/NLP/format_alignments.py @@ -1,9 +1,10 @@ +import copy +import re + import pandas as pd import processing -import re -import copy -# input file name +# input file name file_path = "alignments_unformatted_headlines.txt" # output file path @@ -15,20 +16,22 @@ chunked_path2 = "test_goldStandard/headlines/STSint.testinput.headlines.sent1.ch alignment_path = "test_goldStandard/headlines/STSint.testinput.headlines.wa" # paths to students andsewrs database -#chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" -#chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" -#alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" +# chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" +# chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" +# alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" # load data goldstandard_chunked = processing.load_chunked(chunked_path1, chunked_path2) goldstandard_alignment = processing.load_alignment(alignment_path) # get a nice table -data = pd.merge(goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True) +data = pd.merge( + goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True +) # open generated alignments -with open(file_path, 'r') as file: - responses = [eval(line.strip()) for line in file.readlines()] +with open(file_path) as file: + responses = [eval(line.strip()) for line in file] for i, r in enumerate(responses): print("\nresponse number " + str(i)) @@ -40,12 +43,12 @@ for i, response in enumerate(responses): temp = response.lstrip("\n") temp = temp.rstrip("\n") - temp = re.sub(r'(?<==)>', '> ', temp) # add space after > - temp = re.sub(r'(?", "> ", temp) # add space after > + temp = re.sub(r"(? ", " <==> ") temp = temp.replace(" => ", " <==> ") temp = temp.replace(" <=> ", " <==> ") @@ -56,7 +59,7 @@ for i, response in enumerate(responses): temp = temp.replace("\n// NOALI", "\n0 <==> 0 // NOALI") temp = temp.replace("\n // ", "\n0 <==> ") temp = temp.replace("\n// ", "\n0 <==> ") - temp = re.sub(r'(^|[^<])(==>+)', r' <==>', temp) + temp = re.sub(r"(^|[^<])(==>+)", r" <==>", temp) temp = temp.replace("// - //", "// 0 //") temp = temp.replace("// score", "// ") temp = temp.replace("// alignment type", "// NOALI") @@ -64,15 +67,15 @@ for i, response in enumerate(responses): temp = temp.replace("- <", "0 <") temp = temp.replace("// //", "// NOALI //") temp = temp.replace("equi", "EQUI") - temp = re.sub(r'\d\. ', '', temp) # remove 1., 2. ... + temp = re.sub(r"\d\. ", "", temp) # remove 1., 2. ... temp = temp.upper() - - temp = re.sub(r'^(<==>)', r'0 \1', temp) - temp = re.sub(r'(?)", r"0 \1", temp) + temp = re.sub(r"(?" not in temp[k]: + if "<==>" not in t: temp[k] = "0 <==> " + temp[k] temp[k] = temp[k].split("<==>") temp[k] = [temp[k][0], *temp[k][1].split("//")] @@ -81,11 +84,11 @@ for i, response in enumerate(responses): q = 1 numberList = [] for chunk in chunk1arr: - chunk = re.sub(r'(?= 4: temp[k][3] = temp[k][3].replace("NOALI", "0") @@ -117,12 +122,20 @@ for i, response in enumerate(responses): temp[k][3] = " 0 " if temp[k][3] == " ": temp[k][3] = " 0 " - temp[k] = temp[k][0] + " <==> " + temp[k][1] + " // " + temp[k][2] + " // " + temp[k][3] + temp[k] = ( + temp[k][0] + + " <==> " + + temp[k][1] + + " // " + + temp[k][2] + + " // " + + temp[k][3] + ) elif len(temp[k]) == 3: - temp[k] = temp[k][0] + " <==> " + temp[k][1] + " // " + temp[k][2] + " // 0" + temp[k] = temp[k][0] + " <==> " + temp[k][1] + " // " + temp[k][2] + " // 0" + + temp[k] = re.sub(r"\s{2,}", " ", temp[k]) # remove double space - temp[k] = re.sub(r'\s{2,}', ' ', temp[k]) # remove double space - temp = [x for x in temp if x != ""] responses[i] = "\n".join(temp).strip() @@ -130,22 +143,22 @@ indexes = [] responses_final = [] rejected_indexes = [] for n, r in enumerate(responses): - if r == '': - rejected_indexes.append(n+1) + if r == "": + rejected_indexes.append(n + 1) continue - if r == '\n': - rejected_indexes.append(n+1) + if r == "\n": + rejected_indexes.append(n + 1) continue - indexes.append(n+1) + indexes.append(n + 1) responses_final.append(r) print("rejected indexes:") print(rejected_indexes) -with open(output_file_path, 'w') as file: +with open(output_file_path, "w") as file: for i, r in zip(indexes, responses_final): - file.write("\n") + file.write('\n') file.write("\n") file.write(r) file.write("\n\n") - file.write("\n\n") \ No newline at end of file + file.write("\n\n") diff --git a/Programming/NLP/format_alignments_with_training.py b/Programming/NLP/format_alignments_with_training.py index b27b2b28..e8efb893 100644 --- a/Programming/NLP/format_alignments_with_training.py +++ b/Programming/NLP/format_alignments_with_training.py @@ -1,22 +1,37 @@ +import copy +import re + import pandas as pd import processing -import re -import copy # paths to students andsewrs database -studentAnswers1_path = "test_goldStandard/student/STSint.testinput.answers-students.sent1.txt" -studentAnswers2_path = "test_goldStandard/student/STSint.testinput.answers-students.sent2.txt" -studentAnsewrs_chunked_path1 = "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" -studentAnsewrs_chunked_path2 = "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" -studentsAnsewrs_alignment_path = "test_goldStandard/student/STSint.testinput.answers-students.wa" +studentAnswers1_path = ( + "test_goldStandard/student/STSint.testinput.answers-students.sent1.txt" +) +studentAnswers2_path = ( + "test_goldStandard/student/STSint.testinput.answers-students.sent2.txt" +) +studentAnsewrs_chunked_path1 = ( + "test_goldStandard/student/STSint.testinput.answers-students.sent1.chunk.txt" +) +studentAnsewrs_chunked_path2 = ( + "test_goldStandard/student/STSint.testinput.answers-students.sent2.chunk.txt" +) +studentsAnsewrs_alignment_path = ( + "test_goldStandard/student/STSint.testinput.answers-students.wa" +) # load data studentAnserws = processing.load_sentences(studentAnswers1_path, studentAnswers1_path) -goldstandard_chunked = processing.load_chunked(studentAnsewrs_chunked_path1, studentAnsewrs_chunked_path2) +goldstandard_chunked = processing.load_chunked( + studentAnsewrs_chunked_path1, studentAnsewrs_chunked_path2 +) goldstandard_alignment = processing.load_alignment(studentsAnsewrs_alignment_path) # get a nice anwser-student table -data = pd.merge(goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True) +data = pd.merge( + goldstandard_chunked, goldstandard_alignment, left_index=True, right_index=True +) file_path = "alignments_with_training_student_K.wa" @@ -35,8 +50,8 @@ for i, response in enumerate(responses): temp = response.lstrip("\n") temp = temp.rstrip("\n") - temp = re.sub(r'(?<==)>', '> ', temp) # add space after > - temp = re.sub(r'(?", "> ", temp) # add space after > + temp = re.sub(r"(?", "0 <==>") temp = temp.replace("<==> NOALI", "<==> 0") temp = temp.replace("\n// NOALI", "\n0 <==> 0 // NOALI") - temp = re.sub(r'(^|[^<])(==>+)', r' <==>', temp) - + temp = re.sub(r"(^|[^<])(==>+)", r" <==>", temp) - temp = re.sub(r'^(<==>)', r'0 \1', temp) + temp = re.sub(r"^(<==>)", r"0 \1", temp) - temp = re.sub(r'(?") temp[k][1] = temp[k][1].split("//") - #for j, chunk in enumerate(data.iloc[i]["chunked_sentance1"].sorted(key=len, reverse=True)): - for j, chunk in enumerate(sorted(data.iloc[i]["chunked_sentance1"], key=lambda x: len(x), reverse=True)): + # for j, chunk in enumerate(data.iloc[i]["chunked_sentance1"].sorted(key=len, reverse=True)): + for j, chunk in enumerate( + sorted( + data.iloc[i]["chunked_sentance1"], key=lambda x: len(x), reverse=True + ) + ): pattern = re.compile(chunk, re.IGNORECASE) - temp[k][0] = pattern.sub(str(j+1), temp[k][0]) - #temp[k][0] = temp[k][0].replace(chunk, str(j+1)) - for j, chunk in enumerate(sorted(data.iloc[i]["chunked_sentance2"], key=lambda x: len(x), reverse=True)): + temp[k][0] = pattern.sub(str(j + 1), temp[k][0]) + # temp[k][0] = temp[k][0].replace(chunk, str(j+1)) + for j, chunk in enumerate( + sorted( + data.iloc[i]["chunked_sentance2"], key=lambda x: len(x), reverse=True + ) + ): pattern = re.compile(chunk, re.IGNORECASE) - temp[k][1][0] = pattern.sub(str(j+1), temp[k][1][0]) - #temp[k][1][0] = temp[k][1][0].replace(chunk, str(j+1)) + temp[k][1][0] = pattern.sub(str(j + 1), temp[k][1][0]) + # temp[k][1][0] = temp[k][1][0].replace(chunk, str(j+1)) if len(temp[k][1]) >= 3: - temp[k] = temp[k][0] + " <==> " + temp[k][1][0] + " // " + temp[k][1][1] + " // " + temp[k][1][2] + temp[k] = ( + temp[k][0] + + " <==> " + + temp[k][1][0] + + " // " + + temp[k][1][1] + + " // " + + temp[k][1][2] + ) elif len(temp[k][1]) == 2: - temp[k] = temp[k][0] + " <==> " + temp[k][1][0] + " // " + temp[k][1][1] + " // 0" - temp[k] = re.sub(r'\s{2,}', ' ', temp[k]) # remove double space - + temp[k] = ( + temp[k][0] + " <==> " + temp[k][1][0] + " // " + temp[k][1][1] + " // 0" + ) + temp[k] = re.sub(r"\s{2,}", " ", temp[k]) # remove double space + responses[i] = "\n".join(temp) @@ -88,14 +120,13 @@ for i, r in enumerate(responses): print(unformatted[i]) - -# write to file +# write to file file_path = "student_fixed_format_with_training.txt" -with open(file_path, 'w') as file: +with open(file_path, "w") as file: for i, r in enumerate(responses): - file.write("\n") + file.write('\n') file.write("\n") file.write(r) file.write("\n\n") - file.write("\n\n") \ No newline at end of file + file.write("\n\n") diff --git a/Programming/NLP/gpt_alignment.py b/Programming/NLP/gpt_alignment.py index e845d4cc..8a367b87 100644 --- a/Programming/NLP/gpt_alignment.py +++ b/Programming/NLP/gpt_alignment.py @@ -1,9 +1,7 @@ -""" -Get chunk alignment from chatGPT -""" -from openai import OpenAI +"""Get chunk alignment from chatGPT.""" + import pandas as pd -import processing +from openai import OpenAI thePrompt = """You are a machine designed to align chunks from 2 sentences. This means you will be taking each chunk from one sentence and comparing it to every chunk from the other sentence. Choose the chunks with the strongest relation and assign them a score. @@ -54,28 +52,28 @@ def createGPT() -> OpenAI: return client -def callApi(client:OpenAI, chunks:str): +def callApi(client: OpenAI, chunks: str): response = client.chat.completions.create( - model="gpt-3.5-turbo-1106", - messages=[ + model="gpt-3.5-turbo-1106", + messages=[ {"role": "user", "content": thePrompt}, - {"role": "user", "content": chunks} - ] + {"role": "user", "content": chunks}, + ], ) return response.choices[0].message.content.strip() -def callApi_examples(client:OpenAI, examples:pd.DataFrame, alignment:str): +def callApi_examples(client: OpenAI, examples: pd.DataFrame, alignment: str): user_input = [] assistant_output = [] for index, row in examples.iterrows(): chunks = "" for chunk in row["chunked_sentance1"]: - chunks = chunks + "[ " + chunk + " ] " + chunks = chunks + "[ " + chunk + " ] " chunks = chunks + "\n" for chunk in row["chunked_sentance2"]: - chunks = chunks + "[ " + chunk + " ] " + chunks = chunks + "[ " + chunk + " ] " user_input.append(chunks) assistant_output.append(row["alignment_text"]) @@ -84,11 +82,10 @@ def callApi_examples(client:OpenAI, examples:pd.DataFrame, alignment:str): for u, a in zip(user_input, assistant_output): messages.append({"role": "user", "content": u}) messages.append({"role": "assistant", "content": a}) - + messages.append({"role": "user", "content": alignment}) response = client.chat.completions.create( - model="gpt-3.5-turbo-1106", - messages=messages + model="gpt-3.5-turbo-1106", messages=messages ) return response.choices[0].message.content.strip() diff --git a/Programming/NLP/gpt_chunks.py b/Programming/NLP/gpt_chunks.py index b0fec3a1..38cb88c8 100644 --- a/Programming/NLP/gpt_chunks.py +++ b/Programming/NLP/gpt_chunks.py @@ -1,25 +1,27 @@ -""" -Orders gpt to chunkify files from task -""" -import re +"""Orders gpt to chunkify files from task.""" + import os +import re + from openai import OpenAI def sentence_loop(sentences): - """ - Processes a list of sentences, applying chunking and formatting to each. + """Processes a list of sentences, applying chunking and formatting to each. This function iterates through a list of sentences, processing each one using the `process_sentence` function. It collects the results into a list. If processing fails for any sentence, the function halts and returns None. - Parameters: + Parameters + ---------- sentences (list of str): A list of sentences to be processed. - Returns: + Returns + ------- list of str: A list of processed and formatted sentences, or None if processing of any sentence fails. + """ # Process each sentence chunked_sentences = [] @@ -31,26 +33,27 @@ def sentence_loop(sentences): def remove_leading_numbering(input_string): - """ - Removes any leading numbering from the string. + """Removes any leading numbering from the string. This function is designed to strip off numbering that typically appears at the beginning of a string, such as '1. ', '2. ', etc. It's useful for processing strings where such numbering is irrelevant or should be omitted for formatting purposes. - Parameters: + Parameters + ---------- string (str): The string from which leading numbering should be removed. - Returns: + Returns + ------- str: The string with leading numbering removed. + """ return re.sub(r"^\d+\.\s*", "", input_string) def process_brackets(input_string): - """ - Processes a string to extract and adjust parts enclosed in square brackets. + """Processes a string to extract and adjust parts enclosed in square brackets. This function identifies and extracts all segments within square brackets from the input string. It also adjusts the last part by handling any trailing punctuation, @@ -58,11 +61,14 @@ def process_brackets(input_string): This is particularly useful for strings that contain multiple bracketed segments that need individual processing. - Parameters: + Parameters + ---------- input_string (str): The string to be processed for bracketed segments. - Returns: + Returns + ------- list of str: A list containing the extracted and adjusted bracketed segments from the string. + """ parts = re.findall(r"\[[^\]]*\]", input_string) if parts: @@ -71,20 +77,22 @@ def process_brackets(input_string): def handle_trailing_punctuation(input_string, last_part): - """ - Handles and appends trailing punctuation to the last part of a bracketed segment. + """Handles and appends trailing punctuation to the last part of a bracketed segment. This function checks the given string for any punctuation (like '.', '!', or '?') immediately following the last bracketed segment. If found, it appends this punctuation to the end of the last part inside the brackets. This is used to maintain sentence punctuation integrity after processing bracketed segments. - Parameters: + Parameters + ---------- input_string (str): The original string containing the bracketed segments. last_part (str): The last bracketed segment extracted from the string. - Returns: + Returns + ------- str: The last part with any trailing punctuation correctly appended. + """ trailing_punctuation = re.search(r"\]\s*([.!?])\s*$", input_string) if trailing_punctuation: @@ -93,32 +101,36 @@ def handle_trailing_punctuation(input_string, last_part): def process_non_bracketed_parts(input_string): - """ - Split the input string by hyphens surrounded by optional whitespace and return + """Split the input string by hyphens surrounded by optional whitespace and return a list of non-empty parts enclosed in square brackets. - Parameters: + Parameters + ---------- input_string (str): The input string to be processed. - Returns: + Returns + ------- list of str: A list of non-empty parts from the input string, enclosed in square brackets. + """ parts = re.split(r"\s*-\s*", input_string) return [f"[ {part.strip()} ]" for part in parts if part.strip()] def reformat_list(strings): - """ - Reformat a list of strings by processing each string element. For strings containing + """Reformat a list of strings by processing each string element. For strings containing square brackets, it calls 'process_brackets' to handle the content within the brackets, and for other strings, it calls 'process_non_bracketed_parts' to handle them. The resulting parts are joined into a single string with spaces. - Parameters: + Parameters + ---------- strings (list of str): A list of strings to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of input strings. + """ result = [] for string in strings: @@ -131,18 +143,20 @@ def reformat_list(strings): def reformat_square_not_full(input_string): - """ - Reformat the input string by splitting it into parts that are either enclosed in + """Reformat the input string by splitting it into parts that are either enclosed in square brackets or not. For parts not in brackets, it removes trailing periods, trims spaces, and encloses them in square brackets. For parts already in brackets, it adds spaces inside the brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Split the string into parts that are either in brackets or not parts = re.split(r"(\[[^\]]*\])", input_string) @@ -163,16 +177,18 @@ def reformat_square_not_full(input_string): def reformat_slash(input_string): - """ - Reformat the input string by splitting it on slashes, trimming spaces from each + """Reformat the input string by splitting it on slashes, trimming spaces from each part, and enclosing each part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Split the string on slashes and strip spaces parts = [part.strip() for part in input_string.split("/")] @@ -182,16 +198,18 @@ def reformat_slash(input_string): def reformat_pipe(input_string): - """ - Reformat the input string by splitting it on ' | ' (pipe symbol with spaces), + """Reformat the input string by splitting it on ' | ' (pipe symbol with spaces), trimming spaces from each part, and enclosing each part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Split the string on ' | ' and strip spaces parts = [part.strip() for part in input_string.split("|")] @@ -201,14 +219,16 @@ def reformat_pipe(input_string): def is_pipe_format(input_string): - """ - Check if the input string follows the ' | ' (pipe symbol with spaces) pattern. + """Check if the input string follows the ' | ' (pipe symbol with spaces) pattern. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Regular expression to check for ' | ' pattern pattern = r"^(?:[^|]+\|)+[^|]+$" @@ -216,20 +236,22 @@ def is_pipe_format(input_string): def is_numbered_line_format(input_string): - """ - Check if the input string(input_string) follows the pattern of lines starting with + """Check if the input string(input_string) follows the pattern of lines starting with 'number. word/phrase \n'. - Parameters: + Parameters + ---------- input_string (str or list of str): The input string(input_string) to be checked. It can be a single string or a list of strings. - Returns: - bool or list of bool: + Returns + ------- + bool or list of bool: If the input is a single string, it returns True if it matches the pattern, otherwise False. If the input is a list of strings it returns a list of booleans indicating if each string matches the pattern. + """ # Regular expression to match lines starting with 'number. word/phrase \n' pattern = r"^(?:\d+\.\s+.+\n?)+$" @@ -243,17 +265,19 @@ def is_numbered_line_format(input_string): def reformat_ists(input_string): - """ - Reformat the input string by finding all occurrences of the (iSTS n) pattern + """Reformat the input string by finding all occurrences of the (iSTS n) pattern (where 'n' is a number), extracting the text inside the parentheses, and enclosing each extracted part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Use regular expression to find all occurrences of the iSTS pattern and # extract the text @@ -264,15 +288,17 @@ def reformat_ists(input_string): def is_ists_format(input_string): - """ - Check if the input string follows the pattern of (iSTS n) followed by text, with + """Check if the input string follows the pattern of (iSTS n) followed by text, with multiple occurrences allowed, where 'n' is a number. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Regular expression to check for the iSTS pattern pattern = r"^(?:\(iSTS \d+\)\s*[^)]+\s*)+$" @@ -280,17 +306,19 @@ def is_ists_format(input_string): def reformat_with_sections(input_string): - """ - Reformat the input string by using regular expressions to split it into parts + """Reformat the input string by using regular expressions to split it into parts following section labels (e.g., [S1], [S2], etc.), removing empty strings, and enclosing each part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Use regular expression to extract text following the section labels parts = re.split(r"\s*\[\s*S\d+\s*\]\s*", input_string) @@ -300,15 +328,17 @@ def reformat_with_sections(input_string): def is_section_format(input_string): - """ - Check if the input string follows the pattern of section labels (e.g., [S1], [S2], etc.) + """Check if the input string follows the pattern of section labels (e.g., [S1], [S2], etc.) followed by text, with multiple occurrences allowed. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Regular expression to check for the section label pattern pattern = r"^(?:\s*\[\s*S\d+\s*\]\s*.+)+$" @@ -316,34 +346,37 @@ def is_section_format(input_string): def is_chunk_format(input_string): - """ - Check if the input string follows the pattern of 'Chunk X: text' or '[Chunk X] text' + """Check if the input string follows the pattern of 'Chunk X: text' or '[Chunk X] text' patterns, where 'X' is a number, with multiple occurrences allowed. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Further optimized regular expression pattern = r"^(Chunk \d+:|\[\s*Chunk \d+\s*\]).*?(?:\s+(?:Chunk \d+:|\[\s*Chunk \d+\s*\]).*?)*$" return bool(re.match(pattern, input_string)) - def reformat_chunks(input_string): - """ - Reformat the input string by finding all occurrences of both + """Reformat the input string by finding all occurrences of both 'Chunk X: text' and '[Chunk X] text' patterns, extracting the text within them, and enclosing each extracted part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Find all occurrences of both chunk patterns and extract the text parts = re.findall( @@ -360,16 +393,18 @@ def reformat_chunks(input_string): def reformat_ists_markers(input_string): - """ - Reformat the input string by splitting it at '[ iSTS ]' markers, removing empty strings, + """Reformat the input string by splitting it at '[ iSTS ]' markers, removing empty strings, and enclosing each part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Split the string at '[ iSTS ]', remove empty strings, and enclose each # part in square brackets @@ -383,15 +418,17 @@ def reformat_ists_markers(input_string): def is_ists_marker_format(input_string): - """ - Check if the input string follows the pattern with '[ iSTS ]' markers followed by text, + """Check if the input string follows the pattern with '[ iSTS ]' markers followed by text, with multiple occurrences allowed. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Regular expression to check for the pattern with '[ iSTS ]' pattern = r"^(?:\[\s*iSTS\s*\].+)+$" @@ -399,16 +436,18 @@ def is_ists_marker_format(input_string): def reformat_with_dashes(input_string): - """ - Reformat the input string by splitting it on ' - ' (dash with spaces), + """Reformat the input string by splitting it on ' - ' (dash with spaces), removing empty strings, and enclosing each part in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Split the string on ' - ', remove empty strings, and enclose each part # in square brackets @@ -418,15 +457,17 @@ def reformat_with_dashes(input_string): def is_dash_format(input_string): - """ - Check if the input string follows the pattern with ' - ' (dash with spaces) + """Check if the input string follows the pattern with ' - ' (dash with spaces) separating non-empty parts. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Regular expression to check for the pattern with ' - ' pattern = r"^[^-]+(?:\s*-\s*[^-]+)+$" @@ -434,19 +475,21 @@ def is_dash_format(input_string): def preliminary_reformat(input_string): - """ - Remove common prefixes like + """Remove common prefixes like 'iSTS chunks:' and 'Here are the iSTS chunks for the given sentence:' from the input string. This function is intended as a preliminary step before further reformatting. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: The input string with common prefixes stripped. + """ input_string = input_string.strip("iSTS chunks:") input_string = input_string.strip( @@ -456,17 +499,19 @@ def preliminary_reformat(input_string): def reformat_brackets_and_text(input_string): - """ - Reformat the input string by removing trailing dots, splitting it into parts that are either + """Reformat the input string by removing trailing dots, splitting it into parts that are either enclosed in square brackets or not, removing commas and trimming spaces for non-bracket parts, and enclosing non-bracket parts in square brackets. The formatted parts are then joined into a single string with spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string containing the processed parts of the input string. + """ # Remove the trailing dot if present input_string = re.sub(r"\.$", "", input_string) @@ -490,16 +535,18 @@ def reformat_brackets_and_text(input_string): def is_brackets_and_text_format(input_string): - """ - Check if the input string follows the pattern of parts + """Check if the input string follows the pattern of parts that are either enclosed in square brackets or not, with optional commas and spaces between the parts. - Parameters: + Parameters + ---------- input_string (str): The input string to be checked. - Returns: + Returns + ------- bool: True if the input string matches the pattern, otherwise False. + """ # Regular expression to check the pattern pattern = r"^(\[[^\]]*\]|\s*[^[\]]+\s*)(,\s*|\s+|$)+$" @@ -507,18 +554,20 @@ def is_brackets_and_text_format(input_string): def is_correct_format(input_string): - """ - Checks if a given string matches a specific format of sequences enclosed in square brackets. + """Checks if a given string matches a specific format of sequences enclosed in square brackets. This function uses a regular expression to determine if the input string adheres to a format where sequences of text are enclosed within square brackets. Each sequence must be separated by spaces, and the entire string must only consist of these bracketed sequences. - Parameters: + Parameters + ---------- input_string (str): The string to be checked against the format. - Returns: + Returns + ------- bool: True if the string matches the format, False otherwise. + """ # Regular expression to match sequences of '[ some words ]' pattern = r"^(\[\s*[^]]+\s*\]\s*)+$" @@ -528,8 +577,7 @@ def is_correct_format(input_string): def check_format(input_data): - """ - Checks if the input data (input_data or list of strings) matches a specific format. + """Checks if the input data (input_data or list of strings) matches a specific format. This function determines if the input data, which can be either a single string or a list of strings, @@ -537,16 +585,19 @@ def check_format(input_data): The format check is performed by the 'is_correct_format' function. For a list, the check is applied to each string in the list. - Parameters: - input_data (str or list of str): - The input data to be checked. + Parameters + ---------- + input_data (str or list of str): + The input data to be checked. Can be a single string or a list of strings. - Returns: + Returns + ------- bool or list of bool: If input_data is a string, returns True if it matches the format, False otherwise. If input_data is a list, returns a list of booleans corresponding to each string. Returns None if input_data is neither a string nor a list. + """ # Check if the input is a string, and if so, process it directly if isinstance(input_data, str): @@ -559,27 +610,29 @@ def check_format(input_data): # Return None or raise an error if the input is neither a string nor a list return None + def check_and_reformat(input_string, check_func, format_func): - """ - Checks what type of string got inputed and formats it accordingly - """ + """Checks what type of string got inputed and formats it accordingly.""" if check_func(input_string): print(f"formatted by {format_func.__name__}", input_string) return format_func(input_string) return None + def reformat(input_string): - """ - Reformat the input string by applying a series of format checks and corresponding + """Reformat the input string by applying a series of format checks and corresponding reformatting functions. This function attempts to identify the format of the input string and apply the appropriate reformatting. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str or bool: The reformatted string if a suitable format is recognized, or False if the format is not recognized and cannot be reformatted. + """ input_string = preliminary_reformat(input_string) if check_format(input_string): @@ -608,15 +661,17 @@ def reformat(input_string): def remove_empty_brackets(input_string): - """ - Remove empty brackets (with or without spaces) from the input string and + """Remove empty brackets (with or without spaces) from the input string and eliminate any extra spaces that might be left after removing brackets. - Parameters: + Parameters + ---------- input_string (str): The input string from which empty brackets should be removed. - Returns: + Returns + ------- str: The input string with empty brackets removed and extra spaces eliminated. + """ # Regular expression to match empty brackets (with any number of spaces) pattern = r"\[\s*\]" @@ -631,30 +686,34 @@ def remove_empty_brackets(input_string): def reformat_brackets(input_string): - """ - Reformat the input string by enclosing text inside square brackets. + """Reformat the input string by enclosing text inside square brackets. This function searches for text enclosed in square brackets (with or without spaces) and ensures that the text inside the brackets is surrounded by spaces. - Parameters: + Parameters + ---------- input_string (str): The input string to be reformatted. - Returns: + Returns + ------- str: A reformatted string with text inside square brackets and spaces around it. + """ formatted_string = re.sub(r"\[\s*(.*?)\s*\]", r"[ \1 ]", input_string) return formatted_string def generate_instruction(sentence): - """ - Generate an instruction for dividing a given sentence into iSTS chunks. + """Generate an instruction for dividing a given sentence into iSTS chunks. - Parameters: + Parameters + ---------- sentence (str): The sentence to be divided into chunks. - Returns: + Returns + ------- str: A formatted instruction with the provided sentence. + """ return ( "Please divide the following sentence into iSTS chunks." @@ -665,15 +724,17 @@ def generate_instruction(sentence): def call_api(instruction): - """ - Call the API using the provided instruction to generate a response. + """Call the API using the provided instruction to generate a response. - Parameters: + Parameters + ---------- client (OpenAI.Client): An OpenAI API client object. instruction (str): The instruction to provide to the API. - Returns: + Returns + ------- dict: The response generated by the API. + """ return client.chat.completions.create( messages=[ @@ -687,29 +748,33 @@ def call_api(instruction): def process_response(response): - """ - Process the response received from the API and extract the chunked sentence. + """Process the response received from the API and extract the chunked sentence. - Parameters: + Parameters + ---------- response (dict): The response generated by the API. - Returns: + Returns + ------- str: The chunked sentence extracted from the response. + """ chunked_sentence = response.choices[0].message.content.strip() return chunked_sentence def check_and_format_sentence(chunked_sentence): - """ - Check and format a chunked sentence by applying a series of checks and reformatting. + """Check and format a chunked sentence by applying a series of checks and reformatting. - Parameters: + Parameters + ---------- chunked_sentence (str): The chunked sentence to be checked and formatted. - Returns: + Returns + ------- str or bool: The formatted chunked sentence if it passes all checks, or False if there are errors. + """ reformated = reformat(chunked_sentence) if not reformated: @@ -723,15 +788,17 @@ def check_and_format_sentence(chunked_sentence): def finalize_formatting(reformatted): - """ - Finalize the formatting of a reformatted string by removing empty brackets and ensuring + """Finalize the formatting of a reformatted string by removing empty brackets and ensuring that text inside brackets is surrounded by spaces. - Parameters: + Parameters + ---------- reformatted (str): The reformatted string to be finalized. - Returns: + Returns + ------- str: The finalized and formatted string. + """ reformatted = remove_empty_brackets(reformatted) reformatted = reformat_brackets(reformatted) @@ -739,18 +806,20 @@ def finalize_formatting(reformatted): def process_sentence(sentence, chunked_sentences): - """ - Process a sentence by generating iSTS chunks and + """Process a sentence by generating iSTS chunks and adding them to the provided list of chunked sentences. - Parameters: + Parameters + ---------- sentence (str): The input sentence to be processed. chunked_sentences (list): A list of previously chunked sentences to which the processed sentence will be appended. client (OpenAI.Client): An OpenAI API client object for calling the API. - Returns: + Returns + ------- list or bool: The updated list of chunked sentences if successful, or False if there are errors. + """ instruction = generate_instruction(sentence) response = call_api(instruction) @@ -764,19 +833,21 @@ def process_sentence(sentence, chunked_sentences): def chunk_sentences(input_file, output): - """ - Read sentences from a file, process each sentence to generate iSTS chunks, + """Read sentences from a file, process each sentence to generate iSTS chunks, and write the chunked sentences to an output file. - Parameters: + Parameters + ---------- file_path (str): The path to the input file containing sentences to be processed. output_path (str): The path to the output file where chunked sentences will be written. - Returns: + Returns + ------- None + """ # Read the sentences from the file - with open(input_file, "r", encoding="utf-8") as file: + with open(input_file, encoding="utf-8") as file: sentences = file.readlines() # Process each sentence @@ -785,8 +856,7 @@ def chunk_sentences(input_file, output): # Write the chunked sentences to a new file with open(output, "w", encoding="utf-8") as output_file: - for sentence in chunked_sentences: - output_file.write(sentence + "\n") + output_file.writelines(sentence + "\n" for sentence in chunked_sentences) # Usage @@ -797,7 +867,7 @@ input_files = [ "test_goldStandard/images/STSint.testinput.images.sent1.txt", "test_goldStandard/images/STSint.testinput.images.sent2.txt", "test_goldStandard/headlines/STSint.testinput.headlines.sent1.txt", - "test_goldStandard/headlines/STSint.testinput.headlines.sent2.txt" + "test_goldStandard/headlines/STSint.testinput.headlines.sent2.txt", ] output_files = [ @@ -806,11 +876,11 @@ output_files = [ "test_goldStandard/images/images-chunks-gpt-one.txt", "test_goldStandard/images/images-chunks-gpt-two.txt", "test_goldStandard/headlines/headlines-chunks-gpt-one.txt", - "test_goldStandard/headlines/headlines-chunks-gpt-two.txt" + "test_goldStandard/headlines/headlines-chunks-gpt-two.txt", ] # Change me to os.environ['API_KEY'] -client = OpenAI(api_key=os.environ['API_KEY']) +client = OpenAI(api_key=os.environ["API_KEY"]) for input_path, output_path in zip(input_files, output_files): chunk_sentences(input_path, output_path) diff --git a/Programming/NLP/processing.py b/Programming/NLP/processing.py index 1a50f028..f7b5bf92 100644 --- a/Programming/NLP/processing.py +++ b/Programming/NLP/processing.py @@ -1,154 +1,146 @@ -import pandas as pd -import numpy as np -from lxml import etree import re +import numpy as np +import pandas as pd +from lxml import etree + def load_sentences(senteance1_path: str, sentance2_path: str) -> pd.DataFrame: - """ - Loads the sentences from the given paths and outputs in a 2 columns dataframe - """ - sentance1 = pd.read_csv(senteance1_path, - dtype=str, - delimiter="}", - header=None) - sentance1.columns = ["sentance1"] + """Loads the sentences from the given paths and outputs in a 2 columns dataframe.""" + sentance1 = pd.read_csv(senteance1_path, dtype=str, delimiter="}", header=None) + sentance1.columns = ["sentance1"] - sentance2 = pd.read_csv(sentance2_path, - dtype=str, - delimiter="}", - header=None) - sentance2.columns = ["sentance2"] + sentance2 = pd.read_csv(sentance2_path, dtype=str, delimiter="}", header=None) + sentance2.columns = ["sentance2"] - sentances = pd.concat([sentance1, sentance2], axis=1) - return sentances + sentances = pd.concat([sentance1, sentance2], axis=1) + return sentances def chunk2list(chunks: str) -> list: - """ - Takes str that is all chunks from a chunked sentance and returns a list of all the chunks as seperate items - """ - chunks = chunks.replace('[', '') - chunks = chunks.replace(']', '') - chunks = chunks.replace(' ', '|') - split = chunks.split('|') + """Takes str that is all chunks from a chunked sentance and returns a list of all the chunks as seperate items.""" + chunks = chunks.replace("[", "") + chunks = chunks.replace("]", "") + chunks = chunks.replace(" ", "|") + split = chunks.split("|") - split = [re.sub(r'^\s+|\s+$', '', s) for s in split] # remove spaces at the beggining and end of chunks - split = [re.sub(r'[^\w\s]', '', s) for s in split] # remove punctuation - return split + split = [ + re.sub(r"^\s+|\s+$", "", s) for s in split + ] # remove spaces at the beggining and end of chunks + split = [re.sub(r"[^\w\s]", "", s) for s in split] # remove punctuation + return split def load_chunked(chunked_path1: str, chunked_path2: str) -> pd.DataFrame: - """ - Loads chunked sentances in [ chunk1 ] [ chunk2 ] format into dataframe with lists of chunks - """ - chunked_sentance1 = pd.read_csv(chunked_path1, - dtype=str, - delimiter="}", - header=None) - chunked_sentance1.columns = ["chunked_sentance1"] + """Loads chunked sentances in [ chunk1 ] [ chunk2 ] format into dataframe with lists of chunks.""" + chunked_sentance1 = pd.read_csv( + chunked_path1, dtype=str, delimiter="}", header=None + ) + chunked_sentance1.columns = ["chunked_sentance1"] - chunked_sentance2 = pd.read_csv(chunked_path2, - dtype=str, - delimiter="}", - header=None) - chunked_sentance2.columns = ["chunked_sentance2"] + chunked_sentance2 = pd.read_csv( + chunked_path2, dtype=str, delimiter="}", header=None + ) + chunked_sentance2.columns = ["chunked_sentance2"] - headlines_chunked = pd.concat([chunked_sentance1, chunked_sentance2], axis=1) + headlines_chunked = pd.concat([chunked_sentance1, chunked_sentance2], axis=1) - # convert chunks from str to list - headlines_chunked['chunked_sentance1'] = headlines_chunked[ - 'chunked_sentance1'].apply(chunk2list) - headlines_chunked['chunked_sentance2'] = headlines_chunked[ - 'chunked_sentance2'].apply(chunk2list) + # convert chunks from str to list + headlines_chunked["chunked_sentance1"] = headlines_chunked[ + "chunked_sentance1" + ].apply(chunk2list) + headlines_chunked["chunked_sentance2"] = headlines_chunked[ + "chunked_sentance2" + ].apply(chunk2list) - return headlines_chunked + return headlines_chunked def return_characteers(cell: str) -> str: - """ - converts the alignment data to restore the <==> and & tokens - """ - cell = cell.replace('ARROWS_PLACEHOLDER', '<==>') - cell = cell.replace('AMPERSAND_PLACEHOLDER', '&') - return cell + """Converts the alignment data to restore the <==> and & tokens.""" + cell = cell.replace("ARROWS_PLACEHOLDER", "<==>") + cell = cell.replace("AMPERSAND_PLACEHOLDER", "&") + return cell def load_alignment(alignment_path: str) -> pd.DataFrame: - """ - Loads the alignment file. Parses only the tag and puts the data into a dataframe - """ - with open(alignment_path, 'r') as file: - file_content = file.read() + """Loads the alignment file. Parses only the tag and puts the data into a dataframe.""" + with open(alignment_path) as file: + file_content = file.read() - # <==> and & break xml loaders so it needs to be replaces with something else - modified_content = file_content.replace('<==>', 'ARROWS_PLACEHOLDER').replace('&', 'AMPERSAND_PLACEHOLDER') - # it also needs a root wrapped to function properly - modified_content = f'{modified_content}' + # <==> and & break xml loaders so it needs to be replaces with something else + modified_content = file_content.replace("<==>", "ARROWS_PLACEHOLDER").replace( + "&", "AMPERSAND_PLACEHOLDER" + ) + # it also needs a root wrapped to function properly + modified_content = f"{modified_content}" - modified_file_path = 'temp.wa' - with open(modified_file_path, 'w') as modified_file: - modified_file.write(modified_content) + modified_file_path = "temp.wa" + with open(modified_file_path, "w") as modified_file: + modified_file.write(modified_content) - # Parse the modified file using ElementTree - tree = etree.parse(modified_file_path) - root = tree.getroot() + # Parse the modified file using ElementTree + tree = etree.parse(modified_file_path) + root = tree.getroot() - # get ansewrs - alignments_data = [] - - for alignment in root.xpath('//alignment'): - # Extract relevant information from the alignment element - data = { - 'sentence_id': alignment.xpath('ancestor::sentence/@id')[0], - 'alignment_text': alignment.text - } - alignments_data.append(data) + # get ansewrs + alignments_data = [] - y = pd.DataFrame(alignments_data) - y = y.drop(columns=["sentence_id"]) - y["alignment_text"] = y["alignment_text"].apply(return_characteers) - return y + for alignment in root.xpath("//alignment"): + # Extract relevant information from the alignment element + data = { + "sentence_id": alignment.xpath("ancestor::sentence/@id")[0], + "alignment_text": alignment.text, + } + alignments_data.append(data) + + y = pd.DataFrame(alignments_data) + y = y.drop(columns=["sentence_id"]) + y["alignment_text"] = y["alignment_text"].apply(return_characteers) + return y def prettyprint(element, **kwargs): - xml = etree.tostring(element, pretty_print=True, **kwargs) - print(xml.decode(), end='') + xml = etree.tostring(element, pretty_print=True, **kwargs) + print(xml.decode(), end="") def test_XML(): - # test out the format - print(alignments_data[0]["alignment_text"]) + # test out the format + print(alignments_data[0]["alignment_text"]) + def generate_train_test_split(xy: pd.DataFrame): - """ - Generates a train, validate, test split of the given dataframes in a 60% 20% 20% ratio - """ - #data = pd.merge(x, y, left_index=True, right_index=True) - data = xy - train, test = np.split(data.sample(frac=1, random_state=42), [int(.03*len(data))]) + """Generates a train, validate, test split of the given dataframes in a 60% 20% 20% ratio.""" + # data = pd.merge(x, y, left_index=True, right_index=True) + data = xy + train, test = np.split( + data.sample(frac=1, random_state=42), [int(0.03 * len(data))] + ) - return train, test + return train, test -def generate_alignment_format(dataFrame:pd.DataFrame, id:int) -> str: - output = "seq1:\n" - chunks1 = dataFrame["chunked_sentance1"][id] - for i, chunk in enumerate(chunks1): - output = output + str(i+1) + ") " + str(chunk) + "\n" - output = output + "\nseq2:\n" - chunks2 = dataFrame["chunked_sentance2"][id] - for i, chunk in enumerate(chunks2): - output = output + str(i+1) + ") " + str(chunk) + "\n" - return output -def get_chunks_as_text(data:pd.DataFrame) -> str: - output = [] - for index, row in data.iterrows(): - chunks = "" - for chunk in row["chunked_sentance1"]: - chunks = chunks + "[ " + chunk + " ] " - chunks = chunks + "\n" - for chunk in row["chunked_sentance2"]: - chunks = chunks + "[ " + chunk + " ] " - output.append(chunks) - return output, data.index +def generate_alignment_format(dataFrame: pd.DataFrame, id: int) -> str: + output = "seq1:\n" + chunks1 = dataFrame["chunked_sentance1"][id] + for i, chunk in enumerate(chunks1): + output = output + str(i + 1) + ") " + str(chunk) + "\n" + output = output + "\nseq2:\n" + chunks2 = dataFrame["chunked_sentance2"][id] + for i, chunk in enumerate(chunks2): + output = output + str(i + 1) + ") " + str(chunk) + "\n" + return output + + +def get_chunks_as_text(data: pd.DataFrame) -> str: + output = [] + for index, row in data.iterrows(): + chunks = "" + for chunk in row["chunked_sentance1"]: + chunks = chunks + "[ " + chunk + " ] " + chunks = chunks + "\n" + for chunk in row["chunked_sentance2"]: + chunks = chunks + "[ " + chunk + " ] " + output.append(chunks) + return output, data.index diff --git a/Programming/NLP/rating.py b/Programming/NLP/rating.py index 33fb24aa..6e9de591 100644 --- a/Programming/NLP/rating.py +++ b/Programming/NLP/rating.py @@ -1,5 +1,5 @@ from openai import OpenAI -def rate_similarity(c1:list, c2:list) -> str: - client = OpenAI(api_key='REDACTED_OPENAI_API_KEY') +def rate_similarity(c1: list, c2: list) -> str: + client = OpenAI(api_key="REDACTED_OPENAI_API_KEY") diff --git a/Programming/NLP/test_goldStandard/chunks.py b/Programming/NLP/test_goldStandard/chunks.py index 5b32c7f7..681b2e1c 100644 --- a/Programming/NLP/test_goldStandard/chunks.py +++ b/Programming/NLP/test_goldStandard/chunks.py @@ -1,5 +1,6 @@ from openai import OpenAI + def sentence_loop(sentences): # Process each sentence chunked_sentences = [] @@ -7,12 +8,16 @@ def sentence_loop(sentences): chunked_sentences = process_sentence(sentence, chunked_sentences) return chunked_sentences + def process_sentence(sentence, chunked_sentences): response = client.chat.completions.create( - messages=[{ - "role": "user", - "content": "Please divide the following sentence into iSTS chunks. Format the response as: [[chunk 1]] [[chunk 2]] ... Ensure each chunk is enclosed in double square brackets and separated by a space. If you cannot correctly chunkify the sentence, return an error message in the format: 'ERROR: Cannot chunkify sentence: '. Here is the sentence: " + sentence, - }], + messages=[ + { + "role": "user", + "content": "Please divide the following sentence into iSTS chunks. Format the response as: [[chunk 1]] [[chunk 2]] ... Ensure each chunk is enclosed in double square brackets and separated by a space. If you cannot correctly chunkify the sentence, return an error message in the format: 'ERROR: Cannot chunkify sentence: '. Here is the sentence: " + + sentence, + } + ], model="gpt-3.5-turbo", ) @@ -21,24 +26,25 @@ def process_sentence(sentence, chunked_sentences): chunked_sentences.append(chunked_sentence) return chunked_sentences + def chunk_sentences(file_path, output_path, api_key): # Read the sentences from the file - with open(file_path, 'r') as file: + with open(file_path) as file: sentences = file.readlines() # Process each sentence chunk_sentences = sentence_loop(sentences) print(chunk_sentences) # Write the chunked sentences to a new file - with open(output_path, 'w') as output_file: - for sentence in chunk_sentences: - output_file.write(sentence + '\n') + with open(output_path, "w") as output_file: + output_file.writelines(sentence + "\n" for sentence in chunk_sentences) + # Usage -file_path = 'student/STSint.testinput.answers-students.sent2.txt' -output_path = 'chunks_gpt_two.txt' +file_path = "student/STSint.testinput.answers-students.sent2.txt" +output_path = "chunks_gpt_two.txt" # Replace me with os.environ['API_KEY'] -api_key = 'REDACTED_OPENAI_API_KEY' +api_key = "REDACTED_OPENAI_API_KEY" client = OpenAI(api_key=api_key) chunk_sentences(file_path, output_path, api_key) diff --git a/Programming/PBAD/data.ipynb b/Programming/PBAD/data.ipynb index 2b984661..2d84fa1f 100644 --- a/Programming/PBAD/data.ipynb +++ b/Programming/PBAD/data.ipynb @@ -22,7 +22,7 @@ "metadata": {}, "outputs": [], "source": [ - "pd.options.display.float_format = '{:.2f}'.format" + "pd.options.display.float_format = \"{:.2f}\".format" ] }, { @@ -48,21 +48,33 @@ ], "source": [ "df_dofinansowanie = pd.read_csv(\n", - " 'umowy_pelna_lista_krajowe.csv',\n", - " encoding='ISO-8859-2',\n", - " converters={'TERYT pe?ny': str},\n", - " thousands=',')\n", + " \"umowy_pelna_lista_krajowe.csv\",\n", + " encoding=\"ISO-8859-2\",\n", + " converters={\"TERYT pe?ny\": str},\n", + " thousands=\",\",\n", + ")\n", "\n", - "df_dofinansowanie = df_dofinansowanie.loc[df_dofinansowanie['TERYT pe?ny'] != ''].reset_index(drop=True)\n", + "df_dofinansowanie = df_dofinansowanie.loc[\n", + " df_dofinansowanie[\"TERYT pe?ny\"] != \"\"\n", + "].reset_index(drop=True)\n", "\n", - "df_dofinansowanie['Dofinansowanie UE (PLN)'] = \\\n", - " df_dofinansowanie['Dofinansowanie UE (PLN)'].apply(pd.to_numeric)\n", + "df_dofinansowanie[\"Dofinansowanie UE (PLN)\"] = df_dofinansowanie[\n", + " \"Dofinansowanie UE (PLN)\"\n", + "].apply(pd.to_numeric)\n", "\n", - "df_dofinansowanie['Data rozpocz?cia realizacji'] = pd.to_datetime(df_dofinansowanie['Data rozpocz?cia realizacji'])\n", - "df_dofinansowanie['Rok rozpocz?cia realizacji'] = df_dofinansowanie['Data rozpocz?cia realizacji'].dt.year\n", + "df_dofinansowanie[\"Data rozpocz?cia realizacji\"] = pd.to_datetime(\n", + " df_dofinansowanie[\"Data rozpocz?cia realizacji\"]\n", + ")\n", + "df_dofinansowanie[\"Rok rozpocz?cia realizacji\"] = df_dofinansowanie[\n", + " \"Data rozpocz?cia realizacji\"\n", + "].dt.year\n", "\n", - "df_dofinansowanie['Data podpisania umowy pierwotnej'] = pd.to_datetime(df_dofinansowanie['Data podpisania umowy pierwotnej'])\n", - "df_dofinansowanie['Rok podpisania umowy pierwotnej'] = df_dofinansowanie['Data podpisania umowy pierwotnej'].dt.year" + "df_dofinansowanie[\"Data podpisania umowy pierwotnej\"] = pd.to_datetime(\n", + " df_dofinansowanie[\"Data podpisania umowy pierwotnej\"]\n", + ")\n", + "df_dofinansowanie[\"Rok podpisania umowy pierwotnej\"] = df_dofinansowanie[\n", + " \"Data podpisania umowy pierwotnej\"\n", + "].dt.year" ] }, { @@ -84,7 +96,7 @@ } ], "source": [ - "print(df_dofinansowanie['Program operacyjny'].drop_duplicates().values)" + "print(df_dofinansowanie[\"Program operacyjny\"].drop_duplicates().values)" ] }, { @@ -103,12 +115,24 @@ "metadata": {}, "outputs": [], "source": [ - "df_dofinansowanie_agg = df_dofinansowanie \\\n", - " .groupby(['TERYT pe?ny', 'Program operacyjny', 'Rok rozpocz?cia realizacji'])['Dofinansowanie UE (PLN)'].sum().reset_index()\n", - "df_dofinansowanie_agg = df_dofinansowanie_agg \\\n", - " .rename(columns={'TERYT pe?ny': 'Kod', 'Rok rozpocz?cia realizacji': 'Rok', 'Program operacyjny': 'Program_operacyjny', 'Dofinansowanie UE (PLN)': 'Suma'})\n", - "df_dofinansowanie_agg = df_dofinansowanie_agg \\\n", - " .loc[df_dofinansowanie_agg['Kod'].str.len() == 7].reset_index(drop=True)" + "df_dofinansowanie_agg = (\n", + " df_dofinansowanie.groupby(\n", + " [\"TERYT pe?ny\", \"Program operacyjny\", \"Rok rozpocz?cia realizacji\"]\n", + " )[\"Dofinansowanie UE (PLN)\"]\n", + " .sum()\n", + " .reset_index()\n", + ")\n", + "df_dofinansowanie_agg = df_dofinansowanie_agg.rename(\n", + " columns={\n", + " \"TERYT pe?ny\": \"Kod\",\n", + " \"Rok rozpocz?cia realizacji\": \"Rok\",\n", + " \"Program operacyjny\": \"Program_operacyjny\",\n", + " \"Dofinansowanie UE (PLN)\": \"Suma\",\n", + " }\n", + ")\n", + "df_dofinansowanie_agg = df_dofinansowanie_agg.loc[\n", + " df_dofinansowanie_agg[\"Kod\"].str.len() == 7\n", + "].reset_index(drop=True)" ] }, { @@ -124,15 +148,11 @@ "metadata": {}, "outputs": [], "source": [ - "df_podz = pd.read_csv(\n", - " 'PODZ_1410_CREL.csv',\n", - " sep=';',\n", - " converters={'Kod': str})\n", - "df_podz = df_podz[['Kod', 'Rok', 'Wartosc']]\n", - "df_podz = df_podz.loc[df_podz['Kod'].str.endswith(('1', '2', '3'))]\n", + "df_podz = pd.read_csv(\"PODZ_1410_CREL.csv\", sep=\";\", converters={\"Kod\": str})\n", + "df_podz = df_podz[[\"Kod\", \"Rok\", \"Wartosc\"]]\n", + "df_podz = df_podz.loc[df_podz[\"Kod\"].str.endswith((\"1\", \"2\", \"3\"))]\n", "df_podz = df_podz.dropna()\n", - "df_podz = df_podz.rename(columns={\n", - " 'Wartosc': 'Powierzchnia'})" + "df_podz = df_podz.rename(columns={\"Wartosc\": \"Powierzchnia\"})" ] }, { @@ -142,16 +162,19 @@ "outputs": [], "source": [ "df_wyna = pd.read_csv(\n", - " 'WYNA_2497_CREL.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_wyna = df_wyna[['Kod', 'Wyszczególnienie', 'Rok', 'Wartosc']]\n", + " \"WYNA_2497_CREL.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_wyna = df_wyna[[\"Kod\", \"Wyszczególnienie\", \"Rok\", \"Wartosc\"]]\n", "df_wyna = df_wyna.dropna()\n", - "df_wyna = df_wyna.pivot_table(index=['Kod', 'Rok'], columns='Wyszczególnienie', values='Wartosc').reset_index()\n", - "df_wyna = df_wyna.rename(columns={\n", - " 'ogółem': 'Wynagrodzenie_ogolem',\n", - " 'przeciętne miesięczne wynagrodzenia brutto w relacji do średniej krajowej (Polska=100)': 'Wynagrodzenie_w_relacji_do_sredniej'})" + "df_wyna = df_wyna.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Wyszczególnienie\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_wyna = df_wyna.rename(\n", + " columns={\n", + " \"ogółem\": \"Wynagrodzenie_ogolem\",\n", + " \"przeciętne miesięczne wynagrodzenia brutto w relacji do średniej krajowej (Polska=100)\": \"Wynagrodzenie_w_relacji_do_sredniej\",\n", + " }\n", + ")" ] }, { @@ -463,23 +486,26 @@ ], "source": [ "df_fina_1 = pd.read_csv(\n", - " 'FINA_2622_CREL_1.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_fina_1 = df_fina_1[['Kod', 'Rodzaje dochodów', 'Rok', 'Wartosc']]\n", + " \"FINA_2622_CREL_1.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_fina_1 = df_fina_1[[\"Kod\", \"Rodzaje dochodów\", \"Rok\", \"Wartosc\"]]\n", "df_fina_1 = df_fina_1.dropna()\n", - "df_fina_1 = df_fina_1.pivot_table(index=['Kod', 'Rok'], columns='Rodzaje dochodów', values='Wartosc').reset_index()\n", - "df_fina_1 = df_fina_1.rename(columns={\n", - " 'dochody podatkowe - podatek leśny': 'Dochody_podatek_lesny',\n", - " 'dochody podatkowe - podatek od czynności cywilnoprawnych': 'Dochody_podatek_PCC',\n", - " 'dochody podatkowe - podatek od działalności gospodarczej osób fizycznych, opłacany w formie karty podatkowej': 'Dochody_podatek_od_dzialalnosci_gospodarczej',\n", - " 'dochody podatkowe - podatek od nieruchomości': 'Dochody_podatek_od_nieruchomosci',\n", - " 'dochody podatkowe - podatek od spadków i darowizn': 'Dochody_podatek_od_spadkow',\n", - " 'dochody podatkowe - podatek od środków transportowych': 'Dochody_podatek_od_srodkow_transportowych',\n", - " 'dochody podatkowe - podatek rolny': 'Dochody_podatek_rolny',\n", - " 'dochody podatkowe - ustalone i pobierane na podstawie odrębnych ustaw': 'Dochody_podatek_odrebne_ustawy',\n", - " 'razem': 'Dochody_razem'})\n", + "df_fina_1 = df_fina_1.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Rodzaje dochodów\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_fina_1 = df_fina_1.rename(\n", + " columns={\n", + " \"dochody podatkowe - podatek leśny\": \"Dochody_podatek_lesny\",\n", + " \"dochody podatkowe - podatek od czynności cywilnoprawnych\": \"Dochody_podatek_PCC\",\n", + " \"dochody podatkowe - podatek od działalności gospodarczej osób fizycznych, opłacany w formie karty podatkowej\": \"Dochody_podatek_od_dzialalnosci_gospodarczej\",\n", + " \"dochody podatkowe - podatek od nieruchomości\": \"Dochody_podatek_od_nieruchomosci\",\n", + " \"dochody podatkowe - podatek od spadków i darowizn\": \"Dochody_podatek_od_spadkow\",\n", + " \"dochody podatkowe - podatek od środków transportowych\": \"Dochody_podatek_od_srodkow_transportowych\",\n", + " \"dochody podatkowe - podatek rolny\": \"Dochody_podatek_rolny\",\n", + " \"dochody podatkowe - ustalone i pobierane na podstawie odrębnych ustaw\": \"Dochody_podatek_odrebne_ustawy\",\n", + " \"razem\": \"Dochody_razem\",\n", + " }\n", + ")\n", "\n", "df_fina_1" ] @@ -868,26 +894,29 @@ ], "source": [ "df_fina_2 = pd.read_csv(\n", - " 'FINA_2622_CREL_2.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_fina_2 = df_fina_2[['Kod', 'Rodzaje dochodów', 'Rok', 'Wartosc']]\n", + " \"FINA_2622_CREL_2.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_fina_2 = df_fina_2[[\"Kod\", \"Rodzaje dochodów\", \"Rok\", \"Wartosc\"]]\n", "df_fina_2 = df_fina_2.dropna()\n", - "df_fina_2 = df_fina_2.pivot_table(index=['Kod', 'Rok'], columns='Rodzaje dochodów', values='Wartosc').reset_index()\n", - "df_fina_2 = df_fina_2.rename(columns={\n", - " 'dochody z majątku': 'Dochody_z_majatku',\n", - " 'dochody z majątku - dochody z najmu i dzierżawy składników majątkowych JST oraz innych umów o podobnym charakterze': 'Dochody_z_najmu_i_dzierzawy',\n", - " 'pozostałe dochody - wpływy z usług': 'Dochody_z_uslug',\n", - " 'pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z innych źródeł - inwestycyjne': 'Dochody_dofinansowanie_inwestycyjne',\n", - " 'pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z innych źródeł - razem': 'Dochody_dofinansowanie_razem',\n", - " 'udziały w podatkach stanowiących dochody budżetu państwa podatek dochodowy od osób fizycznych': 'Udzialy_w_podatkach_dochodowych_od_osob_fizycznych',\n", - " 'udziały w podatkach stanowiących dochody budżetu państwa podatek dochodowy od osób prawnych': 'Udzialy_w_podatkach_dochodowych_od_osob_prywatnych',\n", - " 'udziały w podatkach stanowiących dochody budżetu państwa razem': 'Udzialy_w_podatkach_dochodowych_razem',\n", - " 'wpływy z innych lokalnych opłat pobieranych przez jednostki samorządu terytorialnego na podstawie odrębnych ustaw': 'Wplywy_z_innych_lokalnych_oplat',\n", - " 'wpływy z opłaty eksploatacyjnej': 'Wplywy_z_oplaty_eksploatacyjnej',\n", - " 'wpływy z opłaty skarbowej': 'Wplywy_z_oplaty_skarbowej',\n", - " 'wpływy z opłaty targowej': 'Wplywy_z_oplaty_targowej'})\n", + "df_fina_2 = df_fina_2.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Rodzaje dochodów\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_fina_2 = df_fina_2.rename(\n", + " columns={\n", + " \"dochody z majątku\": \"Dochody_z_majatku\",\n", + " \"dochody z majątku - dochody z najmu i dzierżawy składników majątkowych JST oraz innych umów o podobnym charakterze\": \"Dochody_z_najmu_i_dzierzawy\",\n", + " \"pozostałe dochody - wpływy z usług\": \"Dochody_z_uslug\",\n", + " \"pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z innych źródeł - inwestycyjne\": \"Dochody_dofinansowanie_inwestycyjne\",\n", + " \"pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z innych źródeł - razem\": \"Dochody_dofinansowanie_razem\",\n", + " \"udziały w podatkach stanowiących dochody budżetu państwa podatek dochodowy od osób fizycznych\": \"Udzialy_w_podatkach_dochodowych_od_osob_fizycznych\",\n", + " \"udziały w podatkach stanowiących dochody budżetu państwa podatek dochodowy od osób prawnych\": \"Udzialy_w_podatkach_dochodowych_od_osob_prywatnych\",\n", + " \"udziały w podatkach stanowiących dochody budżetu państwa razem\": \"Udzialy_w_podatkach_dochodowych_razem\",\n", + " \"wpływy z innych lokalnych opłat pobieranych przez jednostki samorządu terytorialnego na podstawie odrębnych ustaw\": \"Wplywy_z_innych_lokalnych_oplat\",\n", + " \"wpływy z opłaty eksploatacyjnej\": \"Wplywy_z_oplaty_eksploatacyjnej\",\n", + " \"wpływy z opłaty skarbowej\": \"Wplywy_z_oplaty_skarbowej\",\n", + " \"wpływy z opłaty targowej\": \"Wplywy_z_oplaty_targowej\",\n", + " }\n", + ")\n", "\n", "df_fina_2" ] @@ -1118,21 +1147,24 @@ ], "source": [ "df_ludn_1 = pd.read_csv( # ogolem\n", - " 'LUDN_1342_CREL_1.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_ludn_1 = df_ludn_1[['Kod', 'Wiek', 'Rok', 'Wartosc']]\n", + " \"LUDN_1342_CREL_1.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_ludn_1 = df_ludn_1[[\"Kod\", \"Wiek\", \"Rok\", \"Wartosc\"]]\n", "df_ludn_1 = df_ludn_1.dropna()\n", - "df_ludn_1 = df_ludn_1.loc[df_ludn_1['Kod'].str.endswith(('1', '2', '3'))]\n", - "df_ludn_1 = df_ludn_1.pivot_table(index=['Kod', 'Rok'], columns='Wiek', values='Wartosc').reset_index()\n", - "df_ludn_1 = df_ludn_1.rename(columns={\n", - " 'ogółem': 'Ludnosc_ogolem',\n", - " 'w wieku poprodukcyjnym': 'Ludnosc_w_wieku_poprodukcyjnym',\n", - " 'w wieku produkcyjnym': 'Ludnosc_w_wieku_produkcyjnym',\n", - " 'w wieku produkcyjnym mobilnym': 'Ludnosc_w_wieku_produkcyjnym_mobilnym',\n", - " 'w wieku produkcyjnym niemobilnym': 'Ludnosc_w_wieku_produkcyjnym_niemobilnym',\n", - " 'w wieku przedprodukcyjnym': 'Ludnosc_w_wieku_przedprodukcyjnym'})\n", + "df_ludn_1 = df_ludn_1.loc[df_ludn_1[\"Kod\"].str.endswith((\"1\", \"2\", \"3\"))]\n", + "df_ludn_1 = df_ludn_1.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Wiek\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_ludn_1 = df_ludn_1.rename(\n", + " columns={\n", + " \"ogółem\": \"Ludnosc_ogolem\",\n", + " \"w wieku poprodukcyjnym\": \"Ludnosc_w_wieku_poprodukcyjnym\",\n", + " \"w wieku produkcyjnym\": \"Ludnosc_w_wieku_produkcyjnym\",\n", + " \"w wieku produkcyjnym mobilnym\": \"Ludnosc_w_wieku_produkcyjnym_mobilnym\",\n", + " \"w wieku produkcyjnym niemobilnym\": \"Ludnosc_w_wieku_produkcyjnym_niemobilnym\",\n", + " \"w wieku przedprodukcyjnym\": \"Ludnosc_w_wieku_przedprodukcyjnym\",\n", + " }\n", + ")\n", "\n", "df_ludn_1" ] @@ -1389,21 +1421,24 @@ ], "source": [ "df_ludn_2 = pd.read_csv( # mezczyzni\n", - " 'LUDN_1342_CREL_2.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_ludn_2 = df_ludn_2[['Kod', 'Wiek', 'Rok', 'Wartosc']]\n", + " \"LUDN_1342_CREL_2.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_ludn_2 = df_ludn_2[[\"Kod\", \"Wiek\", \"Rok\", \"Wartosc\"]]\n", "df_ludn_2 = df_ludn_2.dropna()\n", - "df_ludn_2 = df_ludn_2.loc[df_ludn_2['Kod'].str.endswith(('1', '2', '3'))]\n", - "df_ludn_2 = df_ludn_2.pivot_table(index=['Kod', 'Rok'], columns='Wiek', values='Wartosc').reset_index()\n", - "df_ludn_2 = df_ludn_2.rename(columns={\n", - " 'ogółem': 'Ludnosc_mezczyzni',\n", - " 'w wieku poprodukcyjnym': 'Ludnosc_mezczyzni_w_wieku_poprodukcyjnym',\n", - " 'w wieku produkcyjnym': 'Ludnosc_mezczyzni_w_wieku_produkcyjnym',\n", - " 'w wieku produkcyjnym mobilnym': 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym',\n", - " 'w wieku produkcyjnym niemobilnym': 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym',\n", - " 'w wieku przedprodukcyjnym': 'Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym'})\n", + "df_ludn_2 = df_ludn_2.loc[df_ludn_2[\"Kod\"].str.endswith((\"1\", \"2\", \"3\"))]\n", + "df_ludn_2 = df_ludn_2.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Wiek\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_ludn_2 = df_ludn_2.rename(\n", + " columns={\n", + " \"ogółem\": \"Ludnosc_mezczyzni\",\n", + " \"w wieku poprodukcyjnym\": \"Ludnosc_mezczyzni_w_wieku_poprodukcyjnym\",\n", + " \"w wieku produkcyjnym\": \"Ludnosc_mezczyzni_w_wieku_produkcyjnym\",\n", + " \"w wieku produkcyjnym mobilnym\": \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym\",\n", + " \"w wieku produkcyjnym niemobilnym\": \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym\",\n", + " \"w wieku przedprodukcyjnym\": \"Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym\",\n", + " }\n", + ")\n", "\n", "df_ludn_2" ] @@ -1647,21 +1682,24 @@ ], "source": [ "df_ludn_3 = pd.read_csv( # kobiety\n", - " 'LUDN_1342_CREL_3.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_ludn_3 = df_ludn_3[['Kod', 'Wiek', 'Rok', 'Wartosc']]\n", + " \"LUDN_1342_CREL_3.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_ludn_3 = df_ludn_3[[\"Kod\", \"Wiek\", \"Rok\", \"Wartosc\"]]\n", "df_ludn_3 = df_ludn_3.dropna()\n", - "df_ludn_3 = df_ludn_3.loc[df_ludn_3['Kod'].str.endswith(('1', '2', '3'))]\n", - "df_ludn_3 = df_ludn_3.pivot_table(index=['Kod', 'Rok'], columns='Wiek', values='Wartosc').reset_index()\n", - "df_ludn_3 = df_ludn_3.rename(columns={\n", - " 'ogółem': 'Ludnosc_kobiety',\n", - " 'w wieku poprodukcyjnym': 'Ludnosc_kobiety_w_wieku_poprodukcyjnym',\n", - " 'w wieku produkcyjnym': 'Ludnosc_kobiety_w_wieku_produkcyjnym',\n", - " 'w wieku produkcyjnym mobilnym': 'Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym',\n", - " 'w wieku produkcyjnym niemobilnym': 'Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym',\n", - " 'w wieku przedprodukcyjnym': 'Ludnosc_kobiety_w_wieku_przedprodukcyjnym'})\n", + "df_ludn_3 = df_ludn_3.loc[df_ludn_3[\"Kod\"].str.endswith((\"1\", \"2\", \"3\"))]\n", + "df_ludn_3 = df_ludn_3.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Wiek\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_ludn_3 = df_ludn_3.rename(\n", + " columns={\n", + " \"ogółem\": \"Ludnosc_kobiety\",\n", + " \"w wieku poprodukcyjnym\": \"Ludnosc_kobiety_w_wieku_poprodukcyjnym\",\n", + " \"w wieku produkcyjnym\": \"Ludnosc_kobiety_w_wieku_produkcyjnym\",\n", + " \"w wieku produkcyjnym mobilnym\": \"Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym\",\n", + " \"w wieku produkcyjnym niemobilnym\": \"Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym\",\n", + " \"w wieku przedprodukcyjnym\": \"Ludnosc_kobiety_w_wieku_przedprodukcyjnym\",\n", + " }\n", + ")\n", "\n", "df_ludn_3" ] @@ -1673,33 +1711,39 @@ "outputs": [], "source": [ "df_ludn_4 = pd.read_csv(\n", - " 'LUDN_2425_CREL.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_ludn_4 = df_ludn_4[['Kod', 'Wskaźniki', 'Rok', 'Wartosc']]\n", + " \"LUDN_2425_CREL.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_ludn_4 = df_ludn_4[[\"Kod\", \"Wskaźniki\", \"Rok\", \"Wartosc\"]]\n", "df_ludn_4 = df_ludn_4.dropna()\n", - "df_ludn_4 = df_ludn_4.loc[df_ludn_4['Kod'].str.endswith(('1', '2', '3'))]\n", - "df_ludn_4 = df_ludn_4.pivot_table(index=['Kod', 'Rok'], columns='Wskaźniki', values='Wartosc').reset_index()\n", - "df_ludn_4 = df_ludn_4.rename(columns={\n", - " 'gęstość zaludnienia powierzchni zabudowanej i zurbanizowanej (osoby/km2)': 'Gestosc_zaludnienia',\n", - " 'ludność na 1 km2': 'Ludnosc_na_1_km2',\n", - " 'ludność w tysiącach': 'Ludnosc',\n", - " 'ludność w tysiącach kobiety': 'Ludnosc_kobiety',\n", - " 'ludność w tysiącach mężczyźni': 'Ludnosc_mezczyzni',\n", - " 'wskaźnik urbanizacji': 'Wskaznik_urbanizacji',\n", - " 'zmiana liczby ludności na 1000 mieszkańców': 'Zmiana_liczby_ludnosci'})\n", + "df_ludn_4 = df_ludn_4.loc[df_ludn_4[\"Kod\"].str.endswith((\"1\", \"2\", \"3\"))]\n", + "df_ludn_4 = df_ludn_4.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Wskaźniki\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_ludn_4 = df_ludn_4.rename(\n", + " columns={\n", + " \"gęstość zaludnienia powierzchni zabudowanej i zurbanizowanej (osoby/km2)\": \"Gestosc_zaludnienia\",\n", + " \"ludność na 1 km2\": \"Ludnosc_na_1_km2\",\n", + " \"ludność w tysiącach\": \"Ludnosc\",\n", + " \"ludność w tysiącach kobiety\": \"Ludnosc_kobiety\",\n", + " \"ludność w tysiącach mężczyźni\": \"Ludnosc_mezczyzni\",\n", + " \"wskaźnik urbanizacji\": \"Wskaznik_urbanizacji\",\n", + " \"zmiana liczby ludności na 1000 mieszkańców\": \"Zmiana_liczby_ludnosci\",\n", + " }\n", + ")\n", "\n", - "df_ludn_4 = df_ludn_4[[\n", - " 'Kod',\n", - " 'Rok',\n", - " # 'Gestosc_zaludnienia',\n", - " 'Ludnosc_na_1_km2',\n", - " 'Ludnosc',\n", - " 'Ludnosc_kobiety',\n", - " 'Ludnosc_mezczyzni',\n", - " 'Wskaznik_urbanizacji',\n", - " 'Zmiana_liczby_ludnosci']]" + "df_ludn_4 = df_ludn_4[\n", + " [\n", + " \"Kod\",\n", + " \"Rok\",\n", + " # 'Gestosc_zaludnienia',\n", + " \"Ludnosc_na_1_km2\",\n", + " \"Ludnosc\",\n", + " \"Ludnosc_kobiety\",\n", + " \"Ludnosc_mezczyzni\",\n", + " \"Wskaznik_urbanizacji\",\n", + " \"Zmiana_liczby_ludnosci\",\n", + " ]\n", + "]" ] }, { @@ -2188,39 +2232,42 @@ ], "source": [ "df_ludn_5 = pd.read_csv(\n", - " 'LUDN_1355_CREL.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_ludn_5['Kierunki migracji'] = df_ludn_5['Kierunki migracji'] + df_ludn_5['Płeć']\n", - "df_ludn_5 = df_ludn_5[['Kod', 'Kierunki migracji', 'Rok', 'Wartosc']]\n", + " \"LUDN_1355_CREL.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_ludn_5[\"Kierunki migracji\"] = df_ludn_5[\"Kierunki migracji\"] + df_ludn_5[\"Płeć\"]\n", + "df_ludn_5 = df_ludn_5[[\"Kod\", \"Kierunki migracji\", \"Rok\", \"Wartosc\"]]\n", "df_ludn_5 = df_ludn_5.dropna()\n", - "df_ludn_5 = df_ludn_5.loc[df_ludn_5['Kod'].str.endswith(('1', '2', '3'))]\n", - "df_ludn_5 = df_ludn_5.pivot_table(index=['Kod', 'Rok'], columns='Kierunki migracji', values='Wartosc').reset_index()\n", - "df_ludn_5 = df_ludn_5.rename(columns={\n", - " 'saldo migracji na 1000 ludnościogółem': 'Saldo_migracji_na_1000_ludnosci',\n", - " 'saldo migracjiogółem': 'Saldo_migracji',\n", - " 'wymeldowania do miastkobiety': 'Wymeldowania_do_miast_kobiety',\n", - " 'wymeldowania do miastmężczyźni': 'Wymeldowania_do_miast_mezczyzni',\n", - " 'wymeldowania do miastogółem': 'Wymeldowania_do_miast_ogolem',\n", - " 'wymeldowania na wieśkobiety': 'Wymeldowania_na_wies_kobiety',\n", - " 'wymeldowania na wieśmężczyźni': 'Wymeldowania_na_wies_mezczyzni',\n", - " 'wymeldowania na wieśogółem': 'Wymeldowania_na_wies_ogolem',\n", - " 'wymeldowania ogółemkobiety': 'Wymeldowania_kobiety',\n", - " 'wymeldowania ogółemmężczyźni': 'Wymeldowania_mezczyzni',\n", - " 'wymeldowania ogółemogółem': 'Wymeldowania_ogolem',\n", - " 'wymeldowania za granicękobiety': 'Wymeldowania_za_granice_kobiety',\n", - " 'wymeldowania za granicęmężczyźni': 'Wymeldowania_za_granice_mezczyzni',\n", - " 'wymeldowania za granicęogółem': 'Wymeldowania_za_granice_ogolem',\n", - " 'zameldowania ogółemkobiety': 'Zameldowania_kobiety',\n", - " 'zameldowania ogółemmężczyźni': 'Zameldowania_mezczyzni',\n", - " 'zameldowania ogółemogółem': 'Zameldowania_ogolem',\n", - " 'zameldowania z miastkobiety': 'Zameldowania_z_miast_kobiety',\n", - " 'zameldowania z miastmężczyźni': 'Zameldowania_z_miast_mezczyzni',\n", - " 'zameldowania z miastogółem': 'Zameldowania_z_miast_ogolem',\n", - " 'zameldowania ze wsikobiety': 'Zameldowania_ze_wsi_kobiety',\n", - " 'zameldowania ze wsimężczyźni': 'Zameldowania_ze_wsi_mezczyzni',\n", - " 'zameldowania ze wsiogółem': 'Zameldowania_ze_wsi_ogolem'})\n", + "df_ludn_5 = df_ludn_5.loc[df_ludn_5[\"Kod\"].str.endswith((\"1\", \"2\", \"3\"))]\n", + "df_ludn_5 = df_ludn_5.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Kierunki migracji\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_ludn_5 = df_ludn_5.rename(\n", + " columns={\n", + " \"saldo migracji na 1000 ludnościogółem\": \"Saldo_migracji_na_1000_ludnosci\",\n", + " \"saldo migracjiogółem\": \"Saldo_migracji\",\n", + " \"wymeldowania do miastkobiety\": \"Wymeldowania_do_miast_kobiety\",\n", + " \"wymeldowania do miastmężczyźni\": \"Wymeldowania_do_miast_mezczyzni\",\n", + " \"wymeldowania do miastogółem\": \"Wymeldowania_do_miast_ogolem\",\n", + " \"wymeldowania na wieśkobiety\": \"Wymeldowania_na_wies_kobiety\",\n", + " \"wymeldowania na wieśmężczyźni\": \"Wymeldowania_na_wies_mezczyzni\",\n", + " \"wymeldowania na wieśogółem\": \"Wymeldowania_na_wies_ogolem\",\n", + " \"wymeldowania ogółemkobiety\": \"Wymeldowania_kobiety\",\n", + " \"wymeldowania ogółemmężczyźni\": \"Wymeldowania_mezczyzni\",\n", + " \"wymeldowania ogółemogółem\": \"Wymeldowania_ogolem\",\n", + " \"wymeldowania za granicękobiety\": \"Wymeldowania_za_granice_kobiety\",\n", + " \"wymeldowania za granicęmężczyźni\": \"Wymeldowania_za_granice_mezczyzni\",\n", + " \"wymeldowania za granicęogółem\": \"Wymeldowania_za_granice_ogolem\",\n", + " \"zameldowania ogółemkobiety\": \"Zameldowania_kobiety\",\n", + " \"zameldowania ogółemmężczyźni\": \"Zameldowania_mezczyzni\",\n", + " \"zameldowania ogółemogółem\": \"Zameldowania_ogolem\",\n", + " \"zameldowania z miastkobiety\": \"Zameldowania_z_miast_kobiety\",\n", + " \"zameldowania z miastmężczyźni\": \"Zameldowania_z_miast_mezczyzni\",\n", + " \"zameldowania z miastogółem\": \"Zameldowania_z_miast_ogolem\",\n", + " \"zameldowania ze wsikobiety\": \"Zameldowania_ze_wsi_kobiety\",\n", + " \"zameldowania ze wsimężczyźni\": \"Zameldowania_ze_wsi_mezczyzni\",\n", + " \"zameldowania ze wsiogółem\": \"Zameldowania_ze_wsi_ogolem\",\n", + " }\n", + ")\n", "\n", "df_ludn_5" ] @@ -2451,20 +2498,23 @@ ], "source": [ "df_tury = pd.read_csv(\n", - " 'TURY_2017_CREL.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_tury = df_tury[['Kod', 'Turystyczne obiekty noclegowe', 'Rok', 'Wartosc']]\n", + " \"TURY_2017_CREL.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_tury = df_tury[[\"Kod\", \"Turystyczne obiekty noclegowe\", \"Rok\", \"Wartosc\"]]\n", "df_tury = df_tury.dropna()\n", - "df_tury = df_tury.pivot_table(index=['Kod', 'Rok'], columns='Turystyczne obiekty noclegowe', values='Wartosc').reset_index()\n", - "df_tury = df_tury.rename(columns={\n", - " 'miejsca noclegowe całoroczne lipiec': 'Miejsca_noclegowe_caloroczne',\n", - " 'miejsca noclegowe ogółem lipiec': 'Miejsca_noclegowe_ogolem',\n", - " 'obiekty całoroczne lipiec': 'Obiekty_caloroczne',\n", - " 'obiekty ogółem lipiec': 'Obiekty_ogolem',\n", - " 'turyści (korzystający) ogółem styczeń-grudzień': 'Turysci_ogolem',\n", - " 'turyści zagraniczni (korzystający) - nierezydenci styczeń-grudzień': 'Turysci_zagraniczni'})\n", + "df_tury = df_tury.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Turystyczne obiekty noclegowe\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_tury = df_tury.rename(\n", + " columns={\n", + " \"miejsca noclegowe całoroczne lipiec\": \"Miejsca_noclegowe_caloroczne\",\n", + " \"miejsca noclegowe ogółem lipiec\": \"Miejsca_noclegowe_ogolem\",\n", + " \"obiekty całoroczne lipiec\": \"Obiekty_caloroczne\",\n", + " \"obiekty ogółem lipiec\": \"Obiekty_ogolem\",\n", + " \"turyści (korzystający) ogółem styczeń-grudzień\": \"Turysci_ogolem\",\n", + " \"turyści zagraniczni (korzystający) - nierezydenci styczeń-grudzień\": \"Turysci_zagraniczni\",\n", + " }\n", + ")\n", "\n", "df_tury" ] @@ -2707,21 +2757,24 @@ ], "source": [ "df_ryne = pd.read_csv(\n", - " 'RYNE_3733_CREL.csv',\n", - " sep=';',\n", - " converters={'Kod': str},\n", - " decimal=',')\n", - "df_ryne = df_ryne[['Kod', 'Bezrobotni', 'Rok', 'Wartosc']]\n", + " \"RYNE_3733_CREL.csv\", sep=\";\", converters={\"Kod\": str}, decimal=\",\"\n", + ")\n", + "df_ryne = df_ryne[[\"Kod\", \"Bezrobotni\", \"Rok\", \"Wartosc\"]]\n", "df_ryne = df_ryne.dropna()\n", - "df_ryne = df_ryne.pivot_table(index=['Kod', 'Rok'], columns='Bezrobotni', values='Wartosc').reset_index()\n", - "df_ryne = df_ryne.rename(columns={\n", - " 'do 25 roku życia': 'Bezrobotni_do_25_roku_zycia',\n", - " 'do 30 roku życia': 'Bezrobotni_do_30_roku_zycia',\n", - " 'długotrwale bezrobotni': 'Dlugotrwale_bezrobotni',\n", - " 'kobiety': 'Bezrobotne_kobiety',\n", - " 'mężczyźni': 'Bezrobotni_mezczyzni',\n", - " 'ogółem': 'Bezrobotni_ogolem',\n", - " 'powyżej 50 roku życia': 'Bezrobotni_powyzej_50_roku_zycia'})\n", + "df_ryne = df_ryne.pivot_table(\n", + " index=[\"Kod\", \"Rok\"], columns=\"Bezrobotni\", values=\"Wartosc\"\n", + ").reset_index()\n", + "df_ryne = df_ryne.rename(\n", + " columns={\n", + " \"do 25 roku życia\": \"Bezrobotni_do_25_roku_zycia\",\n", + " \"do 30 roku życia\": \"Bezrobotni_do_30_roku_zycia\",\n", + " \"długotrwale bezrobotni\": \"Dlugotrwale_bezrobotni\",\n", + " \"kobiety\": \"Bezrobotne_kobiety\",\n", + " \"mężczyźni\": \"Bezrobotni_mezczyzni\",\n", + " \"ogółem\": \"Bezrobotni_ogolem\",\n", + " \"powyżej 50 roku życia\": \"Bezrobotni_powyzej_50_roku_zycia\",\n", + " }\n", + ")\n", "\n", "df_ryne" ] @@ -2758,24 +2811,34 @@ "outputs": [], "source": [ "wojewodztwo_dictionary = {\n", - "'02': 'Dolnoslaskie',\n", - "'04': 'Kujawsko_Pomorskie',\n", - "'06': 'Lubelskie',\n", - "'08': 'Lubuskie',\n", - "'10': 'Lodzkie',\n", - "'12': 'Malopolskie',\n", - "'14': 'Mazowieckie',\n", - "'16': 'Opolskie',\n", - "'18': 'Podkarpackie',\n", - "'20': 'Podlaskie',\n", - "'22': 'Pomorskie',\n", - "'24': 'Slaskie',\n", - "'26': 'Swietokrzyskie',\n", - "'28': 'Warminsko_Mazurskie',\n", - "'30': 'Wielkopolskie',\n", - "'32': 'Zachodniopomorskie'}\n", + " \"02\": \"Dolnoslaskie\",\n", + " \"04\": \"Kujawsko_Pomorskie\",\n", + " \"06\": \"Lubelskie\",\n", + " \"08\": \"Lubuskie\",\n", + " \"10\": \"Lodzkie\",\n", + " \"12\": \"Malopolskie\",\n", + " \"14\": \"Mazowieckie\",\n", + " \"16\": \"Opolskie\",\n", + " \"18\": \"Podkarpackie\",\n", + " \"20\": \"Podlaskie\",\n", + " \"22\": \"Pomorskie\",\n", + " \"24\": \"Slaskie\",\n", + " \"26\": \"Swietokrzyskie\",\n", + " \"28\": \"Warminsko_Mazurskie\",\n", + " \"30\": \"Wielkopolskie\",\n", + " \"32\": \"Zachodniopomorskie\",\n", + "}\n", "\n", - "df_data = pd.concat([df_data, pd.get_dummies(df_data['Kod'].apply(lambda x: wojewodztwo_dictionary.get(x[:2], None)), prefix='Wojewodztwo').astype(int)], axis=1)" + "df_data = pd.concat(\n", + " [\n", + " df_data,\n", + " pd.get_dummies(\n", + " df_data[\"Kod\"].apply(lambda x: wojewodztwo_dictionary.get(x[:2])),\n", + " prefix=\"Wojewodztwo\",\n", + " ).astype(int),\n", + " ],\n", + " axis=1,\n", + ")" ] }, { @@ -2785,11 +2848,20 @@ "outputs": [], "source": [ "rodzaj_gminy_dictionary = {\n", - "'1': 'Gmina_miejska',\n", - "'2': 'Gmina_wiejska',\n", - "'3': 'Gmina_miejsko_wiejska'}\n", + " \"1\": \"Gmina_miejska\",\n", + " \"2\": \"Gmina_wiejska\",\n", + " \"3\": \"Gmina_miejsko_wiejska\",\n", + "}\n", "\n", - "df_data = pd.concat([df_data, pd.get_dummies(df_data['Kod'].apply(lambda x: rodzaj_gminy_dictionary.get(x[-1], None))).astype(int)], axis=1)" + "df_data = pd.concat(\n", + " [\n", + " df_data,\n", + " pd.get_dummies(\n", + " df_data[\"Kod\"].apply(lambda x: rodzaj_gminy_dictionary.get(x[-1]))\n", + " ).astype(int),\n", + " ],\n", + " axis=1,\n", + ")" ] }, { @@ -2806,8 +2878,14 @@ } ], "source": [ - "df_data = df_data.merge(df_podz, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_podz['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_podz'))\n", - "df_data = df_data.drop(['key_0', 'Kod_podz'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_podz,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_podz[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_podz\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_podz\"], axis=1)\n", "print(len(df_data))" ] }, @@ -2825,8 +2903,14 @@ } ], "source": [ - "df_data = df_data.merge(df_wyna, left_on=[df_data['Kod'].str.slice(stop=-3), 'Rok'], right_on=[df_wyna['Kod'].str.slice(stop=-3), 'Rok'], how='left', suffixes=(None, '_wyna'))\n", - "df_data = df_data.drop(['key_0', 'Kod_wyna'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_wyna,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-3), \"Rok\"],\n", + " right_on=[df_wyna[\"Kod\"].str.slice(stop=-3), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_wyna\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_wyna\"], axis=1)\n", "print(len(df_data))" ] }, @@ -2845,12 +2929,24 @@ } ], "source": [ - "df_data = df_data.merge(df_fina_1, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_fina_1['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_fina_1'))\n", - "df_data = df_data.drop(['key_0', 'Kod_fina_1'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_fina_1,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_fina_1[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_fina_1\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_fina_1\"], axis=1)\n", "print(len(df_data))\n", "\n", - "df_data = df_data.merge(df_fina_2, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_fina_2['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_fina_2'))\n", - "df_data = df_data.drop(['key_0', 'Kod_fina_2'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_fina_2,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_fina_2[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_fina_2\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_fina_2\"], axis=1)\n", "print(len(df_data))" ] }, @@ -2873,28 +2969,58 @@ ], "source": [ "# df_data = df_data.merge(df_ludn_1, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_1'))\n", - "df_data = df_data.merge(df_ludn_1, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_ludn_1['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_ludn_1'))\n", - "df_data = df_data.drop(['key_0', 'Kod_ludn_1'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_ludn_1,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_ludn_1[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_ludn_1\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_ludn_1\"], axis=1)\n", "print(len(df_data))\n", "\n", "# df_data = df_data.merge(df_ludn_2, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_2'))\n", - "df_data = df_data.merge(df_ludn_2, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_ludn_2['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_ludn_2'))\n", - "df_data = df_data.drop(['key_0', 'Kod_ludn_2'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_ludn_2,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_ludn_2[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_ludn_2\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_ludn_2\"], axis=1)\n", "print(len(df_data))\n", "\n", "# df_data = df_data.merge(df_ludn_3, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_3'))\n", - "df_data = df_data.merge(df_ludn_3, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_ludn_3['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_ludn_3'))\n", - "df_data = df_data.drop(['key_0', 'Kod_ludn_3'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_ludn_3,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_ludn_3[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_ludn_3\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_ludn_3\"], axis=1)\n", "print(len(df_data))\n", "\n", "# df_data = df_data.merge(df_ludn_4, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_4'))\n", - "df_data = df_data.merge(df_ludn_4, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_ludn_4['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_ludn_4'))\n", - "df_data = df_data.drop(['key_0', 'Kod_ludn_4'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_ludn_4,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_ludn_4[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_ludn_4\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_ludn_4\"], axis=1)\n", "print(len(df_data))\n", "\n", "# df_data = df_data.merge(df_ludn_5, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_5'))\n", - "df_data = df_data.merge(df_ludn_5, left_on=[df_data['Kod'].str.slice(stop=-1), 'Rok'], right_on=[df_ludn_5['Kod'].str.slice(stop=-1), 'Rok'], how='left', suffixes=(None, '_ludn_5'))\n", - "df_data = df_data.drop(['key_0', 'Kod_ludn_5'], axis=1)\n", + "df_data = df_data.merge(\n", + " df_ludn_5,\n", + " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " right_on=[df_ludn_5[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_ludn_5\"),\n", + ")\n", + "df_data = df_data.drop([\"key_0\", \"Kod_ludn_5\"], axis=1)\n", "print(len(df_data))\n", "\n", "# df_data['Kod'].value_counts()" @@ -2914,7 +3040,13 @@ } ], "source": [ - "df_data = df_data.merge(df_tury, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_tury'))\n", + "df_data = df_data.merge(\n", + " df_tury,\n", + " left_on=[\"Kod\", \"Rok\"],\n", + " right_on=[\"Kod\", \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_tury\"),\n", + ")\n", "# df_data = df_data.drop(['key_0', 'Kod_tury'], axis=1)\n", "print(len(df_data))" ] @@ -2933,7 +3065,13 @@ } ], "source": [ - "df_data = df_data.merge(df_ryne, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ryne'))\n", + "df_data = df_data.merge(\n", + " df_ryne,\n", + " left_on=[\"Kod\", \"Rok\"],\n", + " right_on=[\"Kod\", \"Rok\"],\n", + " how=\"left\",\n", + " suffixes=(None, \"_ryne\"),\n", + ")\n", "# df_data = df_data.drop(['key_0', 'Kod_ryne'], axis=1)\n", "print(len(df_data))" ] @@ -2955,9 +3093,9 @@ } ], "source": [ - "df_data['Gestosc_zaludnienia'] = df_data['Ludnosc'] / df_data['Powierzchnia']\n", + "df_data[\"Gestosc_zaludnienia\"] = df_data[\"Ludnosc\"] / df_data[\"Powierzchnia\"]\n", "\n", - "df_data['Gestosc_zaludnienia'].mean()" + "df_data[\"Gestosc_zaludnienia\"].mean()" ] }, { @@ -3016,7 +3154,7 @@ "# 'Wymeldowania_za_granice_mezczyzni',\n", "# 'Wymeldowania_za_granice_ogolem',\n", "# 'Bezrobotni_do_30_roku_zycia',\n", - " \n", + "\n", "# 'Wojewodztwo_Dolnoslaskie', # 43\n", "# 'Wojewodztwo_Kujawsko_Pomorskie', # 44\n", "# 'Wojewodztwo_Lubelskie', # 45\n", @@ -3184,14 +3322,19 @@ ], "source": [ "# Usunięcie niepotrzebnych kolumn...\n", - "df_data.drop(columns=[\n", - " # 'Program_operacyjny',\n", - " 'Wymeldowania_za_granice_kobiety',\n", - " 'Wymeldowania_za_granice_mezczyzni',\n", - " 'Wymeldowania_za_granice_ogolem',\n", - " 'Bezrobotni_do_30_roku_zycia'], inplace=True, errors='ignore')\n", + "df_data.drop(\n", + " columns=[\n", + " # 'Program_operacyjny',\n", + " \"Wymeldowania_za_granice_kobiety\",\n", + " \"Wymeldowania_za_granice_mezczyzni\",\n", + " \"Wymeldowania_za_granice_ogolem\",\n", + " \"Bezrobotni_do_30_roku_zycia\",\n", + " ],\n", + " inplace=True,\n", + " errors=\"ignore\",\n", + ")\n", "\n", - "df_data.to_csv('dane1.csv', index=False)\n", + "df_data.to_csv(\"dane1.csv\", index=False)\n", "\n", "for c in df_data.columns:\n", " print(c)" @@ -3260,153 +3403,167 @@ } ], "source": [ - "from sklearn.model_selection import train_test_split\n", - "from sklearn.tree import DecisionTreeRegressor, plot_tree, export_text\n", - "from sklearn.metrics import mean_squared_error\n", "import matplotlib.pyplot as plt\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeRegressor, export_text\n", "\n", - "df_data[[\n", - " 'Miejsca_noclegowe_caloroczne',\n", - " 'Miejsca_noclegowe_ogolem',\n", - " 'Obiekty_caloroczne',\n", - " 'Obiekty_ogolem',\n", - " 'Turysci_ogolem',\n", - " 'Turysci_zagraniczni']] = df_data[[\n", - " 'Miejsca_noclegowe_caloroczne',\n", - " 'Miejsca_noclegowe_ogolem',\n", - " 'Obiekty_caloroczne',\n", - " 'Obiekty_ogolem',\n", - " 'Turysci_ogolem',\n", - " 'Turysci_zagraniczni']].fillna(0)\n", + "df_data[\n", + " [\n", + " \"Miejsca_noclegowe_caloroczne\",\n", + " \"Miejsca_noclegowe_ogolem\",\n", + " \"Obiekty_caloroczne\",\n", + " \"Obiekty_ogolem\",\n", + " \"Turysci_ogolem\",\n", + " \"Turysci_zagraniczni\",\n", + " ]\n", + "] = df_data[\n", + " [\n", + " \"Miejsca_noclegowe_caloroczne\",\n", + " \"Miejsca_noclegowe_ogolem\",\n", + " \"Obiekty_caloroczne\",\n", + " \"Obiekty_ogolem\",\n", + " \"Turysci_ogolem\",\n", + " \"Turysci_zagraniczni\",\n", + " ]\n", + "].fillna(0)\n", "\n", "feature_names = [\n", - " 'Powierzchnia', # 1\n", - " 'Wynagrodzenie_ogolem', # 2\n", - " 'Wynagrodzenie_w_relacji_do_sredniej', # 3\n", - " 'Dochody_podatek_lesny', # 4\n", - " 'Dochody_podatek_PCC', # 5\n", - " 'Dochody_podatek_od_dzialalnosci_gospodarczej', # 6\n", - " 'Dochody_podatek_od_nieruchomosci', # 7\n", - " 'Dochody_podatek_od_spadkow', # 8\n", - " 'Dochody_podatek_od_srodkow_transportowych', # 9\n", - " 'Dochody_podatek_rolny', # 10\n", - " 'Dochody_podatek_odrebne_ustawy', # 11\n", - " 'Dochody_razem', # 12\n", - " 'Dochody_z_majatku', # 13\n", - " 'Dochody_z_najmu_i_dzierzawy', # 14\n", - " 'Dochody_z_uslug', # 15\n", - " 'Dochody_dofinansowanie_inwestycyjne', # 16\n", - " 'Dochody_dofinansowanie_razem', # 17\n", - " 'Udzialy_w_podatkach_dochodowych_od_osob_fizycznych', # 18\n", - " 'Udzialy_w_podatkach_dochodowych_od_osob_prywatnych', # 19\n", - " 'Udzialy_w_podatkach_dochodowych_razem', # 20\n", - " 'Wplywy_z_innych_lokalnych_oplat', # 21\n", - " 'Wplywy_z_oplaty_eksploatacyjnej', # 22\n", - " 'Wplywy_z_oplaty_skarbowej', # 23\n", - " 'Wplywy_z_oplaty_targowej', # 24\n", - " 'Ludnosc_ogolem', # 25\n", - " 'Ludnosc_w_wieku_poprodukcyjnym', # 26\n", - " 'Ludnosc_w_wieku_produkcyjnym', # 27\n", - " 'Ludnosc_w_wieku_produkcyjnym_mobilnym', # 28\n", - " 'Ludnosc_w_wieku_produkcyjnym_niemobilnym', # 29\n", - " 'Ludnosc_w_wieku_przedprodukcyjnym', # 30\n", - " 'Ludnosc_mezczyzni', # 31\n", - " 'Ludnosc_mezczyzni_w_wieku_poprodukcyjnym', # 32\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym', # 33\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym', # 34\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym', # 35\n", - " 'Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym', # 36\n", - " 'Ludnosc_kobiety', # 37\n", - " 'Ludnosc_kobiety_w_wieku_poprodukcyjnym', # 38\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym', # 39\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym', # 40\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym', # 41\n", - " 'Ludnosc_kobiety_w_wieku_przedprodukcyjnym', # 42\n", - " 'Wojewodztwo_Dolnoslaskie', # 43\n", - " 'Wojewodztwo_Kujawsko_Pomorskie', # 44\n", - " 'Wojewodztwo_Lubelskie', # 45\n", - " 'Wojewodztwo_Lubuskie', # 46\n", - " 'Wojewodztwo_Lodzkie', # 47\n", - " 'Wojewodztwo_Malopolskie', # 48\n", - " 'Wojewodztwo_Mazowieckie', # 49\n", - " 'Wojewodztwo_Opolskie', # 50\n", - " 'Wojewodztwo_Podkarpackie', # 51\n", - " 'Wojewodztwo_Podlaskie', # 52\n", - " 'Wojewodztwo_Pomorskie', # 53\n", - " 'Wojewodztwo_Slaskie', # 54\n", - " 'Wojewodztwo_Swietokrzyskie', # 55\n", - " 'Wojewodztwo_Warminsko_Mazurskie', # 56\n", - " 'Wojewodztwo_Wielkopolskie', # 57\n", - " 'Wojewodztwo_Zachodniopomorskie', # 58\n", - " 'Gestosc_zaludnienia', # 59\n", - " 'Ludnosc_na_1_km2', # 60\n", - " 'Ludnosc', # 61\n", - " 'Ludnosc_kobiety', # 62\n", - " 'Ludnosc_mezczyzni', # 63\n", - " 'Wskaznik_urbanizacji', # 64\n", - " 'Zmiana_liczby_ludnosci', # 65\n", - " 'Saldo_migracji_na_1000_ludnosci', # 66\n", - " 'Saldo_migracji', # 67\n", - " 'Wymeldowania_do_miast_kobiety', # 68\n", - " 'Wymeldowania_do_miast_mezczyzni', # 69\n", - " 'Wymeldowania_do_miast_ogolem', # 70\n", - " 'Wymeldowania_na_wies_kobiety', # 71\n", - " 'Wymeldowania_na_wies_mezczyzni', # 72\n", - " 'Wymeldowania_na_wies_ogolem', # 73\n", - " 'Wymeldowania_kobiety', # 74\n", - " 'Wymeldowania_mezczyzni', # 75\n", - " 'Wymeldowania_ogolem', # 76\n", - " 'Zameldowania_kobiety', # 77\n", - " 'Zameldowania_mezczyzni', # 78\n", - " 'Zameldowania_ogolem', # 79\n", - " 'Zameldowania_z_miast_kobiety', # 80\n", - " 'Zameldowania_z_miast_mezczyzni', # 81\n", - " 'Zameldowania_z_miast_ogolem', # 82\n", - " 'Zameldowania_ze_wsi_kobiety', # 83\n", - " 'Zameldowania_ze_wsi_mezczyzni', # 84\n", - " 'Zameldowania_ze_wsi_ogolem', # 85\n", - " 'Miejsca_noclegowe_caloroczne', # 86\n", - " 'Miejsca_noclegowe_ogolem', # 87\n", - " 'Obiekty_caloroczne', # 88\n", - " 'Obiekty_ogolem', # 89\n", - " 'Turysci_ogolem', # 90\n", - " 'Turysci_zagraniczni', # 91\n", - " 'Bezrobotni_do_25_roku_zycia', # 92\n", - " 'Dlugotrwale_bezrobotni', # 93\n", - " 'Bezrobotne_kobiety', # 94\n", - " 'Bezrobotni_mezczyzni', # 95\n", - " 'Bezrobotni_ogolem', # 96\n", - " 'Bezrobotni_powyzej_50_roku_zycia', # 97\n", - " 'Gmina_miejska', # 98\n", - " 'Gmina_miejsko_wiejska', # 99\n", - " 'Gmina_wiejska'] # 100\n", + " \"Powierzchnia\", # 1\n", + " \"Wynagrodzenie_ogolem\", # 2\n", + " \"Wynagrodzenie_w_relacji_do_sredniej\", # 3\n", + " \"Dochody_podatek_lesny\", # 4\n", + " \"Dochody_podatek_PCC\", # 5\n", + " \"Dochody_podatek_od_dzialalnosci_gospodarczej\", # 6\n", + " \"Dochody_podatek_od_nieruchomosci\", # 7\n", + " \"Dochody_podatek_od_spadkow\", # 8\n", + " \"Dochody_podatek_od_srodkow_transportowych\", # 9\n", + " \"Dochody_podatek_rolny\", # 10\n", + " \"Dochody_podatek_odrebne_ustawy\", # 11\n", + " \"Dochody_razem\", # 12\n", + " \"Dochody_z_majatku\", # 13\n", + " \"Dochody_z_najmu_i_dzierzawy\", # 14\n", + " \"Dochody_z_uslug\", # 15\n", + " \"Dochody_dofinansowanie_inwestycyjne\", # 16\n", + " \"Dochody_dofinansowanie_razem\", # 17\n", + " \"Udzialy_w_podatkach_dochodowych_od_osob_fizycznych\", # 18\n", + " \"Udzialy_w_podatkach_dochodowych_od_osob_prywatnych\", # 19\n", + " \"Udzialy_w_podatkach_dochodowych_razem\", # 20\n", + " \"Wplywy_z_innych_lokalnych_oplat\", # 21\n", + " \"Wplywy_z_oplaty_eksploatacyjnej\", # 22\n", + " \"Wplywy_z_oplaty_skarbowej\", # 23\n", + " \"Wplywy_z_oplaty_targowej\", # 24\n", + " \"Ludnosc_ogolem\", # 25\n", + " \"Ludnosc_w_wieku_poprodukcyjnym\", # 26\n", + " \"Ludnosc_w_wieku_produkcyjnym\", # 27\n", + " \"Ludnosc_w_wieku_produkcyjnym_mobilnym\", # 28\n", + " \"Ludnosc_w_wieku_produkcyjnym_niemobilnym\", # 29\n", + " \"Ludnosc_w_wieku_przedprodukcyjnym\", # 30\n", + " \"Ludnosc_mezczyzni\", # 31\n", + " \"Ludnosc_mezczyzni_w_wieku_poprodukcyjnym\", # 32\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym\", # 33\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym\", # 34\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym\", # 35\n", + " \"Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym\", # 36\n", + " \"Ludnosc_kobiety\", # 37\n", + " \"Ludnosc_kobiety_w_wieku_poprodukcyjnym\", # 38\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym\", # 39\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym\", # 40\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym\", # 41\n", + " \"Ludnosc_kobiety_w_wieku_przedprodukcyjnym\", # 42\n", + " \"Wojewodztwo_Dolnoslaskie\", # 43\n", + " \"Wojewodztwo_Kujawsko_Pomorskie\", # 44\n", + " \"Wojewodztwo_Lubelskie\", # 45\n", + " \"Wojewodztwo_Lubuskie\", # 46\n", + " \"Wojewodztwo_Lodzkie\", # 47\n", + " \"Wojewodztwo_Malopolskie\", # 48\n", + " \"Wojewodztwo_Mazowieckie\", # 49\n", + " \"Wojewodztwo_Opolskie\", # 50\n", + " \"Wojewodztwo_Podkarpackie\", # 51\n", + " \"Wojewodztwo_Podlaskie\", # 52\n", + " \"Wojewodztwo_Pomorskie\", # 53\n", + " \"Wojewodztwo_Slaskie\", # 54\n", + " \"Wojewodztwo_Swietokrzyskie\", # 55\n", + " \"Wojewodztwo_Warminsko_Mazurskie\", # 56\n", + " \"Wojewodztwo_Wielkopolskie\", # 57\n", + " \"Wojewodztwo_Zachodniopomorskie\", # 58\n", + " \"Gestosc_zaludnienia\", # 59\n", + " \"Ludnosc_na_1_km2\", # 60\n", + " \"Ludnosc\", # 61\n", + " \"Ludnosc_kobiety\", # 62\n", + " \"Ludnosc_mezczyzni\", # 63\n", + " \"Wskaznik_urbanizacji\", # 64\n", + " \"Zmiana_liczby_ludnosci\", # 65\n", + " \"Saldo_migracji_na_1000_ludnosci\", # 66\n", + " \"Saldo_migracji\", # 67\n", + " \"Wymeldowania_do_miast_kobiety\", # 68\n", + " \"Wymeldowania_do_miast_mezczyzni\", # 69\n", + " \"Wymeldowania_do_miast_ogolem\", # 70\n", + " \"Wymeldowania_na_wies_kobiety\", # 71\n", + " \"Wymeldowania_na_wies_mezczyzni\", # 72\n", + " \"Wymeldowania_na_wies_ogolem\", # 73\n", + " \"Wymeldowania_kobiety\", # 74\n", + " \"Wymeldowania_mezczyzni\", # 75\n", + " \"Wymeldowania_ogolem\", # 76\n", + " \"Zameldowania_kobiety\", # 77\n", + " \"Zameldowania_mezczyzni\", # 78\n", + " \"Zameldowania_ogolem\", # 79\n", + " \"Zameldowania_z_miast_kobiety\", # 80\n", + " \"Zameldowania_z_miast_mezczyzni\", # 81\n", + " \"Zameldowania_z_miast_ogolem\", # 82\n", + " \"Zameldowania_ze_wsi_kobiety\", # 83\n", + " \"Zameldowania_ze_wsi_mezczyzni\", # 84\n", + " \"Zameldowania_ze_wsi_ogolem\", # 85\n", + " \"Miejsca_noclegowe_caloroczne\", # 86\n", + " \"Miejsca_noclegowe_ogolem\", # 87\n", + " \"Obiekty_caloroczne\", # 88\n", + " \"Obiekty_ogolem\", # 89\n", + " \"Turysci_ogolem\", # 90\n", + " \"Turysci_zagraniczni\", # 91\n", + " \"Bezrobotni_do_25_roku_zycia\", # 92\n", + " \"Dlugotrwale_bezrobotni\", # 93\n", + " \"Bezrobotne_kobiety\", # 94\n", + " \"Bezrobotni_mezczyzni\", # 95\n", + " \"Bezrobotni_ogolem\", # 96\n", + " \"Bezrobotni_powyzej_50_roku_zycia\", # 97\n", + " \"Gmina_miejska\", # 98\n", + " \"Gmina_miejsko_wiejska\", # 99\n", + " \"Gmina_wiejska\",\n", + "] # 100\n", "\n", - "df_data.drop(columns=[\n", - " 'Program_operacyjny',\n", - " 'Wymeldowania_za_granice_kobiety',\n", - " 'Wymeldowania_za_granice_mezczyzni',\n", - " 'Wymeldowania_za_granice_ogolem',\n", - " 'Bezrobotni_do_30_roku_zycia'], inplace=True, errors='ignore')\n", + "df_data.drop(\n", + " columns=[\n", + " \"Program_operacyjny\",\n", + " \"Wymeldowania_za_granice_kobiety\",\n", + " \"Wymeldowania_za_granice_mezczyzni\",\n", + " \"Wymeldowania_za_granice_ogolem\",\n", + " \"Bezrobotni_do_30_roku_zycia\",\n", + " ],\n", + " inplace=True,\n", + " errors=\"ignore\",\n", + ")\n", "\n", "print(len(df_data))\n", "df_data.dropna(inplace=True)\n", - "df_data = df_data[df_data['Suma'] > 0]\n", + "df_data = df_data[df_data[\"Suma\"] > 0]\n", "print(len(df_data))\n", "\n", "X = df_data[feature_names]\n", - "y = df_data['Suma']\n", + "y = df_data[\"Suma\"]\n", "\n", - "color_column = df_data['Gestosc_zaludnienia']\n", - "color_column = (df_data['Gestosc_zaludnienia'] > 1.5).astype(int)\n", + "color_column = df_data[\"Gestosc_zaludnienia\"]\n", + "color_column = (df_data[\"Gestosc_zaludnienia\"] > 1.5).astype(int)\n", "\n", - "X_train, X_test, y_train, y_test, color_column_train, color_column_test = train_test_split(X, y, color_column, test_size=0.2, random_state=1)\n", + "X_train, X_test, y_train, y_test, color_column_train, color_column_test = (\n", + " train_test_split(X, y, color_column, test_size=0.2, random_state=1)\n", + ")\n", "\n", "model = DecisionTreeRegressor(random_state=1)\n", "model.fit(X_train, y_train)\n", "\n", "y_pred = model.predict(X_test)\n", "mse = mean_squared_error(y_test, y_pred)\n", - "print('Mean Squared Error:', mse)" + "print(\"Mean Squared Error:\", mse)" ] }, { @@ -3452,7 +3609,7 @@ ], "source": [ "correlation_matrix = df_data.corr()\n", - "print(correlation_matrix['Suma'].sort_values(ascending=False)[1:29])" + "print(correlation_matrix[\"Suma\"].sort_values(ascending=False)[1:29])" ] }, { @@ -3472,10 +3629,10 @@ } ], "source": [ - "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap='viridis')\n", - "plt.xlabel('Actual')\n", - "plt.ylabel('Predicted')\n", - "plt.title('Actual vs Predicted')\n", + "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap=\"viridis\")\n", + "plt.xlabel(\"Actual\")\n", + "plt.ylabel(\"Predicted\")\n", + "plt.title(\"Actual vs Predicted\")\n", "\n", "plt.xlim(0, max(max(y_test), max(y_pred)))\n", "plt.ylim(0, max(max(y_test), max(y_pred)))\n", @@ -3500,13 +3657,13 @@ } ], "source": [ - "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap='viridis')\n", - "plt.xlabel('Actual')\n", - "plt.ylabel('Predicted')\n", - "plt.title('Actual vs Predicted')\n", + "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap=\"viridis\")\n", + "plt.xlabel(\"Actual\")\n", + "plt.ylabel(\"Predicted\")\n", + "plt.title(\"Actual vs Predicted\")\n", "\n", - "plt.xlim(0, 3*10**7)\n", - "plt.ylim(0, 3*10**7)\n", + "plt.xlim(0, 3 * 10**7)\n", + "plt.ylim(0, 3 * 10**7)\n", "\n", "plt.show()" ] @@ -3528,13 +3685,13 @@ } ], "source": [ - "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap='viridis')\n", - "plt.xlabel('Actual')\n", - "plt.ylabel('Predicted')\n", - "plt.title('Actual vs Predicted')\n", + "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap=\"viridis\")\n", + "plt.xlabel(\"Actual\")\n", + "plt.ylabel(\"Predicted\")\n", + "plt.title(\"Actual vs Predicted\")\n", "\n", - "plt.xlim(0, 3*10**6)\n", - "plt.ylim(0, 3*10**6)\n", + "plt.xlim(0, 3 * 10**6)\n", + "plt.ylim(0, 3 * 10**6)\n", "\n", "plt.show()" ] @@ -3963,8 +4120,10 @@ ], "source": [ "feature_importance = dict(zip(feature_names, model.feature_importances_))\n", - "for feature, importance in sorted(feature_importance.items(), key=lambda x: x[1], reverse=True):\n", - " print(f'{importance:.5f} \\u2014 {feature}')" + "for feature, importance in sorted(\n", + " feature_importance.items(), key=lambda x: x[1], reverse=True\n", + "):\n", + " print(f\"{importance:.5f} \\u2014 {feature}\")" ] }, { diff --git a/Programming/PBAD/main.ipynb b/Programming/PBAD/main.ipynb index 85ca9d7e..10bd7945 100644 --- a/Programming/PBAD/main.ipynb +++ b/Programming/PBAD/main.ipynb @@ -18,139 +18,140 @@ } ], "source": [ - "import pandas as pd\n", "from statistics import median\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.feature_selection import RFE\n", - "from sklearn.tree import DecisionTreeRegressor, plot_tree\n", - "import numpy as np\n", - "from sklearn.metrics import mean_squared_error\n", + "\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", "import seaborn as sns\n", + "from sklearn.feature_selection import RFE\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeRegressor\n", "\n", "feature_names = [\n", - " 'Powierzchnia', # 1\n", - " 'Wynagrodzenie_ogolem', # 2\n", - " 'Wynagrodzenie_w_relacji_do_sredniej', # 3\n", - " 'Dochody_podatek_lesny', # 4\n", - " 'Dochody_podatek_PCC', # 5\n", - " 'Dochody_podatek_od_dzialalnosci_gospodarczej', # 6\n", - " 'Dochody_podatek_od_nieruchomosci', # 7\n", - " 'Dochody_podatek_od_spadkow', # 8\n", - " 'Dochody_podatek_od_srodkow_transportowych', # 9\n", - " 'Dochody_podatek_rolny', # 10\n", - " 'Dochody_podatek_odrebne_ustawy', # 11\n", - " 'Dochody_razem', # 12\n", - " 'Dochody_z_majatku', # 13\n", - " 'Dochody_z_najmu_i_dzierzawy', # 14\n", - " 'Dochody_z_uslug', # 15\n", - " 'Dochody_dofinansowanie_inwestycyjne', # 16\n", - " 'Dochody_dofinansowanie_razem', # 17\n", - " 'Udzialy_w_podatkach_dochodowych_od_osob_fizycznych', # 18\n", - " 'Udzialy_w_podatkach_dochodowych_od_osob_prywatnych', # 19\n", - " 'Udzialy_w_podatkach_dochodowych_razem', # 20\n", - " 'Wplywy_z_innych_lokalnych_oplat', # 21\n", - " 'Wplywy_z_oplaty_eksploatacyjnej', # 22\n", - " 'Wplywy_z_oplaty_skarbowej', # 23\n", - " 'Wplywy_z_oplaty_targowej', # 24\n", - " 'Ludnosc_ogolem', # 25\n", - " 'Ludnosc_w_wieku_poprodukcyjnym', # 26\n", - " 'Ludnosc_w_wieku_produkcyjnym', # 27\n", - " 'Ludnosc_w_wieku_produkcyjnym_mobilnym', # 28\n", - " 'Ludnosc_w_wieku_produkcyjnym_niemobilnym', # 29\n", - " 'Ludnosc_w_wieku_przedprodukcyjnym', # 30\n", - " 'Ludnosc_mezczyzni', # 31\n", - " 'Ludnosc_mezczyzni_w_wieku_poprodukcyjnym', # 32\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym', # 33\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym', # 34\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym', # 35\n", - " 'Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym', # 36\n", - " 'Ludnosc_kobiety', # 37\n", - " 'Ludnosc_kobiety_w_wieku_poprodukcyjnym', # 38\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym', # 39\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym', # 40\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym', # 41\n", - " 'Ludnosc_kobiety_w_wieku_przedprodukcyjnym', # 42\n", - " 'Wojewodztwo_Dolnoslaskie', # 43\n", - " 'Wojewodztwo_Kujawsko_Pomorskie', # 44\n", - " 'Wojewodztwo_Lubelskie', # 45\n", - " 'Wojewodztwo_Lubuskie', # 46\n", - " 'Wojewodztwo_Lodzkie', # 47\n", - " 'Wojewodztwo_Malopolskie', # 48\n", - " 'Wojewodztwo_Mazowieckie', # 49\n", - " 'Wojewodztwo_Opolskie', # 50\n", - " 'Wojewodztwo_Podkarpackie', # 51\n", - " 'Wojewodztwo_Podlaskie', # 52\n", - " 'Wojewodztwo_Pomorskie', # 53\n", - " 'Wojewodztwo_Slaskie', # 54\n", - " 'Wojewodztwo_Swietokrzyskie', # 55\n", - " 'Wojewodztwo_Warminsko_Mazurskie', # 56\n", - " 'Wojewodztwo_Wielkopolskie', # 57\n", - " 'Wojewodztwo_Zachodniopomorskie', # 58\n", - " 'Gestosc_zaludnienia', # 59\n", - " 'Ludnosc_na_1_km2', # 60\n", - " 'Ludnosc', # 61\n", - " 'Wskaznik_urbanizacji', # 62\n", - " 'Zmiana_liczby_ludnosci', # 63\n", - " 'Saldo_migracji_na_1000_ludnosci', # 64\n", - " 'Saldo_migracji', # 65\n", - " 'Wymeldowania_do_miast_kobiety', # 66\n", - " 'Wymeldowania_do_miast_mezczyzni', # 67\n", - " 'Wymeldowania_do_miast_ogolem', # 68\n", - " 'Wymeldowania_na_wies_kobiety', # 69\n", - " 'Wymeldowania_na_wies_mezczyzni', # 70\n", - " 'Wymeldowania_na_wies_ogolem', # 71\n", - " 'Wymeldowania_kobiety', # 72\n", - " 'Wymeldowania_mezczyzni', # 73\n", - " 'Wymeldowania_ogolem', # 74\n", - " 'Zameldowania_kobiety', # 75\n", - " 'Zameldowania_mezczyzni', # 76\n", - " 'Zameldowania_ogolem', # 77\n", - " 'Zameldowania_z_miast_kobiety', # 78\n", - " 'Zameldowania_z_miast_mezczyzni', # 79\n", - " 'Zameldowania_z_miast_ogolem', # 80\n", - " 'Zameldowania_ze_wsi_kobiety', # 81\n", - " 'Zameldowania_ze_wsi_mezczyzni', # 82\n", - " 'Zameldowania_ze_wsi_ogolem', # 83\n", - " 'Miejsca_noclegowe_caloroczne', # 84\n", - " 'Miejsca_noclegowe_ogolem', # 85\n", - " 'Obiekty_caloroczne', # 86\n", - " 'Obiekty_ogolem', # 87\n", - " 'Turysci_ogolem', # 88\n", - " 'Turysci_zagraniczni', # 89\n", - " 'Bezrobotni_do_25_roku_zycia', # 90\n", - " 'Dlugotrwale_bezrobotni', # 91\n", - " 'Bezrobotne_kobiety', # 92\n", - " 'Bezrobotni_mezczyzni', # 93\n", - " 'Bezrobotni_ogolem', # 94\n", - " 'Bezrobotni_powyzej_50_roku_zycia', # 95\n", - " 'Gmina_miejska', # 96\n", - " 'Gmina_miejsko_wiejska', # 97\n", - " 'Gmina_wiejska', # 98\n", - " 'Odleglosc_Warszawa', # 99\n", - " 'Odleglosc_od_centrum_decyzyjnego'] # 100\n", + " \"Powierzchnia\", # 1\n", + " \"Wynagrodzenie_ogolem\", # 2\n", + " \"Wynagrodzenie_w_relacji_do_sredniej\", # 3\n", + " \"Dochody_podatek_lesny\", # 4\n", + " \"Dochody_podatek_PCC\", # 5\n", + " \"Dochody_podatek_od_dzialalnosci_gospodarczej\", # 6\n", + " \"Dochody_podatek_od_nieruchomosci\", # 7\n", + " \"Dochody_podatek_od_spadkow\", # 8\n", + " \"Dochody_podatek_od_srodkow_transportowych\", # 9\n", + " \"Dochody_podatek_rolny\", # 10\n", + " \"Dochody_podatek_odrebne_ustawy\", # 11\n", + " \"Dochody_razem\", # 12\n", + " \"Dochody_z_majatku\", # 13\n", + " \"Dochody_z_najmu_i_dzierzawy\", # 14\n", + " \"Dochody_z_uslug\", # 15\n", + " \"Dochody_dofinansowanie_inwestycyjne\", # 16\n", + " \"Dochody_dofinansowanie_razem\", # 17\n", + " \"Udzialy_w_podatkach_dochodowych_od_osob_fizycznych\", # 18\n", + " \"Udzialy_w_podatkach_dochodowych_od_osob_prywatnych\", # 19\n", + " \"Udzialy_w_podatkach_dochodowych_razem\", # 20\n", + " \"Wplywy_z_innych_lokalnych_oplat\", # 21\n", + " \"Wplywy_z_oplaty_eksploatacyjnej\", # 22\n", + " \"Wplywy_z_oplaty_skarbowej\", # 23\n", + " \"Wplywy_z_oplaty_targowej\", # 24\n", + " \"Ludnosc_ogolem\", # 25\n", + " \"Ludnosc_w_wieku_poprodukcyjnym\", # 26\n", + " \"Ludnosc_w_wieku_produkcyjnym\", # 27\n", + " \"Ludnosc_w_wieku_produkcyjnym_mobilnym\", # 28\n", + " \"Ludnosc_w_wieku_produkcyjnym_niemobilnym\", # 29\n", + " \"Ludnosc_w_wieku_przedprodukcyjnym\", # 30\n", + " \"Ludnosc_mezczyzni\", # 31\n", + " \"Ludnosc_mezczyzni_w_wieku_poprodukcyjnym\", # 32\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym\", # 33\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym\", # 34\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym\", # 35\n", + " \"Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym\", # 36\n", + " \"Ludnosc_kobiety\", # 37\n", + " \"Ludnosc_kobiety_w_wieku_poprodukcyjnym\", # 38\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym\", # 39\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym\", # 40\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym\", # 41\n", + " \"Ludnosc_kobiety_w_wieku_przedprodukcyjnym\", # 42\n", + " \"Wojewodztwo_Dolnoslaskie\", # 43\n", + " \"Wojewodztwo_Kujawsko_Pomorskie\", # 44\n", + " \"Wojewodztwo_Lubelskie\", # 45\n", + " \"Wojewodztwo_Lubuskie\", # 46\n", + " \"Wojewodztwo_Lodzkie\", # 47\n", + " \"Wojewodztwo_Malopolskie\", # 48\n", + " \"Wojewodztwo_Mazowieckie\", # 49\n", + " \"Wojewodztwo_Opolskie\", # 50\n", + " \"Wojewodztwo_Podkarpackie\", # 51\n", + " \"Wojewodztwo_Podlaskie\", # 52\n", + " \"Wojewodztwo_Pomorskie\", # 53\n", + " \"Wojewodztwo_Slaskie\", # 54\n", + " \"Wojewodztwo_Swietokrzyskie\", # 55\n", + " \"Wojewodztwo_Warminsko_Mazurskie\", # 56\n", + " \"Wojewodztwo_Wielkopolskie\", # 57\n", + " \"Wojewodztwo_Zachodniopomorskie\", # 58\n", + " \"Gestosc_zaludnienia\", # 59\n", + " \"Ludnosc_na_1_km2\", # 60\n", + " \"Ludnosc\", # 61\n", + " \"Wskaznik_urbanizacji\", # 62\n", + " \"Zmiana_liczby_ludnosci\", # 63\n", + " \"Saldo_migracji_na_1000_ludnosci\", # 64\n", + " \"Saldo_migracji\", # 65\n", + " \"Wymeldowania_do_miast_kobiety\", # 66\n", + " \"Wymeldowania_do_miast_mezczyzni\", # 67\n", + " \"Wymeldowania_do_miast_ogolem\", # 68\n", + " \"Wymeldowania_na_wies_kobiety\", # 69\n", + " \"Wymeldowania_na_wies_mezczyzni\", # 70\n", + " \"Wymeldowania_na_wies_ogolem\", # 71\n", + " \"Wymeldowania_kobiety\", # 72\n", + " \"Wymeldowania_mezczyzni\", # 73\n", + " \"Wymeldowania_ogolem\", # 74\n", + " \"Zameldowania_kobiety\", # 75\n", + " \"Zameldowania_mezczyzni\", # 76\n", + " \"Zameldowania_ogolem\", # 77\n", + " \"Zameldowania_z_miast_kobiety\", # 78\n", + " \"Zameldowania_z_miast_mezczyzni\", # 79\n", + " \"Zameldowania_z_miast_ogolem\", # 80\n", + " \"Zameldowania_ze_wsi_kobiety\", # 81\n", + " \"Zameldowania_ze_wsi_mezczyzni\", # 82\n", + " \"Zameldowania_ze_wsi_ogolem\", # 83\n", + " \"Miejsca_noclegowe_caloroczne\", # 84\n", + " \"Miejsca_noclegowe_ogolem\", # 85\n", + " \"Obiekty_caloroczne\", # 86\n", + " \"Obiekty_ogolem\", # 87\n", + " \"Turysci_ogolem\", # 88\n", + " \"Turysci_zagraniczni\", # 89\n", + " \"Bezrobotni_do_25_roku_zycia\", # 90\n", + " \"Dlugotrwale_bezrobotni\", # 91\n", + " \"Bezrobotne_kobiety\", # 92\n", + " \"Bezrobotni_mezczyzni\", # 93\n", + " \"Bezrobotni_ogolem\", # 94\n", + " \"Bezrobotni_powyzej_50_roku_zycia\", # 95\n", + " \"Gmina_miejska\", # 96\n", + " \"Gmina_miejsko_wiejska\", # 97\n", + " \"Gmina_wiejska\", # 98\n", + " \"Odleglosc_Warszawa\", # 99\n", + " \"Odleglosc_od_centrum_decyzyjnego\",\n", + "] # 100\n", "\n", - "all_columns = ['Kod', 'Rok'] + feature_names\n", + "all_columns = [\"Kod\", \"Rok\"] + feature_names\n", "\n", - "df_data = pd.read_csv(\n", - " 'dane1.csv',\n", - " encoding='ISO-8859-2',\n", - " converters={'Kod': str})\n", + "df_data = pd.read_csv(\"dane1.csv\", encoding=\"ISO-8859-2\", converters={\"Kod\": str})\n", "\n", "df_odl = pd.read_csv(\n", - " 'gminy_centroid.csv',\n", - " encoding='ISO-8859-2',\n", - " converters={'TERYT': str})\n", - "df_odl['TERYT'] = df_odl['TERYT'].astype('str')\n", - "df_odl = df_odl[['TERYT', 'odl_Wawa', 'odl_woj']]\n", - "df_odl = df_odl.rename(columns={\n", - " 'TERYT': 'Kod',\n", - " 'odl_Wawa': 'Odleglosc_Warszawa',\n", - " 'odl_woj': 'Odleglosc_od_centrum_decyzyjnego'})\n", + " \"gminy_centroid.csv\", encoding=\"ISO-8859-2\", converters={\"TERYT\": str}\n", + ")\n", + "df_odl[\"TERYT\"] = df_odl[\"TERYT\"].astype(\"str\")\n", + "df_odl = df_odl[[\"TERYT\", \"odl_Wawa\", \"odl_woj\"]]\n", + "df_odl = df_odl.rename(\n", + " columns={\n", + " \"TERYT\": \"Kod\",\n", + " \"odl_Wawa\": \"Odleglosc_Warszawa\",\n", + " \"odl_woj\": \"Odleglosc_od_centrum_decyzyjnego\",\n", + " }\n", + ")\n", "\n", - "df_data = df_data.merge(df_odl, on=['Kod'], how='left')\n", + "df_data = df_data.merge(df_odl, on=[\"Kod\"], how=\"left\")\n", "\n", - "print(df_data['Program_operacyjny'].drop_duplicates().values)" + "print(df_data[\"Program_operacyjny\"].drop_duplicates().values)" ] }, { @@ -181,37 +182,45 @@ ], "source": [ "# Wybór programu operacyjnego...\n", - "df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Polska Cyfrowa'].reset_index(drop=True)\n", + "df_data = df_data.loc[\n", + " df_data[\"Program_operacyjny\"] == \"Program Operacyjny Polska Cyfrowa\"\n", + "].reset_index(drop=True)\n", "\n", "# Uzupełnienie brakujących danych...\n", - "df_data[[\n", - " 'Miejsca_noclegowe_caloroczne',\n", - " 'Miejsca_noclegowe_ogolem',\n", - " 'Obiekty_caloroczne',\n", - " 'Obiekty_ogolem',\n", - " 'Turysci_ogolem',\n", - " 'Turysci_zagraniczni']] = df_data[[\n", - " 'Miejsca_noclegowe_caloroczne',\n", - " 'Miejsca_noclegowe_ogolem',\n", - " 'Obiekty_caloroczne',\n", - " 'Obiekty_ogolem',\n", - " 'Turysci_ogolem',\n", - " 'Turysci_zagraniczni']].fillna(0)\n", + "df_data[\n", + " [\n", + " \"Miejsca_noclegowe_caloroczne\",\n", + " \"Miejsca_noclegowe_ogolem\",\n", + " \"Obiekty_caloroczne\",\n", + " \"Obiekty_ogolem\",\n", + " \"Turysci_ogolem\",\n", + " \"Turysci_zagraniczni\",\n", + " ]\n", + "] = df_data[\n", + " [\n", + " \"Miejsca_noclegowe_caloroczne\",\n", + " \"Miejsca_noclegowe_ogolem\",\n", + " \"Obiekty_caloroczne\",\n", + " \"Obiekty_ogolem\",\n", + " \"Turysci_ogolem\",\n", + " \"Turysci_zagraniczni\",\n", + " ]\n", + "].fillna(0)\n", "\n", "# Usunięcie niepotrzebnych rzędów...\n", "df_data.dropna(inplace=True)\n", - "df_data = df_data[df_data['Suma'] > 0]\n", + "df_data = df_data[df_data[\"Suma\"] > 0]\n", "\n", - "df_data = df_data.groupby(all_columns)['Suma'].sum().reset_index()\n", + "df_data = df_data.groupby(all_columns)[\"Suma\"].sum().reset_index()\n", "\n", "# ...\n", "# df_data['Suma'] = df_data['Suma'] / df_data['Ludnosc']\n", "\n", - "print('Median', median(df_data['Suma']))\n", - "print('Mean', sum(df_data['Suma'])/len(df_data['Suma']))\n", + "print(\"Median\", median(df_data[\"Suma\"]))\n", + "print(\"Mean\", sum(df_data[\"Suma\"]) / len(df_data[\"Suma\"]))\n", "\n", "X = df_data[feature_names]\n", - "y = df_data['Suma']\n", + "y = df_data[\"Suma\"]\n", "\n", "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)" ] @@ -724,8 +733,55 @@ ], "source": [ "param_grid = {\n", - " 'max_depth': [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28],\n", - " 'n_features_to_select': [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2]\n", + " \"max_depth\": [\n", + " 3,\n", + " 4,\n", + " 5,\n", + " 6,\n", + " 7,\n", + " 8,\n", + " 9,\n", + " 10,\n", + " 11,\n", + " 12,\n", + " 13,\n", + " 14,\n", + " 15,\n", + " 16,\n", + " 17,\n", + " 18,\n", + " 19,\n", + " 20,\n", + " 21,\n", + " 22,\n", + " 23,\n", + " 24,\n", + " 25,\n", + " 26,\n", + " 27,\n", + " 28,\n", + " ],\n", + " \"n_features_to_select\": [\n", + " 20,\n", + " 19,\n", + " 18,\n", + " 17,\n", + " 16,\n", + " 15,\n", + " 14,\n", + " 13,\n", + " 12,\n", + " 11,\n", + " 10,\n", + " 9,\n", + " 8,\n", + " 7,\n", + " 6,\n", + " 5,\n", + " 4,\n", + " 3,\n", + " 2,\n", + " ],\n", "}\n", "\n", "best_mse = np.inf\n", @@ -735,41 +791,51 @@ "results_train = []\n", "results_test = []\n", "\n", - "for params in [{'max_depth': max_depth, 'n_features_to_select': n_features} \n", - " for max_depth in param_grid['max_depth'] \n", - " for n_features in param_grid['n_features_to_select']]:\n", - "\n", - " selector = RFE(estimator=DecisionTreeRegressor(random_state=0, max_depth=params['max_depth']), n_features_to_select=params['n_features_to_select'])\n", + "for params in [\n", + " {\"max_depth\": max_depth, \"n_features_to_select\": n_features}\n", + " for max_depth in param_grid[\"max_depth\"]\n", + " for n_features in param_grid[\"n_features_to_select\"]\n", + "]:\n", + " selector = RFE(\n", + " estimator=DecisionTreeRegressor(random_state=0, max_depth=params[\"max_depth\"]),\n", + " n_features_to_select=params[\"n_features_to_select\"],\n", + " )\n", " selector.fit(X_train, y_train)\n", " X_train_selected = selector.transform(X_train)\n", " X_test_selected = selector.transform(X_test)\n", - " \n", - " model = DecisionTreeRegressor(random_state=0, max_depth=params['max_depth'])\n", + "\n", + " model = DecisionTreeRegressor(random_state=0, max_depth=params[\"max_depth\"])\n", " model.fit(X_train_selected, y_train)\n", - " \n", + "\n", " y_pred_train = model.predict(X_train_selected)\n", " y_pred_test = model.predict(X_test_selected)\n", "\n", " mse_train = mean_squared_error(y_pred_train, y_train)\n", " mse_test = mean_squared_error(y_pred_test, y_test)\n", "\n", - " results_train.append((params['max_depth'], params['n_features_to_select'], mse_train))\n", - " results_test.append((params['max_depth'], params['n_features_to_select'], mse_test))\n", + " results_train.append(\n", + " (params[\"max_depth\"], params[\"n_features_to_select\"], mse_train)\n", + " )\n", + " results_test.append((params[\"max_depth\"], params[\"n_features_to_select\"], mse_test))\n", "\n", " if mse_test < best_mse:\n", " best_mse = mse_test\n", " best_model = (model, selector)\n", - " best_params = (params['max_depth'], params['n_features_to_select'])\n", + " best_params = (params[\"max_depth\"], params[\"n_features_to_select\"])\n", "\n", - " print(f\"max_depth: {params['max_depth']:>{2}}, \"\n", - " f\"n_features: {params['n_features_to_select']:>{2}}, \"\n", - " f\"mse_train: {mse_train:>{20}.1f}, \"\n", - " f\"mse_test: {mse_test:>{20}.1f} <-\")\n", + " print(\n", + " f\"max_depth: {params['max_depth']:>{2}}, \"\n", + " f\"n_features: {params['n_features_to_select']:>{2}}, \"\n", + " f\"mse_train: {mse_train:>{20}.1f}, \"\n", + " f\"mse_test: {mse_test:>{20}.1f} <-\"\n", + " )\n", " else:\n", - " print(f\"max_depth: {params['max_depth']:>{2}}, \"\n", - " f\"n_features: {params['n_features_to_select']:>{2}}, \"\n", - " f\"mse_train: {mse_train:>{20}.1f}, \"\n", - " f\"mse_test: {mse_test:>{20}.1f}\")" + " print(\n", + " f\"max_depth: {params['max_depth']:>{2}}, \"\n", + " f\"n_features: {params['n_features_to_select']:>{2}}, \"\n", + " f\"mse_train: {mse_train:>{20}.1f}, \"\n", + " f\"mse_test: {mse_test:>{20}.1f}\"\n", + " )" ] }, { @@ -806,20 +872,22 @@ } ], "source": [ - "results_train_df = pd.DataFrame(results_train, columns=['max_depth', 'n_features_to_select', 'mse_train'])\n", + "results_train_df = pd.DataFrame(\n", + " results_train, columns=[\"max_depth\", \"n_features_to_select\", \"mse_train\"]\n", + ")\n", "\n", "fig = plt.figure()\n", - "ax = fig.add_subplot(111, projection='3d')\n", + "ax = fig.add_subplot(111, projection=\"3d\")\n", "\n", - "xs = results_train_df['max_depth']\n", - "ys = results_train_df['n_features_to_select']\n", - "zs = results_train_df['mse_train']\n", + "xs = results_train_df[\"max_depth\"]\n", + "ys = results_train_df[\"n_features_to_select\"]\n", + "zs = results_train_df[\"mse_train\"]\n", "\n", - "ax.scatter(xs, ys, zs, c=zs, marker='o')\n", + "ax.scatter(xs, ys, zs, c=zs, marker=\"o\")\n", "\n", - "ax.set_xlabel('Max Depth')\n", - "ax.set_ylabel('Number of Features')\n", - "ax.set_zlabel('MSE')\n", + "ax.set_xlabel(\"Max Depth\")\n", + "ax.set_ylabel(\"Number of Features\")\n", + "ax.set_zlabel(\"MSE\")\n", "\n", "plt.show()" ] @@ -841,20 +909,22 @@ } ], "source": [ - "results_test_df = pd.DataFrame(results_test, columns=['max_depth', 'n_features_to_select', 'mse_test'])\n", + "results_test_df = pd.DataFrame(\n", + " results_test, columns=[\"max_depth\", \"n_features_to_select\", \"mse_test\"]\n", + ")\n", "\n", "fig = plt.figure()\n", - "ax = fig.add_subplot(111, projection='3d')\n", + "ax = fig.add_subplot(111, projection=\"3d\")\n", "\n", - "xs = results_test_df['max_depth']\n", - "ys = results_test_df['n_features_to_select']\n", - "zs = results_test_df['mse_test']\n", + "xs = results_test_df[\"max_depth\"]\n", + "ys = results_test_df[\"n_features_to_select\"]\n", + "zs = results_test_df[\"mse_test\"]\n", "\n", - "ax.scatter(xs, ys, zs, c=zs, marker='o')\n", + "ax.scatter(xs, ys, zs, c=zs, marker=\"o\")\n", "\n", - "ax.set_xlabel('Max Depth')\n", - "ax.set_ylabel('Number of Features')\n", - "ax.set_zlabel('MSE')\n", + "ax.set_xlabel(\"Max Depth\")\n", + "ax.set_ylabel(\"Number of Features\")\n", + "ax.set_zlabel(\"MSE\")\n", "\n", "plt.show()" ] @@ -879,21 +949,27 @@ "X_test_selected = best_model[1].transform(X_test)\n", "y_pred_test = best_model[0].predict(X_test_selected)\n", "\n", - "color_column = df_data['Gmina_miejska'] * 0 + df_data['Gmina_miejsko_wiejska'] * 10 + df_data['Gmina_wiejska'] * 5\n", + "color_column = (\n", + " df_data[\"Gmina_miejska\"] * 0\n", + " + df_data[\"Gmina_miejsko_wiejska\"] * 10\n", + " + df_data[\"Gmina_wiejska\"] * 5\n", + ")\n", "\n", - "color_column_train, color_column_test = train_test_split(color_column, test_size=0.2, random_state=0)\n", + "color_column_train, color_column_test = train_test_split(\n", + " color_column, test_size=0.2, random_state=0\n", + ")\n", "\n", "min_val = min(min(y_test), min(y_pred_test))\n", "max_val = max(max(y_test), max(y_pred_test))\n", - "plt.plot([min_val, max_val], [min_val, max_val], 'r--')\n", + "plt.plot([min_val, max_val], [min_val, max_val], \"r--\")\n", "\n", - "plt.scatter(y_test, y_pred_test, alpha=0.5, c=color_column_test, cmap='rainbow')\n", - "plt.xlabel('Actual')\n", - "plt.ylabel('Predicted')\n", - "plt.title('Actual vs Predicted')\n", + "plt.scatter(y_test, y_pred_test, alpha=0.5, c=color_column_test, cmap=\"rainbow\")\n", + "plt.xlabel(\"Actual\")\n", + "plt.ylabel(\"Predicted\")\n", + "plt.title(\"Actual vs Predicted\")\n", "\n", - "plt.xlim(0, 0.75*10**7)\n", - "plt.ylim(min(y_pred_test), 0.75*10**7)\n", + "plt.xlim(0, 0.75 * 10**7)\n", + "plt.ylim(min(y_pred_test), 0.75 * 10**7)\n", "\n", "plt.show()" ] @@ -915,15 +991,19 @@ } ], "source": [ - "results_train_df = pd.DataFrame(results_train, columns=['max_depth', 'n_features_to_select', 'mse_train'])\n", + "results_train_df = pd.DataFrame(\n", + " results_train, columns=[\"max_depth\", \"n_features_to_select\", \"mse_train\"]\n", + ")\n", "\n", - "heatmap_data = results_train_df.pivot(index='max_depth', columns='n_features_to_select', values='mse_train')\n", + "heatmap_data = results_train_df.pivot(\n", + " index=\"max_depth\", columns=\"n_features_to_select\", values=\"mse_train\"\n", + ")\n", "\n", "sns.heatmap(heatmap_data, annot=False, fmt=\".2f\", cmap=\"viridis\")\n", "\n", - "plt.title('MSE')\n", - "plt.xlabel('Number of Features')\n", - "plt.ylabel('Max Depth')\n", + "plt.title(\"MSE\")\n", + "plt.xlabel(\"Number of Features\")\n", + "plt.ylabel(\"Max Depth\")\n", "plt.show()" ] }, @@ -944,15 +1024,19 @@ } ], "source": [ - "results_test_df = pd.DataFrame(results_test, columns=['max_depth', 'n_features_to_select', 'mse_test'])\n", + "results_test_df = pd.DataFrame(\n", + " results_test, columns=[\"max_depth\", \"n_features_to_select\", \"mse_test\"]\n", + ")\n", "\n", - "heatmap_data = results_test_df.pivot(index='max_depth', columns='n_features_to_select', values='mse_test')\n", + "heatmap_data = results_test_df.pivot(\n", + " index=\"max_depth\", columns=\"n_features_to_select\", values=\"mse_test\"\n", + ")\n", "\n", "sns.heatmap(heatmap_data, annot=False, fmt=\".2f\", cmap=\"viridis\")\n", "\n", - "plt.title('MSE')\n", - "plt.xlabel('Number of Features')\n", - "plt.ylabel('Max Depth')\n", + "plt.title(\"MSE\")\n", + "plt.xlabel(\"Number of Features\")\n", + "plt.ylabel(\"Max Depth\")\n", "plt.show()" ] }, @@ -983,8 +1067,10 @@ ], "source": [ "feature_importance = dict(zip(feature_names, best_model[0].feature_importances_))\n", - "for feature, importance in sorted(feature_importance.items(), key=lambda x: x[1], reverse=True):\n", - " print(f'{importance:.4f} \\u2014 {feature}')" + "for feature, importance in sorted(\n", + " feature_importance.items(), key=lambda x: x[1], reverse=True\n", + "):\n", + " print(f\"{importance:.4f} \\u2014 {feature}\")" ] }, { @@ -1478,10 +1564,10 @@ } ], "source": [ - "X_test['Przewidziana Suma'] = y_pred_test\n", - "X_test['Suma'] = y_test\n", + "X_test[\"Przewidziana Suma\"] = y_pred_test\n", + "X_test[\"Suma\"] = y_test\n", "display(X_test[np.abs(y_test - y_pred_test) < 1])\n", - "display(X_test['Suma'][np.abs(y_test - y_pred_test) < 1].drop_duplicates().values)" + "display(X_test[\"Suma\"][np.abs(y_test - y_pred_test) < 1].drop_duplicates().values)" ] }, { @@ -1510,7 +1596,10 @@ } ], "source": [ - "selector = RFE(estimator=DecisionTreeRegressor(random_state=0, max_depth=best_params[0]), n_features_to_select=best_params[1])\n", + "selector = RFE(\n", + " estimator=DecisionTreeRegressor(random_state=0, max_depth=best_params[0]),\n", + " n_features_to_select=best_params[1],\n", + ")\n", "selector.fit(X, y)\n", "X_selected = selector.transform(X)\n", "\n", @@ -1518,8 +1607,10 @@ "model.fit(X_selected, y)\n", "\n", "feature_importance = dict(zip(feature_names, model.feature_importances_))\n", - "for feature, importance in sorted(feature_importance.items(), key=lambda x: x[1], reverse=True):\n", - " print(f'{importance:.4f} \\u2014 {feature}')" + "for feature, importance in sorted(\n", + " feature_importance.items(), key=lambda x: x[1], reverse=True\n", + "):\n", + " print(f\"{importance:.4f} \\u2014 {feature}\")" ] }, { diff --git a/Programming/PBAD/pbad.ipynb b/Programming/PBAD/pbad.ipynb index 9107b3fb..eaf5bfe3 100644 --- a/Programming/PBAD/pbad.ipynb +++ b/Programming/PBAD/pbad.ipynb @@ -6,14 +6,11 @@ "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.tree import DecisionTreeRegressor, plot_tree, export_text\n", - "from sklearn.metrics import mean_squared_error\n", "import matplotlib.pyplot as plt\n", - "from sklearn.preprocessing import StandardScaler\n", - "from sklearn.preprocessing import MinMaxScaler\n", - "from sklearn.decomposition import PCA" + "import pandas as pd\n", + "from sklearn.metrics import mean_squared_error\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeRegressor, export_text" ] }, { @@ -23,108 +20,109 @@ "outputs": [], "source": [ "feature_names = [\n", - " 'Powierzchnia', # 1\n", - " 'Wynagrodzenie_ogolem', # 2\n", - " 'Wynagrodzenie_w_relacji_do_sredniej', # 3\n", - " 'Dochody_podatek_lesny', # 4\n", - " 'Dochody_podatek_PCC', # 5\n", - " 'Dochody_podatek_od_dzialalnosci_gospodarczej', # 6\n", - " 'Dochody_podatek_od_nieruchomosci', # 7\n", - " 'Dochody_podatek_od_spadkow', # 8\n", - " 'Dochody_podatek_od_srodkow_transportowych', # 9\n", - " 'Dochody_podatek_rolny', # 10\n", - " 'Dochody_podatek_odrebne_ustawy', # 11\n", - " 'Dochody_razem', # 12\n", - " 'Dochody_z_majatku', # 13\n", - " 'Dochody_z_najmu_i_dzierzawy', # 14\n", - " 'Dochody_z_uslug', # 15\n", - " 'Dochody_dofinansowanie_inwestycyjne', # 16\n", - " 'Dochody_dofinansowanie_razem', # 17\n", - " 'Udzialy_w_podatkach_dochodowych_od_osob_fizycznych', # 18\n", - " 'Udzialy_w_podatkach_dochodowych_od_osob_prywatnych', # 19\n", - " 'Udzialy_w_podatkach_dochodowych_razem', # 20\n", - " 'Wplywy_z_innych_lokalnych_oplat', # 21\n", - " 'Wplywy_z_oplaty_eksploatacyjnej', # 22\n", - " 'Wplywy_z_oplaty_skarbowej', # 23\n", - " 'Wplywy_z_oplaty_targowej', # 24\n", - " 'Ludnosc_ogolem', # 25\n", - " 'Ludnosc_w_wieku_poprodukcyjnym', # 26\n", - " 'Ludnosc_w_wieku_produkcyjnym', # 27\n", - " 'Ludnosc_w_wieku_produkcyjnym_mobilnym', # 28\n", - " 'Ludnosc_w_wieku_produkcyjnym_niemobilnym', # 29\n", - " 'Ludnosc_w_wieku_przedprodukcyjnym', # 30\n", - " 'Ludnosc_mezczyzni', # 31\n", - " 'Ludnosc_mezczyzni_w_wieku_poprodukcyjnym', # 32\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym', # 33\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym', # 34\n", - " 'Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym', # 35\n", - " 'Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym', # 36\n", - " 'Ludnosc_kobiety', # 37\n", - " 'Ludnosc_kobiety_w_wieku_poprodukcyjnym', # 38\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym', # 39\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym', # 40\n", - " 'Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym', # 41\n", - " 'Ludnosc_kobiety_w_wieku_przedprodukcyjnym', # 42\n", - " 'Wojewodztwo_Dolnoslaskie', # 43\n", - " 'Wojewodztwo_Kujawsko_Pomorskie', # 44\n", - " 'Wojewodztwo_Lubelskie', # 45\n", - " 'Wojewodztwo_Lubuskie', # 46\n", - " 'Wojewodztwo_Lodzkie', # 47\n", - " 'Wojewodztwo_Malopolskie', # 48\n", - " 'Wojewodztwo_Mazowieckie', # 49\n", - " 'Wojewodztwo_Opolskie', # 50\n", - " 'Wojewodztwo_Podkarpackie', # 51\n", - " 'Wojewodztwo_Podlaskie', # 52\n", - " 'Wojewodztwo_Pomorskie', # 53\n", - " 'Wojewodztwo_Slaskie', # 54\n", - " 'Wojewodztwo_Swietokrzyskie', # 55\n", - " 'Wojewodztwo_Warminsko_Mazurskie', # 56\n", - " 'Wojewodztwo_Wielkopolskie', # 57\n", - " 'Wojewodztwo_Zachodniopomorskie', # 58\n", - " 'Gestosc_zaludnienia', # 59\n", - " 'Ludnosc_na_1_km2', # 60\n", - " 'Ludnosc', # 61\n", - " 'Wskaznik_urbanizacji', # 62\n", - " 'Zmiana_liczby_ludnosci', # 63\n", - " 'Saldo_migracji_na_1000_ludnosci', # 64\n", - " 'Saldo_migracji', # 65\n", - " 'Wymeldowania_do_miast_kobiety', # 66\n", - " 'Wymeldowania_do_miast_mezczyzni', # 67\n", - " 'Wymeldowania_do_miast_ogolem', # 68\n", - " 'Wymeldowania_na_wies_kobiety', # 69\n", - " 'Wymeldowania_na_wies_mezczyzni', # 70\n", - " 'Wymeldowania_na_wies_ogolem', # 71\n", - " 'Wymeldowania_kobiety', # 72\n", - " 'Wymeldowania_mezczyzni', # 73\n", - " 'Wymeldowania_ogolem', # 74\n", - " 'Zameldowania_kobiety', # 75\n", - " 'Zameldowania_mezczyzni', # 76\n", - " 'Zameldowania_ogolem', # 77\n", - " 'Zameldowania_z_miast_kobiety', # 78\n", - " 'Zameldowania_z_miast_mezczyzni', # 79\n", - " 'Zameldowania_z_miast_ogolem', # 80\n", - " 'Zameldowania_ze_wsi_kobiety', # 81\n", - " 'Zameldowania_ze_wsi_mezczyzni', # 82\n", - " 'Zameldowania_ze_wsi_ogolem', # 83\n", - " 'Miejsca_noclegowe_caloroczne', # 84\n", - " 'Miejsca_noclegowe_ogolem', # 85\n", - " 'Obiekty_caloroczne', # 86\n", - " 'Obiekty_ogolem', # 87\n", - " 'Turysci_ogolem', # 88\n", - " 'Turysci_zagraniczni', # 89\n", - " 'Bezrobotni_do_25_roku_zycia', # 90\n", - " 'Dlugotrwale_bezrobotni', # 91\n", - " 'Bezrobotne_kobiety', # 92\n", - " 'Bezrobotni_mezczyzni', # 93\n", - " 'Bezrobotni_ogolem', # 94\n", - " 'Bezrobotni_powyzej_50_roku_zycia', # 95\n", - " 'Gmina_miejska', # 96\n", - " 'Gmina_miejsko_wiejska', # 97\n", - " 'Gmina_wiejska', # 98\n", - " 'Odleglosc_Warszawa', # 99\n", - " 'Odleglosc_od_centrum_decyzyjnego'] # 100\n", + " \"Powierzchnia\", # 1\n", + " \"Wynagrodzenie_ogolem\", # 2\n", + " \"Wynagrodzenie_w_relacji_do_sredniej\", # 3\n", + " \"Dochody_podatek_lesny\", # 4\n", + " \"Dochody_podatek_PCC\", # 5\n", + " \"Dochody_podatek_od_dzialalnosci_gospodarczej\", # 6\n", + " \"Dochody_podatek_od_nieruchomosci\", # 7\n", + " \"Dochody_podatek_od_spadkow\", # 8\n", + " \"Dochody_podatek_od_srodkow_transportowych\", # 9\n", + " \"Dochody_podatek_rolny\", # 10\n", + " \"Dochody_podatek_odrebne_ustawy\", # 11\n", + " \"Dochody_razem\", # 12\n", + " \"Dochody_z_majatku\", # 13\n", + " \"Dochody_z_najmu_i_dzierzawy\", # 14\n", + " \"Dochody_z_uslug\", # 15\n", + " \"Dochody_dofinansowanie_inwestycyjne\", # 16\n", + " \"Dochody_dofinansowanie_razem\", # 17\n", + " \"Udzialy_w_podatkach_dochodowych_od_osob_fizycznych\", # 18\n", + " \"Udzialy_w_podatkach_dochodowych_od_osob_prywatnych\", # 19\n", + " \"Udzialy_w_podatkach_dochodowych_razem\", # 20\n", + " \"Wplywy_z_innych_lokalnych_oplat\", # 21\n", + " \"Wplywy_z_oplaty_eksploatacyjnej\", # 22\n", + " \"Wplywy_z_oplaty_skarbowej\", # 23\n", + " \"Wplywy_z_oplaty_targowej\", # 24\n", + " \"Ludnosc_ogolem\", # 25\n", + " \"Ludnosc_w_wieku_poprodukcyjnym\", # 26\n", + " \"Ludnosc_w_wieku_produkcyjnym\", # 27\n", + " \"Ludnosc_w_wieku_produkcyjnym_mobilnym\", # 28\n", + " \"Ludnosc_w_wieku_produkcyjnym_niemobilnym\", # 29\n", + " \"Ludnosc_w_wieku_przedprodukcyjnym\", # 30\n", + " \"Ludnosc_mezczyzni\", # 31\n", + " \"Ludnosc_mezczyzni_w_wieku_poprodukcyjnym\", # 32\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym\", # 33\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym\", # 34\n", + " \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym\", # 35\n", + " \"Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym\", # 36\n", + " \"Ludnosc_kobiety\", # 37\n", + " \"Ludnosc_kobiety_w_wieku_poprodukcyjnym\", # 38\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym\", # 39\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym\", # 40\n", + " \"Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym\", # 41\n", + " \"Ludnosc_kobiety_w_wieku_przedprodukcyjnym\", # 42\n", + " \"Wojewodztwo_Dolnoslaskie\", # 43\n", + " \"Wojewodztwo_Kujawsko_Pomorskie\", # 44\n", + " \"Wojewodztwo_Lubelskie\", # 45\n", + " \"Wojewodztwo_Lubuskie\", # 46\n", + " \"Wojewodztwo_Lodzkie\", # 47\n", + " \"Wojewodztwo_Malopolskie\", # 48\n", + " \"Wojewodztwo_Mazowieckie\", # 49\n", + " \"Wojewodztwo_Opolskie\", # 50\n", + " \"Wojewodztwo_Podkarpackie\", # 51\n", + " \"Wojewodztwo_Podlaskie\", # 52\n", + " \"Wojewodztwo_Pomorskie\", # 53\n", + " \"Wojewodztwo_Slaskie\", # 54\n", + " \"Wojewodztwo_Swietokrzyskie\", # 55\n", + " \"Wojewodztwo_Warminsko_Mazurskie\", # 56\n", + " \"Wojewodztwo_Wielkopolskie\", # 57\n", + " \"Wojewodztwo_Zachodniopomorskie\", # 58\n", + " \"Gestosc_zaludnienia\", # 59\n", + " \"Ludnosc_na_1_km2\", # 60\n", + " \"Ludnosc\", # 61\n", + " \"Wskaznik_urbanizacji\", # 62\n", + " \"Zmiana_liczby_ludnosci\", # 63\n", + " \"Saldo_migracji_na_1000_ludnosci\", # 64\n", + " \"Saldo_migracji\", # 65\n", + " \"Wymeldowania_do_miast_kobiety\", # 66\n", + " \"Wymeldowania_do_miast_mezczyzni\", # 67\n", + " \"Wymeldowania_do_miast_ogolem\", # 68\n", + " \"Wymeldowania_na_wies_kobiety\", # 69\n", + " \"Wymeldowania_na_wies_mezczyzni\", # 70\n", + " \"Wymeldowania_na_wies_ogolem\", # 71\n", + " \"Wymeldowania_kobiety\", # 72\n", + " \"Wymeldowania_mezczyzni\", # 73\n", + " \"Wymeldowania_ogolem\", # 74\n", + " \"Zameldowania_kobiety\", # 75\n", + " \"Zameldowania_mezczyzni\", # 76\n", + " \"Zameldowania_ogolem\", # 77\n", + " \"Zameldowania_z_miast_kobiety\", # 78\n", + " \"Zameldowania_z_miast_mezczyzni\", # 79\n", + " \"Zameldowania_z_miast_ogolem\", # 80\n", + " \"Zameldowania_ze_wsi_kobiety\", # 81\n", + " \"Zameldowania_ze_wsi_mezczyzni\", # 82\n", + " \"Zameldowania_ze_wsi_ogolem\", # 83\n", + " \"Miejsca_noclegowe_caloroczne\", # 84\n", + " \"Miejsca_noclegowe_ogolem\", # 85\n", + " \"Obiekty_caloroczne\", # 86\n", + " \"Obiekty_ogolem\", # 87\n", + " \"Turysci_ogolem\", # 88\n", + " \"Turysci_zagraniczni\", # 89\n", + " \"Bezrobotni_do_25_roku_zycia\", # 90\n", + " \"Dlugotrwale_bezrobotni\", # 91\n", + " \"Bezrobotne_kobiety\", # 92\n", + " \"Bezrobotni_mezczyzni\", # 93\n", + " \"Bezrobotni_ogolem\", # 94\n", + " \"Bezrobotni_powyzej_50_roku_zycia\", # 95\n", + " \"Gmina_miejska\", # 96\n", + " \"Gmina_miejsko_wiejska\", # 97\n", + " \"Gmina_wiejska\", # 98\n", + " \"Odleglosc_Warszawa\", # 99\n", + " \"Odleglosc_od_centrum_decyzyjnego\",\n", + "] # 100\n", "\n", - "all_columns = ['Kod', 'Rok'] + feature_names" + "all_columns = [\"Kod\", \"Rok\"] + feature_names" ] }, { @@ -133,10 +131,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_data = pd.read_csv(\n", - " 'dane1.csv',\n", - " encoding='ISO-8859-2',\n", - " converters={'Kod': str})" + "df_data = pd.read_csv(\"dane1.csv\", encoding=\"ISO-8859-2\", converters={\"Kod\": str})" ] }, { @@ -146,15 +141,17 @@ "outputs": [], "source": [ "df_odl = pd.read_csv(\n", - " 'gminy_centroid.csv',\n", - " encoding='ISO-8859-2',\n", - " converters={'TERYT': str})\n", - "df_odl['TERYT'] = df_odl['TERYT'].astype('str')\n", - "df_odl = df_odl[['TERYT', 'odl_Wawa', 'odl_woj']]\n", - "df_odl = df_odl.rename(columns={\n", - " 'TERYT': 'Kod',\n", - " 'odl_Wawa': 'Odleglosc_Warszawa',\n", - " 'odl_woj': 'Odleglosc_od_centrum_decyzyjnego'})" + " \"gminy_centroid.csv\", encoding=\"ISO-8859-2\", converters={\"TERYT\": str}\n", + ")\n", + "df_odl[\"TERYT\"] = df_odl[\"TERYT\"].astype(\"str\")\n", + "df_odl = df_odl[[\"TERYT\", \"odl_Wawa\", \"odl_woj\"]]\n", + "df_odl = df_odl.rename(\n", + " columns={\n", + " \"TERYT\": \"Kod\",\n", + " \"odl_Wawa\": \"Odleglosc_Warszawa\",\n", + " \"odl_woj\": \"Odleglosc_od_centrum_decyzyjnego\",\n", + " }\n", + ")" ] }, { @@ -163,7 +160,7 @@ "metadata": {}, "outputs": [], "source": [ - "df_data = df_data.merge(df_odl, on=['Kod'], how='left')" + "df_data = df_data.merge(df_odl, on=[\"Kod\"], how=\"left\")" ] }, { @@ -184,7 +181,7 @@ } ], "source": [ - "print(df_data['Program_operacyjny'].drop_duplicates().values)" + "print(df_data[\"Program_operacyjny\"].drop_duplicates().values)" ] }, { @@ -207,35 +204,44 @@ ], "source": [ "# Wybór programu operacyjnego...\n", - "df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Infrastruktura i ?rodowisko 2014-2020'].reset_index(drop=True)\n", + "df_data = df_data.loc[\n", + " df_data[\"Program_operacyjny\"]\n", + " == \"Program Operacyjny Infrastruktura i ?rodowisko 2014-2020\"\n", + "].reset_index(drop=True)\n", "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Inteligentny RozwĂłj'].reset_index(drop=True)\n", "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Polska Cyfrowa'].reset_index(drop=True)\n", "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Wiedza Edukacja RozwĂłj'].reset_index(drop=True)\n", "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Polska Wschodnia'].reset_index(drop=True)\n", "\n", "# Uzupełnienie brakujących danych...\n", - "df_data[[\n", - " 'Miejsca_noclegowe_caloroczne',\n", - " 'Miejsca_noclegowe_ogolem',\n", - " 'Obiekty_caloroczne',\n", - " 'Obiekty_ogolem',\n", - " 'Turysci_ogolem',\n", - " 'Turysci_zagraniczni']] = df_data[[\n", - " 'Miejsca_noclegowe_caloroczne',\n", - " 'Miejsca_noclegowe_ogolem',\n", - " 'Obiekty_caloroczne',\n", - " 'Obiekty_ogolem',\n", - " 'Turysci_ogolem',\n", - " 'Turysci_zagraniczni']].fillna(0)\n", + "df_data[\n", + " [\n", + " \"Miejsca_noclegowe_caloroczne\",\n", + " \"Miejsca_noclegowe_ogolem\",\n", + " \"Obiekty_caloroczne\",\n", + " \"Obiekty_ogolem\",\n", + " \"Turysci_ogolem\",\n", + " \"Turysci_zagraniczni\",\n", + " ]\n", + "] = df_data[\n", + " [\n", + " \"Miejsca_noclegowe_caloroczne\",\n", + " \"Miejsca_noclegowe_ogolem\",\n", + " \"Obiekty_caloroczne\",\n", + " \"Obiekty_ogolem\",\n", + " \"Turysci_ogolem\",\n", + " \"Turysci_zagraniczni\",\n", + " ]\n", + "].fillna(0)\n", "\n", "# Usunięcie niepotrzebnych rzędów...\n", "df_data.dropna(inplace=True)\n", - "df_data = df_data[df_data['Suma'] > 0]\n", + "df_data = df_data[df_data[\"Suma\"] > 0]\n", "\n", - "df_data = df_data.groupby(all_columns)['Suma'].sum().reset_index()\n", + "df_data = df_data.groupby(all_columns)[\"Suma\"].sum().reset_index()\n", "\n", "# ...\n", - "df_data['Suma'] = df_data['Suma'] / df_data['Ludnosc']" + "df_data[\"Suma\"] = df_data[\"Suma\"] / df_data[\"Ludnosc\"]" ] }, { @@ -257,18 +263,20 @@ "# scaler = MinMaxScaler()\n", "# X = scaler.fit_transform(X)\n", "\n", - "y = df_data['Suma']\n", + "y = df_data[\"Suma\"]\n", "\n", - "color_column = (df_data['Gestosc_zaludnienia'] > 1.5).astype(int)\n", + "color_column = (df_data[\"Gestosc_zaludnienia\"] > 1.5).astype(int)\n", "\n", - "X_train, X_test, y_train, y_test, color_column_train, color_column_test = train_test_split(X, y, color_column, test_size=0.2, random_state=99)\n", + "X_train, X_test, y_train, y_test, color_column_train, color_column_test = (\n", + " train_test_split(X, y, color_column, test_size=0.2, random_state=99)\n", + ")\n", "\n", "model = DecisionTreeRegressor(random_state=99)\n", "model.fit(X_train, y_train)\n", "\n", "y_pred = model.predict(X_test)\n", "mse = mean_squared_error(y_test, y_pred)\n", - "print('Mean Squared Error:', mse)" + "print(\"Mean Squared Error:\", mse)" ] }, { @@ -288,10 +296,10 @@ } ], "source": [ - "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap='viridis')\n", - "plt.xlabel('Actual')\n", - "plt.ylabel('Predicted')\n", - "plt.title('Actual vs Predicted')\n", + "plt.scatter(y_test, y_pred, alpha=0.5, c=color_column_test, cmap=\"viridis\")\n", + "plt.xlabel(\"Actual\")\n", + "plt.ylabel(\"Predicted\")\n", + "plt.title(\"Actual vs Predicted\")\n", "\n", "# plt.xlim(0, max(max(y_test), max(y_pred)))\n", "# plt.ylim(0, max(max(y_test), max(y_pred)))\n", @@ -351,7 +359,7 @@ ], "source": [ "correlation_matrix = df_data.corr()\n", - "print(correlation_matrix['Suma'].sort_values(ascending=False)[1:29])" + "print(correlation_matrix[\"Suma\"].sort_values(ascending=False)[1:29])" ] }, { @@ -792,8 +800,10 @@ ], "source": [ "feature_importance = dict(zip(feature_names, model.feature_importances_))\n", - "for feature, importance in sorted(feature_importance.items(), key=lambda x: x[1], reverse=True):\n", - " print(f'{importance:.4f} \\u2014 {feature}')" + "for feature, importance in sorted(\n", + " feature_importance.items(), key=lambda x: x[1], reverse=True\n", + "):\n", + " print(f\"{importance:.4f} \\u2014 {feature}\")" ] } ], diff --git a/Programming/PSD/flink/anomaly_detector.py b/Programming/PSD/flink/anomaly_detector.py index 7a4a8d91..6f3ad9a5 100644 --- a/Programming/PSD/flink/anomaly_detector.py +++ b/Programming/PSD/flink/anomaly_detector.py @@ -1,18 +1,21 @@ +import json + +from pyflink.common.serialization import SimpleStringSchema from pyflink.datastream import StreamExecutionEnvironment from pyflink.datastream.connectors.kafka import FlinkKafkaConsumer -from pyflink.common.serialization import SimpleStringSchema -import json + def detect_anomalies(transaction): transaction_data = json.loads(transaction) # Add anomaly detection logic here return transaction_data + env = StreamExecutionEnvironment.get_execution_environment() kafka_consumer = FlinkKafkaConsumer( - topics='transactions', + topics="transactions", deserialization_schema=SimpleStringSchema(), - properties={'bootstrap.servers': 'localhost:9092', 'group.id': 'anomaly_detection'} + properties={"bootstrap.servers": "localhost:9092", "group.id": "anomaly_detection"}, ) data_stream = env.add_source(kafka_consumer).map(detect_anomalies) diff --git a/Programming/PSD/kafka/generate_and_send.py b/Programming/PSD/kafka/generate_and_send.py index 9c12d84d..b32901b1 100644 --- a/Programming/PSD/kafka/generate_and_send.py +++ b/Programming/PSD/kafka/generate_and_send.py @@ -1,32 +1,41 @@ -from confluent_kafka import Producer import json import random import time +from confluent_kafka import Producer + # Configuration for Kafka Producer -conf = {'bootstrap.servers': "localhost:9092"} +conf = {"bootstrap.servers": "localhost:9092"} producer = Producer(**conf) + def generate_transaction(): return { "card_id": random.randint(1000, 9999), "user_id": random.randint(100, 999), "location": { "latitude": round(random.uniform(-90, 90), 6), - "longitude": round(random.uniform(-180, 180), 6) + "longitude": round(random.uniform(-180, 180), 6), }, "transaction_value": round(random.uniform(1, 1000), 2), - "spending_limit": round(random.uniform(1000, 5000), 2) + "spending_limit": round(random.uniform(1000, 5000), 2), } + def delivery_report(err, msg): if err is not None: - print('Message delivery failed: {}'.format(err)) + print(f"Message delivery failed: {err}") else: - print('Message delivered to {} [{}]'.format(msg.topic(), msg.partition())) + print(f"Message delivered to {msg.topic()} [{msg.partition()}]") + while True: transaction = generate_transaction() - producer.produce('transactions', key=str(transaction["card_id"]), value=json.dumps(transaction), callback=delivery_report) + producer.produce( + "transactions", + key=str(transaction["card_id"]), + value=json.dumps(transaction), + callback=delivery_report, + ) producer.poll(1) time.sleep(1) # Adjust the sleep time to control the frequency of transactions diff --git a/Programming/PSD/kafka/kafka_consumer.py b/Programming/PSD/kafka/kafka_consumer.py index 21a79f9d..5c25bf81 100644 --- a/Programming/PSD/kafka/kafka_consumer.py +++ b/Programming/PSD/kafka/kafka_consumer.py @@ -1,11 +1,13 @@ from confluent_kafka import Consumer, KafkaError -conf = {'bootstrap.servers': "localhost:9092", - 'group.id': "test_group", - 'auto.offset.reset': 'earliest'} +conf = { + "bootstrap.servers": "localhost:9092", + "group.id": "test_group", + "auto.offset.reset": "earliest", +} consumer = Consumer(**conf) -consumer.subscribe(['transactions']) +consumer.subscribe(["transactions"]) while True: msg = consumer.poll(timeout=1.0) @@ -17,5 +19,5 @@ while True: else: print(msg.error()) break - print('Received message: {}'.format(msg.value().decode('utf-8'))) + print("Received message: {}".format(msg.value().decode("utf-8"))) consumer.close() diff --git a/Programming/PSD/webinterface/webinterface.py b/Programming/PSD/webinterface/webinterface.py index c08d8959..15babc54 100644 --- a/Programming/PSD/webinterface/webinterface.py +++ b/Programming/PSD/webinterface/webinterface.py @@ -1,17 +1,20 @@ -from flask import Flask, jsonify from confluent_kafka import Consumer, KafkaError +from flask import Flask, jsonify app = Flask(__name__) -@app.route('/alarms', methods=['GET']) + +@app.route("/alarms", methods=["GET"]) def get_alarms(): alarms = [] - conf = {'bootstrap.servers': "localhost:9092", - 'group.id': "alarm_group", - 'auto.offset.reset': 'earliest'} + conf = { + "bootstrap.servers": "localhost:9092", + "group.id": "alarm_group", + "auto.offset.reset": "earliest", + } consumer = Consumer(**conf) - consumer.subscribe(['alarms']) + consumer.subscribe(["alarms"]) while True: msg = consumer.poll(timeout=1.0) @@ -20,12 +23,12 @@ def get_alarms(): if msg.error(): if msg.error().code() == KafkaError._PARTITION_EOF: continue - else: - return str(msg.error()), 500 - alarms.append(json.loads(msg.value().decode('utf-8'))) - + return str(msg.error()), 500 + alarms.append(json.loads(msg.value().decode("utf-8"))) + consumer.close() return jsonify(alarms) -if __name__ == '__main__': + +if __name__ == "__main__": app.run(debug=True) diff --git a/Programming/PSD/zin1/konsument.py b/Programming/PSD/zin1/konsument.py index 89e14898..b199289d 100644 --- a/Programming/PSD/zin1/konsument.py +++ b/Programming/PSD/zin1/konsument.py @@ -1,12 +1,14 @@ import json + from kafka import KafkaConsumer # Thresholds TEMP_TOO_COLD = -10 TEMP_TOO_HOT = 35 + def process_temperature_reading(reading): - temperature = reading['temperature'] + temperature = reading["temperature"] if temperature < TEMP_TOO_COLD: alert = f"WARNING: Temperature is too cold! ({temperature}°C)" elif temperature > TEMP_TOO_HOT: @@ -15,11 +17,12 @@ def process_temperature_reading(reading): alert = f"Temperature is normal. ({temperature}°C)" return alert -if __name__ == '__main__': + +if __name__ == "__main__": consumer = KafkaConsumer( - 'temperature_readings', - bootstrap_servers='localhost:9092', - auto_offset_reset='earliest' + "temperature_readings", + bootstrap_servers="localhost:9092", + auto_offset_reset="earliest", ) for message in consumer: diff --git a/Programming/PSD/zin1/producent.py b/Programming/PSD/zin1/producent.py index 089cd336..86cd6579 100644 --- a/Programming/PSD/zin1/producent.py +++ b/Programming/PSD/zin1/producent.py @@ -1,20 +1,22 @@ import json import random import time + from kafka import KafkaProducer from simulate_temperature_sensor import generate_temperature_reading + def serializer(message): - return json.dumps(message).encode('utf-8') + return json.dumps(message).encode("utf-8") + producer = KafkaProducer( - bootstrap_servers=['localhost:9092'], - value_serializer=serializer + bootstrap_servers=["localhost:9092"], value_serializer=serializer ) -if __name__ == '__main__': +if __name__ == "__main__": while True: reading = generate_temperature_reading() print(f"Sending reading: {reading}") - producer.send('temperature_readings', reading) + producer.send("temperature_readings", reading) time.sleep(random.randint(1, 5)) # Simulate readings sent at random intervals diff --git a/Programming/PSD/zin1/simulate_temperature_sensor.py b/Programming/PSD/zin1/simulate_temperature_sensor.py index 5f730d0b..1f24f73d 100644 --- a/Programming/PSD/zin1/simulate_temperature_sensor.py +++ b/Programming/PSD/zin1/simulate_temperature_sensor.py @@ -1,12 +1,15 @@ import random import time + def generate_temperature_reading(): location_id = random.randint(1, 10) # Simulate 10 different locations - temperature = random.uniform(-20, 40) # Temperature range from -20 to 40 degrees Celsius + temperature = random.uniform( + -20, 40 + ) # Temperature range from -20 to 40 degrees Celsius timestamp = time.time() # Current Unix timestamp return { - 'location_id': location_id, - 'temperature': round(temperature, 2), - 'timestamp': timestamp + "location_id": location_id, + "temperature": round(temperature, 2), + "timestamp": timestamp, } diff --git a/Programming/PSD/zin3/python/code/generator/temperature_generator.py b/Programming/PSD/zin3/python/code/generator/temperature_generator.py index 250e9b41..e3eee2d9 100644 --- a/Programming/PSD/zin3/python/code/generator/temperature_generator.py +++ b/Programming/PSD/zin3/python/code/generator/temperature_generator.py @@ -1,66 +1,68 @@ import json -import random -import time -from confluent_kafka import Producer -import sys import os +import random +import sys +import time + +from confluent_kafka import Producer # Add parent directory to path to import model sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from model.temperature_reading import TemperatureReading + def delivery_report(err, msg): - """Callback for message delivery reports""" + """Callback for message delivery reports.""" if err is not None: - print(f'Message delivery failed: {err}') + print(f"Message delivery failed: {err}") else: pass # Successfully delivered + def main(): # Set up Kafka producer with confluent-kafka - producer_config = { - 'bootstrap.servers': 'localhost:9092' - } + producer_config = {"bootstrap.servers": "localhost:9092"} producer = Producer(producer_config) - + # Generate a fixed number of thermometer IDs thermometer_ids = ["Therm-1", "Therm-2", "Therm-3", "Therm-4", "Therm-5"] - + try: while True: for thermometer_id in thermometer_ids: # Generate a random temperature between -10 and 30 degrees temperature = random.uniform(-10, 30) - + # Create reading object reading = TemperatureReading( thermometer_id=thermometer_id, timestamp=int(time.time() * 1000), - temperature=temperature + temperature=temperature, ) - + # Convert to dictionary for JSON serialization reading_dict = reading.to_dict() - payload = json.dumps(reading_dict).encode('utf-8') - + payload = json.dumps(reading_dict).encode("utf-8") + # Send to Kafka topic "Temperatura" producer.produce( - "Temperatura", - key=thermometer_id.encode('utf-8'), + "Temperatura", + key=thermometer_id.encode("utf-8"), value=payload, - callback=delivery_report + callback=delivery_report, ) producer.flush(timeout=1) # print(f"Sent: {json.dumps(reading_dict)}") - + # Sleep for a second time.sleep(1) - + except KeyboardInterrupt: print("Stopping temperature generator") finally: # Wait for any outstanding messages to be delivered producer.flush() + if __name__ == "__main__": main() diff --git a/Programming/PSD/zin3/python/code/model/temperature_alarm.py b/Programming/PSD/zin3/python/code/model/temperature_alarm.py index d37da56d..da253701 100644 --- a/Programming/PSD/zin3/python/code/model/temperature_alarm.py +++ b/Programming/PSD/zin3/python/code/model/temperature_alarm.py @@ -3,18 +3,18 @@ class TemperatureAlarm: self.thermometer_id = thermometer_id self.timestamp = timestamp self.temperature = temperature - + def to_dict(self): return { "thermometerId": self.thermometer_id, "timestamp": self.timestamp, - "temperature": self.temperature + "temperature": self.temperature, } - + @classmethod def from_dict(cls, data): return cls( thermometer_id=data.get("thermometerId"), timestamp=data.get("timestamp"), - temperature=data.get("temperature") + temperature=data.get("temperature"), ) diff --git a/Programming/PSD/zin3/python/code/model/temperature_reading.py b/Programming/PSD/zin3/python/code/model/temperature_reading.py index 01c15f24..e3d5a04a 100644 --- a/Programming/PSD/zin3/python/code/model/temperature_reading.py +++ b/Programming/PSD/zin3/python/code/model/temperature_reading.py @@ -3,18 +3,18 @@ class TemperatureReading: self.thermometer_id = thermometer_id self.timestamp = timestamp self.temperature = temperature - + def to_dict(self): return { "thermometerId": self.thermometer_id, "timestamp": self.timestamp, - "temperature": self.temperature + "temperature": self.temperature, } - + @classmethod def from_dict(cls, data): return cls( thermometer_id=data.get("thermometerId"), timestamp=data.get("timestamp"), - temperature=data.get("temperature") + temperature=data.get("temperature"), ) diff --git a/Programming/PSD/zin3/python/code/processor/temperature_anomaly_detector.py b/Programming/PSD/zin3/python/code/processor/temperature_anomaly_detector.py index 89e2e569..b7871fe2 100644 --- a/Programming/PSD/zin3/python/code/processor/temperature_anomaly_detector.py +++ b/Programming/PSD/zin3/python/code/processor/temperature_anomaly_detector.py @@ -1,62 +1,63 @@ -import os import json +import os import sys -from pyflink.datastream import StreamExecutionEnvironment + from pyflink.common.serialization import SimpleStringSchema -from pyflink.datastream.connectors import FlinkKafkaConsumer, FlinkKafkaProducer -from pyflink.common.time import Time from pyflink.common.typeinfo import Types from pyflink.common.watermark_strategy import WatermarkStrategy -from pyflink.datastream.functions import MapFunction, KeyedProcessFunction -from pyflink.datastream.state import ValueStateDescriptor +from pyflink.datastream import StreamExecutionEnvironment +from pyflink.datastream.connectors import FlinkKafkaConsumer, FlinkKafkaProducer +from pyflink.datastream.functions import KeyedProcessFunction, MapFunction # Add parent directory to path to import model sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + class ReadingMapFunction(MapFunction): def map(self, value): data = json.loads(value) return (data["thermometerId"], data["timestamp"], data["temperature"]) + class AnomalyDetectionFunction(KeyedProcessFunction): def __init__(self): self.window_size = 10 * 1000 # 10 seconds in milliseconds def process_element(self, value, ctx): thermometer_id, timestamp, temperature = value - + # Check if temperature is below zero (anomaly) if temperature < 0.0: # Create alarm alarm = { "thermometerId": thermometer_id, "timestamp": timestamp, - "temperature": temperature + "temperature": temperature, } yield json.dumps(alarm) + def add_kafka_dependencies(env): - """Add Kafka connector dependencies to the environment""" - + """Add Kafka connector dependencies to the environment.""" # User-specific paths where JAR files are located kafka_path = "/home/psd/Downloads/kafka_2.13-3.4.0/libs" flink_path = "/home/psd/Downloads/flink-1.17.0/lib" - + # Required JAR files kafka_jars = { "kafka-clients": "kafka-clients-3.4.0.jar", - "flink-connector-kafka": None # Will be detected or reported as missing + "flink-connector-kafka": None, # Will be detected or reported as missing } - + jar_paths = [] - + # Add kafka-clients JAR clients_jar = os.path.join(kafka_path, kafka_jars["kafka-clients"]) if os.path.exists(clients_jar): jar_paths.append(f"file://{clients_jar}") else: print(f"Warning: Could not find Kafka clients JAR at {clients_jar}") - + # Try to find Flink Kafka connector JAR connector_found = False for file in os.listdir(flink_path): @@ -65,65 +66,66 @@ def add_kafka_dependencies(env): jar_paths.append(f"file://{os.path.join(flink_path, file)}") connector_found = True break - + if not connector_found: print("Warning: No Flink Kafka connector JAR found! You need to download it.") print("You can download it from Maven Central or the Apache Flink website.") print("The JAR should be named like 'flink-connector-kafka-1.17.0.jar'") print(f"Place it in: {flink_path}") - + if jar_paths: - env.get_config().get_configuration().set_string("pipeline.jars", ";".join(jar_paths)) + env.get_config().get_configuration().set_string( + "pipeline.jars", ";".join(jar_paths) + ) print(f"Added JAR files: {jar_paths}") + def main(): # Create execution environment env = StreamExecutionEnvironment.get_execution_environment() - + # Add Kafka connector dependencies add_kafka_dependencies(env) - + # Configure Kafka consumer consumer_props = { - 'bootstrap.servers': 'localhost:9092', - 'group.id': 'temperature-anomaly-detector' + "bootstrap.servers": "localhost:9092", + "group.id": "temperature-anomaly-detector", } - + # Create Kafka consumer consumer = FlinkKafkaConsumer( - topics=['Temperatura'], + topics=["Temperatura"], deserialization_schema=SimpleStringSchema(), - properties=consumer_props + properties=consumer_props, ) - + # Configure Kafka producer - producer_props = { - 'bootstrap.servers': 'localhost:9092' - } - + producer_props = {"bootstrap.servers": "localhost:9092"} + # Create Kafka producer producer = FlinkKafkaProducer( - topic='Alarm', + topic="Alarm", serialization_schema=SimpleStringSchema(), - producer_config=producer_props + producer_config=producer_props, ) - + # Add source and transformations - env.add_source(consumer) \ - .map(ReadingMapFunction(), output_type=Types.TUPLE([Types.STRING(), Types.LONG(), Types.DOUBLE()])) \ - .assign_timestamps_and_watermarks( - WatermarkStrategy.for_monotonous_timestamps() - .with_timestamp_assigner(lambda event, timestamp: event[1]) - ) \ - .key_by(lambda x: x[0]) \ - .process(AnomalyDetectionFunction()) \ - .add_sink(producer) - + env.add_source(consumer).map( + ReadingMapFunction(), + output_type=Types.TUPLE([Types.STRING(), Types.LONG(), Types.DOUBLE()]), + ).assign_timestamps_and_watermarks( + WatermarkStrategy.for_monotonous_timestamps().with_timestamp_assigner( + lambda event, timestamp: event[1] + ) + ).key_by(lambda x: x[0]).process(AnomalyDetectionFunction()).add_sink(producer) + # Print the execution plan print(env.get_execution_plan()) - + # Execute env.execute("Temperature Anomaly Detector with Time Windows") + if __name__ == "__main__": main() diff --git a/Programming/PSD/zin3/python/code/visualizer/alarm_visualizer.py b/Programming/PSD/zin3/python/code/visualizer/alarm_visualizer.py index 3f9175f3..6dd18b6b 100644 --- a/Programming/PSD/zin3/python/code/visualizer/alarm_visualizer.py +++ b/Programming/PSD/zin3/python/code/visualizer/alarm_visualizer.py @@ -1,57 +1,61 @@ import json +import os +import sys import threading import tkinter as tk -from tkinter import ttk -from confluent_kafka import Consumer -import sys -import os -from datetime import datetime from collections import Counter +from datetime import datetime +from tkinter import ttk + +from confluent_kafka import Consumer # Add parent directory to path to import model sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from model.temperature_alarm import TemperatureAlarm + class AlarmVisualizer: def __init__(self, root): self.root = root self.root.title("Temperature Alarm Visualizer") self.root.geometry("600x400") - + # Create the list model for alarms self.alarm_list = tk.Listbox(root, font=("Courier", 12)) self.alarm_list.pack(fill=tk.BOTH, expand=True) - + # Create scrollbar - scrollbar = ttk.Scrollbar(self.alarm_list, orient="vertical", command=self.alarm_list.yview) + scrollbar = ttk.Scrollbar( + self.alarm_list, orient="vertical", command=self.alarm_list.yview + ) scrollbar.pack(side=tk.RIGHT, fill=tk.Y) self.alarm_list.config(yscrollcommand=scrollbar.set) - + # Create stats panel self.stats_frame = tk.Frame(root) self.stats_frame.pack(fill=tk.X, side=tk.BOTTOM) - + self.stats_label = tk.Label(self.stats_frame, text="No alarms yet") self.stats_label.pack(pady=5) - + # Stats tracking self.thermometer_alarm_count = Counter() self.max_alarms = 100 - + # Start Kafka consumer in a separate thread self.consumer_thread = threading.Thread(target=self.consume_alarms, daemon=True) self.consumer_thread.start() - + def consume_alarms(self): # Set up Kafka consumer with confluent-kafka consumer_config = { - 'bootstrap.servers': 'localhost:9092', - 'group.id': 'alarm-visualizer', - 'auto.offset.reset': 'latest' + "bootstrap.servers": "localhost:9092", + "group.id": "alarm-visualizer", + "auto.offset.reset": "latest", } consumer = Consumer(consumer_config) - consumer.subscribe(['Alarm']) - + consumer.subscribe(["Alarm"]) + try: while True: msg = consumer.poll(1.0) @@ -60,47 +64,51 @@ class AlarmVisualizer: if msg.error(): print(f"Consumer error: {msg.error()}") continue - + # Parse the alarm message - alarm_data = json.loads(msg.value().decode('utf-8')) + alarm_data = json.loads(msg.value().decode("utf-8")) alarm = TemperatureAlarm.from_dict(alarm_data) - + # Format the timestamp timestamp_dt = datetime.fromtimestamp(alarm.timestamp / 1000) - formatted_date = timestamp_dt.strftime('%Y-%m-%d %H:%M:%S') - + formatted_date = timestamp_dt.strftime("%Y-%m-%d %H:%M:%S") + # Create alarm message alarm_message = f"⚠️ ALARM: Thermometer {alarm.thermometer_id} reported {alarm.temperature:.2f}°C at {formatted_date}" - + # Update the UI self.root.after(0, self.update_ui, alarm_message, alarm.thermometer_id) - + except Exception as e: print(f"Error in Kafka consumer: {e}") finally: consumer.close() - + def update_ui(self, alarm_message, thermometer_id): # Add new alarm to the top of the list self.alarm_list.insert(0, alarm_message) - + # Keep list at a reasonable size if self.alarm_list.size() > self.max_alarms: self.alarm_list.delete(self.max_alarms) - + # Update statistics self.thermometer_alarm_count[thermometer_id] += 1 - + # Format stats string if self.thermometer_alarm_count: - stats_parts = [f"{id}={count}" for id, count in self.thermometer_alarm_count.items()] + stats_parts = [ + f"{id}={count}" for id, count in self.thermometer_alarm_count.items() + ] stats_text = "Alarm Counts: " + ", ".join(stats_parts) self.stats_label.config(text=stats_text) + def main(): root = tk.Tk() app = AlarmVisualizer(root) root.mainloop() + if __name__ == "__main__": main() diff --git a/Programming/TRAK/.gitignore b/Programming/TRAK/.gitignore index bb830a31..933bca77 100644 --- a/Programming/TRAK/.gitignore +++ b/Programming/TRAK/.gitignore @@ -167,4 +167,7 @@ cython_debug/ #.idea/ # *.exr -- the environment map below is tracked; see root .gitignore -!*.exr \ No newline at end of file +!*.exr +# Output of `../../run.sh run`, which must never land on a tracked +# deliverable in outputs/. +build/ diff --git a/Programming/TRAK/code/mapa_srodowiska/main.py b/Programming/TRAK/code/mapa_srodowiska/main.py index 311581f8..96c631ff 100644 --- a/Programming/TRAK/code/mapa_srodowiska/main.py +++ b/Programming/TRAK/code/mapa_srodowiska/main.py @@ -1,10 +1,10 @@ -import OpenEXR -import Imath import os + +import Imath +import OpenEXR import OpenGL.GL as gl -import OpenGL.GLUT as glut import OpenGL.GLU as glu -import math +import OpenGL.GLUT as glut # Camera parameters camera_angle_x = 0.0 @@ -14,44 +14,45 @@ mouse_last_x = 0 mouse_last_y = 0 mouse_left_down = False + def load_hdr_environment_map(filepath): - """ - Load an HDR environment map from an OpenEXR file. - + """Load an HDR environment map from an OpenEXR file. + Args: filepath (str): Path to the HDR file. - + Returns: tuple: A tuple containing the width, height, and a bytes object representing the HDR environment map in RGB format. + """ if not os.path.exists(filepath): raise FileNotFoundError(f"File not found: {filepath}") - + # Check file permissions if not os.access(filepath, os.R_OK): raise PermissionError(f"File is not readable: {filepath}") - + # Open the EXR file try: exr_file = OpenEXR.InputFile(filepath) except Exception as e: - raise OSError(f"Unable to open '{filepath}' for read: {str(e)}") - + raise OSError(f"Unable to open '{filepath}' for read: {e!s}") + # Get the image dimensions header = exr_file.header() - dw = header['dataWindow'] + dw = header["dataWindow"] width = dw.max.x - dw.min.x + 1 height = dw.max.y - dw.min.y + 1 # Define the channel names (R, G, B) - channels = ['R', 'G', 'B'] - + channels = ["R", "G", "B"] + # Read the channel data channel_data = { channel: exr_file.channel(channel, Imath.PixelType(Imath.PixelType.FLOAT)) for channel in channels } - + # Combine channel data into a single bytes object hdr_image = bytearray(width * height * 3 * 4) # 3 channels, 4 bytes per float for i, channel in enumerate(channels): @@ -59,7 +60,9 @@ def load_hdr_environment_map(filepath): for j in range(height): for k in range(width): index = (j * width + k) * 3 * 4 + i * 4 - hdr_image[index:index + 4] = channel_buffer[(j * width + k) * 4:(j * width + k + 1) * 4] + hdr_image[index : index + 4] = channel_buffer[ + (j * width + k) * 4 : (j * width + k + 1) * 4 + ] # Flip the image vertically flipped_hdr_image = bytearray(width * height * 3 * 4) @@ -67,51 +70,76 @@ def load_hdr_environment_map(filepath): for j in range(height): src_index = j * row_size dst_index = (height - 1 - j) * row_size - flipped_hdr_image[dst_index:dst_index + row_size] = hdr_image[src_index:src_index + row_size] + flipped_hdr_image[dst_index : dst_index + row_size] = hdr_image[ + src_index : src_index + row_size + ] return width, height, bytes(flipped_hdr_image) + def display_hdr_image(width, height, hdr_image): gl.glClear(gl.GL_COLOR_BUFFER_BIT | gl.GL_DEPTH_BUFFER_BIT) - + # Enable texture mapping gl.glEnable(gl.GL_TEXTURE_2D) texture_id = gl.glGenTextures(1) gl.glBindTexture(gl.GL_TEXTURE_2D, texture_id) - + # Set texture parameters gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_S, gl.GL_CLAMP_TO_EDGE) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_WRAP_T, gl.GL_CLAMP_TO_EDGE) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MIN_FILTER, gl.GL_LINEAR) gl.glTexParameteri(gl.GL_TEXTURE_2D, gl.GL_TEXTURE_MAG_FILTER, gl.GL_LINEAR) - + # Load the texture - gl.glTexImage2D(gl.GL_TEXTURE_2D, 0, gl.GL_RGB32F, width, height, 0, gl.GL_RGB, gl.GL_FLOAT, hdr_image) - + gl.glTexImage2D( + gl.GL_TEXTURE_2D, + 0, + gl.GL_RGB32F, + width, + height, + 0, + gl.GL_RGB, + gl.GL_FLOAT, + hdr_image, + ) + # Set up the viewport and projection - gl.glViewport(0, 0, glut.glutGet(glut.GLUT_WINDOW_WIDTH), glut.glutGet(glut.GLUT_WINDOW_HEIGHT)) + gl.glViewport( + 0, + 0, + glut.glutGet(glut.GLUT_WINDOW_WIDTH), + glut.glutGet(glut.GLUT_WINDOW_HEIGHT), + ) gl.glMatrixMode(gl.GL_PROJECTION) gl.glLoadIdentity() - glu.gluPerspective(45.0, glut.glutGet(glut.GLUT_WINDOW_WIDTH) / float(glut.glutGet(glut.GLUT_WINDOW_HEIGHT)), 0.1, 100.0) + glu.gluPerspective( + 45.0, + glut.glutGet(glut.GLUT_WINDOW_WIDTH) + / float(glut.glutGet(glut.GLUT_WINDOW_HEIGHT)), + 0.1, + 100.0, + ) gl.glMatrixMode(gl.GL_MODELVIEW) gl.glLoadIdentity() - + # Apply camera transformations gl.glTranslatef(0.0, 0.0, -camera_distance) gl.glRotatef(camera_angle_y, 1.0, 0.0, 0.0) gl.glRotatef(camera_angle_x, 0.0, 1.0, 0.0) - + # Draw a textured sphere to simulate being inside the HDR environment quadric = glu.gluNewQuadric() glu.gluQuadricTexture(quadric, gl.GL_TRUE) glu.gluSphere(quadric, 50.0, 50, 50) glu.gluDeleteQuadric(quadric) - + # Disable texture mapping gl.glDisable(gl.GL_TEXTURE_2D) - + glut.glutSwapBuffers() + def mouse_motion(x, y): global mouse_last_x, mouse_last_y, camera_angle_x, camera_angle_y if mouse_left_down: @@ -123,6 +151,7 @@ def mouse_motion(x, y): mouse_last_y = y glut.glutPostRedisplay() + def mouse_button(button, state, x, y): global mouse_left_down if button == glut.GLUT_LEFT_BUTTON: @@ -133,29 +162,31 @@ def mouse_button(button, state, x, y): elif state == glut.GLUT_UP: mouse_left_down = False -def main(filepath): + +def main(filepath): try: width, height, hdr_map = load_hdr_environment_map(filepath) print("HDR Map Loaded. Dimensions:", width, "x", height) - + # Initialize GLUT and create window glut.glutInit() glut.glutInitDisplayMode(glut.GLUT_DOUBLE | glut.GLUT_RGB | glut.GLUT_DEPTH) glut.glutInitWindowSize(800, 600) # Set initial window size glut.glutCreateWindow(b"HDR Environment Map") - + # Set display callback glut.glutDisplayFunc(lambda: display_hdr_image(width, height, hdr_map)) - + # Set mouse callbacks glut.glutMotionFunc(mouse_motion) glut.glutMouseFunc(mouse_button) - + # Start the GLUT main loop glut.glutMainLoop() except Exception as e: print(e) + # Example usage if __name__ == "__main__": filepath = "lilienstein_4k.exr" diff --git a/Programming/TRAK/code/photonmapping/python/main.py b/Programming/TRAK/code/photonmapping/python/main.py index 92567914..99b99523 100644 --- a/Programming/TRAK/code/photonmapping/python/main.py +++ b/Programming/TRAK/code/photonmapping/python/main.py @@ -1,34 +1,37 @@ -import numpy as np -import matplotlib.pyplot as plt import time +import matplotlib.pyplot as plt +import numpy as np + PHOTONS = 0 + # Define basic vector operations class Vector3: def __init__(self, x, y, z): self.x = x self.y = y self.z = z - + def __add__(self, other): - return Vector3(self.x+other.x, self.y+other.y, self.z+other.z) - + return Vector3(self.x + other.x, self.y + other.y, self.z + other.z) + def __sub__(self, other): - return Vector3(self.x-other.x, self.y-other.y, self.z-other.z) - + return Vector3(self.x - other.x, self.y - other.y, self.z - other.z) + def __mul__(self, scalar): - return Vector3(self.x*scalar, self.y*scalar, self.z*scalar) - + return Vector3(self.x * scalar, self.y * scalar, self.z * scalar) + def dot(self, other): - return self.x*other.x + self.y*other.y + self.z*other.z - + return self.x * other.x + self.y * other.y + self.z * other.z + def norm(self): return np.sqrt(self.dot(self)) - + def normalize(self): n = self.norm() - return Vector3(self.x/n, self.y/n, self.z/n) + return Vector3(self.x / n, self.y / n, self.z / n) + # Define the photon class Photon: @@ -37,6 +40,7 @@ class Photon: self.direction = direction self.power = power + # Define a simple sphere class Sphere: def __init__(self, center, radius, color): @@ -49,27 +53,27 @@ class Sphere: oc = ray_origin - self.center a = ray_direction.dot(ray_direction) b = 2.0 * oc.dot(ray_direction) - c = oc.dot(oc) - self.radius*self.radius - discriminant = b*b - 4*a*c + c = oc.dot(oc) - self.radius * self.radius + discriminant = b * b - 4 * a * c if discriminant < 0: return None # No intersection - else: - t = (-b - np.sqrt(discriminant)) / (2.0*a) - if t < 0: - t = (-b + np.sqrt(discriminant)) / (2.0*a) - if t < 0: - return None - hit_point = ray_origin + ray_direction * t - normal = (hit_point - self.center).normalize() - return (t, hit_point, normal) - + t = (-b - np.sqrt(discriminant)) / (2.0 * a) + if t < 0: + t = (-b + np.sqrt(discriminant)) / (2.0 * a) + if t < 0: + return None + hit_point = ray_origin + ray_direction * t + normal = (hit_point - self.center).normalize() + return (t, hit_point, normal) + + # Define a simple plane class Plane: def __init__(self, point, normal, color): self.point = point self.normal = normal.normalize() self.color = color - + def intersect(self, ray_origin, ray_direction): denom = self.normal.dot(ray_direction) if abs(denom) > 1e-6: @@ -79,9 +83,12 @@ class Plane: return (t, hit_point, self.normal) return None + # Scene setup sphere = Sphere(Vector3(0, 0, -5), 1.0, np.array([1, 0, 0])) # Red sphere -plane = Plane(Vector3(0, -1, 0), Vector3(0, 1, 0), np.array([0.5, 0.5, 0.5])) # Gray plane +plane = Plane( + Vector3(0, -1, 0), Vector3(0, 1, 0), np.array([0.5, 0.5, 0.5]) +) # Gray plane objects = [sphere, plane] @@ -97,6 +104,7 @@ num_photons = 10000 # Number of photons to emit max_depth = 5 # Maximum number of bounces gather_radius = 0.5 # Radius for radiance estimation + def emit_photons(): for _ in range(num_photons): # Emit photons in random directions from the light source @@ -105,6 +113,7 @@ def emit_photons(): photon = Photon(light_position, direction, power) trace_photon(photon, 0) + def trace_photon(photon, depth): global PHOTONS PHOTONS += 1 @@ -131,20 +140,23 @@ def trace_photon(photon, depth): photon.direction = new_direction # Absorb some power photon.power = photon.power * 0.8 # Simple absorption - trace_photon(photon, depth+1) + trace_photon(photon, depth + 1) + def random_unit_vector(): - theta = np.random.uniform(0, 2*np.pi) + theta = np.random.uniform(0, 2 * np.pi) z = np.random.uniform(-1, 1) - r = np.sqrt(1 - z*z) + r = np.sqrt(1 - z * z) return Vector3(r * np.cos(theta), r * np.sin(theta), z) + def random_hemisphere_direction(normal): dir = random_unit_vector() if dir.dot(normal) < 0: dir = Vector3(-dir.x, -dir.y, -dir.z) return dir + def render_image(width, height): aspect_ratio = width / height fov = np.pi / 3 # 60 degrees field of view @@ -160,6 +172,7 @@ def render_image(width, height): image[y, x, :] = np.clip(color, 0, 1) return image + def trace_ray(ray_origin, ray_direction): closest_t = np.inf hit_object = None @@ -178,8 +191,8 @@ def trace_ray(ray_origin, ray_direction): direct_light = compute_direct_light(hit_point, normal) indirect_light = estimate_radiance(hit_point, normal) return color * (direct_light + indirect_light) - else: - return np.array([0, 0, 0]) # Background color + return np.array([0, 0, 0]) # Background color + def compute_direct_light(point, normal): # Simple Lambertian reflection from light source @@ -195,9 +208,9 @@ def compute_direct_light(point, normal): break if in_shadow: return np.array([0, 0, 0]) - else: - intensity = max(0, normal.dot(direction_to_light)) - return intensity * light_power / (4 * np.pi * (light_position - point).norm()**2) + intensity = max(0, normal.dot(direction_to_light)) + return intensity * light_power / (4 * np.pi * (light_position - point).norm() ** 2) + def estimate_radiance(point, normal): # Gather photons within the gather_radius @@ -207,29 +220,31 @@ def estimate_radiance(point, normal): if distance < gather_radius: weight = max(0, normal.dot((photon_pos - point).normalize())) accumulated_power += photon_power * weight - area = np.pi * gather_radius ** 2 + area = np.pi * gather_radius**2 return accumulated_power / (area * num_photons) + # Main execution -if __name__ == '__main__': +if __name__ == "__main__": print("Emitting photons...") emit_photons() - + print("Rendering image...") width = 100 height = 100 t0 = time.time() image = render_image(width, height) - - print (f"Render Took: {round(time.time() - t0, 2)}s\n" + + print( + f"Render Took: {round(time.time() - t0, 2)}s\n" f"resolution: {width}x{height}\n" f"photons (emitted): {num_photons}\n" f"photons (reflected): {PHOTONS - num_photons}\n" f"photons (total): {PHOTONS}\n" - f"rays: {width*height}" - ) - + f"rays: {width * height}" + ) + # Display the image plt.imshow(image) - plt.axis('off') + plt.axis("off") plt.show() diff --git a/Programming/TRAK/lab6.py b/Programming/TRAK/lab6.py index 10585d0a..0fe9a6a8 100644 --- a/Programming/TRAK/lab6.py +++ b/Programming/TRAK/lab6.py @@ -1,4 +1,2 @@ -import bpy -import os -script_path = '' # path to script -exec(compile(open(script_path).read(), script_path, 'exec')) +script_path = "" # path to script +exec(compile(open(script_path).read(), script_path, "exec")) diff --git a/Programming/TRAK/main.py b/Programming/TRAK/main.py index 53d8a8db..9aa9c4bc 100644 --- a/Programming/TRAK/main.py +++ b/Programming/TRAK/main.py @@ -1,33 +1,96 @@ import argparse -from rendering import ray_trace -from utils import load_config, parse_resolution import importlib import os + import matplotlib.pyplot as plt from photon_mapping import render_photon_mapping +from rendering import ray_trace +from utils import load_config, parse_resolution + def main(): # default config - config = load_config('config.ini') + config = load_config("config.ini") # Parse parser = argparse.ArgumentParser(description="Rendering Program") - parser.add_argument('--algorithm', type=str, help='Algorithm to use', default=config.get('DEFAULT', 'algorithm')) - parser.add_argument('--scene', type=str, help='Name of the scene to render (without .py).', default=config.get('DEFAULT', 'scene')) - parser.add_argument('--environment', type=str, help='Environment file', default=config.get('DEFAULT', 'environment')) - parser.add_argument('--env_blur', type=str, help='Environment blur', default=config.get('DEFAULT', 'env_blur')) - parser.add_argument('--resolution', type=str, help='Image resolution (WIDTHxHEIGHT)', - default=config.get('DEFAULT', 'resolution')) - parser.add_argument("--samples_per_pixel", type=int, default=config.get('ray_tracing', 'samples_per_pixel'), help="Samples per pixel for rendering.") - parser.add_argument("--output", type=str, default=config.get('DEFAULT', 'output'), help="Output file name.") - - parser.add_argument('--num_spheres', type=int, default=3, help='Number of spheres in the scene for Ray Tracing 0') - parser.add_argument('--num_photons', type=int, default=config.getint('photon_mapping', 'num_photons'), help='Number of photons for photon mapping') - parser.add_argument('--max_depth', type=int, default=config.getint('photon_mapping', 'max_depth'), - help='Maximum depth for photon tracing') - parser.add_argument('--gather_radius', type=float, default=config.getfloat('photon_mapping', 'gather_radius'), - help='Radius for radiance estimation in photon mapping') + parser.add_argument( + "--algorithm", + type=str, + help="Algorithm to use", + default=config.get("DEFAULT", "algorithm"), + ) + parser.add_argument( + "--scene", + type=str, + help="Name of the scene to render (without .py).", + default=config.get("DEFAULT", "scene"), + ) + parser.add_argument( + "--environment", + type=str, + help="Environment file", + default=config.get("DEFAULT", "environment"), + ) + parser.add_argument( + "--env_blur", + type=str, + help="Environment blur", + default=config.get("DEFAULT", "env_blur"), + ) + parser.add_argument( + "--resolution", + type=str, + help="Image resolution (WIDTHxHEIGHT)", + default=config.get("DEFAULT", "resolution"), + ) + parser.add_argument( + "--samples_per_pixel", + type=int, + default=config.get("ray_tracing", "samples_per_pixel"), + help="Samples per pixel for rendering.", + ) + parser.add_argument( + "--output", + type=str, + default=config.get("DEFAULT", "output"), + help="Output file name.", + ) + parser.add_argument( + "--num_spheres", + type=int, + default=3, + help="Number of spheres in the scene for Ray Tracing 0", + ) + parser.add_argument( + "--num_photons", + type=int, + default=config.getint("photon_mapping", "num_photons"), + help="Number of photons for photon mapping", + ) + parser.add_argument( + "--max_depth", + type=int, + default=config.getint("photon_mapping", "max_depth"), + help="Maximum depth for photon tracing", + ) + parser.add_argument( + "--gather_radius", + type=float, + default=config.getfloat("photon_mapping", "gather_radius"), + help="Radius for radiance estimation in photon mapping", + ) + # The render is always written to outputs/; this only controls whether a + # viewer is ALSO popped up. It defaults off because img.show() shells out + # to xdg-open and then blocks until a human closes the window -- which + # left ../../run.sh run Programming/TRAK hanging for 45 minutes after the + # ray trace had actually finished. + parser.add_argument( + "--show", + action="store_true", + help="also open the finished image in a viewer (blocks until closed)", + ) args = parser.parse_args() @@ -37,8 +100,13 @@ def main(): if args.algorithm == "ray_tracing0": print("Starting ray tracing zero...") # ray_trace(args.scene, args.environment_map, image_width=width, image_height=height, output_file="output_ray_traced.png") - ray_trace(args.num_spheres, args.environment_map, image_width=width, image_height=height, # na razie generujemy w kodzie, ale potem trzeba będzie obj wczytywać - output_file="output_ray_traced.png") + ray_trace( + args.num_spheres, + args.environment_map, + image_width=width, + image_height=height, # na razie generujemy w kodzie, ale potem trzeba będzie obj wczytywać + output_file="output_ray_traced.png", + ) elif args.algorithm == "ray_tracing": print("Starting ray tracing...") try: @@ -48,29 +116,40 @@ def main(): print(f"Error: Scene '{args.scene}' not found in the 'scenes' directory.") return try: - scene = scene_module.setup_scene(width=width, height=height, environment=f"{args.environment}") + scene = scene_module.setup_scene( + width=width, height=height, environment=f"{args.environment}" + ) except AttributeError: - print(f"Error: Scene '{args.scene}' does not define a `setup_scene` function.") + print( + f"Error: Scene '{args.scene}' does not define a `setup_scene` function." + ) return # Renderowanie - print(f"Rendering scene '{args.scene}' with {args.samples_per_pixel} samples per pixel...") + print( + f"Rendering scene '{args.scene}' with {args.samples_per_pixel} samples per pixel..." + ) img = scene.render(samples_per_pixel=args.samples_per_pixel) output_path = os.path.join("outputs", args.output) img.save(output_path) print(f"Image saved to {output_path}") - img.show() + if args.show: + img.show() elif args.algorithm == "photon_mapping": print("Starting photon mapping...") - image = render_photon_mapping(width, height, args.num_photons, args.max_depth, args.gather_radius) + image = render_photon_mapping( + width, height, args.num_photons, args.max_depth, args.gather_radius + ) plt.imshow(image) - plt.axis('off') + plt.axis("off") output_path = os.path.join("outputs", args.output) plt.savefig(output_path) print(f"Image saved to {output_path}") - plt.show() + if args.show: + plt.show() else: print(f"Unknown algorithm: {args.algorithm}") return -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/Programming/TRAK/photon_mapping.py b/Programming/TRAK/photon_mapping.py index befe273c..1719b025 100644 --- a/Programming/TRAK/photon_mapping.py +++ b/Programming/TRAK/photon_mapping.py @@ -1,5 +1,5 @@ import numpy as np -import matplotlib.pyplot as plt + # Define basic vector operations class Vector3: @@ -49,15 +49,14 @@ class Sphere: discriminant = b * b - 4 * a * c if discriminant < 0: return None - else: - t = (-b - np.sqrt(discriminant)) / (2.0 * a) - if t < 0: - t = (-b + np.sqrt(discriminant)) / (2.0 * a) - if t < 0: - return None - hit_point = ray_origin + ray_direction * t - normal = (hit_point - self.center).normalize() - return (t, hit_point, normal) + t = (-b - np.sqrt(discriminant)) / (2.0 * a) + if t < 0: + t = (-b + np.sqrt(discriminant)) / (2.0 * a) + if t < 0: + return None + hit_point = ray_origin + ray_direction * t + normal = (hit_point - self.center).normalize() + return (t, hit_point, normal) class Plane: @@ -80,7 +79,9 @@ def render_photon_mapping(width, height, num_photons, max_depth, gather_radius): # Photon mapping logic photon_map = [] sphere = Sphere(Vector3(0, 0, -5), 1.0, np.array([1, 0, 0])) # Red sphere - plane = Plane(Vector3(0, -1, 0), Vector3(0, 1, 0), np.array([0.5, 0.5, 0.5])) # Gray plane + plane = Plane( + Vector3(0, -1, 0), Vector3(0, 1, 0), np.array([0.5, 0.5, 0.5]) + ) # Gray plane objects = [sphere, plane] light_position = Vector3(-5, 5, -5) @@ -145,8 +146,7 @@ def render_photon_mapping(width, height, num_photons, max_depth, gather_radius): direct_light = compute_direct_light(hit_point, normal) indirect_light = estimate_radiance(hit_point, normal) return color * (direct_light + indirect_light) - else: - return np.array([0, 0, 0]) + return np.array([0, 0, 0]) def compute_direct_light(point, normal): direction_to_light = (light_position - point).normalize() @@ -160,9 +160,10 @@ def render_photon_mapping(width, height, num_photons, max_depth, gather_radius): break if in_shadow: return np.array([0, 0, 0]) - else: - intensity = max(0, normal.dot(direction_to_light)) - return intensity * light_power / (4 * np.pi * (light_position - point).norm() ** 2) + intensity = max(0, normal.dot(direction_to_light)) + return ( + intensity * light_power / (4 * np.pi * (light_position - point).norm() ** 2) + ) def estimate_radiance(point, normal): accumulated_power = np.array([0.0, 0.0, 0.0]) @@ -171,7 +172,7 @@ def render_photon_mapping(width, height, num_photons, max_depth, gather_radius): if distance < gather_radius: weight = max(0, normal.dot((photon_pos - point).normalize())) accumulated_power += photon_power * weight - area = np.pi * gather_radius ** 2 + area = np.pi * gather_radius**2 return accumulated_power / area emit_photons() diff --git a/Programming/TRAK/rendering.py b/Programming/TRAK/rendering.py index b0112517..f9b23230 100644 --- a/Programming/TRAK/rendering.py +++ b/Programming/TRAK/rendering.py @@ -1,33 +1,38 @@ #!/usr/bin/env python3 -""" Renders an image using raytracing """ -import numpy as np -import matplotlib.pyplot as plt +"""Renders an image using raytracing.""" + import time -def ray_trace(num_spheres, environment, image_width=400, image_height=300, output_file="fig.png"): +import matplotlib.pyplot as plt +import numpy as np + + +def ray_trace( + num_spheres, environment, image_width=400, image_height=300, output_file="fig.png" +): IMAGE_WIDTH = image_width IMAGE_HEIGHT = image_height - def normalize(vector): - """ - Normalize a vector. + """Normalize a vector. - Parameters: + Parameters + ---------- vector (numpy.ndarray): The input vector to be normalized. - Returns: + Returns + ------- numpy.ndarray: The normalized vector. + """ vector /= np.linalg.norm(vector) return vector - def intersect_plane(ray_origin, ray_direction, plane_point, plane_normal): - """ - Calculate the intersection of a ray with a plane. + """Calculate the intersection of a ray with a plane. - Parameters: + Parameters + ---------- ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. ray_direction (numpy.ndarray): A normalized 3D vector representing the direction of the ray. @@ -35,11 +40,13 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu plane_normal (numpy.ndarray): A normalized 3D vector representing the normal of the plane. - Returns: + Returns + ------- float: The distance from the origin ray_origin to the intersection point with the plane. Returns +inf if there is no intersection or if the intersection is behind the origin. + """ denom = np.dot(ray_direction, plane_normal) if np.abs(denom) < 1e-6: @@ -49,12 +56,11 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu return np.inf return d - def intersect_sphere(ray_origin, ray_direction, sphere_center, sphere_radius): - """ - Calculate the intersection of a ray with a sphere. + """Calculate the intersection of a ray with a sphere. - Parameters: + Parameters + ---------- ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. ray_direction (numpy.ndarray): A normalized 3D vector representing the direction of the ray. @@ -62,11 +68,13 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu the center of the sphere. sphere_radius (float): The radius of the sphere. - Returns: + Returns + ------- float: The distance from the origin ray_origin to the intersection point with the sphere. Returns +inf if there is no intersection or if the intersection is behind the origin. + """ a = np.dot(ray_direction, ray_direction) origin_to_center = ray_origin - sphere_center @@ -76,23 +84,24 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu disc = b * b - 4 * a * c return calculate_sphere_intersection(a, b, c, disc) - def calculate_sphere_intersection(a, b, c, disc): - """ - Calculate the + """Calculate the intersection distance of a ray with a sphere using the quadratic formula. - Parameters: + Parameters + ---------- a (float): Coefficient of t^2 in the quadratic equation. b (float): Coefficient of t in the quadratic equation. c (float): Constant term in the quadratic equation. disc (float): Discriminant of the quadratic equation. - Returns: + Returns + ------- float: The distance from the origin to the intersection point with the sphere. Returns +inf if there is no intersection or if the intersection is behind the origin. + """ if disc > 0: distance_squared = np.sqrt(disc) @@ -108,112 +117,121 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu return t1 if t0 < 0 else t0 return np.inf - def intersect(ray_origin, ray_direction, object_): - """ - Calculate the intersection of a ray with an object. + """Calculate the intersection of a ray with an object. - Parameters: + Parameters + ---------- ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. ray_direction (numpy.ndarray): A normalized 3D vector representing the direction of the ray. obj (dict): A dictionary representing the object with keys 'type', 'position', 'normal' (for planes), and 'radius' (for spheres). - Returns: + Returns + ------- float: The distance from the origin ray_origin to the intersection point with the object. Returns +inf if there is no intersection or if the intersection is behind the origin. - """ - if object_['type'] == 'plane': - return intersect_plane(ray_origin, ray_direction, - object_['position'], object_['normal']) - # object_['type'] == 'sphere': - return intersect_sphere(ray_origin, ray_direction, - object_['position'], object_['radius']) + """ + if object_["type"] == "plane": + return intersect_plane( + ray_origin, ray_direction, object_["position"], object_["normal"] + ) + # object_['type'] == 'sphere': + return intersect_sphere( + ray_origin, ray_direction, object_["position"], object_["radius"] + ) def get_normal(object_, intersection_point): - """ - Calculate the normal at the intersection point on the object. + """Calculate the normal at the intersection point on the object. - Parameters: + Parameters + ---------- obj (dict): A dictionary representing the object with keys 'type' and 'position'. intersection_point (numpy.ndarray): A 3D point representing the intersection point on the object. - Returns: + Returns + ------- numpy.ndarray: The normal vector at the intersection point. + """ - if object_['type'] == 'sphere': - normal = normalize(intersection_point - object_['position']) - elif object_['type'] == 'plane': - normal = object_['normal'] + if object_["type"] == "sphere": + normal = normalize(intersection_point - object_["position"]) + elif object_["type"] == "plane": + normal = object_["normal"] else: raise ValueError(f"Unknown object type: {object_['type']}") return normal - def get_color(object_, intersection_point): - """ - Get the color of the object at the intersection point. + """Get the color of the object at the intersection point. - Parameters: + Parameters + ---------- object_ (dict): A dictionary representing the object with a key 'color'. intersection_point (numpy.ndarray): A 3D point representing the intersection point on the object. - Returns: + Returns + ------- numpy.ndarray: The color of the object at the intersection point. + """ - color = object_['color'] - if not hasattr(color, '__len__'): + color = object_["color"] + if not hasattr(color, "__len__"): color = color(intersection_point) return color - def trace_ray(ray_origin, ray_direction): - """ - Trace a ray and find the color at the intersection point. + """Trace a ray and find the color at the intersection point. - Parameters: + Parameters + ---------- ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. ray_direction (numpy.ndarray): A normalized 3D vector representing the direction of the ray. - Returns: + Returns + ------- tuple: A tuple containing the object, intersection point, normal at the intersection, and the color at the intersection point. Returns None if there is no intersection. + """ t, obj_idx = find_intersection(ray_origin, ray_direction) if t == np.inf: return None object_, intersection_point = get_intersection_details( - ray_origin, ray_direction, t, obj_idx) - normal, color = get_normal(object_, intersection_point), get_color( - object_, intersection_point) + ray_origin, ray_direction, t, obj_idx + ) + normal, color = ( + get_normal(object_, intersection_point), + get_color(object_, intersection_point), + ) if is_shadowed(intersection_point, normal, obj_idx): return None - return compute_color( - object_, intersection_point, normal, color, ray_origin) - + return compute_color(object_, intersection_point, normal, color, ray_origin) def find_intersection(ray_origin, ray_direction): - """ - Find the intersection of a ray with the objects in the scene. + """Find the intersection of a ray with the objects in the scene. - Parameters: + Parameters + ---------- ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. ray_direction (numpy.ndarray): A normalized 3D vector representing the direction of the ray. - Returns: + Returns + ------- tuple: A tuple containing the distance to the intersection point and the index of the intersected object. + """ t = np.inf obj_idx = -1 @@ -223,52 +241,55 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu t, obj_idx = t_obj, index return t, obj_idx - def get_intersection_details(ray_origin, ray_direction, t, obj_idx): - """ - Get the details of the intersection point on the object. + """Get the details of the intersection point on the object. - Parameters: + Parameters + ---------- ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. ray_direction (numpy.ndarray): A normalized 3D vector representing the direction of the ray. t (float): The distance to the intersection point. obj_idx (int): The index of the intersected object in the scene. - Returns: + Returns + ------- tuple: A tuple containing the intersected object and the intersection point. + """ object_ = scene[obj_idx] intersection_point = ray_origin + ray_direction * t return object_, intersection_point - def is_shadowed(intersection_point, normal, obj_idx): - """ - Determine if the intersection point is in shadow. + """Determine if the intersection point is in shadow. - Parameters: + Parameters + ---------- intersection_point (numpy.ndarray): A 3D point representing the intersection point on the object. normal (numpy.ndarray): The normal vector at the intersection point. obj_idx (int): The index of the intersected object in the scene. - Returns: + Returns + ------- bool: True if the intersection point is in shadow, False otherwise. + """ to_light = normalize(L - intersection_point) - shadow_intersections = [intersect( - intersection_point + normal * .0001, to_light, obj_sh) - for k, obj_sh in enumerate(scene) if k != obj_idx] + shadow_intersections = [ + intersect(intersection_point + normal * 0.0001, to_light, obj_sh) + for k, obj_sh in enumerate(scene) + if k != obj_idx + ] return shadow_intersections and min(shadow_intersections) < np.inf - def compute_color(object_, intersection_point, normal, color, ray_origin): - """ - Compute the color at the intersection point using shading techniques. + """Compute the color at the intersection point using shading techniques. - Parameters: + Parameters + ---------- object_ (dict): A dictionary representing the intersected object. intersection_point (numpy.ndarray): A 3D point representing the intersection point on the object. @@ -276,85 +297,94 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu color (numpy.ndarray): The base color of the object. ray_origin (numpy.ndarray): A 3D point representing the origin of the ray. - Returns: + Returns + ------- tuple: A tuple containing the intersected object, intersection point, normal, and the computed color. + """ to_light = normalize(L - intersection_point) to_origin = normalize(ray_origin - intersection_point) color_ray = AMBIENT - diffuse_intensity = object_.get('diffuse_c', DIFFUSE_C) * max( - np.dot(normal, to_light), 0) + diffuse_intensity = object_.get("diffuse_c", DIFFUSE_C) * max( + np.dot(normal, to_light), 0 + ) color_ray += diffuse_intensity * color half_vector = normalize(to_light + to_origin) - specular_intensity = object_.get('specular_c', SPECULAR_C) * max( - np.dot(normal, half_vector), 0) ** SPECULAR_K + specular_intensity = ( + object_.get("specular_c", SPECULAR_C) + * max(np.dot(normal, half_vector), 0) ** SPECULAR_K + ) color_ray += specular_intensity * color_light return object_, intersection_point, normal, color_ray - def add_sphere(position, radius, color): - """ - Create a dictionary representing a sphere object. + """Create a dictionary representing a sphere object. - Parameters: + Parameters + ---------- position (list or numpy.ndarray): A 3D point representing the position of the sphere. radius (float): The radius of the sphere. color (list or numpy.ndarray): The color of the sphere. - Returns: + Returns + ------- dict: A dictionary representing the sphere object. + """ return { - 'type': 'sphere', - 'position': np.array(position), - 'radius': np.array(radius), - 'color': np.array(color), - 'reflection': .5 + "type": "sphere", + "position": np.array(position), + "radius": np.array(radius), + "color": np.array(color), + "reflection": 0.5, } - def add_plane(position, normal): - """ - Create a dictionary representing a plane object. + """Create a dictionary representing a plane object. - Parameters: + Parameters + ---------- position (list or numpy.ndarray): A 3D point representing a point on the plane. normal (list or numpy.ndarray): A normalized 3D vector representing the normal of the plane. - Returns: + Returns + ------- dict: A dictionary representing the plane object. + """ return { - 'type': 'plane', - 'position': np.array(position), - 'normal': np.array(normal), - 'color': lambda M: (color_plane0 - if (int(M[0] * 2) % 2) == (int(M[2] * 2) % 2) - else color_plane1), - 'diffuse_c': .75, - 'specular_c': .5, - 'reflection': .25 + "type": "plane", + "position": np.array(position), + "normal": np.array(normal), + "color": lambda M: ( + color_plane0 + if (int(M[0] * 2) % 2) == (int(M[2] * 2) % 2) + else color_plane1 + ), + "diffuse_c": 0.75, + "specular_c": 0.5, + "reflection": 0.25, } scene = [] # List of objects. - color_plane0 = 1. * np.ones(3) - color_plane1 = 0. * np.ones(3) - scene.append(add_plane([0., -0.5, 0.], [0., 1., 0.])) + color_plane0 = 1.0 * np.ones(3) + color_plane1 = 0.0 * np.ones(3) + scene.append(add_plane([0.0, -0.5, 0.0], [0.0, 1.0, 0.0])) base_radius = 1 / np.sqrt(num_spheres) # Im więcej kul, tym mniejsze - base_distance = 4.5 / num_spheres + base_distance = 4.5 / num_spheres for i in range(num_spheres): # Wyliczanie pozycji każdej kuli x = (i - num_spheres // 2) * base_distance y = 0.1 - z = 1. + i * 0.5 + z = 1.0 + i * 0.5 # Dynamiczny kolor (gradient na podstawie indeksu) color = np.array([i / num_spheres, (num_spheres - i) / num_spheres, 0.5]) @@ -363,24 +393,24 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu scene.append(add_sphere([x, y, z], base_radius, color)) # Light position and color. - L = np.array([5., 5., -10.]) + L = np.array([5.0, 5.0, -10.0]) color_light = np.ones(3) # Default light and material parameters. - AMBIENT = .05 - DIFFUSE_C = 1. - SPECULAR_C = 1. + AMBIENT = 0.05 + DIFFUSE_C = 1.0 + SPECULAR_C = 1.0 SPECULAR_K = 50 DEPTH_MAX = 5 # Maximum number of light reflections. col = np.zeros(3) # Current color. - camera_origin = np.array([0., 0.35, -1.]) # Camera. - Q = np.array([0., 0., 0.]) # Camera pointing to. + camera_origin = np.array([0.0, 0.35, -1.0]) # Camera. + Q = np.array([0.0, 0.0, 0.0]) # Camera pointing to. img = np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH, 3)) r = float(IMAGE_WIDTH) / IMAGE_HEIGHT # Screen coordinates: x0, y0, x1, y1. - S = (-1., -1. / r + .25, 1., 1. / r + .25) + S = (-1.0, -1.0 / r + 0.25, 1.0, 1.0 / r + 0.25) renderTime = time.time() reflections = 0 @@ -396,7 +426,7 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu D = normalize(Q - camera_origin) DEPTH = 0 rayO, rayD = camera_origin, D - REFLECTION = 1. + REFLECTION = 1.0 initialRays += 1 # Loop through initial and secondary rays. while DEPTH < DEPTH_MAX: @@ -407,19 +437,20 @@ def ray_trace(num_spheres, environment, image_width=400, image_height=300, outpu reflections += 1 obj, M, N, col_ray = traced # Reflection: create a new ray. - rayO, rayD = M + \ - N * .0001, normalize(rayD - 2 * np.dot(rayD, N) * N) + rayO, rayD = M + N * 0.0001, normalize(rayD - 2 * np.dot(rayD, N) * N) DEPTH += 1 col += REFLECTION * col_ray - REFLECTION *= obj.get('reflection', 1.) + REFLECTION *= obj.get("reflection", 1.0) img[IMAGE_HEIGHT - j - 1, i, :] = np.clip(col, 0, 1) renderTime = time.time() - renderTime plt.imsave(output_file, img) - print(f"Image saved as {output_file}\n" - f"resolution: {IMAGE_WIDTH}x{IMAGE_HEIGHT}\n" - f"render time: {round(renderTime, 2)} s\n" - f"reflections: {reflections}\n" - f"rays (initial): {initialRays}\n" - f"rays (secondary): {rays - initialRays}\n" - f"rays (total): {rays}") + print( + f"Image saved as {output_file}\n" + f"resolution: {IMAGE_WIDTH}x{IMAGE_HEIGHT}\n" + f"render time: {round(renderTime, 2)} s\n" + f"reflections: {reflections}\n" + f"rays (initial): {initialRays}\n" + f"rays (secondary): {rays - initialRays}\n" + f"rays (total): {rays}" + ) diff --git a/Programming/TRAK/run.sh b/Programming/TRAK/run.sh index 4ae9e700..a865d109 100755 --- a/Programming/TRAK/run.sh +++ b/Programming/TRAK/run.sh @@ -7,5 +7,16 @@ set -euo pipefail # shellcheck source=../../scripts/course_lib.sh . "$(dirname "$(readlink -f "$0")")/../../scripts/course_lib.sh" +# Three main.py files live here (the renderer, plus the two standalone +# experiments under code/), so the harness cannot pick one on its own and +# falls back to syntax-checking. Name the renderer explicitly: a run that +# actually ray-traces cornell_box at 400x300 is a real check, and "every +# module parses" is not. +# --output, because config.ini's default is outputs/output.png and that file +# is a TRACKED deliverable: a plain run silently overwrote what was submitted. +# build/ is gitignored, so a rebuild can never do that again. +mkdir -p "$(dirname "$(readlink -f "$0")")/build" course_main --kind python \ + --entry "$(dirname "$(readlink -f "$0")")/main.py" \ + --entry-args "--output ../build/render.png" \ -- "$@" diff --git a/Programming/TRAK/scenes/cornell_box.py b/Programming/TRAK/scenes/cornell_box.py index 404ac37e..b4254025 100644 --- a/Programming/TRAK/scenes/cornell_box.py +++ b/Programming/TRAK/scenes/cornell_box.py @@ -1,5 +1,6 @@ from sightpy import * + # define materials to use def setup_scene(width=400, height=300, environment=None): diff --git a/Programming/TRAK/scenes/soap_bubble.py b/Programming/TRAK/scenes/soap_bubble.py index 7f022618..9a2386e7 100644 --- a/Programming/TRAK/scenes/soap_bubble.py +++ b/Programming/TRAK/scenes/soap_bubble.py @@ -1,5 +1,6 @@ from sightpy import * + # define materials to use def setup_scene(width=400, height=300, environment="lake.png"): # Set Scene @@ -7,13 +8,24 @@ def setup_scene(width=400, height=300, environment="lake.png"): Sc = Scene(ambient_color=rgb(0.01, 0.01, 0.01)) angle = -np.pi * 0.5 - Sc.add_Camera(screen_height=height, screen_width=width, - look_from=vec3(4.0 * np.sin(angle), 0.00, 4.0 * np.cos(angle)), - look_at=vec3(0., 0.05, 0.0)) + Sc.add_Camera( + screen_height=height, + screen_width=width, + look_from=vec3(4.0 * np.sin(angle), 0.00, 4.0 * np.cos(angle)), + look_at=vec3(0.0, 0.05, 0.0), + ) - soap_bubble = ThinFilmInterference(thickness=330, noise=60.) - Sc.add(Sphere(material=soap_bubble, center=vec3(1., 0.0, 1.5), radius=1.7, shadow=False, max_ray_depth=5)) + soap_bubble = ThinFilmInterference(thickness=330, noise=60.0) + Sc.add( + Sphere( + material=soap_bubble, + center=vec3(1.0, 0.0, 1.5), + radius=1.7, + shadow=False, + max_ray_depth=5, + ) + ) - Sc.add_Background(environment, blur=10.) + Sc.add_Background(environment, blur=10.0) return Sc diff --git a/Programming/TRAK/scenes/three_spheres.py b/Programming/TRAK/scenes/three_spheres.py index 830113a7..0df887ce 100644 --- a/Programming/TRAK/scenes/three_spheres.py +++ b/Programming/TRAK/scenes/three_spheres.py @@ -1,33 +1,76 @@ from sightpy import * + # define materials to use def setup_scene(width=400, height=300, environment="miramar.jpeg"): - blue_glass = Refractive(n=vec3(1.5 + 4e-8j, 1.5 + 4e-8j, 1.5 + 0.j)) # n = index of refraction - green_glass = Refractive(n=vec3(1.5 + 4e-8j, 1.5 + 0.j, 1.5 + 4e-8j)) - red_glass = Refractive(n=vec3(1.5 + 0.j, 1.5 + 5e-8j, 1.5 + 5e-8j)) + blue_glass = Refractive( + n=vec3(1.5 + 4e-8j, 1.5 + 4e-8j, 1.5 + 0.0j) + ) # n = index of refraction + green_glass = Refractive(n=vec3(1.5 + 4e-8j, 1.5 + 0.0j, 1.5 + 4e-8j)) + red_glass = Refractive(n=vec3(1.5 + 0.0j, 1.5 + 5e-8j, 1.5 + 5e-8j)) - floor = Glossy(diff_color=image("checkered_floor.png", repeat=80.), n=vec3(1.2 + 0.3j, 1.2 + 0.3j, 1.1 + 0.3j), - roughness=0.2, spec_coeff=0.3, diff_coeff=0.9) + floor = Glossy( + diff_color=image("checkered_floor.png", repeat=80.0), + n=vec3(1.2 + 0.3j, 1.2 + 0.3j, 1.1 + 0.3j), + roughness=0.2, + spec_coeff=0.3, + diff_coeff=0.9, + ) # Set Scene Sc = Scene(ambient_color=rgb(0.05, 0.05, 0.05)) angle = np.pi / 2 * 0.3 - Sc.add_Camera(look_from=vec3(2.5 * np.sin(angle), 0.25, 2.5 * np.cos(angle) - 1.5), - look_at=vec3(0., 0.25, -1.5), - screen_width=width, - screen_height=height) + Sc.add_Camera( + look_from=vec3(2.5 * np.sin(angle), 0.25, 2.5 * np.cos(angle) - 1.5), + look_at=vec3(0.0, 0.25, -1.5), + screen_width=width, + screen_height=height, + ) Sc.add_DirectionalLight(Ldir=vec3(0.52, 0.45, -0.5), color=rgb(0.15, 0.15, 0.15)) - Sc.add(Sphere(material=blue_glass, center=vec3(-1.2, 0.0, -1.5), radius=.5, shadow=False, max_ray_depth=3)) - Sc.add(Sphere(material=green_glass, center=vec3(0., 0.0, -1.5), radius=.5, shadow=False, max_ray_depth=3)) - Sc.add(Sphere(material=red_glass, center=vec3(1.2, 0.0, -1.5), radius=.5, shadow=False, max_ray_depth=3)) + Sc.add( + Sphere( + material=blue_glass, + center=vec3(-1.2, 0.0, -1.5), + radius=0.5, + shadow=False, + max_ray_depth=3, + ) + ) + Sc.add( + Sphere( + material=green_glass, + center=vec3(0.0, 0.0, -1.5), + radius=0.5, + shadow=False, + max_ray_depth=3, + ) + ) + Sc.add( + Sphere( + material=red_glass, + center=vec3(1.2, 0.0, -1.5), + radius=0.5, + shadow=False, + max_ray_depth=3, + ) + ) - Sc.add(Plane(material=floor, center=vec3(0, -0.5, -3.0), width=120.0, height=120.0, u_axis=vec3(1.0, 0, 0), - v_axis=vec3(0, 0, -1.0), max_ray_depth=3)) + Sc.add( + Plane( + material=floor, + center=vec3(0, -0.5, -3.0), + width=120.0, + height=120.0, + u_axis=vec3(1.0, 0, 0), + v_axis=vec3(0, 0, -1.0), + max_ray_depth=3, + ) + ) # see sightpy/backgrounds Sc.add_Background(environment) diff --git a/Programming/TRAK/scenes/transparent_cuboid.py b/Programming/TRAK/scenes/transparent_cuboid.py index 475cb523..c9e32e18 100644 --- a/Programming/TRAK/scenes/transparent_cuboid.py +++ b/Programming/TRAK/scenes/transparent_cuboid.py @@ -1,27 +1,52 @@ from sightpy import * + # define materials to use def setup_scene(width=400, height=300, environment="stormydays.png"): - floor = Glossy(diff_color=image("checkered_floor.png", repeat=2.), roughness=0.2, spec_coeff=0.3, diff_coeff=0.7, - n=vec3(2.2, 2.2, 2.2)) # n = index of refraction - green_glass = Refractive(n=vec3(1.5 + 4e-8j, 1.5 + 0.j, 1.5 + 4e-8j)) + floor = Glossy( + diff_color=image("checkered_floor.png", repeat=2.0), + roughness=0.2, + spec_coeff=0.3, + diff_coeff=0.7, + n=vec3(2.2, 2.2, 2.2), + ) # n = index of refraction + green_glass = Refractive(n=vec3(1.5 + 4e-8j, 1.5 + 0.0j, 1.5 + 4e-8j)) Sc = Scene() - Sc.add_Camera(look_from=vec3(0., 0.25, 1.), look_at=vec3(0., 0.25, -3.), - screen_width=width, - screen_height=height) + Sc.add_Camera( + look_from=vec3(0.0, 0.25, 1.0), + look_at=vec3(0.0, 0.25, -3.0), + screen_width=width, + screen_height=height, + ) Sc.add_DirectionalLight(Ldir=vec3(0.0, 0.5, 0.5), color=rgb(0.5, 0.5, 0.5)) - Sc.add(Plane(material=floor, center=vec3(0, -0.5, -3.0), width=6.0, height=6.0, u_axis=vec3(1.0, 0, 0), - v_axis=vec3(0, 0, -1.0), max_ray_depth=5)) + Sc.add( + Plane( + material=floor, + center=vec3(0, -0.5, -3.0), + width=6.0, + height=6.0, + u_axis=vec3(1.0, 0, 0), + v_axis=vec3(0, 0, -1.0), + max_ray_depth=5, + ) + ) - cb = Cuboid(material=green_glass, center=vec3(0.00, 0.0001, -0.8), width=0.9, height=1.0, length=0.4, shadow=False, - max_ray_depth=5) + cb = Cuboid( + material=green_glass, + center=vec3(0.00, 0.0001, -0.8), + width=0.9, + height=1.0, + length=0.4, + shadow=False, + max_ray_depth=5, + ) cb.rotate(θ=30, u=vec3(0, 1, 0)) Sc.add(cb) # see sightpy/backgrounds Sc.add_Background(environment) - return Sc \ No newline at end of file + return Sc diff --git a/Programming/TRAK/scenes/two_spheres.py b/Programming/TRAK/scenes/two_spheres.py index 846bea3c..5f08b654 100644 --- a/Programming/TRAK/scenes/two_spheres.py +++ b/Programming/TRAK/scenes/two_spheres.py @@ -1,38 +1,74 @@ from sightpy import * + # define materials to use def setup_scene(width=400, height=300, environment="stormydays.png"): - gold_metal = Glossy(diff_color = rgb(1., .572, .184), n = vec3(0.15+3.58j, 0.4+2.37j, 1.54+1.91j), roughness = 0.0, spec_coeff = 0.2, diff_coeff= 0.8) # n = index of refraction - bluish_metal = Glossy(diff_color = rgb(0.0, 0, 0.1), n = vec3(1.3+1.91j, 1.3+1.91j, 1.4+2.91j), roughness = 0.2,spec_coeff = 0.5, diff_coeff= 0.3) - - floor = Glossy(diff_color = image("checkered_floor.png", repeat = 80.), - n = vec3(1.2+ 0.3j, 1.2+ 0.3j, 1.1+ 0.3j), roughness = 0.2, spec_coeff = 0.3, diff_coeff= 0.9 ) - - - + gold_metal = Glossy( + diff_color=rgb(1.0, 0.572, 0.184), + n=vec3(0.15 + 3.58j, 0.4 + 2.37j, 1.54 + 1.91j), + roughness=0.0, + spec_coeff=0.2, + diff_coeff=0.8, + ) # n = index of refraction + bluish_metal = Glossy( + diff_color=rgb(0.0, 0, 0.1), + n=vec3(1.3 + 1.91j, 1.3 + 1.91j, 1.4 + 2.91j), + roughness=0.2, + spec_coeff=0.5, + diff_coeff=0.3, + ) + floor = Glossy( + diff_color=image("checkered_floor.png", repeat=80.0), + n=vec3(1.2 + 0.3j, 1.2 + 0.3j, 1.1 + 0.3j), + roughness=0.2, + spec_coeff=0.3, + diff_coeff=0.9, + ) # Set Scene - Sc = Scene(ambient_color = rgb(0.05, 0.05, 0.05)) + Sc = Scene(ambient_color=rgb(0.05, 0.05, 0.05)) + angle = -np.pi / 2 * 0.3 + Sc.add_Camera( + look_from=vec3(2.5 * np.sin(angle), 0.25, 2.5 * np.cos(angle) - 1.5), + look_at=vec3(0.0, 0.25, -3.0), + screen_width=width, + screen_height=height, + ) - angle = -np.pi/2 * 0.3 - Sc.add_Camera(look_from = vec3(2.5*np.sin(angle), 0.25, 2.5*np.cos(angle) -1.5 ), - look_at = vec3(0., 0.25, -3.), - screen_width = width , - screen_height = height) + Sc.add_DirectionalLight(Ldir=vec3(0.52, 0.45, -0.5), color=rgb(0.15, 0.15, 0.15)) + Sc.add( + Sphere( + material=gold_metal, + center=vec3(-0.75, 0.1, -3.0), + radius=0.6, + max_ray_depth=3, + ) + ) + Sc.add( + Sphere( + material=bluish_metal, + center=vec3(1.25, 0.1, -3.0), + radius=0.6, + max_ray_depth=3, + ) + ) + Sc.add( + Plane( + material=floor, + center=vec3(0, -0.5, -3.0), + width=120.0, + height=120.0, + u_axis=vec3(1.0, 0, 0), + v_axis=vec3(0, 0, -1.0), + max_ray_depth=3, + ) + ) - Sc.add_DirectionalLight(Ldir = vec3(0.52,0.45, -0.5), color = rgb(0.15, 0.15, 0.15)) - - - Sc.add(Sphere(material = gold_metal, center = vec3(-.75, .1, -3.),radius = .6, max_ray_depth = 3)) - Sc.add(Sphere(material = bluish_metal, center = vec3(1.25, .1, -3.), radius = .6, max_ray_depth = 3)) - - Sc.add(Plane(material = floor, center = vec3(0, -0.5, -3.0), width = 120.0,height = 120.0, u_axis = vec3(1.0, 0, 0), v_axis = vec3(0, 0, -1.0), max_ray_depth = 3)) - - #see sightpy/backgrounds + # see sightpy/backgrounds Sc.add_Background(environment) - return Sc \ No newline at end of file + return Sc diff --git a/Programming/TRAK/utils.py b/Programming/TRAK/utils.py index b211d68c..e59a979e 100644 --- a/Programming/TRAK/utils.py +++ b/Programming/TRAK/utils.py @@ -1,13 +1,17 @@ from configparser import ConfigParser + def load_config(config_path): config = ConfigParser() config.read(config_path) return config + def parse_resolution(resolution): try: - width, height = map(int, resolution.lower().split('x')) + width, height = map(int, resolution.lower().split("x")) return width, height except ValueError: - raise ValueError("Resolution must be in the format WIDTHxHEIGHT, e.g., 1920x1080.") \ No newline at end of file + raise ValueError( + "Resolution must be in the format WIDTHxHEIGHT, e.g., 1920x1080." + ) diff --git a/Programming/WDWR/projekt/data/pareto_front.csv b/Programming/WDWR/projekt/data/pareto_front.csv new file mode 100644 index 00000000..8c7d58b6 --- /dev/null +++ b/Programming/WDWR/projekt/data/pareto_front.csv @@ -0,0 +1,21 @@ +i;minimalAverageProfit;averageProfit;riskMeasureGini;m1_prod_P1;m1_prod_P2;m1_prod_P3;m1_prod_P4;m2_prod_P1;m2_prod_P2;m2_prod_P3;m2_prod_P4;m3_prod_P1;m3_prod_P2;m3_prod_P3;m3_prod_P4;m1_stock_P1;m1_stock_P2;m1_stock_P3;m1_stock_P4;m2_stock_P1;m2_stock_P2;m2_stock_P3;m2_stock_P4;m3_stock_P1;m3_stock_P2;m3_stock_P3;m3_stock_P4 +1;-600;-600;0;0;0;0;0;0;0;0;0;0;0;0;0;50;50;50;50;50;50;50;50;50;50;50;50 +2;-67.59;-67.423334025;12.249866065;0;0;0;1;0;0;0;0;9;8;1;43;41;50;50;7;41;42;49;7;50;50;50;50 +3;464.82;465.647226284;26.015138747;0;0;0;0;0;0;0;29;20;19;9;52;30;50;41;0;30;31;41;0;50;50;50;50 +4;997.23;997.66185313;40.349353614;0;0;0;110;0;0;0;0;30;30;13;50;20;50;37;0;20;20;37;0;50;50;50;50 +5;1529.64;1530.172175866;54.689711726;0;0;0;114;0;0;0;0;42;40;17;92;8;50;33;0;8;10;33;0;50;50;50;50 +6;2062.05;2062.088549477;69.062913221;0;0;0;9;0;0;0;200;50;50;25;50;0;50;25;0;0;0;25;0;50;50;50;50 +7;2594.46;2594.684714046;84.265583873;9;0;0;64;0;10;0;159;50;51;28;51;0;50;22;0;0;0;22;0;50;50;50;50 +8;3126.87;3127.12427492;99.497458717;20;0;0;0;0;30;0;111;50;58;35;249;0;50;15;0;0;0;15;0;50;50;50;50 +9;3659.28;3659.345897427;114.728718529;29;0;0;2;1;31;0;200;50;69;39;249;0;50;11;0;0;0;11;0;50;50;50;50 +10;4191.69;4191.916591732;129.963670504;36;0;0;114;5;32;0;160;50;79;45;210;0;50;5;0;0;0;5;0;50;50;50;50 +11;4724.1;4724.191045823;145.204545226;50;0;0;100;2;34;0;197;50;89;50;250;0;50;0;0;0;0;0;0;50;50;50;50 +12;5256.51;5256.719240584;162.349562672;72;0;34;150;0;3;0;200;50;123;50;210;0;50;0;0;0;0;0;0;50;50;50;50 +13;5788.92;5789.016098305;182.46075748;122;0;0;150;1;1;0;200;50;149;94;210;0;50;0;0;0;0;0;0;50;50;50;50 +14;6321.33;6321.358645932;205.000585625;110;0;50;150;19;33;0;200;50;156;102;250;0;50;0;0;0;0;0;0;50;50;50;50 +15;6853.74;6853.83219788;229.023498807;109;0;24;110;1;0;94;160;50;191;51;250;0;50;0;0;0;0;0;0;50;50;50;50 +16;7386.15;7386.171785273;253.962965038;1;0;1;150;121;30;189;200;50;200;50;210;0;50;0;0;0;0;0;0;50;50;50;50 +17;7918.56;7918.565798346;279.907638363;110;0;30;110;59;48;76;160;50;205;130;210;0;50;0;0;0;0;0;0;50;50;50;50 +18;8450.97;8451.020747071;306.383357869;111;0;0;150;72;0;191;200;50;256;131;210;0;50;0;0;0;0;0;0;50;50;50;50 +19;8983.38;8983.400103901;332.936696297;111;0;49;110;83;15;144;200;50;317;149;250;0;50;0;0;0;0;0;0;50;50;50;50 +20;9515.79;9515.804890325;360.18213126;110;0;30;110;110;28;160;160;50;273;150;250;0;50;0;0;0;0;0;0;50;50;50;50 diff --git a/Programming/WDWR/projekt/graph.py b/Programming/WDWR/projekt/graph.py old mode 100644 new mode 100755 index ce747828..359d6b95 --- a/Programming/WDWR/projekt/graph.py +++ b/Programming/WDWR/projekt/graph.py @@ -1,53 +1,59 @@ +#!/usr/bin/env python3 +"""Plot the Pareto front of the WDWR production model: profit against risk. + +Each row of ``data/pareto_front.csv`` is one solution of the bi-criteria MILP +-- an average profit, its Gini risk measure, and the production and stock +levels that produced them. The scatter shows the trade-off between the two +criteria. + +The data used to be a 20-line CSV embedded in this file as a string literal. +It moved to a sibling file because its header line alone is 342 characters, +and a data record is not something that can be wrapped to fit a line limit. +""" + +import logging +from pathlib import Path + import matplotlib.pyplot as plt import pandas as pd -from io import StringIO -# Your data -data_str = """i;minimalAverageProfit;averageProfit;riskMeasureGini;m1_prod_P1;m1_prod_P2;m1_prod_P3;m1_prod_P4;m2_prod_P1;m2_prod_P2;m2_prod_P3;m2_prod_P4;m3_prod_P1;m3_prod_P2;m3_prod_P3;m3_prod_P4;m1_stock_P1;m1_stock_P2;m1_stock_P3;m1_stock_P4;m2_stock_P1;m2_stock_P2;m2_stock_P3;m2_stock_P4;m3_stock_P1;m3_stock_P2;m3_stock_P3;m3_stock_P4 -1;-600;-600;0;0;0;0;0;0;0;0;0;0;0;0;0;50;50;50;50;50;50;50;50;50;50;50;50 -2;-67.59;-67.423334025;12.249866065;0;0;0;1;0;0;0;0;9;8;1;43;41;50;50;7;41;42;49;7;50;50;50;50 -3;464.82;465.647226284;26.015138747;0;0;0;0;0;0;0;29;20;19;9;52;30;50;41;0;30;31;41;0;50;50;50;50 -4;997.23;997.66185313;40.349353614;0;0;0;110;0;0;0;0;30;30;13;50;20;50;37;0;20;20;37;0;50;50;50;50 -5;1529.64;1530.172175866;54.689711726;0;0;0;114;0;0;0;0;42;40;17;92;8;50;33;0;8;10;33;0;50;50;50;50 -6;2062.05;2062.088549477;69.062913221;0;0;0;9;0;0;0;200;50;50;25;50;0;50;25;0;0;0;25;0;50;50;50;50 -7;2594.46;2594.684714046;84.265583873;9;0;0;64;0;10;0;159;50;51;28;51;0;50;22;0;0;0;22;0;50;50;50;50 -8;3126.87;3127.12427492;99.497458717;20;0;0;0;0;30;0;111;50;58;35;249;0;50;15;0;0;0;15;0;50;50;50;50 -9;3659.28;3659.345897427;114.728718529;29;0;0;2;1;31;0;200;50;69;39;249;0;50;11;0;0;0;11;0;50;50;50;50 -10;4191.69;4191.916591732;129.963670504;36;0;0;114;5;32;0;160;50;79;45;210;0;50;5;0;0;0;5;0;50;50;50;50 -11;4724.1;4724.191045823;145.204545226;50;0;0;100;2;34;0;197;50;89;50;250;0;50;0;0;0;0;0;0;50;50;50;50 -12;5256.51;5256.719240584;162.349562672;72;0;34;150;0;3;0;200;50;123;50;210;0;50;0;0;0;0;0;0;50;50;50;50 -13;5788.92;5789.016098305;182.46075748;122;0;0;150;1;1;0;200;50;149;94;210;0;50;0;0;0;0;0;0;50;50;50;50 -14;6321.33;6321.358645932;205.000585625;110;0;50;150;19;33;0;200;50;156;102;250;0;50;0;0;0;0;0;0;50;50;50;50 -15;6853.74;6853.83219788;229.023498807;109;0;24;110;1;0;94;160;50;191;51;250;0;50;0;0;0;0;0;0;50;50;50;50 -16;7386.15;7386.171785273;253.962965038;1;0;1;150;121;30;189;200;50;200;50;210;0;50;0;0;0;0;0;0;50;50;50;50 -17;7918.56;7918.565798346;279.907638363;110;0;30;110;59;48;76;160;50;205;130;210;0;50;0;0;0;0;0;0;50;50;50;50 -18;8450.97;8451.020747071;306.383357869;111;0;0;150;72;0;191;200;50;256;131;210;0;50;0;0;0;0;0;0;50;50;50;50 -19;8983.38;8983.400103901;332.936696297;111;0;49;110;83;15;144;200;50;317;149;250;0;50;0;0;0;0;0;0;50;50;50;50 -20;9515.79;9515.804890325;360.18213126;110;0;30;110;110;28;160;160;50;273;150;250;0;50;0;0;0;0;0;0;50;50;50;50""" +logger = logging.getLogger(__name__) -# Read the data -df = pd.read_csv(StringIO(data_str), sep=';') +DATA_FILE = Path(__file__).parent / "data" / "pareto_front.csv" -# Create the scatter plot -plt.figure(figsize=(10, 6)) -plt.scatter(df['averageProfit'], df['riskMeasureGini'], alpha=0.7, s=200) +# Everything is sized for a report figure rather than for the screen. +LABEL_FONT_SIZE = 16 +MARKER_AREA = 200 -# Add labels and title -plt.xlabel('Przeciętny Zysk', fontsize=16) -plt.ylabel('Ryzyko', fontsize=16) -plt.xticks(fontsize=16) -plt.yticks(fontsize=16) -# Add grid for better readability -plt.grid(True, alpha=0.3) +def main() -> None: + """Draw the scatter plot and report the range of each axis.""" + df = pd.read_csv(DATA_FILE, sep=";") -# Format the plot -plt.tight_layout() + plt.figure(figsize=(10, 6)) + plt.scatter(df["averageProfit"], df["riskMeasureGini"], alpha=0.7, s=MARKER_AREA) -# Show the plot -plt.show() + plt.xlabel("Przeciętny Zysk", fontsize=LABEL_FONT_SIZE) + plt.ylabel("Ryzyko", fontsize=LABEL_FONT_SIZE) + plt.xticks(fontsize=LABEL_FONT_SIZE) + plt.yticks(fontsize=LABEL_FONT_SIZE) + plt.grid(visible=True, alpha=0.3) + plt.tight_layout() + plt.show() -# Print some basic statistics -print(f"Average Profit range: {df['averageProfit'].min():.2f} to {df['averageProfit'].max():.2f}") -print(f"Gini Risk range: {df['riskMeasureGini'].min():.2f} to {df['riskMeasureGini'].max():.2f}") -print(f"Number of data points: {len(df)}") \ No newline at end of file + logger.info( + "Average Profit range: %.2f to %.2f", + df["averageProfit"].min(), + df["averageProfit"].max(), + ) + logger.info( + "Gini Risk range: %.2f to %.2f", + df["riskMeasureGini"].min(), + df["riskMeasureGini"].max(), + ) + logger.info("Number of data points: %d", len(df)) + + +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + main() diff --git a/Programming/WDWR/projekt/graph2.py b/Programming/WDWR/projekt/graph2.py old mode 100644 new mode 100755 index 24e29667..58af4f2a --- a/Programming/WDWR/projekt/graph2.py +++ b/Programming/WDWR/projekt/graph2.py @@ -1,32 +1,120 @@ +#!/usr/bin/env python3 +"""Plot the inverse distribution of profit for the three WDWR scenarios. + +One curve per scenario: the sorted profits of its 100 sampled realisations +against their empirical cumulative probability, which is how the report shows +which scenario stochastically dominates the others. +""" + +import logging + import matplotlib.pyplot as plt import numpy as np +logger = logging.getLogger(__name__) +logging.basicConfig(format="%(message)s", level=logging.INFO) + +# Sized for a report figure rather than for the screen. +LABEL_FONT_SIZE = 16 + # Data for the three scenarios scenario1_profits = [ - 8283.101223531, 8360.092861464, 8023.436774628, 7996.435353438, 8108.734620296, - 8709.702668322, 9236.429507678, 8492.980846575, 8912.962073143, 8201.079841867, - 7947.08574225, 8426.386242154, 8633.510288097, 7925.419317516, 8216.653093455, - 8162.239336848, 10606.276724793, 7766.753696884, 8902.966530729, 8329.954853003, - 8615.534725088, 9307.905933844, 8711.836200338, 7853.713101162, 8160.850707138, - 8505.74747647, 7807.599624626, 8650.006339759, 7571.333855925, 9103.892851116 + 8283.101223531, + 8360.092861464, + 8023.436774628, + 7996.435353438, + 8108.734620296, + 8709.702668322, + 9236.429507678, + 8492.980846575, + 8912.962073143, + 8201.079841867, + 7947.08574225, + 8426.386242154, + 8633.510288097, + 7925.419317516, + 8216.653093455, + 8162.239336848, + 10606.276724793, + 7766.753696884, + 8902.966530729, + 8329.954853003, + 8615.534725088, + 9307.905933844, + 8711.836200338, + 7853.713101162, + 8160.850707138, + 8505.74747647, + 7807.599624626, + 8650.006339759, + 7571.333855925, + 9103.892851116, ] scenario2_profits = [ - 8779.880622914, 8905.098118171, 8457.038873118, 8476.337700492, 8611.662470696, - 9284.604268643, 9787.525726881, 9033.316521658, 9469.444921654, 8726.555858861, - 8437.173060618, 8935.043125766, 9172.334141304, 8425.564554151, 8725.52174345, - 8672.538442695, 11321.501722433, 8233.9854963, 9465.167737138, 8841.407418574, - 9162.816556878, 9946.670992861, 9281.411111594, 8349.695093959, 8671.731748701, - 9074.152941559, 8296.712958538, 9208.592940681, 8041.287735389, 9707.228511359 + 8779.880622914, + 8905.098118171, + 8457.038873118, + 8476.337700492, + 8611.662470696, + 9284.604268643, + 9787.525726881, + 9033.316521658, + 9469.444921654, + 8726.555858861, + 8437.173060618, + 8935.043125766, + 9172.334141304, + 8425.564554151, + 8725.52174345, + 8672.538442695, + 11321.501722433, + 8233.9854963, + 9465.167737138, + 8841.407418574, + 9162.816556878, + 9946.670992861, + 9281.411111594, + 8349.695093959, + 8671.731748701, + 9074.152941559, + 8296.712958538, + 9208.592940681, + 8041.287735389, + 9707.228511359, ] scenario3_profits = [ - 9268.899026395, 9430.210649623, 8919.631184021, 8944.062758608, 9095.775246456, - 9886.312580673, 10387.393647889, 9637.953603446, 10098.221535588, 9238.913391815, - 8980.977156104, 9414.393022001, 9722.356857713, 8917.726783311, 9178.474614142, - 9167.292074295, 12011.512542804, 8693.048121325, 10039.47692674, 9375.735645214, - 9730.36014247, 10531.376234688, 9750.101324443, 8881.341453044, 9186.289888284, - 9675.754350507, 8773.329747496, 9722.523537949, 8489.277170975, 10325.425491734 + 9268.899026395, + 9430.210649623, + 8919.631184021, + 8944.062758608, + 9095.775246456, + 9886.312580673, + 10387.393647889, + 9637.953603446, + 10098.221535588, + 9238.913391815, + 8980.977156104, + 9414.393022001, + 9722.356857713, + 8917.726783311, + 9178.474614142, + 9167.292074295, + 12011.512542804, + 8693.048121325, + 10039.47692674, + 9375.735645214, + 9730.36014247, + 10531.376234688, + 9750.101324443, + 8881.341453044, + 9186.289888284, + 9675.754350507, + 8773.329747496, + 9722.523537949, + 8489.277170975, + 10325.425491734, ] # Create figure with single plot @@ -42,17 +130,23 @@ num_points = len(scenario1_profits) y_values = [i / num_points for i in range(1, num_points + 1)] # Plot all three scenarios on the same graph -plt.plot(scenario1_sorted, y_values, 'b-o', label='Scenariusz 1', linewidth=2, markersize=4) -plt.plot(scenario2_sorted, y_values, 'g-s', label='Scenariusz 2', linewidth=2, markersize=4) -plt.plot(scenario3_sorted, y_values, 'r-^', label='Scenariusz 3', linewidth=2, markersize=4) +plt.plot( + scenario1_sorted, y_values, "b-o", label="Scenariusz 1", linewidth=2, markersize=4 +) +plt.plot( + scenario2_sorted, y_values, "g-s", label="Scenariusz 2", linewidth=2, markersize=4 +) +plt.plot( + scenario3_sorted, y_values, "r-^", label="Scenariusz 3", linewidth=2, markersize=4 +) # Add labels and title -plt.xlabel('Przeciętny Zysk', fontsize=16) -plt.ylabel('Odwrotna dystrybuanta', fontsize=16) -plt.xticks(fontsize=16) -plt.yticks(fontsize=16) +plt.xlabel("Przeciętny Zysk", fontsize=LABEL_FONT_SIZE) +plt.ylabel("Odwrotna dystrybuanta", fontsize=LABEL_FONT_SIZE) +plt.xticks(fontsize=LABEL_FONT_SIZE) +plt.yticks(fontsize=LABEL_FONT_SIZE) plt.ylim(0, 1) # Set y-axis limits to cut off values outside 0-1 range -plt.grid(True, alpha=0.3) -plt.legend(fontsize=16) +plt.grid(visible=True, alpha=0.3) +plt.legend(fontsize=LABEL_FONT_SIZE) # Format the plot plt.tight_layout() @@ -61,7 +155,13 @@ plt.tight_layout() plt.show() # Print statistics for each scenario -print("Scenario Statistics:") -print(f"Scenario 1 - Mean: {np.mean(scenario1_profits):.2f}, Std: {np.std(scenario1_profits):.2f}") -print(f"Scenario 2 - Mean: {np.mean(scenario2_profits):.2f}, Std: {np.std(scenario2_profits):.2f}") -print(f"Scenario 3 - Mean: {np.mean(scenario3_profits):.2f}, Std: {np.std(scenario3_profits):.2f}") \ No newline at end of file +logger.info("Scenario Statistics:") +for number, profits in enumerate( + (scenario1_profits, scenario2_profits, scenario3_profits), start=1 +): + logger.info( + "Scenario %d - Mean: %.2f, Std: %.2f", + number, + np.mean(profits), + np.std(profits), + ) diff --git a/Programming/WDWR/projekt/graph3.py b/Programming/WDWR/projekt/graph3.py old mode 100644 new mode 100755 index 5c2edd2d..eb5c5246 --- a/Programming/WDWR/projekt/graph3.py +++ b/Programming/WDWR/projekt/graph3.py @@ -1,23 +1,120 @@ +#!/usr/bin/env python3 +"""Plot the inverse distribution of RISK for the three WDWR scenarios. + +The companion of graph2.py: same three scenarios, same empirical cumulative +probability on the y axis, but the Gini risk measure on the x axis instead of +the profit. +""" + +import logging + import matplotlib.pyplot as plt import numpy as np +logger = logging.getLogger(__name__) +logging.basicConfig(format="%(message)s", level=logging.INFO) + +# Sized for a report figure rather than for the screen. +LABEL_FONT_SIZE = 16 + # Data for the three scenarios - Risk values scenario1_risks = [ - 184.458, 195.09, 207.678, 278.26, 220.823, 181.042, 195.281, 199.255, 182.194, 181.295, - 285.773, 185.306, 272.997, 288.405, 223.407, 213.341, 208.693, 354.01, 217.093, 301.892, - 200.499, 637.424, 408.232, 308.314, 181.042, 470.693, 212.039, 184.694, 345.455, 205.291 + 184.458, + 195.09, + 207.678, + 278.26, + 220.823, + 181.042, + 195.281, + 199.255, + 182.194, + 181.295, + 285.773, + 185.306, + 272.997, + 288.405, + 223.407, + 213.341, + 208.693, + 354.01, + 217.093, + 301.892, + 200.499, + 637.424, + 408.232, + 308.314, + 181.042, + 470.693, + 212.039, + 184.694, + 345.455, + 205.291, ] scenario2_risks = [ - 403.784, 414.043, 408.225, 664.379, 413.447, 448.303, 513.763, 439.966, 403.784, 408.366, - 773.926, 459.531, 537.056, 795.835, 542.569, 512.817, 471.741, 879.425, 441.46, 533.521, - 659.742, 990.73, 842.758, 776.278, 404.787, 989.35, 481.833, 409.978, 795.275, 421.261 + 403.784, + 414.043, + 408.225, + 664.379, + 413.447, + 448.303, + 513.763, + 439.966, + 403.784, + 408.366, + 773.926, + 459.531, + 537.056, + 795.835, + 542.569, + 512.817, + 471.741, + 879.425, + 441.46, + 533.521, + 659.742, + 990.73, + 842.758, + 776.278, + 404.787, + 989.35, + 481.833, + 409.978, + 795.275, + 421.261, ] scenario3_risks = [ - 559.839, 576.633, 596.264, 942.092, 560.045, 679.026, 683.47, 627.607, 559.839, 562.478, - 1245.41, 638.37, 675.366, 1132.8, 784.609, 686.634, 680.842, 1115.92, 572.793, 682.114, - 914.647, 1211.67, 1051.95, 1104.87, 583.36, 1360.71, 743.425, 577.734, 1191.09, 581.791 + 559.839, + 576.633, + 596.264, + 942.092, + 560.045, + 679.026, + 683.47, + 627.607, + 559.839, + 562.478, + 1245.41, + 638.37, + 675.366, + 1132.8, + 784.609, + 686.634, + 680.842, + 1115.92, + 572.793, + 682.114, + 914.647, + 1211.67, + 1051.95, + 1104.87, + 583.36, + 1360.71, + 743.425, + 577.734, + 1191.09, + 581.791, ] # Create figure with single plot @@ -33,18 +130,24 @@ num_points = len(scenario1_risks) y_values = [i / num_points for i in range(1, num_points + 1)] # Plot all three scenarios on the same graph -plt.plot(scenario1_sorted, y_values, 'b-o', label='Scenariusz 1', linewidth=2, markersize=4) -plt.plot(scenario2_sorted, y_values, 'g-s', label='Scenariusz 2', linewidth=2, markersize=4) -plt.plot(scenario3_sorted, y_values, 'r-^', label='Scenariusz 3', linewidth=2, markersize=4) +plt.plot( + scenario1_sorted, y_values, "b-o", label="Scenariusz 1", linewidth=2, markersize=4 +) +plt.plot( + scenario2_sorted, y_values, "g-s", label="Scenariusz 2", linewidth=2, markersize=4 +) +plt.plot( + scenario3_sorted, y_values, "r-^", label="Scenariusz 3", linewidth=2, markersize=4 +) # Add labels and title -plt.xlabel('Ryzyko', fontsize=16) -plt.ylabel('Odwrotna dystrybuanta', fontsize=16) -plt.xticks(fontsize=16) -plt.yticks(fontsize=16) +plt.xlabel("Ryzyko", fontsize=LABEL_FONT_SIZE) +plt.ylabel("Odwrotna dystrybuanta", fontsize=LABEL_FONT_SIZE) +plt.xticks(fontsize=LABEL_FONT_SIZE) +plt.yticks(fontsize=LABEL_FONT_SIZE) plt.ylim(0, 1) # Set y-axis limits to cut off values outside 0-1 range -plt.grid(True, alpha=0.3) -plt.legend(fontsize=16) +plt.grid(visible=True, alpha=0.3) +plt.legend(fontsize=LABEL_FONT_SIZE) # Format the plot plt.tight_layout() @@ -53,10 +156,10 @@ plt.tight_layout() plt.show() # Print statistics for each scenario -print("Scenario Statistics:") -print(f"Scenario 1 - Mean: {np.mean(scenario1_risks):.2f}, " - f"Std: {np.std(scenario1_risks):.2f}") -print(f"Scenario 2 - Mean: {np.mean(scenario2_risks):.2f}, " - f"Std: {np.std(scenario2_risks):.2f}") -print(f"Scenario 3 - Mean: {np.mean(scenario3_risks):.2f}, " - f"Std: {np.std(scenario3_risks):.2f}") \ No newline at end of file +logger.info("Scenario Statistics:") +for number, risks in enumerate( + (scenario1_risks, scenario2_risks, scenario3_risks), start=1 +): + logger.info( + "Scenario %d - Mean: %.2f, Std: %.2f", number, np.mean(risks), np.std(risks) + ) diff --git a/Programming/twm_4/TWM_KerasIntro.ipynb b/Programming/twm_4/TWM_KerasIntro.ipynb index 593eea4f..28d7f233 100644 --- a/Programming/twm_4/TWM_KerasIntro.ipynb +++ b/Programming/twm_4/TWM_KerasIntro.ipynb @@ -9,7 +9,7 @@ "outputs": [], "source": [ "%matplotlib inline\n", - "#gjfklg" + "# gjfklg" ] }, { @@ -87,22 +87,21 @@ } ], "source": [ - "import numpy as np # advanced math library\n", - "import matplotlib.pyplot as plt # MATLAB like plotting routines\n", - "import random # for generating random numbers\n", - "\n", - "import tensorflow as tf\n", - "from keras.datasets import mnist # MNIST dataset is included in Keras%\n", - "from keras.models import Sequential # Model type to be used\n", - "\n", - "from keras.layers import Dense, Dropout, Activation # Types of layers to be used in our model\n", - "from keras.utils import to_categorical # NumPy related tools\n", - "\n", - "from keras import optimizers\n", - "\n", - "\n", - "from sklearn.metrics import confusion_matrix\n", "import itertools\n", + "import random # for generating random numbers\n", + "\n", + "import matplotlib.pyplot as plt # MATLAB like plotting routines\n", + "import numpy as np # advanced math library\n", + "import tensorflow as tf\n", + "from keras.datasets import mnist # MNIST dataset is included in Keras%\n", + "from keras.layers import (\n", + " Activation,\n", + " Dense,\n", + " Dropout,\n", + ") # Types of layers to be used in our model\n", + "from keras.models import Sequential # Model type to be used\n", + "from keras.utils import to_categorical # NumPy related tools\n", + "from sklearn.metrics import confusion_matrix\n", "\n", "print(tf.__version__)" ] @@ -161,22 +160,25 @@ }, "outputs": [], "source": [ - "plt.rcParams['figure.figsize'] = (9,9) # Make the figures a bit bigger\n", + "plt.rcParams[\"figure.figsize\"] = (9, 9) # Make the figures a bit bigger\n", + "\n", "\n", "def visualize_classes(X, y):\n", - " for i in range(0, 10):\n", - " img_batch = X[y == i][0:10]\n", - " img_batch = np.reshape(img_batch, (img_batch.shape[0]*img_batch.shape[1], img_batch.shape[2]))\n", - " if i > 0:\n", - " img = np.concatenate([img, img_batch], axis = 1)\n", - " else:\n", - " img = img_batch\n", - " plt.figure(figsize=(10,20))\n", - " plt.axis('off')\n", - " plt.imshow(img, cmap='gray')\n", + " for i in range(10):\n", + " img_batch = X[y == i][0:10]\n", + " img_batch = np.reshape(\n", + " img_batch, (img_batch.shape[0] * img_batch.shape[1], img_batch.shape[2])\n", + " )\n", + " if i > 0:\n", + " img = np.concatenate([img, img_batch], axis=1)\n", + " else:\n", + " img = img_batch\n", + " plt.figure(figsize=(10, 20))\n", + " plt.axis(\"off\")\n", + " plt.imshow(img, cmap=\"gray\")\n", "\n", "\n", - "visualize_classes(X_train, y_train)\n" + "visualize_classes(X_train, y_train)" ] }, { @@ -202,11 +204,12 @@ "source": [ "# just a little function for pretty printing a matrix\n", "def matprint(mat, fmt=\"g\"):\n", - " col_maxes = [max([len((\"{:\"+fmt+\"}\").format(x)) for x in col]) for col in mat.T]\n", + " col_maxes = [max([len((\"{:\" + fmt + \"}\").format(x)) for x in col]) for col in mat.T]\n", " for x in mat:\n", " for i, y in enumerate(x):\n", - " print((\"{:\"+str(col_maxes[i])+fmt+\"}\").format(y), end=\" \")\n", - " print(\"\")\n", + " print((\"{:\" + str(col_maxes[i]) + fmt + \"}\").format(y), end=\" \")\n", + " print()\n", + "\n", "\n", "# now print!\n", "matprint(X_train[-1])" @@ -254,13 +257,14 @@ }, "outputs": [], "source": [ - "from sklearn.manifold import TSNE\n", "import matplotlib.patheffects as PathEffects\n", "import seaborn as sns\n", + "from sklearn.manifold import TSNE\n", "\n", - "RS=19238\n", + "RS = 19238\n", + "\n", + "class_names = [str(clid) for clid in range(10)]\n", "\n", - "class_names = [ str(clid) for clid in range(10) ]\n", "\n", "def scatter(x, colors):\n", " # We choose a color palette with seaborn.\n", @@ -268,13 +272,12 @@ "\n", " # We create a scatter plot.\n", " f = plt.figure(figsize=(8, 8))\n", - " ax = plt.subplot(aspect='equal')\n", - " sc = ax.scatter(x[:,0], x[:,1], lw=0, s=40,\n", - " c=palette[colors.astype(\"int\")])\n", + " ax = plt.subplot(aspect=\"equal\")\n", + " sc = ax.scatter(x[:, 0], x[:, 1], lw=0, s=40, c=palette[colors.astype(\"int\")])\n", " plt.xlim(-25, 25)\n", " plt.ylim(-25, 25)\n", - " ax.axis('off')\n", - " ax.axis('tight')\n", + " ax.axis(\"off\")\n", + " ax.axis(\"tight\")\n", "\n", " # We add the labels for each digit.\n", " txts = []\n", @@ -282,15 +285,16 @@ " # Position of each label.\n", " xtext, ytext = np.median(x[colors == i, :], axis=0)\n", " txt = ax.text(xtext, ytext, class_names[i], fontsize=15)\n", - " txt.set_path_effects([\n", - " PathEffects.Stroke(linewidth=5, foreground=\"w\"),\n", - " PathEffects.Normal()])\n", + " txt.set_path_effects(\n", + " [PathEffects.Stroke(linewidth=5, foreground=\"w\"), PathEffects.Normal()]\n", + " )\n", " txts.append(txt)\n", "\n", + "\n", "def plot_tsne(X, y):\n", - " print('calculating tsne ...')\n", - " proj = TSNE(random_state=RS, learning_rate=200, init='random').fit_transform(X)\n", - " scatter(proj, y)" + " print(\"calculating tsne ...\")\n", + " proj = TSNE(random_state=RS, learning_rate=200, init=\"random\").fit_transform(X)\n", + " scatter(proj, y)" ] }, { @@ -345,13 +349,19 @@ }, "outputs": [], "source": [ - "X_train = X_train.reshape(60000, 784) # reshape 60,000 28 x 28 matrices into 60,000 784-length vectors.\n", - "X_test = X_test.reshape(10000, 784) # reshape 10,000 28 x 28 matrices into 10,000 784-length vectors.\n", + "X_train = X_train.reshape(\n", + " 60000, 784\n", + ") # reshape 60,000 28 x 28 matrices into 60,000 784-length vectors.\n", + "X_test = X_test.reshape(\n", + " 10000, 784\n", + ") # reshape 10,000 28 x 28 matrices into 10,000 784-length vectors.\n", "\n", - "X_train = X_train.astype('float32') # change integers to 32-bit floating point numbers\n", - "X_test = X_test.astype('float32')\n", + "X_train = X_train.astype(\"float32\") # change integers to 32-bit floating point numbers\n", + "X_test = X_test.astype(\"float32\")\n", "\n", - "X_train /= 255 # normalize each value for each pixel for the entire vector for each input\n", + "X_train /= (\n", + " 255 # normalize each value for each pixel for the entire vector for each input\n", + ")\n", "X_test /= 255\n", "\n", "print(\"Training matrix shape\", X_train.shape)\n", @@ -389,7 +399,7 @@ }, "outputs": [], "source": [ - "nb_classes = 10 # number of unique digits\n", + "nb_classes = 10 # number of unique digits\n", "\n", "Y_train = to_categorical(y_train, nb_classes)\n", "Y_test = to_categorical(y_test, nb_classes)" @@ -439,7 +449,9 @@ "# The first hidden layer is a set of 512 nodes (artificial neurons).\n", "# Each node will receive an element from each input vector and apply some weight and bias to it.\n", "\n", - "model.add(Dense(512, input_shape=(784,))) #(784,) is not a typo -- that represents a 784 length vector!" + "model.add(\n", + " Dense(512, input_shape=(784,))\n", + ") # (784,) is not a typo -- that represents a 784 length vector!" ] }, { @@ -456,7 +468,7 @@ "# Those inputs are then not considered to be fired.\n", "# Positive values of a node are unchanged.\n", "\n", - "model.add(Activation('relu'))" + "model.add(Activation(\"relu\"))" ] }, { @@ -504,7 +516,7 @@ "# they receive 512 inputs from the output of the first 512-node layer.\n", "\n", "model.add(Dense(512))\n", - "model.add(Activation('relu'))\n", + "model.add(Activation(\"relu\"))\n", "model.add(Dropout(0.2))" ] }, @@ -541,7 +553,7 @@ "# The \"softmax\" activation represents a probability distribution over K different possible outcomes.\n", "# Its values are all non-negative and sum to 1.\n", "\n", - "model.add(Activation('softmax'))" + "model.add(Activation(\"softmax\"))" ] }, { @@ -624,7 +636,7 @@ "source": [ "# Let's use the Adam optimizer for learning\n", "adam = tf.optimizers.Adam(learning_rate=0.001)\n", - "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" + "model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])" ] }, { @@ -664,9 +676,7 @@ }, "outputs": [], "source": [ - "model.fit(X_train, Y_train,\n", - " batch_size=128, epochs=5,\n", - " verbose=1)" + "model.fit(X_train, Y_train, batch_size=128, epochs=5, verbose=1)" ] }, { @@ -700,8 +710,8 @@ "outputs": [], "source": [ "score = model.evaluate(X_test, Y_test)\n", - "print('Test score:', score[0])\n", - "print('Test accuracy:', score[1])" + "print(\"Test score:\", score[0])\n", + "print(\"Test accuracy:\", score[1])" ] }, { @@ -712,39 +722,40 @@ }, "outputs": [], "source": [ - "def plot_confusion_matrix(cm, classes,\n", - " normalize=False,\n", - " title='Confusion matrix',\n", - " cmap=plt.cm.Blues):\n", - " \"\"\"\n", - " This function prints and plots the confusion matrix.\n", + "def plot_confusion_matrix(\n", + " cm, classes, normalize=False, title=\"Confusion matrix\", cmap=plt.cm.Blues\n", + "):\n", + " \"\"\"This function prints and plots the confusion matrix.\n", " Normalization can be applied by setting `normalize=True`.\n", " \"\"\"\n", " if normalize:\n", - " cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n", + " cm = cm.astype(\"float\") / cm.sum(axis=1)[:, np.newaxis]\n", " print(\"Normalized confusion matrix\")\n", " else:\n", - " print('Confusion matrix, without normalization')\n", + " print(\"Confusion matrix, without normalization\")\n", "\n", " print(cm)\n", "\n", - "\n", - " plt.imshow(cm, interpolation='nearest', cmap=cmap)\n", + " plt.imshow(cm, interpolation=\"nearest\", cmap=cmap)\n", " plt.title(title)\n", " plt.colorbar()\n", " tick_marks = np.arange(len(classes))\n", " plt.xticks(tick_marks, classes, rotation=45)\n", " plt.yticks(tick_marks, classes)\n", "\n", - " fmt = '.2f' if normalize else 'd'\n", - " thresh = cm.max() / 2.\n", + " fmt = \".2f\" if normalize else \"d\"\n", + " thresh = cm.max() / 2.0\n", " for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n", - " plt.text(j, i, format(cm[i, j], fmt),\n", - " horizontalalignment=\"center\",\n", - " color=\"white\" if cm[i, j] > thresh else \"black\")\n", + " plt.text(\n", + " j,\n", + " i,\n", + " format(cm[i, j], fmt),\n", + " horizontalalignment=\"center\",\n", + " color=\"white\" if cm[i, j] > thresh else \"black\",\n", + " )\n", "\n", - " plt.ylabel('True label')\n", - " plt.xlabel('Predicted label')\n", + " plt.ylabel(\"True label\")\n", + " plt.xlabel(\"Predicted label\")\n", " plt.tight_layout()" ] }, @@ -775,7 +786,7 @@ "# The predict_classes function outputs the highest probability class\n", "# according to the trained classifier for each input example.\n", "predict = model.predict(X_test)\n", - "predicted_classes = np.argmax(predict,axis=1)\n", + "predicted_classes = np.argmax(predict, axis=1)\n", "\n", "# Check which items we got right / wrong\n", "correct_indices = np.nonzero(predicted_classes == y_test)[0]\n", @@ -786,10 +797,11 @@ "\n", "# Plot non-normalized confusion matrix\n", "plt.figure()\n", - "plot_confusion_matrix(cnf_matrix, classes=class_names,\n", - " title='Confusion matrix, without normalization')\n", + "plot_confusion_matrix(\n", + " cnf_matrix, classes=class_names, title=\"Confusion matrix, without normalization\"\n", + ")\n", "\n", - "plt.show()\n" + "plt.show()" ] }, { @@ -823,23 +835,28 @@ }, "outputs": [], "source": [ - "def show_samples(indices, preds, images, labels, count=3, names = []):\n", + "def show_samples(indices, preds, images, labels, count=3, names=[]):\n", " plt.figure()\n", - " for i, sample in enumerate(indices[:count**2]):\n", + " for i, sample in enumerate(indices[: count**2]):\n", " pred_id = int(np.argmax(preds[sample]))\n", " real_id = int(labels[sample])\n", " pred_score = preds[sample][pred_id]\n", " real_score = preds[sample][real_id]\n", - " plt.subplot(count,count,i+1)\n", - " plt.imshow(images[sample].reshape(28,28), cmap='gray', interpolation='none')\n", - " plt.axis('off')\n", + " plt.subplot(count, count, i + 1)\n", + " plt.imshow(images[sample].reshape(28, 28), cmap=\"gray\", interpolation=\"none\")\n", + " plt.axis(\"off\")\n", " if len(names) > 0:\n", - " plt.title(\"P: {} ({:.2f})\\nE: {} ({:.2f})\".format(names[pred_id], pred_score, names[real_id], real_score))\n", + " plt.title(\n", + " f\"P: {names[pred_id]} ({pred_score:.2f})\\nE: {names[real_id]} ({real_score:.2f})\"\n", + " )\n", " else:\n", - " plt.title(\"P: {} ({:.2f})\\nE: {} ({:.2f})\".format(pred_id, pred_score, real_id, real_score))\n", + " plt.title(\n", + " f\"P: {pred_id} ({pred_score:.2f})\\nE: {real_id} ({real_score:.2f})\"\n", + " )\n", "\n", " plt.tight_layout()\n", "\n", + "\n", "show_samples(correct_indices, predict, X_test, y_test, 5)" ] }, @@ -988,9 +1005,13 @@ "source": [ "# import some additional tools\n", "\n", - "from keras.preprocessing.image import ImageDataGenerator\n", - "from keras.layers import Conv2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, Flatten\n", - "from keras.layers import BatchNormalization" + "from keras.layers import (\n", + " BatchNormalization,\n", + " Conv2D,\n", + " Flatten,\n", + " MaxPooling2D,\n", + ")\n", + "from keras.preprocessing.image import ImageDataGenerator" ] }, { @@ -1020,13 +1041,17 @@ "# Again, do some formatting\n", "# Except we do not flatten each image into a 784-length vector because we want to perform convolutions first\n", "\n", - "X_train = X_train.reshape(60000, 28, 28, 1) #add an additional dimension to represent the single-channel\n", + "X_train = X_train.reshape(\n", + " 60000, 28, 28, 1\n", + ") # add an additional dimension to represent the single-channel\n", "X_test = X_test.reshape(10000, 28, 28, 1)\n", "\n", - "X_train = X_train.astype('float32') # change integers to 32-bit floating point numbers\n", - "X_test = X_test.astype('float32')\n", + "X_train = X_train.astype(\"float32\") # change integers to 32-bit floating point numbers\n", + "X_test = X_test.astype(\"float32\")\n", "\n", - "X_train /= 255 # normalize each value for each pixel for the entire vector for each input\n", + "X_train /= (\n", + " 255 # normalize each value for each pixel for the entire vector for each input\n", + ")\n", "X_test /= 255\n", "\n", "print(\"Training matrix shape\", X_train.shape)\n", @@ -1043,7 +1068,7 @@ "source": [ "# one-hot format classes\n", "\n", - "nb_classes = 10 # number of unique digits\n", + "nb_classes = 10 # number of unique digits\n", "\n", "Y_train = to_categorical(y_train, nb_classes)\n", "Y_test = to_categorical(y_test, nb_classes)" @@ -1057,27 +1082,29 @@ }, "outputs": [], "source": [ - "model = Sequential() # Linear stacking of layers\n", + "model = Sequential() # Linear stacking of layers\n", "\n", "# Convolution Layer 1\n", - "model.add(Conv2D(16, (5, 5), input_shape=(28,28,1))) # 16 different 5x5 kernels -- so 16 feature maps\n", - "model.add(Activation('relu') ) # activation\n", - "model.add(MaxPooling2D(pool_size=(2,2))) # Pool the max values over a 2x2 kernel\n", + "model.add(\n", + " Conv2D(16, (5, 5), input_shape=(28, 28, 1))\n", + ") # 16 different 5x5 kernels -- so 16 feature maps\n", + "model.add(Activation(\"relu\")) # activation\n", + "model.add(MaxPooling2D(pool_size=(2, 2))) # Pool the max values over a 2x2 kernel\n", "\n", "# Convolution Layer 2\n", - "model.add(Conv2D(32, (5, 5))) # 32 different 5x5 kernels -- so 32 feature maps\n", - "model.add(Activation('relu')) # activation\n", - "model.add(MaxPooling2D(pool_size=(2,2))) # Pool the max values over a 2x2 kernel\n", + "model.add(Conv2D(32, (5, 5))) # 32 different 5x5 kernels -- so 32 feature maps\n", + "model.add(Activation(\"relu\")) # activation\n", + "model.add(MaxPooling2D(pool_size=(2, 2))) # Pool the max values over a 2x2 kernel\n", "\n", - "model.add(Flatten()) # Flatten final output matrix into a vector\n", + "model.add(Flatten()) # Flatten final output matrix into a vector\n", "\n", "# Fully Connected Layer\n", - "model.add(Dense(128)) # 128 FC nodes\n", - "model.add(Activation('relu')) # activation\n", + "model.add(Dense(128)) # 128 FC nodes\n", + "model.add(Activation(\"relu\")) # activation\n", "\n", "# Fully Connected Layer\n", - "model.add(Dense(10)) # final 10 FC nodes\n", - "model.add(Activation('softmax')) # softmax activation" + "model.add(Dense(10)) # final 10 FC nodes\n", + "model.add(Activation(\"softmax\")) # softmax activation" ] }, { @@ -1105,7 +1132,7 @@ "source": [ "# we'll use the same optimizer\n", "adam = tf.optimizers.Adam(learning_rate=0.001)\n", - "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" + "model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])" ] }, { @@ -1119,8 +1146,14 @@ "# data augmentation prevents overfitting by slightly changing the data randomly\n", "# Keras has a great built-in feature to do automatic augmentation\n", "\n", - "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n", - " height_shift_range=0.08, zoom_range=0.08, validation_split=0.2)\n", + "gen = ImageDataGenerator(\n", + " rotation_range=8,\n", + " width_shift_range=0.08,\n", + " shear_range=0.3,\n", + " height_shift_range=0.08,\n", + " zoom_range=0.08,\n", + " validation_split=0.2,\n", + ")\n", "\n", "test_gen = ImageDataGenerator()" ] @@ -1139,8 +1172,8 @@ "\n", "# Before the data was all loaded into memory, but then processed in batches.\n", "\n", - "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset='training')\n", - "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation')\n", + "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"training\")\n", + "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"validation\")\n", "test_generator = test_gen.flow(X_test, Y_test, batch_size=128)" ] }, @@ -1161,7 +1194,14 @@ "\n", "# SIGNIFICANT MEMORY SAVINGS (important for larger, deeper networks)\n", "\n", - "model.fit(train_generator, steps_per_epoch=48000//128, epochs=5, verbose=1, validation_data=valid_generator, validation_steps = 12000 // 128)" + "model.fit(\n", + " train_generator,\n", + " steps_per_epoch=48000 // 128,\n", + " epochs=5,\n", + " verbose=1,\n", + " validation_data=valid_generator,\n", + " validation_steps=12000 // 128,\n", + ")" ] }, { @@ -1177,8 +1217,8 @@ "outputs": [], "source": [ "score = model.evaluate(X_test, Y_test)\n", - "print('Test score:', score[0])\n", - "print('Test accuracy:', score[1])" + "print(\"Test score:\", score[0])\n", + "print(\"Test accuracy:\", score[1])" ] }, { @@ -1197,7 +1237,7 @@ "# The predict_classes function outputs the highest probability class\n", "# according to the trained classifier for each input example.\n", "predict = model.predict(X_test)\n", - "predicted_classes = np.argmax(predict,axis=1)\n", + "predicted_classes = np.argmax(predict, axis=1)\n", "\n", "# Check which items we got right / wrong\n", "correct_indices = np.nonzero(predicted_classes == y_test)[0]\n", @@ -1211,8 +1251,9 @@ "\n", "# Plot non-normalized confusion matrix\n", "plt.figure()\n", - "plot_confusion_matrix(cnf_matrix, classes=class_names,\n", - " title='Confusion matrix, without normalization')\n", + "plot_confusion_matrix(\n", + " cnf_matrix, classes=class_names, title=\"Confusion matrix, without normalization\"\n", + ")\n", "\n", "plt.show()" ] @@ -1269,25 +1310,26 @@ "outputs": [], "source": [ "def visualize2(model, layer, img, r, c):\n", - " from keras import Model\n", - " # expand dimensions so that it represents a single 'sample'\n", - " img = np.expand_dims(img, axis=0)\n", - " outputs = [layer.output]\n", - " model = Model(inputs=model.inputs, outputs=outputs)\n", - " # get feature map for first hidden layer\n", - " feature_maps = model.predict(img)\n", - " for fmap in feature_maps:\n", - " # plot all 64 maps in an 8x8 squares\n", - " ix = 1\n", - " for _ in range(r):\n", - " for _ in range(c):\n", - " # specify subplot and turn of axis\n", - " ax = plt.subplot(r, c, ix)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " # plot filter channel in grayscale\n", - " plt.imshow(fmap[:, :, ix-1], cmap='gray')\n", - " ix += 1" + " from keras import Model\n", + "\n", + " # expand dimensions so that it represents a single 'sample'\n", + " img = np.expand_dims(img, axis=0)\n", + " outputs = [layer.output]\n", + " model = Model(inputs=model.inputs, outputs=outputs)\n", + " # get feature map for first hidden layer\n", + " feature_maps = model.predict(img)\n", + " for fmap in feature_maps:\n", + " # plot all 64 maps in an 8x8 squares\n", + " ix = 1\n", + " for _ in range(r):\n", + " for _ in range(c):\n", + " # specify subplot and turn of axis\n", + " ax = plt.subplot(r, c, ix)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " # plot filter channel in grayscale\n", + " plt.imshow(fmap[:, :, ix - 1], cmap=\"gray\")\n", + " ix += 1" ] }, { @@ -1305,7 +1347,7 @@ "source": [ "plt.figure()\n", "img = X_test[11]\n", - "plt.imshow(img[:,:,0], cmap='gray', interpolation='none')" + "plt.imshow(img[:, :, 0], cmap=\"gray\", interpolation=\"none\")" ] }, { @@ -1368,26 +1410,30 @@ "source": [ "# Loading the specified subset of digits from the MNIST dataset\n", "def get_mnist(digits):\n", - " # Reload the MNIST data\n", - " (X_train, y_train), (X_test, y_test) = mnist.load_data()\n", + " # Reload the MNIST data\n", + " (X_train, y_train), (X_test, y_test) = mnist.load_data()\n", "\n", - " train_ids = np.in1d(y_train, digits)\n", - " test_ids = np.in1d(y_test, digits)\n", - " X_train = X_train[train_ids]\n", - " X_test = X_test[test_ids]\n", - " y_train = y_train[train_ids]\n", - " y_test = y_test[test_ids]\n", + " train_ids = np.in1d(y_train, digits)\n", + " test_ids = np.in1d(y_test, digits)\n", + " X_train = X_train[train_ids]\n", + " X_test = X_test[test_ids]\n", + " y_train = y_train[train_ids]\n", + " y_test = y_test[test_ids]\n", "\n", - " X_train = np.expand_dims(X_train, axis=-1)\n", - " X_test = np.expand_dims(X_test, axis=-1)\n", + " X_train = np.expand_dims(X_train, axis=-1)\n", + " X_test = np.expand_dims(X_test, axis=-1)\n", "\n", - " X_train = X_train.astype('float32') # change integers to 32-bit floating point numbers\n", - " X_test = X_test.astype('float32')\n", + " X_train = X_train.astype(\n", + " \"float32\"\n", + " ) # change integers to 32-bit floating point numbers\n", + " X_test = X_test.astype(\"float32\")\n", "\n", - " X_train /= 255 # normalize each value for each pixel for the entire vector for each input\n", - " X_test /= 255\n", + " X_train /= (\n", + " 255 # normalize each value for each pixel for the entire vector for each input\n", + " )\n", + " X_test /= 255\n", "\n", - " return X_train, y_train, X_test, y_test" + " return X_train, y_train, X_test, y_test" ] }, { @@ -1400,34 +1446,34 @@ "source": [ "# Preparing the base model\n", "def prepare_model(input_shape, class_count):\n", - " model = Sequential() # Linear stacking of layers\n", + " model = Sequential() # Linear stacking of layers\n", "\n", - " # Convolution Layer 1\n", - " model.add(Conv2D(16, (3, 3), padding=\"same\", input_shape=input_shape))\n", - " model.add(Activation('relu') )\n", - " model.add(MaxPooling2D(pool_size=(2,2)))\n", + " # Convolution Layer 1\n", + " model.add(Conv2D(16, (3, 3), padding=\"same\", input_shape=input_shape))\n", + " model.add(Activation(\"relu\"))\n", + " model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", - " # Convolution Layer 2\n", - " model.add(Conv2D(32, (3, 3), padding=\"same\"))\n", - " model.add(Activation('relu'))\n", - " model.add(MaxPooling2D(pool_size=(2,2)))\n", + " # Convolution Layer 2\n", + " model.add(Conv2D(32, (3, 3), padding=\"same\"))\n", + " model.add(Activation(\"relu\"))\n", + " model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", - " # Convolution Layer 3\n", - " model.add(Conv2D(64, (3, 3), padding=\"same\"))\n", - " model.add(Activation('relu'))\n", - " model.add(MaxPooling2D(pool_size=(2,2)))\n", + " # Convolution Layer 3\n", + " model.add(Conv2D(64, (3, 3), padding=\"same\"))\n", + " model.add(Activation(\"relu\"))\n", + " model.add(MaxPooling2D(pool_size=(2, 2)))\n", "\n", - " model.add(Flatten())\n", + " model.add(Flatten())\n", "\n", - " # Fully Connected Layer\n", - " model.add(Dense(64))\n", - " model.add(Activation('relu'))\n", + " # Fully Connected Layer\n", + " model.add(Dense(64))\n", + " model.add(Activation(\"relu\"))\n", "\n", - " # Fully Connected Layer\n", - " model.add(Dense(class_count))\n", - " model.add(Activation('softmax'))\n", + " # Fully Connected Layer\n", + " model.add(Dense(class_count))\n", + " model.add(Activation(\"softmax\"))\n", "\n", - " return model" + " return model" ] }, { @@ -1440,11 +1486,12 @@ "source": [ "# One-hot encoding of non-consecutive labels\n", "def one_hot(labels):\n", - " from sklearn.preprocessing import OneHotEncoder\n", - " encoder = OneHotEncoder(categories='auto')\n", - " l = labels.reshape(-1, 1)\n", - " output = encoder.fit_transform(l)\n", - " return output.toarray()" + " from sklearn.preprocessing import OneHotEncoder\n", + "\n", + " encoder = OneHotEncoder(categories=\"auto\")\n", + " l = labels.reshape(-1, 1)\n", + " output = encoder.fit_transform(l)\n", + " return output.toarray()" ] }, { @@ -1456,33 +1503,33 @@ "outputs": [], "source": [ "def test_model(model, X_test, Y_test, y_test, digits):\n", - " score = model.evaluate(X_test, Y_test)\n", - " print('Test score:', score[0])\n", - " print('Test accuracy:', score[1])\n", + " score = model.evaluate(X_test, Y_test)\n", + " print(\"Test score:\", score[0])\n", + " print(\"Test accuracy:\", score[1])\n", "\n", - " # The predict_classes function outputs the highest probability class\n", - " # according to the trained classifier for each input example.\n", - " predicted = model.predict(X_test)\n", - " predicted_classes = np.argmax(predicted, axis=1)\n", + " # The predict_classes function outputs the highest probability class\n", + " # according to the trained classifier for each input example.\n", + " predicted = model.predict(X_test)\n", + " predicted_classes = np.argmax(predicted, axis=1)\n", "\n", - " actual_classes = [digits[x] for x in predicted_classes]\n", + " actual_classes = [digits[x] for x in predicted_classes]\n", "\n", - " # Check which items we got right / wrong\n", - " correct_indices = np.nonzero(actual_classes == y_test)[0]\n", + " # Check which items we got right / wrong\n", + " correct_indices = np.nonzero(actual_classes == y_test)[0]\n", "\n", - " incorrect_indices = np.nonzero(actual_classes != y_test)[0]\n", + " incorrect_indices = np.nonzero(actual_classes != y_test)[0]\n", "\n", + " cnf_matrix = confusion_matrix(y_test, actual_classes)\n", "\n", - " cnf_matrix = confusion_matrix(y_test, actual_classes)\n", + " class_names = [str(i) for i in digits]\n", "\n", - " class_names = [str(i) for i in digits]\n", + " # Plot non-normalized confusion matrix\n", + " plt.figure()\n", + " plot_confusion_matrix(\n", + " cnf_matrix, classes=class_names, title=\"Confusion matrix, without normalization\"\n", + " )\n", "\n", - " # Plot non-normalized confusion matrix\n", - " plt.figure()\n", - " plot_confusion_matrix(cnf_matrix, classes=class_names,\n", - " title='Confusion matrix, without normalization')\n", - "\n", - " plt.show()" + " plt.show()" ] }, { @@ -1513,13 +1560,12 @@ "Y_train = one_hot(y_train)\n", "Y_test = one_hot(y_test)\n", "\n", - "model = prepare_model( (28, 28, 1), 5)\n", + "model = prepare_model((28, 28, 1), 5)\n", "\n", "adam = tf.optimizers.Adam(learning_rate=0.001)\n", - "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n", + "model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])\n", "\n", - "model.summary()\n", - "\n" + "model.summary()" ] }, { @@ -1530,13 +1576,19 @@ }, "outputs": [], "source": [ - "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n", - " height_shift_range=0.08, zoom_range=0.08, validation_split=0.2)\n", + "gen = ImageDataGenerator(\n", + " rotation_range=8,\n", + " width_shift_range=0.08,\n", + " shear_range=0.3,\n", + " height_shift_range=0.08,\n", + " zoom_range=0.08,\n", + " validation_split=0.2,\n", + ")\n", "\n", "test_gen = ImageDataGenerator()\n", "\n", - "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset='training')\n", - "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation')\n", + "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"training\")\n", + "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"validation\")\n", "test_generator = test_gen.flow(X_test, Y_test, batch_size=128)" ] }, @@ -1552,7 +1604,14 @@ }, "outputs": [], "source": [ - "model.fit_generator(train_generator, steps_per_epoch=25000//128, epochs=5, verbose=1, validation_data=valid_generator, validation_steps = 5000 // 128)" + "model.fit_generator(\n", + " train_generator,\n", + " steps_per_epoch=25000 // 128,\n", + " epochs=5,\n", + " verbose=1,\n", + " validation_data=valid_generator,\n", + " validation_steps=5000 // 128,\n", + ")" ] }, { @@ -1590,13 +1649,13 @@ "source": [ "# Extract specified range of layers to a new sequential model\n", "def extract_layers(main_model, starting_layer_ix, ending_layer_ix):\n", - " # create an empty model\n", - " new_model = Sequential()\n", - " for ix in range(starting_layer_ix, ending_layer_ix + 1):\n", - " curr_layer = main_model.get_layer(index=ix)\n", - " # copy this layer over to the new model\n", - " new_model.add(curr_layer)\n", - " return new_model" + " # create an empty model\n", + " new_model = Sequential()\n", + " for ix in range(starting_layer_ix, ending_layer_ix + 1):\n", + " curr_layer = main_model.get_layer(index=ix)\n", + " # copy this layer over to the new model\n", + " new_model.add(curr_layer)\n", + " return new_model" ] }, { @@ -1626,14 +1685,14 @@ "\n", "# Fully Connected Layer\n", "new_model.add(Dense(64))\n", - "new_model.add(Activation('relu'))\n", + "new_model.add(Activation(\"relu\"))\n", "\n", "# Fully Connected Layer\n", "new_model.add(Dense(5))\n", - "new_model.add(Activation('softmax'))\n", + "new_model.add(Activation(\"softmax\"))\n", "\n", - "for ix in range(0, 9+1):\n", - " new_model.get_layer(index=ix).trainable=False\n", + "for ix in range(9 + 1):\n", + " new_model.get_layer(index=ix).trainable = False\n", "\n", "new_digits = [5, 6, 7, 8, 9]\n", "\n", @@ -1642,7 +1701,11 @@ "Y_train = one_hot(y_train)\n", "Y_test = one_hot(y_test)\n", "\n", - "new_model.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.legacy.Adam(), metrics=['accuracy'])\n", + "new_model.compile(\n", + " loss=\"categorical_crossentropy\",\n", + " optimizer=tf.keras.optimizers.legacy.Adam(),\n", + " metrics=[\"accuracy\"],\n", + ")\n", "\n", "new_model.summary()" ] @@ -1655,13 +1718,19 @@ }, "outputs": [], "source": [ - "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n", - " height_shift_range=0.08, zoom_range=0.08, validation_split=0.2)\n", + "gen = ImageDataGenerator(\n", + " rotation_range=8,\n", + " width_shift_range=0.08,\n", + " shear_range=0.3,\n", + " height_shift_range=0.08,\n", + " zoom_range=0.08,\n", + " validation_split=0.2,\n", + ")\n", "\n", "test_gen = ImageDataGenerator()\n", "\n", - "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset='training')\n", - "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation')\n", + "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"training\")\n", + "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"validation\")\n", "test_generator = test_gen.flow(X_test, Y_test, batch_size=128)" ] }, @@ -1686,7 +1755,14 @@ }, "outputs": [], "source": [ - "new_model.fit_generator(train_generator, steps_per_epoch=24000//128, epochs=5, verbose=1, validation_data=valid_generator, validation_steps = 6000 // 128)" + "new_model.fit_generator(\n", + " train_generator,\n", + " steps_per_epoch=24000 // 128,\n", + " epochs=5,\n", + " verbose=1,\n", + " validation_data=valid_generator,\n", + " validation_steps=6000 // 128,\n", + ")" ] }, { @@ -1735,10 +1811,14 @@ }, "outputs": [], "source": [ - "for ix in range(0, 9+1):\n", - " new_model.get_layer(index=ix).trainable=True\n", + "for ix in range(9 + 1):\n", + " new_model.get_layer(index=ix).trainable = True\n", "\n", - "new_model.compile(loss='categorical_crossentropy', optimizer=tf.keras.optimizers.legacy.Adam(), metrics=['accuracy'])\n", + "new_model.compile(\n", + " loss=\"categorical_crossentropy\",\n", + " optimizer=tf.keras.optimizers.legacy.Adam(),\n", + " metrics=[\"accuracy\"],\n", + ")\n", "new_model.summary()" ] }, @@ -1754,7 +1834,14 @@ }, "outputs": [], "source": [ - "new_model.fit_generator(train_generator, steps_per_epoch=24000//128, epochs=2, verbose=1, validation_data=valid_generator, validation_steps = 6000 // 128)" + "new_model.fit_generator(\n", + " train_generator,\n", + " steps_per_epoch=24000 // 128,\n", + " epochs=2,\n", + " verbose=1,\n", + " validation_data=valid_generator,\n", + " validation_steps=6000 // 128,\n", + ")" ] }, { @@ -1808,60 +1895,61 @@ "import os\n", "import urllib\n", "\n", - "def download_and_load(class_names, test_split = 0.2, max_items_per_class = 10000):\n", - " root = 'data'\n", - " if not os.path.exists(root):\n", - " os.makedirs(root)\n", "\n", - " print('downloading ...')\n", - " base = 'https://storage.googleapis.com/quickdraw_dataset/full/numpy_bitmap/'\n", - " for c in class_names:\n", - " cc = c.replace('_', '%20')\n", - " path = base+cc+'.npy'\n", - " print(path)\n", - " if not os.path.exists(f'{root}/{cc}.npy'):\n", - " urllib.request.urlretrieve(path, f'{root}/{cc}.npy')\n", - " else:\n", - " print(\"Already downloaded\")\n", - " print('loading ...')\n", + "def download_and_load(class_names, test_split=0.2, max_items_per_class=10000):\n", + " root = \"data\"\n", + " if not os.path.exists(root):\n", + " os.makedirs(root)\n", "\n", - " #initialize variables\n", - " x = np.empty([0, 784])\n", - " y = np.empty([0])\n", + " print(\"downloading ...\")\n", + " base = \"https://storage.googleapis.com/quickdraw_dataset/full/numpy_bitmap/\"\n", + " for c in class_names:\n", + " cc = c.replace(\"_\", \"%20\")\n", + " path = base + cc + \".npy\"\n", + " print(path)\n", + " if not os.path.exists(f\"{root}/{cc}.npy\"):\n", + " urllib.request.urlretrieve(path, f\"{root}/{cc}.npy\")\n", + " else:\n", + " print(\"Already downloaded\")\n", + " print(\"loading ...\")\n", "\n", - " #load each data file\n", - " for idx, file in enumerate(class_names):\n", - " file = file.replace('_', '%20')\n", - " data = np.load(f'{root}/{file}.npy')\n", - " data = data[0: max_items_per_class, :]\n", - " labels = np.full(data.shape[0], idx)\n", + " # initialize variables\n", + " x = np.empty([0, 784])\n", + " y = np.empty([0])\n", "\n", - " x = np.concatenate((x, data), axis=0)\n", - " y = np.append(y, labels)\n", + " # load each data file\n", + " for idx, file in enumerate(class_names):\n", + " file = file.replace(\"_\", \"%20\")\n", + " data = np.load(f\"{root}/{file}.npy\")\n", + " data = data[0:max_items_per_class, :]\n", + " labels = np.full(data.shape[0], idx)\n", "\n", - " data = None\n", - " labels = None\n", + " x = np.concatenate((x, data), axis=0)\n", + " y = np.append(y, labels)\n", "\n", - " #randomize the dataset\n", - " permutation = np.random.permutation(y.shape[0])\n", - " x = x[permutation, :]\n", - " y = y[permutation]\n", + " data = None\n", + " labels = None\n", "\n", - " #reshape and inverse the colors\n", - " x = 255 - np.reshape(x, (x.shape[0], 28, 28))\n", + " # randomize the dataset\n", + " permutation = np.random.permutation(y.shape[0])\n", + " x = x[permutation, :]\n", + " y = y[permutation]\n", "\n", - " #separate into training and testing\n", - " test_size = int(x.shape[0]/100*(test_split*100))\n", + " # reshape and inverse the colors\n", + " x = 255 - np.reshape(x, (x.shape[0], 28, 28))\n", "\n", - " x_test = x[0:test_size, :]\n", - " y_test = y[0:test_size]\n", + " # separate into training and testing\n", + " test_size = int(x.shape[0] / 100 * (test_split * 100))\n", "\n", - " x_train = x[test_size:x.shape[0], :]\n", - " y_train = y[test_size:y.shape[0]]\n", + " x_test = x[0:test_size, :]\n", + " y_test = y[0:test_size]\n", "\n", - " print('Training Data : ', x_train.shape[0])\n", - " print('Testing Data : ', x_test.shape[0])\n", - " return x_train, y_train, x_test, y_test" + " x_train = x[test_size : x.shape[0], :]\n", + " y_train = y[test_size : y.shape[0]]\n", + "\n", + " print(\"Training Data : \", x_train.shape[0])\n", + " print(\"Testing Data : \", x_test.shape[0])\n", + " return x_train, y_train, x_test, y_test" ] }, { @@ -1889,15 +1977,13 @@ }, "outputs": [], "source": [ - "import random\n", - "\n", - "with open('categories.txt') as f:\n", + "with open(\"categories.txt\") as f:\n", " all_class_names = f.readlines()\n", "\n", "\n", "random.seed(RANDOM_SEED)\n", "\n", - "all_class_names = [x.strip().replace(' ', '_') for x in all_class_names]\n", + "all_class_names = [x.strip().replace(\" \", \"_\") for x in all_class_names]\n", "\n", "nb_classes = 10\n", "\n", @@ -1905,19 +1991,23 @@ "class_names = random.sample(all_class_names, nb_classes)\n", "\n", "# you can also try those \"hard\" classes instead\n", - "#class_names = ['wheel', 'pizza', 'smiley_face', 'apple', 'potato', 'basketball', 'soccer_ball', 'brain', 'clock', 'circle']\n", + "# class_names = ['wheel', 'pizza', 'smiley_face', 'apple', 'potato', 'basketball', 'soccer_ball', 'brain', 'clock', 'circle']\n", "\n", "print(class_names)\n", "\n", "X_train, y_train, X_test, y_test = download_and_load(class_names)\n", "\n", - "X_train = X_train.reshape(X_train.shape[0], 28, 28, 1) #add an additional dimension to represent the single-channel\n", + "X_train = X_train.reshape(\n", + " X_train.shape[0], 28, 28, 1\n", + ") # add an additional dimension to represent the single-channel\n", "X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)\n", "\n", - "X_train = X_train.astype('float32') # change integers to 32-bit floating point numbers\n", - "X_test = X_test.astype('float32')\n", + "X_train = X_train.astype(\"float32\") # change integers to 32-bit floating point numbers\n", + "X_test = X_test.astype(\"float32\")\n", "\n", - "X_train /= 255 # normalize each value for each pixel for the entire vector for each input\n", + "X_train /= (\n", + " 255 # normalize each value for each pixel for the entire vector for each input\n", + ")\n", "X_test /= 255\n", "\n", "Y_train = to_categorical(y_train, nb_classes)\n", @@ -1943,35 +2033,37 @@ }, "outputs": [], "source": [ - "model = Sequential() # Linear stacking of layers\n", + "model = Sequential() # Linear stacking of layers\n", "\n", "# Convolution Layer 1\n", - "model.add(Conv2D(64, (5, 5), input_shape=(28,28,1))) # 16 different 5x5 kernels -- so 16 feature maps\n", - "model.add(BatchNormalization()) # weights normalization\n", - "model.add(Activation('relu') ) # activation\n", - "model.add(MaxPooling2D(pool_size=(2,2))) # Pool the max values over a 2x2 kernel\n", + "model.add(\n", + " Conv2D(64, (5, 5), input_shape=(28, 28, 1))\n", + ") # 16 different 5x5 kernels -- so 16 feature maps\n", + "model.add(BatchNormalization()) # weights normalization\n", + "model.add(Activation(\"relu\")) # activation\n", + "model.add(MaxPooling2D(pool_size=(2, 2))) # Pool the max values over a 2x2 kernel\n", "\n", "\n", "# Convolution Layer 2\n", - "model.add(Conv2D(32, (5, 5))) # 32 different 5x5 kernels -- so 32 feature maps\n", - "model.add(BatchNormalization()) # weights normalization\n", - "model.add(Activation('relu')) # activation\n", - "model.add(MaxPooling2D(pool_size=(2,2))) # Pool the max values over a 2x2 kernel\n", + "model.add(Conv2D(32, (5, 5))) # 32 different 5x5 kernels -- so 32 feature maps\n", + "model.add(BatchNormalization()) # weights normalization\n", + "model.add(Activation(\"relu\")) # activation\n", + "model.add(MaxPooling2D(pool_size=(2, 2))) # Pool the max values over a 2x2 kernel\n", "\n", - "model.add(Flatten()) # Flatten final output matrix into a vector\n", + "model.add(Flatten()) # Flatten final output matrix into a vector\n", "\n", "# Fully Connected Layer\n", - "model.add(Dense(128)) # 128 FC nodes\n", - "model.add(Activation('relu')) # activation\n", + "model.add(Dense(128)) # 128 FC nodes\n", + "model.add(Activation(\"relu\")) # activation\n", "\n", "# Fully Connected Layer\n", - "model.add(Dense(10)) # final 10 FC nodes\n", - "model.add(Activation('softmax')) # softmax activation\n", + "model.add(Dense(10)) # final 10 FC nodes\n", + "model.add(Activation(\"softmax\")) # softmax activation\n", "\n", "\n", "# we'll use the same optimizer\n", "adam = tf.optimizers.Adam(learning_rate=0.001)\n", - "model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])" + "model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])" ] }, { @@ -1985,13 +2077,19 @@ "# data augmentation prevents overfitting by slightly changing the data randomly\n", "# Keras has a great built-in feature to do automatic augmentation\n", "\n", - "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n", - " height_shift_range=0.08, zoom_range=0.08, validation_split=0.2)\n", + "gen = ImageDataGenerator(\n", + " rotation_range=8,\n", + " width_shift_range=0.08,\n", + " shear_range=0.3,\n", + " height_shift_range=0.08,\n", + " zoom_range=0.08,\n", + " validation_split=0.2,\n", + ")\n", "\n", "test_gen = ImageDataGenerator()\n", "\n", - "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset='training')\n", - "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation')\n", + "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"training\")\n", + "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"validation\")\n", "test_generator = test_gen.flow(X_test, Y_test, batch_size=128)" ] }, @@ -2008,7 +2106,14 @@ }, "outputs": [], "source": [ - "model.fit(train_generator, steps_per_epoch=64000//128, epochs=5, verbose=1, validation_data=valid_generator, validation_steps = 16000 // 128)" + "model.fit(\n", + " train_generator,\n", + " steps_per_epoch=64000 // 128,\n", + " epochs=5,\n", + " verbose=1,\n", + " validation_data=valid_generator,\n", + " validation_steps=16000 // 128,\n", + ")" ] }, { @@ -2020,8 +2125,8 @@ "outputs": [], "source": [ "score = model.evaluate(X_test, Y_test)\n", - "print('Test score:', score[0])\n", - "print('Test accuracy:', score[1])" + "print(\"Test score:\", score[0])\n", + "print(\"Test accuracy:\", score[1])" ] }, { @@ -2046,8 +2151,9 @@ "\n", "# Plot non-normalized confusion matrix\n", "plt.figure()\n", - "plot_confusion_matrix(cnf_matrix, classes=class_names,\n", - " title='Confusion matrix, without normalization')\n", + "plot_confusion_matrix(\n", + " cnf_matrix, classes=class_names, title=\"Confusion matrix, without normalization\"\n", + ")\n", "\n", "plt.show()" ] @@ -2104,19 +2210,20 @@ }, "outputs": [], "source": [ - "from tensorflow import keras\n", "from keras.datasets import cifar10\n", "\n", "(X_train, y_train), (X_test, y_test) = cifar10.load_data()\n", "\n", - "X_train = X_train.astype('float32') # change integers to 32-bit floating point numbers\n", - "X_test = X_test.astype('float32')\n", + "X_train = X_train.astype(\"float32\") # change integers to 32-bit floating point numbers\n", + "X_test = X_test.astype(\"float32\")\n", "\n", - "X_train /= 255 # normalize each value for each pixel for the entire vector for each input\n", + "X_train /= (\n", + " 255 # normalize each value for each pixel for the entire vector for each input\n", + ")\n", "X_test /= 255\n", "\n", - "y_train = y_train.reshape((1,-1))[0]\n", - "y_test = y_test.reshape((1,-1))[0]\n", + "y_train = y_train.reshape((1, -1))[0]\n", + "y_test = y_test.reshape((1, -1))[0]\n", "\n", "print(\"Training matrix shape\", X_train.shape, y_train.shape)\n", "print(\"Testing matrix shape\", X_test.shape, y_test.shape)\n", @@ -2128,7 +2235,18 @@ "Y_train = to_categorical(y_train, nb_classes)\n", "Y_test = to_categorical(y_test, nb_classes)\n", "\n", - "cifar_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']" + "cifar_names = [\n", + " \"airplane\",\n", + " \"automobile\",\n", + " \"bird\",\n", + " \"cat\",\n", + " \"deer\",\n", + " \"dog\",\n", + " \"frog\",\n", + " \"horse\",\n", + " \"ship\",\n", + " \"truck\",\n", + "]" ] }, { @@ -2153,16 +2271,23 @@ }, "outputs": [], "source": [ - "for i in range(0, 10):\n", - " img_batch = X_train[y_train == i][0:10]\n", - " img_batch = np.reshape(img_batch, (img_batch.shape[0]*img_batch.shape[1], img_batch.shape[2], img_batch.shape[3]))\n", - " if i > 0:\n", - " img = np.concatenate([img, img_batch], axis = 1)\n", - " else:\n", - " img = img_batch\n", - "plt.figure(figsize=(10,20))\n", - "plt.axis('off')\n", - "plt.imshow(img, cmap='gray')\n" + "for i in range(10):\n", + " img_batch = X_train[y_train == i][0:10]\n", + " img_batch = np.reshape(\n", + " img_batch,\n", + " (\n", + " img_batch.shape[0] * img_batch.shape[1],\n", + " img_batch.shape[2],\n", + " img_batch.shape[3],\n", + " ),\n", + " )\n", + " if i > 0:\n", + " img = np.concatenate([img, img_batch], axis=1)\n", + " else:\n", + " img = img_batch\n", + "plt.figure(figsize=(10, 20))\n", + "plt.axis(\"off\")\n", + "plt.imshow(img, cmap=\"gray\")" ] }, { @@ -2186,98 +2311,103 @@ "outputs": [], "source": [ "def generate_model():\n", - " model = Sequential() # Linear stacking of layers\n", + " model = Sequential() # Linear stacking of layers\n", "\n", - " # Convolution Layer 1\n", - " model.add(Conv2D(16, (3, 3), input_shape=(32,32,3)))\n", - " model.add(Activation('relu') )\n", + " # Convolution Layer 1\n", + " model.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3)))\n", + " model.add(Activation(\"relu\"))\n", "\n", - " # ...\n", + " # ...\n", "\n", - " model.add(Flatten()) # Flatten final output matrix into a vector\n", + " model.add(Flatten()) # Flatten final output matrix into a vector\n", "\n", - " # ...\n", + " # ...\n", "\n", - " # Fully Connected Layer\n", - " model.add(Dense(10)) # final 10 FC nodes\n", - " model.add(Activation('softmax')) # softmax activation\n", + " # Fully Connected Layer\n", + " model.add(Dense(10)) # final 10 FC nodes\n", + " model.add(Activation(\"softmax\")) # softmax activation\n", "\n", - " model.summary()\n", + " model.summary()\n", "\n", - " adam = tf.optimizers.Adam(learning_rate=0.001)\n", - " model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n", + " adam = tf.optimizers.Adam(learning_rate=0.001)\n", + " model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])\n", + "\n", + " return model\n", "\n", - " return model\n", "\n", "def generate_model_default():\n", - " model = Sequential() # Linear stacking of layers\n", + " model = Sequential() # Linear stacking of layers\n", "\n", - " # Convolution Layer 1\n", - " model.add(Conv2D(16, (3, 3), input_shape=(32,32,3)))\n", - " model.add(Activation('relu') )\n", + " # Convolution Layer 1\n", + " model.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3)))\n", + " model.add(Activation(\"relu\"))\n", "\n", - " # ...\n", + " # ...\n", "\n", - " model.add(Flatten()) # Flatten final output matrix into a vector\n", + " model.add(Flatten()) # Flatten final output matrix into a vector\n", "\n", - " # ...\n", + " # ...\n", "\n", - " # Fully Connected Layer\n", - " model.add(Dense(10)) # final 10 FC nodes\n", - " model.add(Activation('softmax')) # softmax activation\n", + " # Fully Connected Layer\n", + " model.add(Dense(10)) # final 10 FC nodes\n", + " model.add(Activation(\"softmax\")) # softmax activation\n", "\n", - " model.summary()\n", + " model.summary()\n", "\n", - " adam = tf.optimizers.Adam(learning_rate=0.001)\n", - " model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n", + " adam = tf.optimizers.Adam(learning_rate=0.001)\n", + " model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])\n", + "\n", + " return model\n", "\n", - " return model\n", "\n", "def generate_model_gemini():\n", " model = Sequential()\n", "\n", " # Convolutional Layers with Max Pooling\n", - " model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)))\n", + " model.add(Conv2D(32, (3, 3), activation=\"relu\", input_shape=(32, 32, 3)))\n", " model.add(MaxPooling2D(pool_size=(2, 2)))\n", - " model.add(Conv2D(64, (3, 3), activation='relu'))\n", + " model.add(Conv2D(64, (3, 3), activation=\"relu\"))\n", " model.add(MaxPooling2D(pool_size=(2, 2)))\n", " model.add(Dropout(0.25)) # Regularization\n", "\n", " # Flatten and Fully Connected Layers\n", " model.add(Flatten())\n", - " model.add(Dense(128, activation='relu'))\n", - " model.add(Dropout(0.5)) # Regularization\n", - " model.add(Dense(10, activation='softmax'))\n", + " model.add(Dense(128, activation=\"relu\"))\n", + " model.add(Dropout(0.5)) # Regularization\n", + " model.add(Dense(10, activation=\"softmax\"))\n", "\n", " # Model Compilation\n", - " model.compile(loss='categorical_crossentropy',\n", - " optimizer='adam', # Consider trying other optimizers\n", - " metrics=['accuracy'])\n", + " model.compile(\n", + " loss=\"categorical_crossentropy\",\n", + " optimizer=\"adam\", # Consider trying other optimizers\n", + " metrics=[\"accuracy\"],\n", + " )\n", "\n", " return model\n", "\n", + "\n", "def generate_model_chat():\n", - " model = Sequential() # Linear stacking of layers\n", + " model = Sequential() # Linear stacking of layers\n", "\n", " # Convolution Layer 1\n", - " model.add(Conv2D(32, (3, 3), padding='same', input_shape=(32, 32, 3)))\n", - " model.add(Activation('relu'))\n", + " model.add(Conv2D(32, (3, 3), padding=\"same\", input_shape=(32, 32, 3)))\n", + " model.add(Activation(\"relu\"))\n", " model.add(BatchNormalization())\n", "\n", " # Convolution Layer 2\n", " model.add(Conv2D(32, (3, 3)))\n", - " model.add(Activation('relu'))\n", + " model.add(Activation(\"relu\"))\n", " model.add(MaxPooling2D(pool_size=(2, 2)))\n", " model.add(Dropout(0.6))\n", "\n", " # Convolution Layer 3\n", - " model.add(Conv2D(64, (3, 3), padding='same'))\n", - " model.add(Activation('relu'))\n", + " model.add(Conv2D(64, (3, 3), padding=\"same\"))\n", + " model.add(Activation(\"relu\"))\n", " model.add(BatchNormalization())\n", "\n", " # Convolution Layer 4\n", " model.add(Conv2D(64, (3, 3)))\n", - " model.add(Activation('relu'))\n", + " model.add(Activation(\"relu\"))\n", " model.add(MaxPooling2D(pool_size=(2, 2)))\n", " model.add(Dropout(0.6))\n", "\n", @@ -2285,20 +2415,20 @@ " model.add(Flatten())\n", "\n", " # Fully Connected Layer\n", - " model.add(Dense(512)) # Large fully connected layer\n", - " model.add(Activation('relu'))\n", + " model.add(Dense(512)) # Large fully connected layer\n", + " model.add(Activation(\"relu\"))\n", " model.add(BatchNormalization())\n", " model.add(Dropout(0.6))\n", "\n", " # Output Layer\n", - " model.add(Dense(10)) # final 10 FC nodes\n", - " model.add(Activation('softmax')) # softmax activation\n", + " model.add(Dense(10)) # final 10 FC nodes\n", + " model.add(Activation(\"softmax\")) # softmax activation\n", "\n", " model.summary()\n", "\n", " # Compile the model\n", " adam = tf.optimizers.Adam(learning_rate=0.001)\n", - " model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy'])\n", + " model.compile(loss=\"categorical_crossentropy\", optimizer=adam, metrics=[\"accuracy\"])\n", "\n", " return model" ] @@ -2342,11 +2472,17 @@ }, "outputs": [], "source": [ - "gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3,\n", - " height_shift_range=0.08, zoom_range=0.08, validation_split=0.2)\n", + "gen = ImageDataGenerator(\n", + " rotation_range=8,\n", + " width_shift_range=0.08,\n", + " shear_range=0.3,\n", + " height_shift_range=0.08,\n", + " zoom_range=0.08,\n", + " validation_split=0.2,\n", + ")\n", "\n", - "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset='training')\n", - "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation')" + "train_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"training\")\n", + "valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset=\"validation\")" ] }, { @@ -2363,7 +2499,14 @@ "source": [ "# Max 20 epoch\n", "for model in models:\n", - " model.fit(train_generator, steps_per_epoch=40000//128, epochs=20, verbose=1, validation_data=valid_generator, validation_steps = 10000 // 128)" + " model.fit(\n", + " train_generator,\n", + " steps_per_epoch=40000 // 128,\n", + " epochs=20,\n", + " verbose=1,\n", + " validation_data=valid_generator,\n", + " validation_steps=10000 // 128,\n", + " )" ] }, { @@ -2388,31 +2531,31 @@ "outputs": [], "source": [ "for model in models:\n", - " score = model.evaluate(X_test, Y_test)\n", - " print('Test score:', score[0])\n", - " print('Test accuracy:', score[1])\n", + " score = model.evaluate(X_test, Y_test)\n", + " print(\"Test score:\", score[0])\n", + " print(\"Test accuracy:\", score[1])\n", "\n", - " # The predict_classes function outputs the highest probability class\n", - " # according to the trained classifier for each input example.\n", - " predicted = model.predict(X_test)\n", - " predicted_classes = np.argmax(predicted, axis=1)\n", + " # The predict_classes function outputs the highest probability class\n", + " # according to the trained classifier for each input example.\n", + " predicted = model.predict(X_test)\n", + " predicted_classes = np.argmax(predicted, axis=1)\n", "\n", - " # Check which items we got right / wrong\n", - " correct_indices = np.nonzero(predicted_classes == y_test)[0]\n", + " # Check which items we got right / wrong\n", + " correct_indices = np.nonzero(predicted_classes == y_test)[0]\n", "\n", - " incorrect_indices = np.nonzero(predicted_classes != y_test)[0]\n", + " incorrect_indices = np.nonzero(predicted_classes != y_test)[0]\n", "\n", + " cnf_matrix = confusion_matrix(y_test, predicted_classes)\n", "\n", - " cnf_matrix = confusion_matrix(y_test, predicted_classes)\n", + " class_names = [str(i) for i in range(10)]\n", "\n", - " class_names = [str(i) for i in range(10)]\n", + " # Plot non-normalized confusion matrix\n", + " plt.figure()\n", + " plot_confusion_matrix(\n", + " cnf_matrix, classes=class_names, title=\"Confusion matrix, without normalization\"\n", + " )\n", "\n", - " # Plot non-normalized confusion matrix\n", - " plt.figure()\n", - " plot_confusion_matrix(cnf_matrix, classes=class_names,\n", - " title='Confusion matrix, without normalization')\n", - "\n", - " plt.show()" + " plt.show()" ] }, { @@ -2426,20 +2569,24 @@ }, "outputs": [], "source": [ - "def show_samples_rgb(indices, preds, images, labels, count=3, names = []):\n", + "def show_samples_rgb(indices, preds, images, labels, count=3, names=[]):\n", " plt.figure()\n", - " for i, sample in enumerate(indices[:count**2]):\n", + " for i, sample in enumerate(indices[: count**2]):\n", " pred_id = int(np.argmax(preds[sample]))\n", " real_id = int(labels[sample])\n", " pred_score = preds[sample][pred_id]\n", " real_score = preds[sample][real_id]\n", - " plt.subplot(count,count,i+1)\n", - " plt.imshow(images[sample], interpolation='none')\n", - " plt.axis('off')\n", + " plt.subplot(count, count, i + 1)\n", + " plt.imshow(images[sample], interpolation=\"none\")\n", + " plt.axis(\"off\")\n", " if len(names) > 0:\n", - " plt.title(\"P: {} ({:.2f})\\nE: {} ({:.2f})\".format(names[pred_id], pred_score, names[real_id], real_score))\n", + " plt.title(\n", + " f\"P: {names[pred_id]} ({pred_score:.2f})\\nE: {names[real_id]} ({real_score:.2f})\"\n", + " )\n", " else:\n", - " plt.title(\"P: {} ({:.2f})\\nE: {} ({:.2f})\".format(pred_id, pred_score, real_id, real_score))\n", + " plt.title(\n", + " f\"P: {pred_id} ({pred_score:.2f})\\nE: {real_id} ({real_score:.2f})\"\n", + " )\n", "\n", " plt.tight_layout()" ] diff --git a/Programming/twm_4/TWM_KerasIntro.py b/Programming/twm_4/TWM_KerasIntro.py index 20972354..3c17c1d5 100644 --- a/Programming/twm_4/TWM_KerasIntro.py +++ b/Programming/twm_4/TWM_KerasIntro.py @@ -4,75 +4,88 @@ # In[ ]: import sys + import tensorflow as tf - # Check if GPU is available -print(tf.config.list_physical_devices('GPU')) -if tf.config.list_physical_devices('GPU'): +print(tf.config.list_physical_devices("GPU")) +if tf.config.list_physical_devices("GPU"): print("GPU is available") else: print("GPU is not available") sys.exit() -from tensorflow import keras -from keras.datasets import cifar10 -from keras.utils import to_categorical -import numpy as np import itertools -import matplotlib.pyplot as plt -from keras.models import Sequential -from sklearn.metrics import confusion_matrix -from tensorflow.keras.preprocessing.image import ImageDataGenerator -from keras.layers import BatchNormalization, Conv2D, MaxPooling2D, ZeroPadding2D, GlobalAveragePooling2D, Flatten, Dense, Dropout, Activation -from tensorflow.keras.optimizers import Adam -def plot_confusion_matrix(cm, classes, - normalize=False, - title='Confusion matrix', - cmap=plt.cm.Blues): - """ - This function prints and plots the confusion matrix. +import matplotlib.pyplot as plt +import numpy as np +from keras.datasets import cifar10 +from keras.layers import ( + Activation, + BatchNormalization, + Conv2D, + Dense, + Dropout, + Flatten, + MaxPooling2D, +) +from keras.models import Sequential +from keras.utils import to_categorical +from sklearn.metrics import confusion_matrix +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.preprocessing.image import ImageDataGenerator + + +def plot_confusion_matrix( + cm, classes, normalize=False, title="Confusion matrix", cmap=plt.cm.Blues +): + """This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ if normalize: - cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + cm = cm.astype("float") / cm.sum(axis=1)[:, np.newaxis] print("Normalized confusion matrix") else: - print('Confusion matrix, without normalization') + print("Confusion matrix, without normalization") print(cm) - - plt.imshow(cm, interpolation='nearest', cmap=cmap) + plt.imshow(cm, interpolation="nearest", cmap=cmap) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) - fmt = '.2f' if normalize else 'd' - thresh = cm.max() / 2. + fmt = ".2f" if normalize else "d" + thresh = cm.max() / 2.0 for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): - plt.text(j, i, format(cm[i, j], fmt), - horizontalalignment="center", - color="white" if cm[i, j] > thresh else "black") + plt.text( + j, + i, + format(cm[i, j], fmt), + horizontalalignment="center", + color="white" if cm[i, j] > thresh else "black", + ) - plt.ylabel('True label') - plt.xlabel('Predicted label') + plt.ylabel("True label") + plt.xlabel("Predicted label") plt.tight_layout() + (X_train, y_train), (X_test, y_test) = cifar10.load_data() -X_train = X_train.astype('float32') # change integers to 32-bit floating point numbers -X_test = X_test.astype('float32') +X_train = X_train.astype("float32") # change integers to 32-bit floating point numbers +X_test = X_test.astype("float32") -X_train /= 255 # normalize each value for each pixel for the entire vector for each input +X_train /= ( + 255 # normalize each value for each pixel for the entire vector for each input +) X_test /= 255 -y_train = y_train.reshape((1,-1))[0] -y_test = y_test.reshape((1,-1))[0] +y_train = y_train.reshape((1, -1))[0] +y_test = y_test.reshape((1, -1))[0] print("Training matrix shape", X_train.shape, y_train.shape) print("Testing matrix shape", X_test.shape, y_test.shape) @@ -84,7 +97,18 @@ nb_classes = 10 Y_train = to_categorical(y_train, nb_classes) Y_test = to_categorical(y_test, nb_classes) -cifar_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck'] +cifar_names = [ + "airplane", + "automobile", + "bird", + "cat", + "deer", + "dog", + "frog", + "horse", + "ship", + "truck", +] # ## Podgląd zbioru treningowego @@ -92,16 +116,23 @@ cifar_names = ['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog', 'frog', ' # In[ ]: -for i in range(0, 10): - img_batch = X_train[y_train == i][0:10] - img_batch = np.reshape(img_batch, (img_batch.shape[0]*img_batch.shape[1], img_batch.shape[2], img_batch.shape[3])) - if i > 0: - img = np.concatenate([img, img_batch], axis = 1) - else: - img = img_batch -plt.figure(figsize=(10,20)) -plt.axis('off') -plt.imshow(img, cmap='gray') +for i in range(10): + img_batch = X_train[y_train == i][0:10] + img_batch = np.reshape( + img_batch, + ( + img_batch.shape[0] * img_batch.shape[1], + img_batch.shape[2], + img_batch.shape[3], + ), + ) + if i > 0: + img = np.concatenate([img, img_batch], axis=1) + else: + img = img_batch +plt.figure(figsize=(10, 20)) +plt.axis("off") +plt.imshow(img, cmap="gray") # ## Przygotowanie modelu @@ -110,98 +141,103 @@ plt.imshow(img, cmap='gray') def generate_model(): - model = Sequential() # Linear stacking of layers + model = Sequential() # Linear stacking of layers - # Convolution Layer 1 - model.add(Conv2D(16, (3, 3), input_shape=(32,32,3))) - model.add(Activation('relu') ) + # Convolution Layer 1 + model.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3))) + model.add(Activation("relu")) - # ... + # ... - model.add(Flatten()) # Flatten final output matrix into a vector + model.add(Flatten()) # Flatten final output matrix into a vector - # ... + # ... - # Fully Connected Layer - model.add(Dense(10)) # final 10 FC nodes - model.add(Activation('softmax')) # softmax activation + # Fully Connected Layer + model.add(Dense(10)) # final 10 FC nodes + model.add(Activation("softmax")) # softmax activation - model.summary() + model.summary() - adam = tf.optimizers.Adam(learning_rate=0.001) - model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) + adam = tf.optimizers.Adam(learning_rate=0.001) + model.compile(loss="categorical_crossentropy", optimizer=adam, metrics=["accuracy"]) + + return model - return model def generate_model_default(): - model = Sequential() # Linear stacking of layers + model = Sequential() # Linear stacking of layers - # Convolution Layer 1 - model.add(Conv2D(16, (3, 3), input_shape=(32,32,3))) - model.add(Activation('relu') ) + # Convolution Layer 1 + model.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3))) + model.add(Activation("relu")) - # ... + # ... - model.add(Flatten()) # Flatten final output matrix into a vector + model.add(Flatten()) # Flatten final output matrix into a vector - # ... + # ... - # Fully Connected Layer - model.add(Dense(10)) # final 10 FC nodes - model.add(Activation('softmax')) # softmax activation + # Fully Connected Layer + model.add(Dense(10)) # final 10 FC nodes + model.add(Activation("softmax")) # softmax activation - model.summary() + model.summary() - adam = tf.optimizers.Adam(learning_rate=0.001) - model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) + adam = tf.optimizers.Adam(learning_rate=0.001) + model.compile(loss="categorical_crossentropy", optimizer=adam, metrics=["accuracy"]) + + return model - return model def generate_model_gemini(): model = Sequential() # Convolutional Layers with Max Pooling - model.add(Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3))) + model.add(Conv2D(32, (3, 3), activation="relu", input_shape=(32, 32, 3))) model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Conv2D(64, (3, 3), activation='relu')) + model.add(Conv2D(64, (3, 3), activation="relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) # Regularization # Flatten and Fully Connected Layers model.add(Flatten()) - model.add(Dense(128, activation='relu')) - model.add(Dropout(0.5)) # Regularization - model.add(Dense(10, activation='softmax')) + model.add(Dense(128, activation="relu")) + model.add(Dropout(0.5)) # Regularization + model.add(Dense(10, activation="softmax")) # Model Compilation - model.compile(loss='categorical_crossentropy', - optimizer='adam', # Consider trying other optimizers - metrics=['accuracy']) + model.compile( + loss="categorical_crossentropy", + optimizer="adam", # Consider trying other optimizers + metrics=["accuracy"], + ) return model + def generate_model_chat(): - model = Sequential() # Linear stacking of layers + model = Sequential() # Linear stacking of layers # Convolution Layer 1 - model.add(Conv2D(32, (3, 3), padding='same', input_shape=(32, 32, 3))) - model.add(Activation('relu')) + model.add(Conv2D(32, (3, 3), padding="same", input_shape=(32, 32, 3))) + model.add(Activation("relu")) model.add(BatchNormalization()) # Convolution Layer 2 model.add(Conv2D(32, (3, 3))) - model.add(Activation('relu')) + model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.3)) # Convolution Layer 3 - model.add(Conv2D(64, (3, 3), padding='same')) - model.add(Activation('relu')) + model.add(Conv2D(64, (3, 3), padding="same")) + model.add(Activation("relu")) model.add(BatchNormalization()) # Convolution Layer 4 model.add(Conv2D(64, (3, 3))) - model.add(Activation('relu')) + model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.3)) @@ -209,20 +245,20 @@ def generate_model_chat(): model.add(Flatten()) # Fully Connected Layer - model.add(Dense(512)) # Large fully connected layer - model.add(Activation('relu')) + model.add(Dense(512)) # Large fully connected layer + model.add(Activation("relu")) model.add(BatchNormalization()) model.add(Dropout(0.6)) # Output Layer - model.add(Dense(10)) # final 10 FC nodes - model.add(Activation('softmax')) # softmax activation + model.add(Dense(10)) # final 10 FC nodes + model.add(Activation("softmax")) # softmax activation model.summary() # Compile the model adam = Adam(learning_rate=0.001) - model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) + model.compile(loss="categorical_crossentropy", optimizer=adam, metrics=["accuracy"]) return model @@ -242,11 +278,17 @@ models = [model_chat] # In[ ]: -gen = ImageDataGenerator(rotation_range=8, width_shift_range=0.08, shear_range=0.3, - height_shift_range=0.08, zoom_range=0.08, validation_split=0.2) +gen = ImageDataGenerator( + rotation_range=8, + width_shift_range=0.08, + shear_range=0.3, + height_shift_range=0.08, + zoom_range=0.08, + validation_split=0.2, +) -train_generator = gen.flow(X_train, Y_train, batch_size=128, subset='training') -valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation') +train_generator = gen.flow(X_train, Y_train, batch_size=128, subset="training") +valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset="validation") # In[ ]: @@ -254,7 +296,14 @@ valid_generator = gen.flow(X_train, Y_train, batch_size=128, subset='validation' # Max 20 epoch for model in models: - model.fit(train_generator, steps_per_epoch=40000//128, epochs=20, verbose=1, validation_data=valid_generator, validation_steps = 10000 // 128) + model.fit( + train_generator, + steps_per_epoch=40000 // 128, + epochs=20, + verbose=1, + validation_data=valid_generator, + validation_steps=10000 // 128, + ) # ## Test @@ -262,50 +311,54 @@ for model in models: # In[ ]: for model in models: - score = model.evaluate(X_test, Y_test) - print('Test score:', score[0]) - print('Test accuracy:', score[1]) + score = model.evaluate(X_test, Y_test) + print("Test score:", score[0]) + print("Test accuracy:", score[1]) - # The predict_classes function outputs the highest probability class - # according to the trained classifier for each input example. - predicted = model.predict(X_test) - predicted_classes = np.argmax(predicted, axis=1) + # The predict_classes function outputs the highest probability class + # according to the trained classifier for each input example. + predicted = model.predict(X_test) + predicted_classes = np.argmax(predicted, axis=1) - # Check which items we got right / wrong - correct_indices = np.nonzero(predicted_classes == y_test)[0] + # Check which items we got right / wrong + correct_indices = np.nonzero(predicted_classes == y_test)[0] - incorrect_indices = np.nonzero(predicted_classes != y_test)[0] + incorrect_indices = np.nonzero(predicted_classes != y_test)[0] + cnf_matrix = confusion_matrix(y_test, predicted_classes) - cnf_matrix = confusion_matrix(y_test, predicted_classes) + class_names = [str(i) for i in range(10)] - class_names = [str(i) for i in range(10)] + # Plot non-normalized confusion matrix + plt.figure() + plot_confusion_matrix( + cnf_matrix, classes=class_names, title="Confusion matrix, without normalization" + ) - # Plot non-normalized confusion matrix - plt.figure() - plot_confusion_matrix(cnf_matrix, classes=class_names, - title='Confusion matrix, without normalization') - - plt.show() + plt.show() # In[ ]: -def show_samples_rgb(indices, preds, images, labels, count=3, names = []): +def show_samples_rgb(indices, preds, images, labels, count=3, names=[]): plt.figure() - for i, sample in enumerate(indices[:count**2]): + for i, sample in enumerate(indices[: count**2]): pred_id = int(np.argmax(preds[sample])) real_id = int(labels[sample]) pred_score = preds[sample][pred_id] real_score = preds[sample][real_id] - plt.subplot(count,count,i+1) - plt.imshow(images[sample], interpolation='none') - plt.axis('off') + plt.subplot(count, count, i + 1) + plt.imshow(images[sample], interpolation="none") + plt.axis("off") if len(names) > 0: - plt.title("P: {} ({:.2f})\nE: {} ({:.2f})".format(names[pred_id], pred_score, names[real_id], real_score)) + plt.title( + f"P: {names[pred_id]} ({pred_score:.2f})\nE: {names[real_id]} ({real_score:.2f})" + ) else: - plt.title("P: {} ({:.2f})\nE: {} ({:.2f})".format(pred_id, pred_score, real_id, real_score)) + plt.title( + f"P: {pred_id} ({pred_score:.2f})\nE: {real_id} ({real_score:.2f})" + ) plt.tight_layout() @@ -324,4 +377,3 @@ show_samples_rgb(correct_indices, predicted, X_test, y_test, 5, cifar_names) show_samples_rgb(incorrect_indices, predicted, X_test, y_test, 5, cifar_names) - diff --git a/Programming/twm_4/twm.py b/Programming/twm_4/twm.py index 7752f1fd..e73990bc 100644 --- a/Programming/twm_4/twm.py +++ b/Programming/twm_4/twm.py @@ -1,19 +1,3 @@ import tensorflow as tf -import numpy as np # advanced math library -import matplotlib.pyplot as plt # MATLAB like plotting routines -import random # for generating random numbers - -from keras.datasets import mnist # MNIST dataset is included in Keras% -from keras.models import Sequential # Model type to be used - -from keras.layers import Dense, Dropout, Activation # Types of layers to be used in our model -from keras.utils import to_categorical # NumPy related tools - -from keras import optimizers - - -from sklearn.metrics import confusion_matrix -import itertools - -print(tf.__version__) \ No newline at end of file +print(tf.__version__) diff --git a/gates/runnability-local.tsv b/gates/runnability-local.tsv index d61d5f21..ffec41ae 100644 --- a/gates/runnability-local.tsv +++ b/gates/runnability-local.tsv @@ -37,7 +37,7 @@ Programming/PBAD blocked jupyter is on PATH but broken (bad interpreter) Programming/PORR runnable python3 /home/kuhy/WUT_Computer_Science/Programming/PORR/code/main.py Programming/PSD runnable starts the Kafka/Flink stack (installs packages with sudo) Programming/psd_project runnable mvn package -Programming/TRAK runnable python3 the entry script +Programming/TRAK runnable python3 /home/kuhy/WUT_Computer_Science/Programming/TRAK/main.py Programming/twm_4 blocked jupyter is on PATH but broken (bad interpreter) Programming/USD no-code README describes a MetaDrive RL assignment; no implementation was submitted Programming/WDWR runnable python3 the entry script diff --git a/gates/vendored.txt b/gates/vendored.txt index 84030ef5..4f8e25a5 100644 --- a/gates/vendored.txt +++ b/gates/vendored.txt @@ -62,3 +62,22 @@ NotProgramming/EMISY/schematic # ControlPanel.java (1852 lines) and Kernel.java (511) are not split. Programming/EOPSY/lab3/task3/work Programming/EOPSY/lab4/task4/work + +# --- Third-party ray tracer ------------------------------------------------ +# sightpy, the rendering engine from rafael-fuente/Python-Raytracer (BSD +# 3-Clause, (c) 2021 Rafael de la Fuente). TRAK's assignment was photon +# mapping ON TOP of it; the engine itself was not written here. +# +# Verified 2026-08-28 against a fresh clone of the upstream repository, after +# normalising line endings (the copy here is LF, upstream ships CRLF): +# 30 of 32 .py files BYTE-IDENTICAL +# sightpy/ray.py +12 lines: RAYS / PRIMARY_RAYS counters and get_rays() +# sightpy/scene.py +13 lines: prints those counters in the render summary +# Both patches are kuhy's instrumentation for the report's ray statistics and +# are the only authored lines inside this tree. +# +# NOT covered by this entry, and still fully gated: TRAK's own main.py, +# photon_mapping.py, rendering.py, lab6.py, utils.py, code/ and scenes/. +# scenes/ is adapted from the upstream examples rather than copied -- the +# closest example differs by more lines than the scene file has. +Programming/TRAK/sightpy diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..72d5af28 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,93 @@ +# Python tooling for the whole archive. There is no package here to build -- +# this file exists so ruff, mypy and coverage read one configuration instead of +# 25 course directories each guessing. +# +# The `exclude` and `omit` lists below are MIRRORS of gates/vendored.txt. +# Edit that manifest and re-run `./run.sh sync-vendored`; never edit a mirror +# on its own, or the gates start disagreeing about what is kuhy's code. + +[tool.ruff] +target-version = "py310" +line-length = 88 +include = ["*.py", "**/*.py", "*.ipynb", "**/*.ipynb"] + +exclude = [ + # --- machine-generated, never authored --- + ".git", + ".venv", + "__pycache__", + "build", + "dist", + "node_modules", + # --- mirrors of gates/vendored.txt --- + "Programming/EGUI/lab2/lab2/wwwroot/lib", + "Programming/E-PSYOPS/theProject/Assets/TextMesh Pro", + "Programming/E-PSYOPS/theProject/Assets/TutorialInfo", + "NotProgramming/EMISY/componentSchematic", + "NotProgramming/EMISY/schematic", + "Programming/EOPSY/lab3/task3/work", + "Programming/EOPSY/lab4/task4/work", + "Programming/TRAK/sightpy", +] + +[tool.ruff.lint] +select = ["ALL"] + +# The only rules turned off, and why. There are no per-file-ignores: the +# coursework is held to the same bar as the tooling, because git history is +# what preserves the version that was handed in. +ignore = [ + # Pairwise-contradictory docstring rules -- ruff ships both halves of each + # pair in ALL and you must pick one. These match the rest of the fleet. + "D203", # ...vs D211, we take D211 (no blank line before class docstring) + "D213", # ...vs D212, we take D212 (summary on the first line) + # Disabled on the formatter's own advice: the formatter already decides + # both of these, and leaving them on makes the two tools fight. + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "COM812", # missing trailing comma + "ISC001", # single-line implicit string concatenation + # A known false-positive generator: it cannot see that an argument list was + # validated upstream. https://github.com/astral-sh/ruff/issues/4045 + "S603", # subprocess call without shell=True + # Ruff began enforcing this outside preview in 0.16; no repo in this fleet + # carries copyright headers, and stamping one onto 137 university + # assignments would be asserting a licence that was never claimed. + "CPY001", # missing copyright notice +] + +fixable = ["ALL"] +unfixable = [] + +[tool.mypy] +strict = true +exclude = [ + "^\\.venv/", + "^build/", + "^Programming/EGUI/lab2/lab2/wwwroot/lib/", + "^Programming/E-PSYOPS/theProject/Assets/TextMesh Pro/", + "^Programming/E-PSYOPS/theProject/Assets/TutorialInfo/", + "^NotProgramming/EMISY/componentSchematic/", + "^NotProgramming/EMISY/schematic/", + "^Programming/EOPSY/lab3/task3/work/", + "^Programming/EOPSY/lab4/task4/work/", + "^Programming/TRAK/sightpy/", +] + +[tool.coverage.run] +branch = true +omit = [ + "*/.venv/*", + "*/build/*", + "Programming/EGUI/lab2/lab2/wwwroot/lib/*", + "Programming/E-PSYOPS/theProject/Assets/TextMesh Pro/*", + "Programming/E-PSYOPS/theProject/Assets/TutorialInfo/*", + "NotProgramming/EMISY/componentSchematic/*", + "NotProgramming/EMISY/schematic/*", + "Programming/EOPSY/lab3/task3/work/*", + "Programming/EOPSY/lab4/task4/work/*", + "Programming/TRAK/sightpy/*", +] + +[tool.coverage.report] +show_missing = true +skip_covered = false diff --git a/run.sh b/run.sh index 80b87699..47d6c0c6 100755 --- a/run.sh +++ b/run.sh @@ -153,6 +153,13 @@ cmd_run() { local runner="${SCRIPT_DIR}/${target}/run.sh" [[ -x "$runner" ]] || { err "no executable run.sh in $target"; return 1; } log "running $target (capped at ${RUN_MEMORY_MAX} RAM, ${RUN_CPU_QUOTA} CPU)" + # Agg, so plt.show() draws nothing and returns. TRAK's renderer finished + # its ray trace in a couple of minutes and then sat there for 45 more: + # plt.show() had opened an image viewer and was waiting for a human to + # close the window. A run that needs a person at a keyboard is not a run. + # export, not an assignment prefix: _run_capped is a shell function, and a + # prefixed assignment on one is not put into the child's environment. + export MPLBACKEND="${MPLBACKEND:-Agg}" _run_capped "$runner" } diff --git a/scripts/course_lib.sh b/scripts/course_lib.sh index 46a1c47f..2c26754d 100644 --- a/scripts/course_lib.sh +++ b/scripts/course_lib.sh @@ -11,7 +11,8 @@ # (no args) actually run it # # A stub calls: -# course_main --kind [--entry ] [--note ] -- "$@" +# course_main --kind [--entry ] [--entry-args ] +# [--note ] [--test-dir ] [--no-bibtex] -- "$@" # Values are passed as arguments rather than set as globals so that a stub has # no assignments a linter has to be told to ignore. @@ -80,252 +81,18 @@ _first_file() { find "$COURSE_ROOT" -name "$1" -not -path '*/node_modules/*' -print -quit 2>/dev/null } -# --- probes ------------------------------------------------------------------ -_probe() { - case "$COURSE_KIND" in - none) _no_code "${COURSE_NOTE:-no executable code in this directory}" ;; - tex) - _probe_any "a LaTeX toolchain" latexmk pdflatex xelatex - _runnable "builds ${COURSE_ENTRY:-the report} with latexmk" - ;; - python) - _have python3 || _blocked "needs python3" - [[ -n "$(_first_file '*.py')" ]] || _no_code "no .py files found" - _runnable "python3 ${COURSE_ENTRY:-the entry script}" - ;; - node) - _have node || _blocked "needs node" - _have npm || _blocked "needs npm" - _runnable "npm install && npm start" - ;; - maven) - _have mvn || _blocked "needs maven" - _have java || _blocked "needs a JDK" - _runnable "mvn package" - ;; - make) - _have make || _blocked "needs make" - _probe_any "a C compiler" gcc cc clang - _runnable "make" - ;; - dotnet) - _have dotnet || _blocked "needs the .NET SDK" - _runnable "dotnet build" - ;; - unity) - _have unity-editor || _have unityhub \ - || _blocked "needs the Unity editor (GUI, not installable on a CI runner)" - _runnable "opens in the Unity editor" - ;; - matlab) - _probe_any "MATLAB or Octave" matlab octave - _runnable "runs the .m scripts" - ;; - cxx) - _probe_any "a C++ compiler" g++ clang++ - [[ -n "$(_first_file '*.cpp')" ]] || _no_code "no .cpp files found" - _runnable "compiles each standalone .cpp" - ;; - notebook) - _have jupyter || _blocked "needs jupyter" - # Being on PATH is not the same as working: this machine has a - # jupyter shim whose pipx venv interpreter is gone, so it fails - # with "bad interpreter". Probe by actually invoking it. - jupyter --version > /dev/null 2>&1 || _blocked "jupyter is on PATH but broken (bad interpreter)" - _runnable "jupyter nbconvert --execute" - ;; - compose) - _have docker || _blocked "needs docker" - docker info > /dev/null 2>&1 || _blocked "docker is installed but the daemon is not running" - _runnable "docker compose up" - ;; - *) _blocked "unknown COURSE_KIND '${COURSE_KIND:-unset}'" ;; - esac -} - -# Use the course venv when one exists, otherwise the system interpreter. -_python_bin() { - if [[ -x "$COURSE_ROOT/.venv/bin/python" ]]; then - printf '%s' "$COURSE_ROOT/.venv/bin/python" - else - printf 'python3' - fi -} - -# Dependencies are the runner's job, not the reader's. Without this, EARIN -# dies on ModuleNotFoundError for gymnasium and the "runnable" status is a lie. -_python_install_requirements() { - local reqs=() - while IFS= read -r r; do reqs+=("$r"); done < <( - find "$COURSE_ROOT" -name 'requirements*.txt' -not -path '*/.venv/*' | sort) - [[ ${#reqs[@]} -gt 0 ]] || return 0 - - if [[ ! -x "$COURSE_ROOT/.venv/bin/python" ]]; then - printf 'creating .venv\n' - python3 -m venv "$COURSE_ROOT/.venv" - fi - "$COURSE_ROOT/.venv/bin/python" -m pip install --quiet --upgrade pip - - # EVERY requirements file, not just the first: EARIN is six independent - # labs plus a project, and installing only one of them left the lab it - # then tried to run missing gymnasium. - local req - for req in "${reqs[@]}"; do - printf 'installing %s\n' "${req#"$COURSE_ROOT"/}" - "$COURSE_ROOT/.venv/bin/python" -m pip install --quiet -r "$req" || { - printf 'warning: some requirements in %s failed\n' "${req#"$COURSE_ROOT"/}" >&2 - } - done -} - -# Where a course ships real tests, running them is a stronger check than -# running the program -- and for an interactive program (ECRYPT_PROJECT prompts -# on stdin) it is the ONLY check that works without a person at a keyboard. -_python_run_tests() { - local suite - suite="$(find "$COURSE_ROOT" -name 'test_*.py' -not -path '*/.venv/*' -print -quit 2>/dev/null)" - [[ -n "$suite" ]] || return 0 - local py - py="$(_python_bin)" - "$py" -c 'import pytest' 2> /dev/null || { - printf 'tests present but pytest is not installed; skipping\n' - return 0 - } - # Which directory pytest runs FROM is course-specific and cannot be - # guessed: ECOTE needs program/ so `from translator.main import` resolves, - # ECRYPT_PROJECT needs sieve/ because its fixtures open P-100000.csv by a - # relative path. So the stub says, and COURSE_ROOT is only the default. - local from_dir="${COURSE_TEST_DIR:-$COURSE_ROOT}" - printf 'running the test suite in %s\n' "${from_dir#"$COURSE_ROOT"/}" - ( cd "$from_dir" && "$py" -m pytest -q ) -} - -# --- runners ----------------------------------------------------------------- - -_run() { - case "$COURSE_KIND" in - none) _no_code "${COURSE_NOTE:-nothing to run}" ;; - tex) - local tex="${COURSE_ENTRY:-}" - [[ -n "$tex" ]] || tex="$(_first_file '*.tex')" - [[ -n "$tex" ]] || _no_code "no .tex found" - local tool - tool="$(_require_any 'a LaTeX toolchain' latexmk pdflatex xelatex)" - # Build into build/, never beside the source. A submitted report.pdf - # is a tracked deliverable, and a rebuild whose output name happens - # to match it would silently overwrite what was handed in. - local dir base - dir="$(dirname "$tex")" - base="$(basename "$tex")" - # A document that declares \bibliography but never cites anything - # makes bibtex exit 2 ("I found no \citation commands"), and - # latexmk turns that into exit 12 even though the PDF is fine. - # --no-bibtex says "this one was handed in without a bibliography", - # which is a fact about the deliverable, not a suppression. - # An `x && y=1` one-liner would be a set -e landmine here: when x - # is false the list returns 1 and the whole runner exits. - local bibflag=() - if [[ -n "${COURSE_TEX_NO_BIBTEX:-}" ]]; then - bibflag=(-bibtex-) - fi - if [[ $tool == latexmk ]]; then - ( cd "$dir" && latexmk -pdf "${bibflag[@]}" \ - -interaction=nonstopmode -outdir=build "$base" ) - else - ( cd "$dir" && mkdir -p build \ - && "$tool" -interaction=nonstopmode -output-directory=build "$base" ) - fi - printf 'output in %s/build\n' "$dir" - ;; - python) - local entry="${COURSE_ENTRY:-}" - # An entry may be given as a directory; resolve it to the module - # inside. Passing the directory itself makes python look for - # __main__.py and fail with a message about the wrong thing. - if [[ -d "$entry" ]]; then - local cand - for cand in main.py project.py __main__.py; do - if [[ -f "$entry/$cand" ]]; then - entry="$entry/$cand" - break - fi - done - fi - # Only auto-pick a main.py when there is exactly ONE. A course - # with six labs has six, and choosing whichever find returns first - # is not "running the project", it is running an arbitrary lab. - if [[ ! -f "$entry" ]]; then - local mains - mains="$(find "$COURSE_ROOT" -name 'main.py' -not -path '*/.venv/*' | wc -l)" - [[ "$mains" -eq 1 ]] && entry="$(_first_file 'main.py')" - fi - - _python_install_requirements - - if [[ -f "$entry" ]]; then - ( cd "$(dirname "$entry")" && "$(_python_bin)" "$(basename "$entry")" ) - else - # A pile of one-shot scripts with no entry contract (NLP is - # the case here). Syntax-check them all rather than picking one - # arbitrarily and calling that "running the project". - printf 'no single entry point; syntax-checking every module instead\n' - find "$COURSE_ROOT" -name '*.py' -not -path '*/.venv/*' -print0 \ - | xargs -0 -r "$(_python_bin)" -m py_compile - printf 'all modules compile\n' - fi - _python_run_tests - ;; - node) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && npm install && npm start ) ;; - maven) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && mvn -q package ) ;; - make) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && make ) ;; - dotnet) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && dotnet build ) ;; - compose) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && docker compose up ) ;; - unity) _blocked "open ${COURSE_ROOT} in the Unity editor; there is no headless path" ;; - cxx) - # Each .cpp here is a self-contained assignment with its own main(), - # so they are compiled individually rather than linked together. - local cxx src rel out - cxx="$(_require_any 'a C++ compiler' g++ clang++)" - # Binaries go to build/, which is gitignored -- compiling in place - # drops untracked executables next to the sources, and an - # extensionless binary is not something .gitignore catches by name. - mkdir -p "$COURSE_ROOT/build" - while IFS= read -r src; do - rel="${src#"$COURSE_ROOT"/}" - if _is_known_incomplete "${src#"${COURSE_GATES%/gates}"/}"; then - printf 'skipping %s (see gates/known-incomplete.txt)\n' "$rel" - continue - fi - out="$COURSE_ROOT/build/$(basename "${rel%.cpp}")" - # Some files here are exam-answer fragments with no main(), so - # they cannot be linked. Compile those to an object instead -- - # that still type-checks them, which is the point. - if grep -qE '\bint[[:space:]]+main[[:space:]]*\(' "$src"; then - printf 'compiling %s\n' "$rel" - "$cxx" -O2 -o "$out" "$src" - else - printf 'checking %s (no main; -c only)\n' "$rel" - "$cxx" -O2 -c -o "${out}.o" "$src" - fi - done < <(find "$COURSE_ROOT" -name '*.cpp' -not -path '*/node_modules/*' -not -path '*/build/*' | sort) - printf 'binaries in %s/build\n' "$COURSE_ROOT" - ;; - notebook) - _have jupyter || _blocked "needs jupyter" - local nb - while IFS= read -r nb; do - jupyter nbconvert --to notebook --execute --inplace "$nb" - done < <(find "$COURSE_ROOT" -name '*.ipynb' -not -path '*/.ipynb_checkpoints/*' | sort) - ;; - matlab) - local tool - tool="$(_require_any 'MATLAB or Octave' matlab octave)" - printf 'run the .m files in %s with %s\n' "$COURSE_ROOT" "$tool" - ;; - *) printf 'unknown COURSE_KIND: %s\n' "${COURSE_KIND:-unset}" >&2; return 1 ;; - esac -} +# The probe and the runner are one case statement each and together are most of +# this file's length, so they live beside it rather than in it. SCRIPTDIR is +# resolved from BASH_SOURCE[0] -- BASH_SOURCE[1] is the *caller*, which is a +# per-course run.sh two directories away. +_COURSE_LIB_DIR="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +# shellcheck source-path=SCRIPTDIR +# shellcheck source=course_probe.sh +. "${_COURSE_LIB_DIR}/course_probe.sh" +# shellcheck source-path=SCRIPTDIR +# shellcheck source=course_run.sh +. "${_COURSE_LIB_DIR}/course_run.sh" course_main() { COURSE_KIND="" @@ -333,6 +100,7 @@ course_main() { COURSE_NOTE="" COURSE_TEST_DIR="" COURSE_TEX_NO_BIBTEX="" + COURSE_ENTRY_ARGS="" while [[ $# -gt 0 ]]; do case "$1" in --kind) COURSE_KIND="$2"; shift 2 ;; @@ -340,6 +108,7 @@ course_main() { --note) COURSE_NOTE="$2"; shift 2 ;; --test-dir) COURSE_TEST_DIR="$2"; shift 2 ;; --no-bibtex) COURSE_TEX_NO_BIBTEX=1; shift ;; + --entry-args) COURSE_ENTRY_ARGS="$2"; shift 2 ;; --) shift; break ;; *) break ;; esac diff --git a/scripts/course_probe.sh b/scripts/course_probe.sh new file mode 100644 index 00000000..e399b149 --- /dev/null +++ b/scripts/course_probe.sh @@ -0,0 +1,70 @@ +# shellcheck shell=bash +# Split out of course_lib.sh, which was over this repo's 250-line cap. Sourced +# by it, never on its own -- the helpers and constants it uses live there. +# +# Answers --probe: can this course run HERE, without running it. + +# --- probes ------------------------------------------------------------------ + +_probe() { + case "$COURSE_KIND" in + none) _no_code "${COURSE_NOTE:-no executable code in this directory}" ;; + tex) + _probe_any "a LaTeX toolchain" latexmk pdflatex xelatex + _runnable "builds ${COURSE_ENTRY:-the report} with latexmk" + ;; + python) + _have python3 || _blocked "needs python3" + [[ -n "$(_first_file '*.py')" ]] || _no_code "no .py files found" + _runnable "python3 ${COURSE_ENTRY:-the entry script}" + ;; + node) + _have node || _blocked "needs node" + _have npm || _blocked "needs npm" + _runnable "npm install && npm start" + ;; + maven) + _have mvn || _blocked "needs maven" + _have java || _blocked "needs a JDK" + _runnable "mvn package" + ;; + make) + _have make || _blocked "needs make" + _probe_any "a C compiler" gcc cc clang + _runnable "make" + ;; + dotnet) + _have dotnet || _blocked "needs the .NET SDK" + _runnable "dotnet build" + ;; + unity) + _have unity-editor || _have unityhub \ + || _blocked "needs the Unity editor (GUI, not installable on a CI runner)" + _runnable "opens in the Unity editor" + ;; + matlab) + _probe_any "MATLAB or Octave" matlab octave + _runnable "runs the .m scripts" + ;; + cxx) + _probe_any "a C++ compiler" g++ clang++ + [[ -n "$(_first_file '*.cpp')" ]] || _no_code "no .cpp files found" + _runnable "compiles each standalone .cpp" + ;; + notebook) + _have jupyter || _blocked "needs jupyter" + # Being on PATH is not the same as working: this machine has a + # jupyter shim whose pipx venv interpreter is gone, so it fails + # with "bad interpreter". Probe by actually invoking it. + jupyter --version > /dev/null 2>&1 || _blocked "jupyter is on PATH but broken (bad interpreter)" + _runnable "jupyter nbconvert --execute" + ;; + compose) + _have docker || _blocked "needs docker" + docker info > /dev/null 2>&1 || _blocked "docker is installed but the daemon is not running" + _runnable "docker compose up" + ;; + *) _blocked "unknown COURSE_KIND '${COURSE_KIND:-unset}'" ;; + esac +} + diff --git a/scripts/course_run.sh b/scripts/course_run.sh new file mode 100644 index 00000000..ee6b39fd --- /dev/null +++ b/scripts/course_run.sh @@ -0,0 +1,198 @@ +# shellcheck shell=bash +# Split out of course_lib.sh, which was over this repo's 250-line cap. Sourced +# by it, never on its own -- the helpers and constants it uses live there. +# +# Answers a bare invocation: actually build or run the course. + + +# Use the course venv when one exists, otherwise the system interpreter. +_python_bin() { + if [[ -x "$COURSE_ROOT/.venv/bin/python" ]]; then + printf '%s' "$COURSE_ROOT/.venv/bin/python" + else + printf 'python3' + fi +} + +# Dependencies are the runner's job, not the reader's. Without this, EARIN +# dies on ModuleNotFoundError for gymnasium and the "runnable" status is a lie. +_python_install_requirements() { + local reqs=() + while IFS= read -r r; do reqs+=("$r"); done < <( + find "$COURSE_ROOT" -name 'requirements*.txt' -not -path '*/.venv/*' | sort) + [[ ${#reqs[@]} -gt 0 ]] || return 0 + + if [[ ! -x "$COURSE_ROOT/.venv/bin/python" ]]; then + printf 'creating .venv\n' + python3 -m venv "$COURSE_ROOT/.venv" + fi + "$COURSE_ROOT/.venv/bin/python" -m pip install --quiet --upgrade pip + + # EVERY requirements file, not just the first: EARIN is six independent + # labs plus a project, and installing only one of them left the lab it + # then tried to run missing gymnasium. + local req + for req in "${reqs[@]}"; do + printf 'installing %s\n' "${req#"$COURSE_ROOT"/}" + "$COURSE_ROOT/.venv/bin/python" -m pip install --quiet -r "$req" || { + printf 'warning: some requirements in %s failed\n' "${req#"$COURSE_ROOT"/}" >&2 + } + done +} + +# Where a course ships real tests, running them is a stronger check than +# running the program -- and for an interactive program (ECRYPT_PROJECT prompts +# on stdin) it is the ONLY check that works without a person at a keyboard. +_python_run_tests() { + local suite + suite="$(find "$COURSE_ROOT" -name 'test_*.py' -not -path '*/.venv/*' -print -quit 2>/dev/null)" + [[ -n "$suite" ]] || return 0 + local py + py="$(_python_bin)" + "$py" -c 'import pytest' 2> /dev/null || { + printf 'tests present but pytest is not installed; skipping\n' + return 0 + } + # Which directory pytest runs FROM is course-specific and cannot be + # guessed: ECOTE needs program/ so `from translator.main import` resolves, + # ECRYPT_PROJECT needs sieve/ because its fixtures open P-100000.csv by a + # relative path. So the stub says, and COURSE_ROOT is only the default. + local from_dir="${COURSE_TEST_DIR:-$COURSE_ROOT}" + printf 'running the test suite in %s\n' "${from_dir#"$COURSE_ROOT"/}" + ( cd "$from_dir" && "$py" -m pytest -q ) +} + +# --- runners ----------------------------------------------------------------- + +_run() { + case "$COURSE_KIND" in + none) _no_code "${COURSE_NOTE:-nothing to run}" ;; + tex) + local tex="${COURSE_ENTRY:-}" + [[ -n "$tex" ]] || tex="$(_first_file '*.tex')" + [[ -n "$tex" ]] || _no_code "no .tex found" + local tool + tool="$(_require_any 'a LaTeX toolchain' latexmk pdflatex xelatex)" + # Build into build/, never beside the source. A submitted report.pdf + # is a tracked deliverable, and a rebuild whose output name happens + # to match it would silently overwrite what was handed in. + local dir base + dir="$(dirname "$tex")" + base="$(basename "$tex")" + # A document that declares \bibliography but never cites anything + # makes bibtex exit 2 ("I found no \citation commands"), and + # latexmk turns that into exit 12 even though the PDF is fine. + # --no-bibtex says "this one was handed in without a bibliography", + # which is a fact about the deliverable, not a suppression. + # An `x && y=1` one-liner would be a set -e landmine here: when x + # is false the list returns 1 and the whole runner exits. + local bibflag=() + if [[ -n "${COURSE_TEX_NO_BIBTEX:-}" ]]; then + bibflag=(-bibtex-) + fi + if [[ $tool == latexmk ]]; then + ( cd "$dir" && latexmk -pdf "${bibflag[@]}" \ + -interaction=nonstopmode -outdir=build "$base" ) + else + ( cd "$dir" && mkdir -p build \ + && "$tool" -interaction=nonstopmode -output-directory=build "$base" ) + fi + printf 'output in %s/build\n' "$dir" + ;; + python) + local entry="${COURSE_ENTRY:-}" + # An entry may be given as a directory; resolve it to the module + # inside. Passing the directory itself makes python look for + # __main__.py and fail with a message about the wrong thing. + if [[ -d "$entry" ]]; then + local cand + for cand in main.py project.py __main__.py; do + if [[ -f "$entry/$cand" ]]; then + entry="$entry/$cand" + break + fi + done + fi + # Only auto-pick a main.py when there is exactly ONE. A course + # with six labs has six, and choosing whichever find returns first + # is not "running the project", it is running an arbitrary lab. + if [[ ! -f "$entry" ]]; then + local mains + mains="$(find "$COURSE_ROOT" -name 'main.py' -not -path '*/.venv/*' | wc -l)" + [[ "$mains" -eq 1 ]] && entry="$(_first_file 'main.py')" + fi + + _python_install_requirements + + if [[ -f "$entry" ]]; then + # Word-split on purpose: --entry-args is a fixed string the + # course stub writes, never user input, and it has to become + # several argv entries. read -a keeps that explicit. + local entry_args=() + if [[ -n "${COURSE_ENTRY_ARGS:-}" ]]; then + read -r -a entry_args <<< "$COURSE_ENTRY_ARGS" + fi + ( cd "$(dirname "$entry")" \ + && "$(_python_bin)" "$(basename "$entry")" "${entry_args[@]}" ) + else + # A pile of one-shot scripts with no entry contract (NLP is + # the case here). Syntax-check them all rather than picking one + # arbitrarily and calling that "running the project". + printf 'no single entry point; syntax-checking every module instead\n' + find "$COURSE_ROOT" -name '*.py' -not -path '*/.venv/*' -print0 \ + | xargs -0 -r "$(_python_bin)" -m py_compile + printf 'all modules compile\n' + fi + _python_run_tests + ;; + node) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && npm install && npm start ) ;; + maven) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && mvn -q package ) ;; + make) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && make ) ;; + dotnet) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && dotnet build ) ;; + compose) ( cd "${COURSE_ENTRY:-$COURSE_ROOT}" && docker compose up ) ;; + unity) _blocked "open ${COURSE_ROOT} in the Unity editor; there is no headless path" ;; + cxx) + # Each .cpp here is a self-contained assignment with its own main(), + # so they are compiled individually rather than linked together. + local cxx src rel out + cxx="$(_require_any 'a C++ compiler' g++ clang++)" + # Binaries go to build/, which is gitignored -- compiling in place + # drops untracked executables next to the sources, and an + # extensionless binary is not something .gitignore catches by name. + mkdir -p "$COURSE_ROOT/build" + while IFS= read -r src; do + rel="${src#"$COURSE_ROOT"/}" + if _is_known_incomplete "${src#"${COURSE_GATES%/gates}"/}"; then + printf 'skipping %s (see gates/known-incomplete.txt)\n' "$rel" + continue + fi + out="$COURSE_ROOT/build/$(basename "${rel%.cpp}")" + # Some files here are exam-answer fragments with no main(), so + # they cannot be linked. Compile those to an object instead -- + # that still type-checks them, which is the point. + if grep -qE '\bint[[:space:]]+main[[:space:]]*\(' "$src"; then + printf 'compiling %s\n' "$rel" + "$cxx" -O2 -o "$out" "$src" + else + printf 'checking %s (no main; -c only)\n' "$rel" + "$cxx" -O2 -c -o "${out}.o" "$src" + fi + done < <(find "$COURSE_ROOT" -name '*.cpp' -not -path '*/node_modules/*' -not -path '*/build/*' | sort) + printf 'binaries in %s/build\n' "$COURSE_ROOT" + ;; + notebook) + _have jupyter || _blocked "needs jupyter" + local nb + while IFS= read -r nb; do + jupyter nbconvert --to notebook --execute --inplace "$nb" + done < <(find "$COURSE_ROOT" -name '*.ipynb' -not -path '*/.ipynb_checkpoints/*' | sort) + ;; + matlab) + local tool + tool="$(_require_any 'MATLAB or Octave' matlab octave)" + printf 'run the .m files in %s with %s\n' "$COURSE_ROOT" "$tool" + ;; + *) printf 'unknown COURSE_KIND: %s\n' "${COURSE_KIND:-unset}" >&2; return 1 ;; + esac +} +