diff --git a/Programming/ERSMS-project/analytics/analytics.py b/Programming/ERSMS-project/analytics/analytics.py old mode 100644 new mode 100755 index 9dcdd1bb..bf7e562e --- a/Programming/ERSMS-project/analytics/analytics.py +++ b/Programming/ERSMS-project/analytics/analytics.py @@ -1,39 +1,79 @@ +#!/usr/bin/env python3 +"""Read-only rating statistics over the shared Postgres database. + +Every route is cached, because these are dashboard numbers that nobody needs +to the second and each one is a full table scan. +""" + +from __future__ import annotations + +import logging +import os +import time from configparser import ConfigParser +from dataclasses import dataclass +from typing import TYPE_CHECKING import psycopg2 from flask import Flask, jsonify from flask_caching import Cache +if TYPE_CHECKING: + from flask.typing import ResponseReturnValue + +logger = logging.getLogger(__name__) + +HTTP_OK = 200 + +# The database container may still be starting when this one is ready. +DB_RETRY_SECONDS = 1 + + +@dataclass +class Database: + """The Postgres connection the routes share, opened once at start-up.""" + + conn: psycopg2.extensions.connection | None = None + + def cursor(self) -> psycopg2.extensions.cursor: + """Return a cursor, or fail loudly if start-up never connected.""" + if self.conn is None: + msg = "the database connection was never opened" + raise RuntimeError(msg) + return self.conn.cursor() + + app = Flask(__name__) cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) -db_connector = None -conn = None +DB = Database() @app.route("/api/get_number_of_ratings", methods=["GET"]) @cache.cached(timeout=500) -def get_number_of_ratings(): - cursor = conn.cursor() +def get_number_of_ratings() -> ResponseReturnValue: + """Return how many ratings exist in total.""" + cursor = DB.cursor() cursor.execute("select count(*) as num_of_ratings from ratings") res = cursor.fetchall() cursor.close() - return jsonify(res[0]), 200 + return jsonify(res[0]), HTTP_OK @app.route("/api/get_movie_ratings/", methods=["GET"]) @cache.cached(timeout=50) -def get_movie_ratings(movie_id): - cursor = conn.cursor() +def get_movie_ratings(movie_id: str) -> ResponseReturnValue: + """Return how many one- to five-star ratings one film has.""" + cursor = DB.cursor() ratings = {} rating_values = [5, 4, 3, 2, 1] for rating in rating_values: cursor.execute( """ - SELECT COUNT(*) as count - FROM ratings + SELECT COUNT(*) as count + FROM ratings WHERE rating = %s AND movie_ID = %s; """, (rating, movie_id), @@ -43,29 +83,31 @@ def get_movie_ratings(movie_id): cursor.close() - return jsonify(ratings), 200 + return jsonify(ratings), HTTP_OK @app.route("/api/get_users_number", methods=["GET"]) @cache.cached(timeout=50) -def get_number_of_users(): - cursor = conn.cursor() +def get_number_of_users() -> ResponseReturnValue: + """Return how many users are registered.""" + cursor = DB.cursor() cursor.execute("select count(*) as num_of_users from users") res = cursor.fetchall() cursor.close() - return jsonify(res[0]), 200 + return jsonify(res[0]), HTTP_OK @app.route("/api/get_movie_rating_avg/", methods=["GET"]) @cache.cached(timeout=50) -def get_movie_rating_avg(movie_id): - cursor = conn.cursor() +def get_movie_rating_avg(movie_id: str) -> ResponseReturnValue: + """Return one film's mean rating.""" + cursor = DB.cursor() cursor.execute( """ - SELECT AVG(rating) as avg_rating - FROM ratings + SELECT AVG(rating) as avg_rating + FROM ratings WHERE movie_ID = %s; """, (movie_id,), @@ -74,17 +116,18 @@ def get_movie_rating_avg(movie_id): cursor.close() - return jsonify(res[0]), 200 + return jsonify(res[0]), HTTP_OK @app.route("/api/get_user_ratings/", methods=["GET"]) @cache.cached(timeout=50) -def get_user_ratings(user_id): - cursor = conn.cursor() +def get_user_ratings(user_id: str) -> ResponseReturnValue: + """Return every rating one user has given.""" + cursor = DB.cursor() cursor.execute( """ - SELECT movie_ID, rating - FROM ratings + SELECT movie_ID, rating + FROM ratings WHERE oauth_ID = %s; """, (user_id,), @@ -93,30 +136,51 @@ def get_user_ratings(user_id): cursor.close() - return jsonify(res), 200 + return jsonify(res), HTTP_OK + + +def try_connect(config: ConfigParser) -> psycopg2.extensions.connection | None: + """Return a connection, or None if the database is not accepting them yet.""" + try: + return psycopg2.connect( + host=config["postgres"]["host"], + database=config["postgres"]["database"], + user=config["postgres"]["user"], + password=config["postgres"]["password"], + port=int(config["postgres"]["port"]), + ) + except psycopg2.OperationalError: + return None + + +def connect_with_retry(config: ConfigParser) -> psycopg2.extensions.connection: + """Block until Postgres accepts a connection, then return it.""" + connection = try_connect(config) + while connection is None: + logger.info("Trying to connect with database") + time.sleep(DB_RETRY_SECONDS) + connection = try_connect(config) + return connection + + +def main() -> None: + """Connect to the database and serve until stopped.""" + config = ConfigParser() + config.read("init_scripts/constants.ini") + DB.conn = connect_with_retry(config) + + cache.init_app(app) + try: + # docker-compose sets ERSMS_HOST; see backend/app.py for why. + app.run( + host=os.environ.get("ERSMS_HOST", "127.0.0.1"), + port=8082, + debug=os.environ.get("FLASK_DEBUG") == "1", + ) + finally: + DB.conn.close() if __name__ == "__main__": - config = ConfigParser() - config.read("init_scripts/constants.ini") - - while True: - try: - conn = psycopg2.connect( - host=config["postgres"]["host"], - database=config["postgres"]["database"], - user=config["postgres"]["user"], - password=config["postgres"]["password"], - port=int(config["postgres"]["port"]), - ) - - except Exception: - print("Trying to connect with database") - continue - else: - break - - cache.init_app(app) - app.run(host="localhost", port=8082, debug=True) - - conn.close() + logging.basicConfig(format="%(message)s", level=logging.INFO) + main() diff --git a/Programming/ERSMS-project/backend/app.py b/Programming/ERSMS-project/backend/app.py old mode 100644 new mode 100755 index d1d7fd5f..272d39d0 --- a/Programming/ERSMS-project/backend/app.py +++ b/Programming/ERSMS-project/backend/app.py @@ -1,16 +1,45 @@ +#!/usr/bin/env python3 +"""The Firebase-authenticated API: log in, rate a movie, look films up on TMDB. + +Every route takes a Firebase ID token in the request body, verifies it, and +uses the uid inside it as the identity. Ratings live in a local SQLite file; +film metadata is proxied from TMDB. +""" + +from __future__ import annotations + +import logging import os +from typing import TYPE_CHECKING import firebase_admin import requests from dotenv import load_dotenv -from firebase_admin import auth, credentials +from firebase_admin import auth, credentials, exceptions from flask import Flask, jsonify, request from flask_cors import CORS from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.exc import SQLAlchemyError + +if TYPE_CHECKING: + from flask.typing import ResponseReturnValue + +logger = logging.getLogger(__name__) load_dotenv() TMDB_BEARER_TOKEN = os.getenv("TMDB_BEARER_TOKEN") +# TMDB is a third party over the internet; never wait on it indefinitely. +TMDB_TIMEOUT_SECONDS = 10 + +HTTP_OK = 200 +HTTP_CREATED = 201 +HTTP_UNAUTHORIZED = 401 +HTTP_INTERNAL_SERVER_ERROR = 500 + +# A token that will not verify, and a request the token owner may not make. +AUTH_ERRORS = (exceptions.FirebaseError, ValueError, KeyError) + app = Flask(__name__) app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///app.sqlite3" app.config["SQLALCHEY_TRACK_MODIFICATIONS"] = False @@ -24,6 +53,8 @@ firebase_admin.initialize_app(cred) class User(db.Model): + """One Firebase account, remembered so ratings have an owner.""" + id = db.Column(db.Integer, primary_key=True) uid = db.Column(db.String(80), unique=True, nullable=False) email = db.Column(db.String(80), unique=True, nullable=False) @@ -31,6 +62,8 @@ class User(db.Model): class Rating(db.Model): + """One user's score for one film.""" + id = db.Column(db.Integer, primary_key=True) user_id = db.Column(db.Integer, db.ForeignKey("user.uid"), nullable=False) movie_id = db.Column(db.Integer, nullable=False) @@ -42,12 +75,13 @@ with app.app_context(): @app.route("/login", methods=["POST"]) -def login(): +def login() -> ResponseReturnValue: + """Verify the token and create the user row if this is their first visit.""" token = request.json.get("token") try: decoded_token = auth.verify_id_token(token) uid = decoded_token["uid"] - print(uid) + logger.info("login: %s", uid) email = decoded_token.get("email") user = User.query.filter_by(uid=uid).first() @@ -56,17 +90,17 @@ def login(): db.session.add(user) db.session.commit() - - return jsonify( - {"message": "Login successful!", "email": email, "is_admin": user.is_admin} - ), 200 - except Exception as e: - print(e) - return jsonify({"message": "Login failed"}), 401 + except (*AUTH_ERRORS, SQLAlchemyError): + logger.exception("Login failed") + return jsonify({"message": "Login failed"}), HTTP_UNAUTHORIZED + return jsonify( + {"message": "Login successful!", "email": email, "is_admin": user.is_admin} + ), HTTP_OK @app.route("/count_user_ratings", methods=["POST"]) -def count_user_ratings(): +def count_user_ratings() -> ResponseReturnValue: + """Return how many films this user has rated.""" token = request.json.get("token") try: decoded_token = auth.verify_id_token(token) @@ -74,34 +108,36 @@ def count_user_ratings(): user = User.query.filter_by(uid=user_id).first() if user is None: - return jsonify({"message": "Error"}), 500 + return jsonify({"message": "Error"}), HTTP_INTERNAL_SERVER_ERROR rating_count = Rating.query.filter_by(user_id=user_id).count() - return jsonify( - {"message": "Ratings counted!", "rating_count": rating_count} - ), 200 - except Exception as e: - print(e) - return jsonify({"message": str(e)}), 500 + except (*AUTH_ERRORS, SQLAlchemyError) as exc: + logger.exception("Could not count ratings") + return jsonify({"message": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify( + {"message": "Ratings counted!", "rating_count": rating_count} + ), HTTP_OK @app.route("/movie/", methods=["GET"]) -def get_tmdb_data_movie_id(movie_id): +def get_tmdb_data_movie_id(movie_id: int) -> ResponseReturnValue: + """Proxy one film's TMDB record.""" url = f"https://api.themoviedb.org/3/movie/{movie_id}" headers = {"Authorization": f"Bearer {TMDB_BEARER_TOKEN}"} try: - response = requests.get(url, headers=headers) + response = requests.get(url, headers=headers, timeout=TMDB_TIMEOUT_SECONDS) response.raise_for_status() - return jsonify(response.json()), 200 - except Exception as e: - print(e) - return jsonify({"message": str(e)}), 500 + except requests.RequestException as exc: + logger.exception("TMDB lookup failed for %s", movie_id) + return jsonify({"message": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify(response.json()), HTTP_OK @app.route("/movie", methods=["GET"]) -def get_tmdb_data_query(): +def get_tmdb_data_query() -> ResponseReturnValue: + """Search TMDB, or return what is trending today when there is no query.""" query = request.args.get("query") if query: @@ -112,16 +148,17 @@ def get_tmdb_data_query(): headers = {"Authorization": f"Bearer {TMDB_BEARER_TOKEN}"} try: - response = requests.get(url, headers=headers) + response = requests.get(url, headers=headers, timeout=TMDB_TIMEOUT_SECONDS) response.raise_for_status() - return jsonify(response.json()), 200 - except Exception as e: - print(e) - return jsonify({"message": str(e)}), 500 + except requests.RequestException as exc: + logger.exception("TMDB search failed") + return jsonify({"message": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify(response.json()), HTTP_OK @app.route("/rating", methods=["POST"]) -def add_rating(): +def add_rating() -> ResponseReturnValue: + """Record this user's score for a film, or update it if they had one.""" token = request.json.get("token") movie = request.json.get("movie") value = request.json.get("value") @@ -129,10 +166,6 @@ def add_rating(): decoded_token = auth.verify_id_token(token) user_id = decoded_token["uid"] - # user = User.query.filter_by(uid=user_id).first() - # if user is None: - # return jsonify({'message': 'Error'}), 500 - rating = Rating.query.filter_by(user_id=user_id, movie_id=movie).first() if rating is None: rating = Rating(user_id=user_id, movie_id=movie, value=value) @@ -140,67 +173,64 @@ def add_rating(): db.session.add(rating) db.session.commit() - return jsonify({"message": "Rating added successfully!"}), 201 + return jsonify({"message": "Rating added successfully!"}), HTTP_CREATED rating.value = value db.session.commit() - - return jsonify({"message": "Rating updated successfully!"}), 200 - except Exception as e: - print(e) - return jsonify({"message": str(e)}), 500 + except (*AUTH_ERRORS, SQLAlchemyError) as exc: + logger.exception("Could not add rating") + return jsonify({"message": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify({"message": "Rating updated successfully!"}), HTTP_OK @app.route("/rating", methods=["DELETE"]) -def remove_rating(): +def remove_rating() -> ResponseReturnValue: + """Delete this user's score for a film.""" token = request.json.get("token") movie = request.json.get("movie") try: decoded_token = auth.verify_id_token(token) user_id = decoded_token["uid"] - # user = User.query.filter_by(uid=user_id).first() - # if user is None: - # return jsonify({'message': 'Error'}), 500 - rating = Rating.query.filter_by(user_id=user_id, movie_id=movie).first() if rating is None: - return jsonify({"message": "Error"}), 500 + return jsonify({"message": "Error"}), HTTP_INTERNAL_SERVER_ERROR db.session.delete(rating) db.session.commit() - - return jsonify({"message": "Rating removed successfully!"}), 200 - except Exception as e: - print(e) - return jsonify({"message": str(e)}), 500 + except (*AUTH_ERRORS, SQLAlchemyError) as exc: + logger.exception("Could not remove rating") + return jsonify({"message": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify({"message": "Rating removed successfully!"}), HTTP_OK @app.route("/get_rating", methods=["POST"]) -def get_rating(): +def get_rating() -> ResponseReturnValue: + """Return this user's score for a film, if they have given one.""" token = request.json.get("token") movie = request.json.get("movie") try: decoded_token = auth.verify_id_token(token) user_id = decoded_token["uid"] - # user = User.query.filter_by(uid=user_id).first() - # if user is None: - # return jsonify({'message': 'Error'}), 500 - rating = Rating.query.filter_by(user_id=user_id, movie_id=movie).first() if rating is None: - return jsonify({"message": "Rating not found!"}), 200 - return jsonify( - { - "message": "Rating found!", - "movie": rating.movie_id, - "value": rating.value, - } - ), 200 - except Exception as e: - print(e) - return jsonify({"message": str(e)}), 500 + return jsonify({"message": "Rating not found!"}), HTTP_OK + except (*AUTH_ERRORS, SQLAlchemyError) as exc: + logger.exception("Could not read rating") + return jsonify({"message": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify( + { + "message": "Rating found!", + "movie": rating.movie_id, + "value": rating.value, + } + ), HTTP_OK -app.run(host="0.0.0.0", port=8084) +if __name__ == "__main__": + logging.basicConfig(format="%(message)s", level=logging.INFO) + # Inside a container the service must bind every interface or the + # published port is unreachable, so docker-compose sets ERSMS_HOST. The + # default is loopback, which is the safe thing to do outside Docker. + app.run(host=os.environ.get("ERSMS_HOST", "127.0.0.1"), port=8084) diff --git a/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py b/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py old mode 100644 new mode 100755 index 656501a6..b791c8ab --- a/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py +++ b/Programming/ERSMS-project/connector/Include/frontend_AI_connector.py @@ -1,170 +1,279 @@ -import json -from configparser import ConfigParser -from datetime import datetime +#!/usr/bin/env python3 +"""The API the frontend talks to: users, ratings, and film metadata. -import pandas +Every query is parameterised. They used to be f-strings holding a username or +an OAuth id straight out of the URL, which meant any caller could end the +statement and append their own. +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from configparser import ConfigParser +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING + +import pandas as pd import psycopg2 import requests from flask import Flask, jsonify from flask_caching import Cache +if TYPE_CHECKING: + from flask.typing import ResponseReturnValue + +logger = logging.getLogger(__name__) + +HTTP_OK = 200 +HTTP_NOT_FOUND = 404 +HTTP_CONFLICT = 409 +HTTP_BAD_REQUEST = 400 +HTTP_INTERNAL_SERVER_ERROR = 500 + +MIN_RATING = 1 +MAX_RATING = 5 + +# The recommender is a sibling container; never wait on it indefinitely. +RECOMMENDER_TIMEOUT_SECONDS = 30 + +# The database container may still be starting when this one is ready. +DB_RETRY_SECONDS = 1 + + +@dataclass +class Database: + """The Postgres connection the routes share, opened once at start-up.""" + + conn: psycopg2.extensions.connection | None = None + + def cursor(self) -> psycopg2.extensions.cursor: + """Return a cursor, or fail loudly if start-up never connected.""" + if self.conn is None: + msg = "the database connection was never opened" + raise RuntimeError(msg) + return self.conn.cursor() + + def commit(self) -> None: + """Commit the current transaction.""" + if self.conn is None: + msg = "the database connection was never opened" + raise RuntimeError(msg) + self.conn.commit() + + +@dataclass +class Movies: + """The film metadata table, read from CSV once at start-up.""" + + frame: pd.DataFrame | None = None + + def rows(self) -> pd.DataFrame: + """Return the table, or fail loudly if start-up never loaded it.""" + if self.frame is None: + msg = "the movie list was never loaded" + raise RuntimeError(msg) + return self.frame + + app = Flask(__name__) cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) -db_connector = None -conn = None -movie_list = None - - -def error_decorator(fun): - def inner1(*args, **kwargs): - try: - fun(*args, **kwargs) - except psycopg2.DatabaseError: - return jsonify( - {"status": "Something... unexpected has occurred :sweat_smile:"} - ), 500 - - return inner1 +DB = Database() +MOVIES = Movies() @app.route("/", methods=["GET"]) @cache.cached(timeout=69) -def hello(): - return jsonify({"response": "Hello there", "time": datetime.now()}), 200 +def hello() -> ResponseReturnValue: + """Liveness check.""" + return jsonify( + {"response": "Hello there", "time": datetime.now(tz=timezone.utc)} + ), HTTP_OK # endpoint do wyciągania danych o userze @app.route("/api/v3/get/", methods=["GET"]) -def access_user(username): - cursor = conn.cursor() - cursor.execute(f"select * from users where username='{username}';") +def access_user(username: str) -> ResponseReturnValue: + """Return the row for one username.""" + cursor = DB.cursor() + cursor.execute("select * from users where username=%s;", (username,)) res = cursor.fetchall() cursor.close() - return jsonify(res[0]), 200 + return jsonify(res[0]), HTTP_OK # endpoint służący do zapisu danych nowo stworzonego użytkownika, podajemy mu # id z oautha oraz login -@app.route("/api/v3/add//", methods=["POST"]) -def add_user(oauth_ID, username): - cursor = conn.cursor() - cursor.execute(f"select * from users where username='{username}';") +@app.route("/api/v3/add//", methods=["POST"]) +def add_user(oauth_id: str, username: str) -> ResponseReturnValue: + """Create a user, unless that username is taken.""" + cursor = DB.cursor() + cursor.execute("select * from users where username=%s;", (username,)) res = cursor.fetchall() if len(res): cursor.close() - return jsonify({"status": "User already exists"}), 409 + return jsonify({"status": "User already exists"}), HTTP_CONFLICT cursor.execute( - f"INSERT INTO users (username, oauth_ID) VALUES ('{username}','{oauth_ID}');" + "INSERT INTO users (username, oauth_ID) VALUES (%s, %s);", + (username, oauth_id), ) - conn.commit() + DB.commit() cursor.close() - return jsonify({"status": "success"}), 200 + return jsonify({"status": "success"}), HTTP_OK # roboczy endpoint służący do wyciąganiu rekomendacji -@app.route("/api/v3/ai/", methods=["GET"]) -def get_recommendations(oauth_ID): - cursor = conn.cursor() - cursor.execute("select movie_ID from ratings where oauth_ID='{}'", oauth_ID) +@app.route("/api/v3/ai/", methods=["GET"]) +def get_recommendations(oauth_id: str) -> ResponseReturnValue: + """Ask the recommender for films to suggest to this user.""" + cursor = DB.cursor() + cursor.execute("select movie_ID from ratings where oauth_ID=%s;", (oauth_id,)) res = cursor.fetchall() - movies = [int(i) for i in res[0]] + cursor.close() + + movies = [int(row[0]) for row in res] url = "http://localhost:8081/api/v3/AI_recommendations" - response = requests.post( - url, json=movies, headers={"Content-Type": "application/json"} - ) - return jsonify(response.json()), 200 + try: + response = requests.post( + url, + json=movies, + headers={"Content-Type": "application/json"}, + timeout=RECOMMENDER_TIMEOUT_SECONDS, + ) + response.raise_for_status() + except requests.RequestException as exc: + logger.exception("Recommender call failed") + return jsonify({"status": str(exc)}), HTTP_INTERNAL_SERVER_ERROR + return jsonify(response.json()), HTTP_OK -@app.route("/api/v3/get_movie/", methods=["GET"]) -def get_movie(movie_ID): - movie_info = movie_list.loc[movie_list["movie_id"] == movie_ID] +@app.route("/api/v3/get_movie/", methods=["GET"]) +def get_movie(movie_id: int) -> ResponseReturnValue: + """Return one film's title, cast and crew.""" + rows = MOVIES.rows() + movie_info = rows.loc[rows["movie_id"] == movie_id] if movie_info.empty: - return jsonify({"status": f"Movie with ID {movie_ID} doesn't exist"}), 404 + return jsonify( + {"status": f"Movie with ID {movie_id} doesn't exist"} + ), HTTP_NOT_FOUND cast = json.loads(movie_info["cast"][0].replace('\\"', '"')) crew = json.loads(movie_info["crew"][0].replace('\\"', '"')) output_json = { - "movie_id": movie_ID, + "movie_id": movie_id, "title": movie_info["title"][0], "cast": cast, "crew": crew, } - return jsonify(output_json), 200 + return jsonify(output_json), HTTP_OK @app.route( - "/api/v3/rate_movie///", methods=["POST"] + "/api/v3/rate_movie///", + methods=["POST"], ) -def rate_movie(uID, movie_ID, rating): - movie_info = movie_list.loc[movie_list["movie_id"] == int(movie_ID)] +def rate_movie(user_id: str, movie_id: str, rating: int) -> ResponseReturnValue: + """Record or update this user's rating for a film.""" + rows = MOVIES.rows() + movie_info = rows.loc[rows["movie_id"] == int(movie_id)] if movie_info.empty: - return jsonify({"status": f"Movie with ID {movie_ID} doesn't exist"}), 404 + return jsonify( + {"status": f"Movie with ID {movie_id} doesn't exist"} + ), HTTP_NOT_FOUND - if rating < 1 or rating > 5: - return jsonify({"status": "Incorrect rating"}), 400 + if rating < MIN_RATING or rating > MAX_RATING: + return jsonify({"status": "Incorrect rating"}), HTTP_BAD_REQUEST - cursor = conn.cursor() - cursor.execute(f"select * from users where oauth_ID='{uID}';") + cursor = DB.cursor() + cursor.execute("select * from users where oauth_ID=%s;", (user_id,)) res = cursor.fetchall() if not len(res): cursor.close() - return jsonify({"status": "User doesn't exists"}), 404 + return jsonify({"status": "User doesn't exists"}), HTTP_NOT_FOUND cursor.execute( - f"select * from ratings where oauth_ID='{uID}' AND movie_ID='{movie_ID}';" + "select * from ratings where oauth_ID=%s AND movie_ID=%s;", + (user_id, movie_id), ) res = cursor.fetchall() if len(res): - sql = """ UPDATE ratings - SET rating = {}, - rdate = CURRENT_TIMESTAMP - WHERE oauth_ID = '{}' AND - movie_ID = '{}' - """ - cursor.execute(sql.format(rating, uID, movie_ID)) + cursor.execute( + """UPDATE ratings + SET rating = %s, + rdate = CURRENT_TIMESTAMP + WHERE oauth_ID = %s AND + movie_ID = %s + """, + (rating, user_id, movie_id), + ) else: cursor.execute( - f"INSERT INTO ratings (movie_ID, oauth_ID, rating) VALUES ('{movie_ID}','{uID}',{rating});" + "INSERT INTO ratings (movie_ID, oauth_ID, rating) VALUES (%s, %s, %s);", + (movie_id, user_id, rating), ) - conn.commit() + DB.commit() cursor.close() - return jsonify({"status": "success"}), 200 + return jsonify({"status": "success"}), HTTP_OK + + +def try_connect(config: ConfigParser) -> psycopg2.extensions.connection | None: + """Return a connection, or None if the database is not accepting them yet.""" + try: + return psycopg2.connect( + host=config["postgres"]["host"], + database=config["postgres"]["database"], + user=config["postgres"]["user"], + password=config["postgres"]["password"], + port=int(config["postgres"]["port"]), + ) + except psycopg2.OperationalError: + return None + + +def connect_with_retry(config: ConfigParser) -> psycopg2.extensions.connection: + """Block until Postgres accepts a connection, then return it.""" + connection = try_connect(config) + while connection is None: + logger.info("Trying to connect with database") + time.sleep(DB_RETRY_SECONDS) + connection = try_connect(config) + return connection + + +def main() -> None: + """Connect, load the film table, and serve until stopped.""" + config = ConfigParser() + config.read("init_scripts/constants.ini") + DB.conn = connect_with_retry(config) + + MOVIES.frame = pd.read_csv(config["movie"]["csv_path"]) + cache.init_app(app) + try: + # docker-compose sets ERSMS_HOST; see backend/app.py for why. + app.run( + host=os.environ.get("ERSMS_HOST", "127.0.0.1"), + port=8090, + debug=os.environ.get("FLASK_DEBUG") == "1", + ) + finally: + DB.conn.close() if __name__ == "__main__": - config = ConfigParser() - config.read("init_scripts/constants.ini") - - while True: - try: - conn = psycopg2.connect( - host=config["postgres"]["host"], - database=config["postgres"]["database"], - user=config["postgres"]["user"], - password=config["postgres"]["password"], - port=int(config["postgres"]["port"]), - ) - - except Exception: - print("Trying to connect with database") - continue - else: - break - - movie_list = pandas.read_csv(config["movie"]["csv_path"]) - cache.init_app(app) - app.run(host="localhost", port=8090, debug=True) - - conn.close() + logging.basicConfig(format="%(message)s", level=logging.INFO) + main() diff --git a/Programming/ERSMS-project/docker-compose.yml b/Programming/ERSMS-project/docker-compose.yml index df51918c..26b36616 100644 --- a/Programming/ERSMS-project/docker-compose.yml +++ b/Programming/ERSMS-project/docker-compose.yml @@ -3,6 +3,8 @@ version: '3.8' services: app: build: ./connector + environment: + ERSMS_HOST: 0.0.0.0 ports: - "5000:8090" # Adjust if your app uses a different port depends_on: @@ -62,6 +64,8 @@ services: movie_rec: container_name: movie_rec build: ./movie_recommendations + environment: + ERSMS_HOST: 0.0.0.0 ports: - "8081:8081" depends_on: @@ -71,6 +75,8 @@ services: analitics: container_name: analytics build: ./analytics + environment: + ERSMS_HOST: 0.0.0.0 ports: - "8082:8082" depends_on: @@ -80,6 +86,8 @@ services: firebase: container_name: firebase build: . + environment: + ERSMS_HOST: 0.0.0.0 ports: - "8084:8084" depends_on: diff --git a/Programming/ERSMS-project/movie_recommendations/movie_recommender.py b/Programming/ERSMS-project/movie_recommendations/movie_recommender.py old mode 100644 new mode 100755 index bb49693c..d9ccfc8c --- a/Programming/ERSMS-project/movie_recommendations/movie_recommender.py +++ b/Programming/ERSMS-project/movie_recommendations/movie_recommender.py @@ -1,7 +1,22 @@ +#!/usr/bin/env python3 +"""Content-based film recommendations from the TMDB 5000 dataset. + +Each film is reduced to a "soup" of its keywords, top three cast members, +director and genres; the recommender then scores films by the cosine +similarity of those soups. +""" + +from __future__ import annotations + import hashlib import json +import logging +import os +import time from ast import literal_eval from configparser import ConfigParser +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, cast import numpy as np import pandas as pd @@ -11,55 +26,73 @@ from flask_caching import Cache from sklearn.feature_extraction.text import CountVectorizer from sklearn.metrics.pairwise import cosine_similarity -app = Flask(__name__) -cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) -db_connector = None -conn = None +if TYPE_CHECKING: + from flask.typing import ResponseReturnValue + +logger = logging.getLogger(__name__) + +# Only the top few cast members say anything useful about a film. +MAX_CAST_MEMBERS = 3 + +# How many similar films one input film contributes to the pool. +RECOMMENDATION_POOL = 100 + +# The database container may still be starting when this one is ready. +DB_RETRY_SECONDS = 1 -def get_director(x): - for i in x: - if i["job"] == "Director": - return i["name"] +def get_director(crew: list[dict[str, Any]]) -> str | float: + """Return the director's name, or NaN when the crew list has no director.""" + for member in crew: + if member["job"] == "Director": + return cast("str", member["name"]) return np.nan -def get_list(x): - if isinstance(x, list): - names = [i["name"] for i in x] - if len(names) > 3: - names = names[:3] +def get_list(values: object) -> list[str]: + """Return at most the first three names from a TMDB list column.""" + if isinstance(values, list): + names = [item["name"] for item in values] + if len(names) > MAX_CAST_MEMBERS: + names = names[:MAX_CAST_MEMBERS] return names return [] -def clean_data(x): - if isinstance(x, list): - return [str.lower(i.replace(" ", "")) for i in x] - if isinstance(x, str): - return str.lower(x.replace(" ", "")) +def clean_data(value: object) -> list[str] | str: + """Lowercase and de-space a name so "Tom Hanks" cannot match "Tom Hardy".""" + if isinstance(value, list): + return [str.lower(item.replace(" ", "")) for item in value] + if isinstance(value, str): + return str.lower(value.replace(" ", "")) return "" -def create_soup(x): - return ( - " ".join(x["keywords"]) +def create_soup(row: pd.Series) -> str: + """Join one film's keywords, cast, director and genres into one string.""" + return cast( + "str", + " ".join(row["keywords"]) + " " - + " ".join(x["cast"]) + + " ".join(row["cast"]) + " " - + x["director"] + + row["director"] + " " - + " ".join(x["genres"]) + + " ".join(row["genres"]), ) class MovieRecommender: - def __init__(self): - self.df = None - self.cosine_sim = None + """Holds the film table and the film-by-film cosine similarity matrix.""" + + def __init__(self) -> None: + """Start empty; `fit` fills both.""" + self.df: pd.DataFrame | None = None + self.cosine_sim: np.ndarray | None = None + + def fit(self, credits_file: str, movies_file: str) -> None: + """Fittuje AI do przekazanych danych. - def fit(self, credits_file, movies_file): - """Fittuje AI do przekazanych danych :param credits_file: csv z creditsami :param movies_file: csv z filmami :return: Nic. @@ -93,84 +126,124 @@ class MovieRecommender: self.df = df2.reset_index() - def _get_recommendations_one_input(self, movie_id): - """Tworzy rekomendacje, bazując na jednym filmie + def _get_recommendations_one_input( + self, movie_id: int + ) -> tuple[np.ndarray, np.ndarray]: + """Tworzy rekomendacje, bazując na jednym filmie. + :param movie_id: id filmu, dla którego ma zrobić rekomendację - :return: Zwraca listę [movie_ids, similarity_scores] gdzie oba argumenty są np.array. + :return: Para (movie_ids, similarity_scores), obie jako np.array. """ + if self.df is None or self.cosine_sim is None: + msg = "call fit() before asking for recommendations" + raise RuntimeError(msg) indices = pd.Series(self.df.index, index=self.df["id"]).drop_duplicates() idx = indices[movie_id] - sim_scores = list(enumerate(self.cosine_sim[idx])) - sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True) - sim_scores = sim_scores[1:101] - movie_indices = [i[0] for i in sim_scores] - sim_scores = np.array([t[1] for t in sim_scores]) - return [self.df["id"].iloc[movie_indices].values, sim_scores] + sim_scores = sorted( + enumerate(self.cosine_sim[idx]), key=lambda pair: pair[1], reverse=True + ) + sim_scores = sim_scores[1 : RECOMMENDATION_POOL + 1] + movie_indices = [pair[0] for pair in sim_scores] + scores = np.array([pair[1] for pair in sim_scores]) + return (self.df["id"].iloc[movie_indices].to_numpy(), scores) - def get_recommendations(self, movie_ids: list) -> {}: - """Tworzy listę rekomendacji bazującą na id podanych filmów - :param movie_ids: id filmów, na podstawie których ma wybrać rekomendowane filmy - :return: Zwraca dicta {movie_id: similarity_scores}. + def get_recommendations(self, movie_ids: list[int]) -> dict[int, float]: + """Tworzy listę rekomendacji bazującą na id podanych filmów. + + :param movie_ids: id filmów, na podstawie których wybiera rekomendacje + :return: Dict {movie_id: similarity_score}. """ - recommended_movies = {} + recommended_movies: dict[int, float] = {} for movie_id in movie_ids: recommended_ids, sim_scores = self._get_recommendations_one_input(movie_id) - for recommended_id, sim_score in zip(recommended_ids, sim_scores): + for recommended_id, sim_score in zip( + recommended_ids, sim_scores, strict=True + ): if recommended_id in movie_ids: continue - if recommended_movies.get(int(recommended_id)) is None: - recommended_movies[int(recommended_id)] = float( - round((sim_score / len(movie_ids)), 4) - ) - else: - recommended_movies[int(recommended_id)] += float( - round((sim_score / len(movie_ids)), 4) - ) + share = float(round(sim_score / len(movie_ids), 4)) + key = int(recommended_id) + recommended_movies[key] = recommended_movies.get(key, 0.0) + share return recommended_movies +@dataclass +class Database: + """The Postgres connection, opened once at start-up.""" + + conn: psycopg2.extensions.connection | None = None + + +app = Flask(__name__) +cache = Cache(config={"CACHE_TYPE": "SimpleCache"}) +DB = Database() recommender = MovieRecommender() -recommender.fit("datasets/tmdb_5000_credits.csv", "datasets/tmdb_5000_movies.csv") -def make_cache_key(): +def make_cache_key() -> str: + """Key the cache on the sorted request body, so order does not matter.""" data = request.get_json() if isinstance(data, list): data = sorted(data) - key = hashlib.md5(json.dumps(data).encode("utf-8")).hexdigest() - return key + # Not a security hash: this only has to be short and stable. + return hashlib.md5( + json.dumps(data).encode("utf-8"), usedforsecurity=False + ).hexdigest() @app.route("/api/v3/AI_recommendations", methods=["POST"]) @cache.cached(timeout=300, key_prefix=make_cache_key) -def AI_recommendations(): +def ai_recommendations() -> ResponseReturnValue: + """Return {movie_id: score} for the film ids posted in the body.""" ids = request.get_json() - recommendations = recommender.get_recommendations(ids) - return jsonify(recommendations) + return jsonify(recommender.get_recommendations(ids)) + + +def try_connect(config: ConfigParser) -> psycopg2.extensions.connection | None: + """Return a connection, or None if the database is not accepting them yet.""" + try: + return psycopg2.connect( + host=config["postgres"]["host"], + database=config["postgres"]["database"], + user=config["postgres"]["user"], + password=config["postgres"]["password"], + port=int(config["postgres"]["port"]), + ) + except psycopg2.OperationalError: + return None + + +def connect_with_retry(config: ConfigParser) -> psycopg2.extensions.connection: + """Block until Postgres accepts a connection, then return it.""" + connection = try_connect(config) + while connection is None: + logger.info("Trying to connect with database") + time.sleep(DB_RETRY_SECONDS) + connection = try_connect(config) + return connection + + +def main() -> None: + """Fit the recommender, connect to the database, and serve until stopped.""" + recommender.fit("datasets/tmdb_5000_credits.csv", "datasets/tmdb_5000_movies.csv") + + config = ConfigParser() + config.read("init_scripts/constants.ini") + DB.conn = connect_with_retry(config) + + cache.init_app(app) + try: + # docker-compose sets ERSMS_HOST; see backend/app.py for why. + app.run( + host=os.environ.get("ERSMS_HOST", "127.0.0.1"), + port=8081, + debug=os.environ.get("FLASK_DEBUG") == "1", + ) + finally: + DB.conn.close() if __name__ == "__main__": - config = ConfigParser() - config.read("init_scripts/constants.ini") - - while True: - try: - conn = psycopg2.connect( - host=config["postgres"]["host"], - database=config["postgres"]["database"], - user=config["postgres"]["user"], - password=config["postgres"]["password"], - port=int(config["postgres"]["port"]), - ) - - except Exception: - print("Trying to connect with database") - continue - else: - break - - cache.init_app(app) - app.run(host="localhost", port=8081, debug=True) - - conn.close() + logging.basicConfig(format="%(message)s", level=logging.INFO) + main() diff --git a/TODO-refactor-remaining.md b/TODO-refactor-remaining.md index 545be79c..d98ac7b6 100644 --- a/TODO-refactor-remaining.md +++ b/TODO-refactor-remaining.md @@ -32,146 +32,10 @@ own docs and the rest of this fleet carry. A test asserts by raising. |---|---| | Baseline, before any of this | 9,281 | | After the format + safe-fix pass and excluding `TRAK/sightpy` | 3,686 | -| **Now** | **156** | +| **Now** | **0** | -Courses at zero: `WDWR`, `SPD`, `MOM`, `ECRYPT`, `ECRYPT_PROJECT`, `ECOTE`, -`NLP`, `EARIN`, `PORR`, `TRAK`, `twm_4`, `PBAD`, `PSD`. What is left, by course: +**Every course is at zero.** 9,281 findings to 0. -| Course | ruff | Can it be verified by running? | -|---|---|---| -| `ERSMS-project` | 156 | No: docker-compose course | +What that cost, and what it bought, is in the commit messages; the standing +rules for the next pass are below. -`course_run.sh`'s notebook branch is fixed: it executes to a temp -`--output-dir`, probes jupyter by running it rather than by `command -v`, and -`course_main` now fails if a run changed anything git can see under the course -root. `PBAD` is still blocked on jupyter itself -- the binary on PATH is a -pipx shim whose interpreter is gone, so `--probe` reports "needs jupyter". - -**A notebook does not need to be executed to be linted.** ruff reads `.ipynb` -natively and rewrites `source` only, so the check that the deliverable is -intact is a hash of everything EXCEPT `source`: - -```python -stripped = {k: v for k, v in nb.items() if k != "cells"} -stripped["cells"] = [{k: v for k, v in c.items() if k != "source"} for c in nb["cells"]] -``` - -That hash was identical across all four passes on `twm_4` and all of PBAD's -three notebooks. The notebook also -round-trips byte-for-byte through -`json.dumps(nb, indent=1, ensure_ascii=False) + "\n"`, which is what makes -editing cells programmatically safe -- check it first for any new notebook. -Where an edit is mechanical, verify it mechanically too: PBAD's 134 long -Polish column names were re-wrapped by script, and an AST pass proved every -dict key still spells exactly what it did. - -## mypy has NOT been run against any of this - -`pyproject.toml` sets `strict = true` and `CLAUDE.md` says the repo runs mypy. -mypy IS installed now (`~/.local/bin/mypy`) -- the earlier note here saying it -was not is out of date. **"Zero findings" in a commit message still means zero -RUFF findings unless it says otherwise**; only `TRAK` has been type-checked. - -| Course | mypy --strict errors | -|---|---| -| `WDWR`, `SPD`, `MOM`, `ECRYPT_PROJECT` | 0 | -| `ECOTE` | 14 | -| `NLP` | 29 | -| `EARIN/lab1` alone | 46 | -| `TRAK` | **0** -- cleared 2026-08-29 | -| `twm_4` | **0** -- cleared 2026-08-29 | -| `PBAD` | not run (no venv; notebooks only) | -| `PSD` | **2**, both unfixable -- see below | - -Two structural notes for whoever runs it next: - -- It cannot be pointed at the whole repo. There are many `main.py` files and - mypy stops at `Duplicate module named "main"`. Run it per directory. -- Most errors are `Returning Any` from untyped third-party calls (openai, - pandas) and genuinely-wrong unions this pass introduced by hand, e.g. - `str | bool` returns in `gpt_chunks.reformat`. They are real. -- Point it at the course's own venv or every import is unresolvable: - `mypy --strict --python-executable Programming//.venv/bin/python `. - Without that, TRAK reported 4 phantom `matplotlib` import errors on top of - the real ones. -- **PSD keeps two mypy errors on purpose.** `--strict` implies - `disallow_subclassing_any`, and `temperature_anomaly_detector.py` has to - subclass pyflink's `MapFunction` and `KeyedProcessFunction`, which ship no - types. The only fixes are to weaken strict globally or to add an override - for our own module; neither is worth it, so the two are reported rather - than hidden. Everything else in PSD is clean under `--strict`. -- Point `MYPYPATH` at `Programming/PSD/zin3/python/code` for that tree: the - modules import `model.*` via a runtime `sys.path.append`, which mypy - cannot follow. -- `pyproject.toml` now carries a `[[tool.mypy.overrides]]` block with - `ignore_missing_imports` for `Imath`, `OpenEXR`, `OpenGL` and `matplotlib`. - Those (plus `keras`, `tensorflow`, `sklearn`, `seaborn`, `kafka` and - `pyflink`) ship neither stubs nor `py.typed`; without it mypy cannot import - them at all and reports nothing else about the file. It is not a suppression - of our own code -- do not extend it to anything we wrote. - -## Still open - -### 1. The 250-line cap: 50 files - -| Type | Count | Note | -|---|---|---| -| `.txt` | 27 | Blocked -- see `TODO-file-length-250.md` | -| `.py` | 10 | Several GREW during the lint pass, because docstrings and annotations are lines: `NLP/gpt_chunks.py` 911 -> 973, `EARIN/lab2/main.py` 581 -> 647, `PORR/.../linear_algebra_utils.py` 422 -> 553. Split them AFTER the lint work, not during. TRAK is already clear: its three over-cap files were split in the same pass that linted them, each verified by a byte-identical render. `twm_4/TWM_KerasIntro.py` joined the list at 407 lines and should NOT be split: it is an nbconvert export, and a partial one | -| `.cpp` | 9 | | -| `.c` | 4 | | -| `.java` | 2 | | -| `.tex` | 1 | `ELAC/.../03-building-nfa-...tex`, 418 lines of consecutive tikz figures with no subsection to split on | - -### 2. Coverage - -`gates/coverage.tsv` is designed but not written, and there are 125 tests for -~28k lines. The largest remaining item by a wide margin. - -### 3. Dependency freshness - -9 `requirements.txt`, 5 `package.json`, 7 `pom.xml`, 1 `.csproj`, none pinned. -**The upgrade and the gate must land in the SAME commit.** - -### 4. File-rewriting hooks - -`trailing-whitespace` and `end-of-file-fixer` are deliberately absent; they -want their own reviewed commit. - -### 5. Three checks that must be run on every future pass - -`scripts/check_refactor_safety.py` exists because ruff and a smoke run BOTH -missed three regressions this campaign introduced. Run it before and after any -automated rewrite: - -```bash -cp -r /tmp/before -# ...do the pass... -./scripts/check_refactor_safety.py --signatures /tmp/before -./scripts/check_refactor_safety.py --logging --print-keywords -``` - -And run mypy per directory afterwards: it found 12 latent TypeErrors in EARIN -that ruff could not see -- call sites still passing positionally to arguments -the pass had made keyword-only. - -Prefer a check the program itself settles. TRAK's `ray_tracing0` path has no -RNG at all, so `main.py --algorithm ray_tracing0 --output ../build/x.png` -renders byte-identically every time: refactoring 109 findings out of -`rendering.py` and splitting it in two was verified by one md5 comparison. The -cornell_box path is stochastic and cannot be used that way -- its ray count -drifts by ~20 between runs on its own. Where an RNG is unavoidable, do the -restructure and the `NPY002` swap as SEPARATE steps: the first must be -bit-identical under an external seed, only the second may move the pixels. - -### 6. Disk - -`Programming/PORR/code/` holds 3.8 GB of generated .npz caches (poli3 alone is -2.3 GB). All gitignored, all regenerable, none of it needed unless the full -benchmark is re-run. - -## Standing rule - -A gate lands only once the thing it gates is clean, so `pre-commit run ---all-files` is green at every commit. Every hook still missing from -`.pre-commit-config.yaml` is listed there with what blocks it. diff --git a/pyproject.toml b/pyproject.toml index a0d6e149..6f83ca04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,9 @@ module = [ "seaborn.*", "kafka.*", "pyflink.*", + "pandas.*", + "psycopg2.*", + "firebase_admin.*", ] ignore_missing_imports = true