diff --git a/Programming/PBAD/data.ipynb b/Programming/PBAD/data.ipynb index 2d84fa1f..3297759b 100644 --- a/Programming/PBAD/data.ipynb +++ b/Programming/PBAD/data.ipynb @@ -13,6 +13,8 @@ "metadata": {}, "outputs": [], "source": [ + "import sys\n", + "\n", "import pandas as pd" ] }, @@ -96,7 +98,9 @@ } ], "source": [ - "print(df_dofinansowanie[\"Program operacyjny\"].drop_duplicates().values)" + "sys.stdout.write(\n", + " f\"{df_dofinansowanie['Program operacyjny'].drop_duplicates().to_numpy()}\\n\"\n", + ")" ] }, { @@ -106,7 +110,8 @@ "outputs": [], "source": [ "# Wybór programu operacyjnego...\n", - "# df_dofinansowanie = df_dofinansowanie.loc[df_dofinansowanie['Program operacyjny'] == 'Program Operacyjny Infrastruktura i ?rodowisko 2014-2020'].reset_index(drop=True)\n" + "# Filtering to a single operational programme happens downstream, in\n", + "# pbad.ipynb and main.ipynb; this notebook builds the full table.\n" ] }, { @@ -130,8 +135,12 @@ " \"Dofinansowanie UE (PLN)\": \"Suma\",\n", " }\n", ")\n", + "# A full TERYT gmina code is seven digits; anything shorter is a county\n", + "# or voivodeship row that does not belong in this table.\n", + "TERYT_GMINA_DIGITS = 7\n", + "\n", "df_dofinansowanie_agg = df_dofinansowanie_agg.loc[\n", - " df_dofinansowanie_agg[\"Kod\"].str.len() == 7\n", + " df_dofinansowanie_agg[\"Kod\"].str.len() == TERYT_GMINA_DIGITS\n", "].reset_index(drop=True)" ] }, @@ -172,7 +181,10 @@ "df_wyna = df_wyna.rename(\n", " columns={\n", " \"ogółem\": \"Wynagrodzenie_ogolem\",\n", - " \"przeciętne miesięczne wynagrodzenia brutto w relacji do średniej krajowej (Polska=100)\": \"Wynagrodzenie_w_relacji_do_sredniej\",\n", + " (\n", + " \"przeciętne miesięczne wynagrodzenia brutto w relacji do średniej krajowej \"\n", + " \"(Polska=100)\"\n", + " ): \"Wynagrodzenie_w_relacji_do_sredniej\",\n", " }\n", ")" ] @@ -496,13 +508,26 @@ "df_fina_1 = df_fina_1.rename(\n", " columns={\n", " \"dochody podatkowe - podatek leśny\": \"Dochody_podatek_lesny\",\n", - " \"dochody podatkowe - podatek od czynności cywilnoprawnych\": \"Dochody_podatek_PCC\",\n", - " \"dochody podatkowe - podatek od działalności gospodarczej osób fizycznych, opłacany w formie karty podatkowej\": \"Dochody_podatek_od_dzialalnosci_gospodarczej\",\n", - " \"dochody podatkowe - podatek od nieruchomości\": \"Dochody_podatek_od_nieruchomosci\",\n", - " \"dochody podatkowe - podatek od spadków i darowizn\": \"Dochody_podatek_od_spadkow\",\n", - " \"dochody podatkowe - podatek od środków transportowych\": \"Dochody_podatek_od_srodkow_transportowych\",\n", + " (\n", + " \"dochody podatkowe - podatek od czynności cywilnoprawnych\"\n", + " ): \"Dochody_podatek_PCC\",\n", + " (\n", + " \"dochody podatkowe - podatek od działalności gospodarczej osób fizycznych, \"\n", + " \"opłacany w formie karty podatkowej\"\n", + " ): \"Dochody_podatek_od_dzialalnosci_gospodarczej\",\n", + " (\n", + " \"dochody podatkowe - podatek od nieruchomości\"\n", + " ): \"Dochody_podatek_od_nieruchomosci\",\n", + " (\n", + " \"dochody podatkowe - podatek od spadków i darowizn\"\n", + " ): \"Dochody_podatek_od_spadkow\",\n", + " (\n", + " \"dochody podatkowe - podatek od środków transportowych\"\n", + " ): \"Dochody_podatek_od_srodkow_transportowych\",\n", " \"dochody podatkowe - podatek rolny\": \"Dochody_podatek_rolny\",\n", - " \"dochody podatkowe - ustalone i pobierane na podstawie odrębnych ustaw\": \"Dochody_podatek_odrebne_ustawy\",\n", + " (\n", + " \"dochody podatkowe - ustalone i pobierane na podstawie odrębnych ustaw\"\n", + " ): \"Dochody_podatek_odrebne_ustawy\",\n", " \"razem\": \"Dochody_razem\",\n", " }\n", ")\n", @@ -904,14 +929,34 @@ "df_fina_2 = df_fina_2.rename(\n", " columns={\n", " \"dochody z majątku\": \"Dochody_z_majatku\",\n", - " \"dochody z majątku - dochody z najmu i dzierżawy składników majątkowych JST oraz innych umów o podobnym charakterze\": \"Dochody_z_najmu_i_dzierzawy\",\n", + " (\n", + " \"dochody z majątku - dochody z najmu i dzierżawy składników majątkowych \"\n", + " \"JST oraz innych umów o podobnym charakterze\"\n", + " ): \"Dochody_z_najmu_i_dzierzawy\",\n", " \"pozostałe dochody - wpływy z usług\": \"Dochody_z_uslug\",\n", - " \"pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z innych źródeł - inwestycyjne\": \"Dochody_dofinansowanie_inwestycyjne\",\n", - " \"pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z innych źródeł - razem\": \"Dochody_dofinansowanie_razem\",\n", - " \"udziały w podatkach stanowiących dochody budżetu państwa podatek dochodowy od osób fizycznych\": \"Udzialy_w_podatkach_dochodowych_od_osob_fizycznych\",\n", - " \"udziały w podatkach stanowiących dochody budżetu państwa podatek dochodowy od osób prawnych\": \"Udzialy_w_podatkach_dochodowych_od_osob_prywatnych\",\n", - " \"udziały w podatkach stanowiących dochody budżetu państwa razem\": \"Udzialy_w_podatkach_dochodowych_razem\",\n", - " \"wpływy z innych lokalnych opłat pobieranych przez jednostki samorządu terytorialnego na podstawie odrębnych ustaw\": \"Wplywy_z_innych_lokalnych_oplat\",\n", + " (\n", + " \"pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z \"\n", + " \"innych źródeł - inwestycyjne\"\n", + " ): \"Dochody_dofinansowanie_inwestycyjne\",\n", + " (\n", + " \"pozostałe dochody - środki na dofinansowanie własnych zadań pozyskane z \"\n", + " \"innych źródeł - razem\"\n", + " ): \"Dochody_dofinansowanie_razem\",\n", + " (\n", + " \"udziały w podatkach stanowiących dochody budżetu państwa podatek \"\n", + " \"dochodowy od osób fizycznych\"\n", + " ): \"Udzialy_w_podatkach_dochodowych_od_osob_fizycznych\",\n", + " (\n", + " \"udziały w podatkach stanowiących dochody budżetu państwa podatek \"\n", + " \"dochodowy od osób prawnych\"\n", + " ): \"Udzialy_w_podatkach_dochodowych_od_osob_prywatnych\",\n", + " (\n", + " \"udziały w podatkach stanowiących dochody budżetu państwa razem\"\n", + " ): \"Udzialy_w_podatkach_dochodowych_razem\",\n", + " (\n", + " \"wpływy z innych lokalnych opłat pobieranych przez jednostki samorządu \"\n", + " \"terytorialnego na podstawie odrębnych ustaw\"\n", + " ): \"Wplywy_z_innych_lokalnych_oplat\",\n", " \"wpływy z opłaty eksploatacyjnej\": \"Wplywy_z_oplaty_eksploatacyjnej\",\n", " \"wpływy z opłaty skarbowej\": \"Wplywy_z_oplaty_skarbowej\",\n", " \"wpływy z opłaty targowej\": \"Wplywy_z_oplaty_targowej\",\n", @@ -1434,8 +1479,12 @@ " \"ogółem\": \"Ludnosc_mezczyzni\",\n", " \"w wieku poprodukcyjnym\": \"Ludnosc_mezczyzni_w_wieku_poprodukcyjnym\",\n", " \"w wieku produkcyjnym\": \"Ludnosc_mezczyzni_w_wieku_produkcyjnym\",\n", - " \"w wieku produkcyjnym mobilnym\": \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym\",\n", - " \"w wieku produkcyjnym niemobilnym\": \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym\",\n", + " (\n", + " \"w wieku produkcyjnym mobilnym\"\n", + " ): \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_mobilnym\",\n", + " (\n", + " \"w wieku produkcyjnym niemobilnym\"\n", + " ): \"Ludnosc_mezczyzni_w_wieku_produkcyjnym_niemobilnym\",\n", " \"w wieku przedprodukcyjnym\": \"Ludnosc_mezczyzni_w_wieku_przedprodukcyjnym\",\n", " }\n", ")\n", @@ -1695,8 +1744,12 @@ " \"ogółem\": \"Ludnosc_kobiety\",\n", " \"w wieku poprodukcyjnym\": \"Ludnosc_kobiety_w_wieku_poprodukcyjnym\",\n", " \"w wieku produkcyjnym\": \"Ludnosc_kobiety_w_wieku_produkcyjnym\",\n", - " \"w wieku produkcyjnym mobilnym\": \"Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym\",\n", - " \"w wieku produkcyjnym niemobilnym\": \"Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym\",\n", + " (\n", + " \"w wieku produkcyjnym mobilnym\"\n", + " ): \"Ludnosc_kobiety_w_wieku_produkcyjnym_mobilnym\",\n", + " (\n", + " \"w wieku produkcyjnym niemobilnym\"\n", + " ): \"Ludnosc_kobiety_w_wieku_produkcyjnym_niemobilnym\",\n", " \"w wieku przedprodukcyjnym\": \"Ludnosc_kobiety_w_wieku_przedprodukcyjnym\",\n", " }\n", ")\n", @@ -1721,7 +1774,9 @@ ").reset_index()\n", "df_ludn_4 = df_ludn_4.rename(\n", " columns={\n", - " \"gęstość zaludnienia powierzchni zabudowanej i zurbanizowanej (osoby/km2)\": \"Gestosc_zaludnienia\",\n", + " (\n", + " \"gęstość zaludnienia powierzchni zabudowanej i zurbanizowanej (osoby/km2)\"\n", + " ): \"Gestosc_zaludnienia\",\n", " \"ludność na 1 km2\": \"Ludnosc_na_1_km2\",\n", " \"ludność w tysiącach\": \"Ludnosc\",\n", " \"ludność w tysiącach kobiety\": \"Ludnosc_kobiety\",\n", @@ -2512,7 +2567,9 @@ " \"obiekty całoroczne lipiec\": \"Obiekty_caloroczne\",\n", " \"obiekty ogółem lipiec\": \"Obiekty_ogolem\",\n", " \"turyści (korzystający) ogółem styczeń-grudzień\": \"Turysci_ogolem\",\n", - " \"turyści zagraniczni (korzystający) - nierezydenci styczeń-grudzień\": \"Turysci_zagraniczni\",\n", + " (\n", + " \"turyści zagraniczni (korzystający) - nierezydenci styczeń-grudzień\"\n", + " ): \"Turysci_zagraniczni\",\n", " }\n", ")\n", "\n", @@ -2801,7 +2858,7 @@ ], "source": [ "df_data = df_dofinansowanie_agg.copy()\n", - "print(len(df_data))" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -2886,7 +2943,7 @@ " suffixes=(None, \"_podz\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_podz\"], axis=1)\n", - "print(len(df_data))" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -2911,7 +2968,7 @@ " suffixes=(None, \"_wyna\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_wyna\"], axis=1)\n", - "print(len(df_data))" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -2937,7 +2994,7 @@ " suffixes=(None, \"_fina_1\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_fina_1\"], axis=1)\n", - "print(len(df_data))\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", "\n", "df_data = df_data.merge(\n", " df_fina_2,\n", @@ -2947,7 +3004,7 @@ " suffixes=(None, \"_fina_2\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_fina_2\"], axis=1)\n", - "print(len(df_data))" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -2968,7 +3025,9 @@ } ], "source": [ - "# df_data = df_data.merge(df_ludn_1, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_1'))\n", + "# Each of these merges was first written joining on Kod directly, next to a\n", + "# df_data['Kod'].value_counts() probe. These tables are published at coarser\n", + "# TERYT levels, so the key is a sliced code instead.\n", "df_data = df_data.merge(\n", " df_ludn_1,\n", " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", @@ -2977,9 +3036,8 @@ " suffixes=(None, \"_ludn_1\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_ludn_1\"], axis=1)\n", - "print(len(df_data))\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", "\n", - "# df_data = df_data.merge(df_ludn_2, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_2'))\n", "df_data = df_data.merge(\n", " df_ludn_2,\n", " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", @@ -2988,9 +3046,8 @@ " suffixes=(None, \"_ludn_2\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_ludn_2\"], axis=1)\n", - "print(len(df_data))\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", "\n", - "# df_data = df_data.merge(df_ludn_3, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_3'))\n", "df_data = df_data.merge(\n", " df_ludn_3,\n", " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", @@ -2999,9 +3056,8 @@ " suffixes=(None, \"_ludn_3\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_ludn_3\"], axis=1)\n", - "print(len(df_data))\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", "\n", - "# df_data = df_data.merge(df_ludn_4, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_4'))\n", "df_data = df_data.merge(\n", " df_ludn_4,\n", " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", @@ -3010,9 +3066,8 @@ " suffixes=(None, \"_ludn_4\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_ludn_4\"], axis=1)\n", - "print(len(df_data))\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", "\n", - "# df_data = df_data.merge(df_ludn_5, left_on=['Kod', 'Rok'], right_on=['Kod', 'Rok'], how='left', suffixes=(None, '_ludn_5'))\n", "df_data = df_data.merge(\n", " df_ludn_5,\n", " left_on=[df_data[\"Kod\"].str.slice(stop=-1), \"Rok\"],\n", @@ -3021,9 +3076,7 @@ " suffixes=(None, \"_ludn_5\"),\n", ")\n", "df_data = df_data.drop([\"key_0\", \"Kod_ludn_5\"], axis=1)\n", - "print(len(df_data))\n", - "\n", - "# df_data['Kod'].value_counts()" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -3040,6 +3093,9 @@ } ], "source": [ + "# This table is published at gmina level, so the merge joins on Kod\n", + "# directly. pandas therefore creates no key_0 column and there is nothing\n", + "# to drop afterwards.\n", "df_data = df_data.merge(\n", " df_tury,\n", " left_on=[\"Kod\", \"Rok\"],\n", @@ -3047,8 +3103,7 @@ " how=\"left\",\n", " suffixes=(None, \"_tury\"),\n", ")\n", - "# df_data = df_data.drop(['key_0', 'Kod_tury'], axis=1)\n", - "print(len(df_data))" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -3065,6 +3120,9 @@ } ], "source": [ + "# This table is published at gmina level, so the merge joins on Kod\n", + "# directly. pandas therefore creates no key_0 column and there is nothing\n", + "# to drop afterwards.\n", "df_data = df_data.merge(\n", " df_ryne,\n", " left_on=[\"Kod\", \"Rok\"],\n", @@ -3072,8 +3130,7 @@ " how=\"left\",\n", " suffixes=(None, \"_ryne\"),\n", ")\n", - "# df_data = df_data.drop(['key_0', 'Kod_ryne'], axis=1)\n", - "print(len(df_data))" + "sys.stdout.write(f\"{len(df_data)}\\n\")" ] }, { @@ -3135,57 +3192,9 @@ "metadata": {}, "outputs": [], "source": [ - "# # df_data[[\n", - "# # 'Miejsca_noclegowe_caloroczne',\n", - "# # 'Miejsca_noclegowe_ogolem',\n", - "# # 'Obiekty_caloroczne',\n", - "# # 'Obiekty_ogolem',\n", - "# # 'Turysci_ogolem',\n", - "# # 'Turysci_zagraniczni']] = df_data[[\n", - "# # 'Miejsca_noclegowe_caloroczne',\n", - "# # 'Miejsca_noclegowe_ogolem',\n", - "# # 'Obiekty_caloroczne',\n", - "# # 'Obiekty_ogolem',\n", - "# # 'Turysci_ogolem',\n", - "# # 'Turysci_zagraniczni']].fillna(0)\n", - "# df_data.drop(columns=[\n", - "# 'Program_operacyjny',\n", - "# 'Wymeldowania_za_granice_kobiety',\n", - "# 'Wymeldowania_za_granice_mezczyzni',\n", - "# 'Wymeldowania_za_granice_ogolem',\n", - "# 'Bezrobotni_do_30_roku_zycia',\n", - "\n", - "# 'Wojewodztwo_Dolnoslaskie', # 43\n", - "# 'Wojewodztwo_Kujawsko_Pomorskie', # 44\n", - "# 'Wojewodztwo_Lubelskie', # 45\n", - "# 'Wojewodztwo_Lubuskie', # 46\n", - "# 'Wojewodztwo_Lodzkie', # 47\n", - "# 'Wojewodztwo_Malopolskie', # 48\n", - "# 'Wojewodztwo_Mazowieckie', # 49\n", - "# 'Wojewodztwo_Opolskie', # 50\n", - "# 'Wojewodztwo_Podkarpackie', # 51\n", - "# 'Wojewodztwo_Podlaskie', # 52\n", - "# 'Wojewodztwo_Pomorskie', # 53\n", - "# 'Wojewodztwo_Slaskie', # 54\n", - "# 'Wojewodztwo_Swietokrzyskie', # 55\n", - "# 'Wojewodztwo_Warminsko_Mazurskie', # 56\n", - "# 'Wojewodztwo_Wielkopolskie', # 57\n", - "# 'Wojewodztwo_Zachodniopomorskie', # 58\n", - "# 'Gmina_miejska', # 98\n", - "# 'Gmina_miejsko_wiejska', # 99\n", - "# 'Gmina_wiejska',\n", - "# 'Wynagrodzenie_ogolem','Wynagrodzenie_w_relacji_do_sredniej','Dochody_podatek_lesny','Dochody_podatek_PCC','Dochody_podatek_od_dzialalnosci_gospodarczej','Dochody_podatek_od_nieruchomosci',\n", - "# 'Dochody_podatek_od_spadkow','Dochody_podatek_od_srodkow_transportowych','Dochody_podatek_rolny','Dochody_podatek_odrebne_ustawy','Dochody_razem','Dochody_z_majatku',\n", - "# 'Dochody_z_najmu_i_dzierzawy','Dochody_z_uslug','Dochody_dofinansowanie_inwestycyjne','Dochody_dofinansowanie_razem','Udzialy_w_podatkach_dochodowych_od_osob_fizycznych','Udzialy_w_podatkach_dochodowych_od_osob_prywatnych',\n", - "# 'Miejsca_noclegowe_caloroczne',\n", - "# 'Miejsca_noclegowe_ogolem',\n", - "# 'Obiekty_caloroczne',\n", - "# 'Obiekty_ogolem',\n", - "# 'Turysci_ogolem',\n", - "# 'Turysci_zagraniczni',\n", - "# 'Udzialy_w_podatkach_dochodowych_razem','Wplywy_z_innych_lokalnych_oplat','Wplywy_z_oplaty_eksploatacyjnej','Wplywy_z_oplaty_skarbowej','Wplywy_z_oplaty_targowej'], inplace=True, errors='ignore')\n", - "\n", - "# df_data[(df_data.isna().any(axis=1)) & (df_data['Rok'] != 2023)].columns # ['Rok'].drop_duplicates().reset_index(drop=True)" + "# An earlier pass filled the tourism columns and dropped a long explicit\n", + "# column list here. Both are done above now, against df_data's real\n", + "# columns rather than a hand-maintained list." ] }, { @@ -3194,7 +3203,9 @@ "metadata": {}, "outputs": [], "source": [ - "# df_data['Suma'] = df_data['Suma'] / df_data['Ludnosc']" + "# pbad.ipynb divides Suma by Ludnosc to get a per-head figure. This\n", + "# notebook writes the raw total to dane1.csv and leaves that choice to\n", + "# whoever reads it." ] }, { @@ -3322,22 +3333,22 @@ ], "source": [ "# Usunięcie niepotrzebnych kolumn...\n", - "df_data.drop(\n", + "# Program_operacyjny is kept here and dropped later, once the per-programme\n", + "# split has been made.\n", + "df_data = df_data.drop(\n", " columns=[\n", - " # 'Program_operacyjny',\n", " \"Wymeldowania_za_granice_kobiety\",\n", " \"Wymeldowania_za_granice_mezczyzni\",\n", " \"Wymeldowania_za_granice_ogolem\",\n", " \"Bezrobotni_do_30_roku_zycia\",\n", " ],\n", - " inplace=True,\n", " errors=\"ignore\",\n", ")\n", "\n", "df_data.to_csv(\"dane1.csv\", index=False)\n", "\n", "for c in df_data.columns:\n", - " print(c)" + " sys.stdout.write(f\"{c}\\n\")" ] }, { @@ -3531,7 +3542,7 @@ " \"Gmina_wiejska\",\n", "] # 100\n", "\n", - "df_data.drop(\n", + "df_data = df_data.drop(\n", " columns=[\n", " \"Program_operacyjny\",\n", " \"Wymeldowania_za_granice_kobiety\",\n", @@ -3539,20 +3550,22 @@ " \"Wymeldowania_za_granice_ogolem\",\n", " \"Bezrobotni_do_30_roku_zycia\",\n", " ],\n", - " inplace=True,\n", " errors=\"ignore\",\n", ")\n", "\n", - "print(len(df_data))\n", - "df_data.dropna(inplace=True)\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", + "df_data = df_data.dropna()\n", "df_data = df_data[df_data[\"Suma\"] > 0]\n", - "print(len(df_data))\n", + "sys.stdout.write(f\"{len(df_data)}\\n\")\n", "\n", "X = df_data[feature_names]\n", "y = df_data[\"Suma\"]\n", "\n", - "color_column = df_data[\"Gestosc_zaludnienia\"]\n", - "color_column = (df_data[\"Gestosc_zaludnienia\"] > 1.5).astype(int)\n", + "# Above this population density a gmina is coloured as the dense group\n", + "# in the scatter below.\n", + "DENSE_GMINA_THRESHOLD = 1.5\n", + "\n", + "color_column = (df_data[\"Gestosc_zaludnienia\"] > DENSE_GMINA_THRESHOLD).astype(int)\n", "\n", "X_train, X_test, y_train, y_test, color_column_train, color_column_test = (\n", " train_test_split(X, y, color_column, test_size=0.2, random_state=1)\n", @@ -3563,7 +3576,7 @@ "\n", "y_pred = model.predict(X_test)\n", "mse = mean_squared_error(y_test, y_pred)\n", - "print(\"Mean Squared Error:\", mse)" + "sys.stdout.write(f\"Mean Squared Error: {mse}\\n\")" ] }, { @@ -3609,7 +3622,7 @@ ], "source": [ "correlation_matrix = df_data.corr()\n", - "print(correlation_matrix[\"Suma\"].sort_values(ascending=False)[1:29])" + "sys.stdout.write(f\"{correlation_matrix['Suma'].sort_values(ascending=False)[1:29]}\\n\")" ] }, { @@ -3634,8 +3647,9 @@ "plt.ylabel(\"Predicted\")\n", "plt.title(\"Actual vs Predicted\")\n", "\n", - "plt.xlim(0, max(max(y_test), max(y_pred)))\n", - "plt.ylim(0, max(max(y_test), max(y_pred)))\n", + "axis_max = max(*y_test, *y_pred)\n", + "plt.xlim(0, axis_max)\n", + "plt.ylim(0, axis_max)\n", "\n", "plt.show()" ] @@ -4005,7 +4019,7 @@ } ], "source": [ - "print(export_text(model, feature_names=feature_names))" + "sys.stdout.write(f\"{export_text(model, feature_names=feature_names)}\\n\")" ] }, { @@ -4119,11 +4133,11 @@ } ], "source": [ - "feature_importance = dict(zip(feature_names, model.feature_importances_))\n", + "feature_importance = dict(zip(feature_names, model.feature_importances_, strict=True))\n", "for feature, importance in sorted(\n", " feature_importance.items(), key=lambda x: x[1], reverse=True\n", "):\n", - " print(f\"{importance:.5f} \\u2014 {feature}\")" + " sys.stdout.write(f\"{importance:.5f} \\u2014 {feature}\\n\")" ] }, { diff --git a/Programming/PBAD/main.ipynb b/Programming/PBAD/main.ipynb index 10bd7945..1aaf53ad 100644 --- a/Programming/PBAD/main.ipynb +++ b/Programming/PBAD/main.ipynb @@ -18,6 +18,7 @@ } ], "source": [ + "import sys\n", "from statistics import median\n", "\n", "import matplotlib.pyplot as plt\n", @@ -132,7 +133,7 @@ " \"Odleglosc_od_centrum_decyzyjnego\",\n", "] # 100\n", "\n", - "all_columns = [\"Kod\", \"Rok\"] + feature_names\n", + "all_columns = [\"Kod\", \"Rok\", *feature_names]\n", "\n", "df_data = pd.read_csv(\"dane1.csv\", encoding=\"ISO-8859-2\", converters={\"Kod\": str})\n", "\n", @@ -151,7 +152,7 @@ "\n", "df_data = df_data.merge(df_odl, on=[\"Kod\"], how=\"left\")\n", "\n", - "print(df_data[\"Program_operacyjny\"].drop_duplicates().values)" + "sys.stdout.write(f\"{df_data['Program_operacyjny'].drop_duplicates().to_numpy()}\\n\")" ] }, { @@ -208,16 +209,16 @@ "].fillna(0)\n", "\n", "# Usunięcie niepotrzebnych rzędów...\n", - "df_data.dropna(inplace=True)\n", + "df_data = df_data.dropna()\n", "df_data = df_data[df_data[\"Suma\"] > 0]\n", "\n", "df_data = df_data.groupby(all_columns)[\"Suma\"].sum().reset_index()\n", "\n", - "# ...\n", - "# df_data['Suma'] = df_data['Suma'] / df_data['Ludnosc']\n", + "# pbad.ipynb divides Suma by Ludnosc at this point to get a per-head\n", + "# figure. This notebook deliberately keeps the raw total.\n", "\n", - "print(\"Median\", median(df_data[\"Suma\"]))\n", - "print(\"Mean\", sum(df_data[\"Suma\"]) / len(df_data[\"Suma\"]))\n", + "sys.stdout.write(f\"Median {median(df_data['Suma'])}\\n\")\n", + "sys.stdout.write(f\"Mean {sum(df_data['Suma']) / len(df_data['Suma'])}\\n\")\n", "\n", "X = df_data[feature_names]\n", "y = df_data[\"Suma\"]\n", @@ -823,18 +824,18 @@ " best_model = (model, selector)\n", " best_params = (params[\"max_depth\"], params[\"n_features_to_select\"])\n", "\n", - " print(\n", + " sys.stdout.write(\n", " f\"max_depth: {params['max_depth']:>{2}}, \"\n", " f\"n_features: {params['n_features_to_select']:>{2}}, \"\n", " f\"mse_train: {mse_train:>{20}.1f}, \"\n", - " f\"mse_test: {mse_test:>{20}.1f} <-\"\n", + " f\"mse_test: {mse_test:>{20}.1f} <-\\n\"\n", " )\n", " else:\n", - " print(\n", + " sys.stdout.write(\n", " f\"max_depth: {params['max_depth']:>{2}}, \"\n", " f\"n_features: {params['n_features_to_select']:>{2}}, \"\n", " f\"mse_train: {mse_train:>{20}.1f}, \"\n", - " f\"mse_test: {mse_test:>{20}.1f}\"\n", + " f\"mse_test: {mse_test:>{20}.1f}\\n\"\n", " )" ] }, @@ -852,7 +853,7 @@ } ], "source": [ - "print(best_params)" + "sys.stdout.write(f\"{best_params}\\n\")" ] }, { @@ -959,8 +960,8 @@ " color_column, test_size=0.2, random_state=0\n", ")\n", "\n", - "min_val = min(min(y_test), min(y_pred_test))\n", - "max_val = max(max(y_test), max(y_pred_test))\n", + "min_val = min(*y_test, *y_pred_test)\n", + "max_val = max(*y_test, *y_pred_test)\n", "plt.plot([min_val, max_val], [min_val, max_val], \"r--\")\n", "\n", "plt.scatter(y_test, y_pred_test, alpha=0.5, c=color_column_test, cmap=\"rainbow\")\n", @@ -995,7 +996,7 @@ " results_train, columns=[\"max_depth\", \"n_features_to_select\", \"mse_train\"]\n", ")\n", "\n", - "heatmap_data = results_train_df.pivot(\n", + "heatmap_data = results_train_df.pivot_table(\n", " index=\"max_depth\", columns=\"n_features_to_select\", values=\"mse_train\"\n", ")\n", "\n", @@ -1028,7 +1029,7 @@ " results_test, columns=[\"max_depth\", \"n_features_to_select\", \"mse_test\"]\n", ")\n", "\n", - "heatmap_data = results_test_df.pivot(\n", + "heatmap_data = results_test_df.pivot_table(\n", " index=\"max_depth\", columns=\"n_features_to_select\", values=\"mse_test\"\n", ")\n", "\n", @@ -1066,11 +1067,13 @@ } ], "source": [ - "feature_importance = dict(zip(feature_names, best_model[0].feature_importances_))\n", + "feature_importance = dict(\n", + " zip(feature_names, best_model[0].feature_importances_, strict=True)\n", + ")\n", "for feature, importance in sorted(\n", " feature_importance.items(), key=lambda x: x[1], reverse=True\n", "):\n", - " print(f\"{importance:.4f} \\u2014 {feature}\")" + " sys.stdout.write(f\"{importance:.4f} \\u2014 {feature}\\n\")" ] }, { @@ -1606,11 +1609,11 @@ "model = DecisionTreeRegressor(random_state=0, max_depth=best_params[0])\n", "model.fit(X_selected, y)\n", "\n", - "feature_importance = dict(zip(feature_names, model.feature_importances_))\n", + "feature_importance = dict(zip(feature_names, model.feature_importances_, strict=True))\n", "for feature, importance in sorted(\n", " feature_importance.items(), key=lambda x: x[1], reverse=True\n", "):\n", - " print(f\"{importance:.4f} \\u2014 {feature}\")" + " sys.stdout.write(f\"{importance:.4f} \\u2014 {feature}\\n\")" ] }, { @@ -1619,7 +1622,9 @@ "metadata": {}, "outputs": [], "source": [ - "# plot_tree(model, fontsize=5)" + "# The fitted tree was also inspected graphically here, with\n", + "# sklearn.tree.plot_tree(model, fontsize=5). At this depth the render is\n", + "# unreadable, so the report uses the feature importances above instead." ] } ], diff --git a/Programming/PBAD/pbad.ipynb b/Programming/PBAD/pbad.ipynb index eaf5bfe3..011fe4ab 100644 --- a/Programming/PBAD/pbad.ipynb +++ b/Programming/PBAD/pbad.ipynb @@ -6,6 +6,8 @@ "metadata": {}, "outputs": [], "source": [ + "import sys\n", + "\n", "import matplotlib.pyplot as plt\n", "import pandas as pd\n", "from sklearn.metrics import mean_squared_error\n", @@ -122,7 +124,7 @@ " \"Odleglosc_od_centrum_decyzyjnego\",\n", "] # 100\n", "\n", - "all_columns = [\"Kod\", \"Rok\"] + feature_names" + "all_columns = [\"Kod\", \"Rok\", *feature_names]" ] }, { @@ -181,7 +183,7 @@ } ], "source": [ - "print(df_data[\"Program_operacyjny\"].drop_duplicates().values)" + "sys.stdout.write(f\"{df_data['Program_operacyjny'].drop_duplicates().to_numpy()}\\n\")" ] }, { @@ -208,10 +210,9 @@ " df_data[\"Program_operacyjny\"]\n", " == \"Program Operacyjny Infrastruktura i ?rodowisko 2014-2020\"\n", "].reset_index(drop=True)\n", - "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Inteligentny RozwĂłj'].reset_index(drop=True)\n", - "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Polska Cyfrowa'].reset_index(drop=True)\n", - "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Wiedza Edukacja RozwĂłj'].reset_index(drop=True)\n", - "# df_data = df_data.loc[df_data['Program_operacyjny'] == 'Program Operacyjny Polska Wschodnia'].reset_index(drop=True)\n", + "# The other four programmes were each run the same way by swapping the\n", + "# name above: Inteligentny Rozwoj, Polska Cyfrowa, Wiedza Edukacja\n", + "# Rozwoj, and Polska Wschodnia.\n", "\n", "# Uzupełnienie brakujących danych...\n", "df_data[\n", @@ -235,7 +236,7 @@ "].fillna(0)\n", "\n", "# Usunięcie niepotrzebnych rzędów...\n", - "df_data.dropna(inplace=True)\n", + "df_data = df_data.dropna()\n", "df_data = df_data[df_data[\"Suma\"] > 0]\n", "\n", "df_data = df_data.groupby(all_columns)[\"Suma\"].sum().reset_index()\n", @@ -258,14 +259,18 @@ } ], "source": [ + "# Above this population density a gmina is coloured as the dense group\n", + "# in the scatter below.\n", + "DENSE_GMINA_THRESHOLD = 1.5\n", + "\n", "X = df_data[feature_names]\n", "\n", - "# scaler = MinMaxScaler()\n", - "# X = scaler.fit_transform(X)\n", + "# Scaling the features with MinMaxScaler was tried here and made no\n", + "# difference to the tree, which splits on order rather than magnitude.\n", "\n", "y = df_data[\"Suma\"]\n", "\n", - "color_column = (df_data[\"Gestosc_zaludnienia\"] > 1.5).astype(int)\n", + "color_column = (df_data[\"Gestosc_zaludnienia\"] > DENSE_GMINA_THRESHOLD).astype(int)\n", "\n", "X_train, X_test, y_train, y_test, color_column_train, color_column_test = (\n", " train_test_split(X, y, color_column, test_size=0.2, random_state=99)\n", @@ -276,7 +281,7 @@ "\n", "y_pred = model.predict(X_test)\n", "mse = mean_squared_error(y_test, y_pred)\n", - "print(\"Mean Squared Error:\", mse)" + "sys.stdout.write(f\"Mean Squared Error: {mse}\\n\")" ] }, { @@ -301,17 +306,9 @@ "plt.ylabel(\"Predicted\")\n", "plt.title(\"Actual vs Predicted\")\n", "\n", - "# plt.xlim(0, max(max(y_test), max(y_pred)))\n", - "# plt.ylim(0, max(max(y_test), max(y_pred)))\n", - "\n", - "# plt.xlim(0, 3*10**7)\n", - "# plt.ylim(0, 3*10**7)\n", - "\n", - "# plt.xlim(0, 3*10**6)\n", - "# plt.ylim(0, 3*10**6)\n", - "\n", - "# plt.xlim(0, 3*10**5)\n", - "# plt.ylim(0, 3*10**5)\n", + "# The plot was also read zoomed in, by capping both axes at the data\n", + "# maximum and then at 3e7, 3e6 and 3e5 in turn. Full range is what the\n", + "# report uses.\n", "\n", "plt.show()" ] @@ -359,7 +356,7 @@ ], "source": [ "correlation_matrix = df_data.corr()\n", - "print(correlation_matrix[\"Suma\"].sort_values(ascending=False)[1:29])" + "sys.stdout.write(f\"{correlation_matrix['Suma'].sort_values(ascending=False)[1:29]}\\n\")" ] }, { @@ -683,7 +680,7 @@ } ], "source": [ - "print(export_text(model, feature_names=feature_names))" + "sys.stdout.write(f\"{export_text(model, feature_names=feature_names)}\\n\")" ] }, { @@ -799,11 +796,11 @@ } ], "source": [ - "feature_importance = dict(zip(feature_names, model.feature_importances_))\n", + "feature_importance = dict(zip(feature_names, model.feature_importances_, strict=True))\n", "for feature, importance in sorted(\n", " feature_importance.items(), key=lambda x: x[1], reverse=True\n", "):\n", - " print(f\"{importance:.4f} \\u2014 {feature}\")" + " sys.stdout.write(f\"{importance:.4f} \\u2014 {feature}\\n\")" ] } ], diff --git a/TODO-refactor-remaining.md b/TODO-refactor-remaining.md index b70517f8..01140663 100644 --- a/TODO-refactor-remaining.md +++ b/TODO-refactor-remaining.md @@ -32,16 +32,15 @@ own docs and the rest of this fleet carry. A test asserts by raising. |---|---| | Baseline, before any of this | 9,281 | | After the format + safe-fix pass and excluding `TRAK/sightpy` | 3,686 | -| **Now** | **423** | +| **Now** | **308** | Courses at zero: `WDWR`, `SPD`, `MOM`, `ECRYPT`, `ECRYPT_PROJECT`, `ECOTE`, -`NLP`, `EARIN`, `PORR`, `TRAK`, `twm_4`. What is left, by course: +`NLP`, `EARIN`, `PORR`, `TRAK`, `twm_4`, `PBAD`. What is left, by course: | Course | ruff | Can it be verified by running? | |---|---|---| | `ERSMS-project` | 156 | No: docker-compose course | | `PSD` | 152 | No: `run.sh` predates the harness and installs packages with sudo | -| `PBAD` | 115 | No: same dead jupyter shim | `course_run.sh`'s notebook branch is fixed: it executes to a temp `--output-dir`, probes jupyter by running it rather than by `command -v`, and @@ -58,10 +57,14 @@ stripped = {k: v for k, v in nb.items() if k != "cells"} stripped["cells"] = [{k: v for k, v in c.items() if k != "source"} for c in nb["cells"]] ``` -That hash was identical across all four passes on `twm_4`. The notebook also +That hash was identical across all four passes on `twm_4` and all of PBAD's +three notebooks. The notebook also round-trips byte-for-byte through `json.dumps(nb, indent=1, ensure_ascii=False) + "\n"`, which is what makes -editing cells programmatically safe -- check that before touching PBAD's three. +editing cells programmatically safe -- check it first for any new notebook. +Where an edit is mechanical, verify it mechanically too: PBAD's 134 long +Polish column names were re-wrapped by script, and an AST pass proved every +dict key still spells exactly what it did. ## mypy has NOT been run against any of this @@ -78,6 +81,7 @@ RUFF findings unless it says otherwise**; only `TRAK` has been type-checked. | `EARIN/lab1` alone | 46 | | `TRAK` | **0** -- cleared 2026-08-29 | | `twm_4` | **0** -- cleared 2026-08-29 | +| `PBAD` | not run (no venv; notebooks only) | | `PSD/zin1` | 5 | Two structural notes for whoever runs it next: