training xgb model not fully accurate

This commit is contained in:
2026-07-22 22:27:36 -04:00
parent ef084b3965
commit fb3173e4ab
2 changed files with 600 additions and 4 deletions

3
.gitignore vendored
View File

@@ -82,4 +82,5 @@ Desktop.ini
*.bak
*.swp
data/*
data/*
models/*

View File

@@ -5694,7 +5694,7 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": 2,
"id": "99be3b8b",
"metadata": {},
"outputs": [
@@ -11068,7 +11068,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 3,
"id": "b58e7aed",
"metadata": {},
"outputs": [
@@ -11281,7 +11281,7 @@
"[5 rows x 26 columns]"
]
},
"execution_count": 5,
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
@@ -11454,6 +11454,601 @@
"\n",
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "be2469ce",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Training rows: 466379\n",
"Testing rows : 116374\n",
"[0]\tvalidation_0-mae:32.71855\tvalidation_1-mae:32.46407\n",
"[100]\tvalidation_0-mae:17.67243\tvalidation_1-mae:17.56379\n",
"[200]\tvalidation_0-mae:15.99231\tvalidation_1-mae:16.02378\n",
"[300]\tvalidation_0-mae:15.81573\tvalidation_1-mae:15.89000\n",
"[400]\tvalidation_0-mae:15.76997\tvalidation_1-mae:15.86225\n",
"[500]\tvalidation_0-mae:15.74462\tvalidation_1-mae:15.84885\n",
"[600]\tvalidation_0-mae:15.72323\tvalidation_1-mae:15.83748\n",
"[700]\tvalidation_0-mae:15.70448\tvalidation_1-mae:15.82844\n",
"[800]\tvalidation_0-mae:15.68914\tvalidation_1-mae:15.82248\n",
"[900]\tvalidation_0-mae:15.67203\tvalidation_1-mae:15.81510\n",
"[1000]\tvalidation_0-mae:15.65865\tvalidation_1-mae:15.80998\n",
"[1100]\tvalidation_0-mae:15.64647\tvalidation_1-mae:15.80701\n",
"[1200]\tvalidation_0-mae:15.63444\tvalidation_1-mae:15.80456\n",
"[1300]\tvalidation_0-mae:15.62326\tvalidation_1-mae:15.80293\n",
"[1400]\tvalidation_0-mae:15.61287\tvalidation_1-mae:15.80151\n",
"[1500]\tvalidation_0-mae:15.60263\tvalidation_1-mae:15.80194\n",
"[1600]\tvalidation_0-mae:15.59322\tvalidation_1-mae:15.80148\n",
"[1700]\tvalidation_0-mae:15.58498\tvalidation_1-mae:15.80152\n",
"[1800]\tvalidation_0-mae:15.57652\tvalidation_1-mae:15.80163\n",
"[1900]\tvalidation_0-mae:15.56848\tvalidation_1-mae:15.80177\n",
"[1999]\tvalidation_0-mae:15.56159\tvalidation_1-mae:15.80210\n",
"\n",
"========================\n",
"Results\n",
"========================\n",
"MAE : 15.802 runs\n",
"RMSE : 22.194 runs\n",
"R² : 0.7054\n",
"\n",
"Feature importance:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>feature</th>\n",
" <th>importance</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>run_rate_factor</td>\n",
" <td>0.477130</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>run_rate</td>\n",
" <td>0.272351</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>projected_score</td>\n",
" <td>0.162744</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>delivery_number</td>\n",
" <td>0.017518</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>score_wicket_combo</td>\n",
" <td>0.013470</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>pressure_index</td>\n",
" <td>0.010688</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>middle_overs</td>\n",
" <td>0.010050</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>score_squared</td>\n",
" <td>0.008396</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>legal_ball</td>\n",
" <td>0.004335</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>score</td>\n",
" <td>0.004045</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>wickets</td>\n",
" <td>0.003218</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>wickets_remaining</td>\n",
" <td>0.002742</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>batting_strength_remaining</td>\n",
" <td>0.002630</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>overs_remaining</td>\n",
" <td>0.002173</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>innings_progress</td>\n",
" <td>0.002061</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>balls_remaining</td>\n",
" <td>0.001827</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>wicket_fraction</td>\n",
" <td>0.001551</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>score_per_ball_remaining</td>\n",
" <td>0.001476</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>score_progress_interaction</td>\n",
" <td>0.001058</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>overs_completed</td>\n",
" <td>0.000537</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>powerplay</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>death_overs</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>progress_squared</td>\n",
" <td>0.000000</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" feature importance\n",
"10 run_rate_factor 0.477130\n",
"8 run_rate 0.272351\n",
"9 projected_score 0.162744\n",
"0 delivery_number 0.017518\n",
"18 score_wicket_combo 0.013470\n",
"19 pressure_index 0.010688\n",
"15 middle_overs 0.010050\n",
"20 score_squared 0.008396\n",
"1 legal_ball 0.004335\n",
"2 score 0.004045\n",
"3 wickets 0.003218\n",
"11 wickets_remaining 0.002742\n",
"13 batting_strength_remaining 0.002630\n",
"7 overs_remaining 0.002173\n",
"4 innings_progress 0.002061\n",
"5 balls_remaining 0.001827\n",
"12 wicket_fraction 0.001551\n",
"17 score_per_ball_remaining 0.001476\n",
"22 score_progress_interaction 0.001058\n",
"6 overs_completed 0.000537\n",
"14 powerplay 0.000000\n",
"16 death_overs 0.000000\n",
"21 progress_squared 0.000000"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Saved model\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"from sklearn.model_selection import GroupShuffleSplit\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"from xgboost import XGBRegressor\n",
"\n",
"import joblib\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# Prepare data\n",
"# =====================================\n",
"\n",
"data = df.copy()\n",
"\n",
"\n",
"# Remove identifiers / leakage\n",
"\n",
"drop_cols = [\n",
" \"target_score\",\n",
" \"match_id\",\n",
" \"over_ball\"\n",
"]\n",
"\n",
"\n",
"X = data.drop(\n",
" columns=drop_cols\n",
")\n",
"\n",
"\n",
"y = data[\"target_score\"]\n",
"\n",
"\n",
"groups = data[\"match_id\"]\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# Match-based split\n",
"# =====================================\n",
"\n",
"splitter = GroupShuffleSplit(\n",
" n_splits=1,\n",
" test_size=0.2,\n",
" random_state=42\n",
")\n",
"\n",
"\n",
"train_idx, test_idx = next(\n",
" splitter.split(\n",
" X,\n",
" y,\n",
" groups\n",
" )\n",
")\n",
"\n",
"\n",
"X_train = X.iloc[train_idx]\n",
"X_test = X.iloc[test_idx]\n",
"\n",
"\n",
"y_train = y.iloc[train_idx]\n",
"y_test = y.iloc[test_idx]\n",
"\n",
"\n",
"\n",
"print(\"Training rows:\", len(X_train))\n",
"print(\"Testing rows :\", len(X_test))\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# XGBoost model\n",
"# =====================================\n",
"\n",
"model = XGBRegressor(\n",
"\n",
" # trees\n",
" n_estimators=2000,\n",
"\n",
" # learning\n",
" learning_rate=0.015,\n",
"\n",
" # tree complexity\n",
" max_depth=6,\n",
" min_child_weight=5,\n",
"\n",
" # randomness\n",
" subsample=0.85,\n",
" colsample_bytree=0.85,\n",
"\n",
" # regularization\n",
" reg_alpha=0.1,\n",
" reg_lambda=2,\n",
"\n",
" objective=\"reg:squarederror\",\n",
"\n",
" eval_metric=\"mae\",\n",
"\n",
" random_state=42,\n",
"\n",
" n_jobs=-1\n",
")\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# Train\n",
"# =====================================\n",
"\n",
"model.fit(\n",
" X_train,\n",
" y_train,\n",
"\n",
" eval_set=[\n",
" (X_train, y_train),\n",
" (X_test, y_test)\n",
" ],\n",
"\n",
" verbose=100\n",
")\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# Evaluate\n",
"# =====================================\n",
"\n",
"pred = model.predict(X_test)\n",
"\n",
"\n",
"mae = mean_absolute_error(\n",
" y_test,\n",
" pred\n",
")\n",
"\n",
"\n",
"rmse = np.sqrt(\n",
" mean_squared_error(\n",
" y_test,\n",
" pred\n",
" )\n",
")\n",
"\n",
"\n",
"r2 = r2_score(\n",
" y_test,\n",
" pred\n",
")\n",
"\n",
"\n",
"print(\"\\n========================\")\n",
"print(\"Results\")\n",
"print(\"========================\")\n",
"\n",
"print(f\"MAE : {mae:.3f} runs\")\n",
"print(f\"RMSE : {rmse:.3f} runs\")\n",
"print(f\"R² : {r2:.4f}\")\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# Feature importance\n",
"# =====================================\n",
"\n",
"importance = pd.DataFrame(\n",
" {\n",
" \"feature\": X.columns,\n",
" \"importance\": model.feature_importances_\n",
" }\n",
")\n",
"\n",
"\n",
"importance = importance.sort_values(\n",
" \"importance\",\n",
" ascending=False\n",
")\n",
"\n",
"\n",
"print(\"\\nFeature importance:\")\n",
"display(importance)\n",
"\n",
"\n",
"\n",
"# =====================================\n",
"# Save model\n",
"# =====================================\n",
"\n",
"joblib.dump(\n",
" model,\n",
" \"models/xgb_run_predictor.pkl\"\n",
")\n",
"\n",
"print(\"\\nSaved model\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "acfb9aa7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"======================\n",
"DLS BASELINE RESULTS\n",
"======================\n",
"Rows evaluated : 116374\n",
"MAE : 23.02 runs\n",
"RMSE : 38.91 runs\n",
"R² : 0.0947\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"\n",
"# ==========================================\n",
"# Create DLS-style resource table from train\n",
"# ==========================================\n",
"\n",
"dls_train = X_train.copy()\n",
"dls_train[\"target_score\"] = y_train.values\n",
"\n",
"dls_train[\"resource_used\"] = (\n",
" dls_train[\"score\"] /\n",
" dls_train[\"target_score\"]\n",
")\n",
"\n",
"# Remove weird cases\n",
"dls_train[\"resource_used\"] = (\n",
" dls_train[\"resource_used\"]\n",
" .replace([np.inf, -np.inf], np.nan)\n",
")\n",
"\n",
"dls_train = dls_train.dropna(\n",
" subset=[\"resource_used\"]\n",
")\n",
"\n",
"dls_train[\"resource_used\"] = (\n",
" dls_train[\"resource_used\"]\n",
" .clip(0.01, 1.0)\n",
")\n",
"\n",
"\n",
"# Average resource remaining at each state\n",
"resource_table = (\n",
" dls_train\n",
" .groupby(\n",
" [\n",
" \"legal_ball\",\n",
" \"wickets\"\n",
" ]\n",
" )[\"resource_used\"]\n",
" .mean()\n",
")\n",
"\n",
"\n",
"# ==========================================\n",
"# DLS prediction function\n",
"# ==========================================\n",
"\n",
"def dls_predict(row):\n",
"\n",
" key = (\n",
" row[\"legal_ball\"],\n",
" row[\"wickets\"]\n",
" )\n",
"\n",
" # exact state\n",
" if key in resource_table.index:\n",
" resource = resource_table[key]\n",
"\n",
" else:\n",
" # nearest legal ball fallback\n",
" subset = resource_table.reset_index()\n",
"\n",
" subset = subset[\n",
" subset[\"wickets\"] == row[\"wickets\"]\n",
" ]\n",
"\n",
" if len(subset) == 0:\n",
" return np.nan\n",
"\n",
" idx = (\n",
" abs(\n",
" subset[\"legal_ball\"]\n",
" - row[\"legal_ball\"]\n",
" )\n",
" .idxmin()\n",
" )\n",
"\n",
" resource = subset.loc[\n",
" idx,\n",
" \"resource_used\"\n",
" ]\n",
"\n",
"\n",
" return row[\"score\"] / resource\n",
"\n",
"\n",
"\n",
"# ==========================================\n",
"# Predict test set\n",
"# ==========================================\n",
"\n",
"dls_predictions = X_test.apply(\n",
" dls_predict,\n",
" axis=1\n",
")\n",
"\n",
"\n",
"# Remove failures\n",
"mask = ~dls_predictions.isna()\n",
"\n",
"dls_predictions = dls_predictions[mask]\n",
"dls_actual = y_test[mask]\n",
"\n",
"\n",
"# ==========================================\n",
"# Metrics\n",
"# ==========================================\n",
"\n",
"mae = mean_absolute_error(\n",
" dls_actual,\n",
" dls_predictions\n",
")\n",
"\n",
"rmse = np.sqrt(\n",
" mean_squared_error(\n",
" dls_actual,\n",
" dls_predictions\n",
" )\n",
")\n",
"\n",
"r2 = r2_score(\n",
" dls_actual,\n",
" dls_predictions\n",
")\n",
"\n",
"\n",
"print(\"======================\")\n",
"print(\"DLS BASELINE RESULTS\")\n",
"print(\"======================\")\n",
"print(f\"Rows evaluated : {len(dls_actual)}\")\n",
"print(f\"MAE : {mae:.2f} runs\")\n",
"print(f\"RMSE : {rmse:.2f} runs\")\n",
"print(f\"R² : {r2:.4f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5853752b",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {