Solution and implementation for Q2 from Machine Learning (ml).
# Import required libraries import pandas as pd import numpy as np import matplotlib.pyplot as plt from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline from sklearn.linear_model import LinearRegression, Ridge, Lasso from sklearn.metrics import r2_score, mean_squared_error
# Load the Uber fares dataset
data = pd.read_csv("uber.csv")
print(data.shape)
data.head()# Remove unnecessary columns and prepare the dataset
data = data.drop(columns=["key", "Unnamed: 0"], errors="ignore")
data["pickup_datetime"] = pd.to_datetime(data["pickup_datetime"], errors="coerce", utc=True)
data = data.dropna(subset=[
"fare_amount", "pickup_datetime", "pickup_longitude", "pickup_latitude",
"dropoff_longitude", "dropoff_latitude", "passenger_count"
])
data = data[
(data["fare_amount"] > 0) &
(data["pickup_longitude"].between(-180, 180)) &
(data["dropoff_longitude"].between(-180, 180)) &
(data["pickup_latitude"].between(-90, 90)) &
(data["dropoff_latitude"].between(-90, 90)) &
(data["pickup_longitude"] != 0) &
(data["pickup_latitude"] != 0) &
(data["dropoff_longitude"] != 0) &
(data["dropoff_latitude"] != 0) &
(data["passenger_count"].between(1, 6))
].copy()
data.head()# Create date and time features data["pickup_hour"] = data["pickup_datetime"].dt.hour data["pickup_day"] = data["pickup_datetime"].dt.day data["pickup_month"] = data["pickup_datetime"].dt.month data["pickup_weekday"] = data["pickup_datetime"].dt.dayofweek
# Calculate trip distance using the Haversine formula lat1 = np.radians(data["pickup_latitude"]) lat2 = np.radians(data["dropoff_latitude"]) lon1 = np.radians(data["pickup_longitude"]) lon2 = np.radians(data["dropoff_longitude"]) dlat = lat2 - lat1 dlon = lon2 - lon1 a = np.sin(dlat / 2) ** 2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon / 2) ** 2 data["distance_km"] = 6371 * 2 * np.arcsin(np.sqrt(a)) data = data[data["distance_km"] > 0].copy() data[["fare_amount", "distance_km"]].head()
# Identify outliers using the IQR method
def get_iqr_bounds(series):
q1 = series.quantile(0.25)
q3 = series.quantile(0.75)
iqr = q3 - q1
return q1 - 1.5 * iqr, q3 + 1.5 * iqr
fare_low, fare_high = get_iqr_bounds(data["fare_amount"])
distance_low, distance_high = get_iqr_bounds(data["distance_km"])
fare_outliers = ~data["fare_amount"].between(fare_low, fare_high)
distance_outliers = ~data["distance_km"].between(distance_low, distance_high)
print("Fare outliers:", fare_outliers.sum())
print("Distance outliers:", distance_outliers.sum())
print("Rows before removing outliers:", len(data))
data = data[
data["fare_amount"].between(fare_low, fare_high) &
data["distance_km"].between(distance_low, distance_high)
].copy()
print("Rows after removing outliers:", len(data))# Check feature correlations
features = [
"pickup_longitude", "pickup_latitude", "dropoff_longitude",
"dropoff_latitude", "passenger_count", "pickup_hour", "pickup_day",
"pickup_month", "pickup_weekday", "distance_km"
]
correlation = data[features + ["fare_amount"]].corr()
print(correlation["fare_amount"].sort_values(ascending=False))
plt.figure(figsize=(10, 7))
plt.imshow(correlation, cmap="coolwarm", vmin=-1, vmax=1)
plt.colorbar(label="Correlation")
plt.xticks(range(len(correlation.columns)), correlation.columns, rotation=90)
plt.yticks(range(len(correlation.columns)), correlation.columns)
plt.title("Correlation Heatmap")
plt.tight_layout()
plt.show()# Split the data into input features and target
X = data[features]
y = data["fare_amount"]
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)# Train linear regression, Ridge, and Lasso models
models = {
"Linear Regression": make_pipeline(StandardScaler(), LinearRegression()),
"Ridge Regression": make_pipeline(StandardScaler(), Ridge(alpha=1.0)),
"Lasso Regression": make_pipeline(StandardScaler(), Lasso(alpha=0.1, max_iter=10000))
}
results = []
for name, model in models.items():
model.fit(X_train, y_train)
predictions = model.predict(X_test)
results.append({
"Model": name,
"R2 Score": r2_score(y_test, predictions),
"RMSE": np.sqrt(mean_squared_error(y_test, predictions))
})
results_df = pd.DataFrame(results)
results_df# Compare model R2 scores and RMSE values
print(results_df.sort_values("R2 Score", ascending=False).to_string(index=False))
plt.figure(figsize=(8, 5))
plt.bar(results_df["Model"], results_df["R2 Score"])
plt.ylabel("R2 Score")
plt.title("Regression Model Comparison")
plt.xticks(rotation=15)
plt.tight_layout()
plt.show()
plt.figure(figsize=(8, 5))
plt.bar(results_df["Model"], results_df["RMSE"])
plt.ylabel("RMSE")
plt.title("RMSE Comparison")
plt.xticks(rotation=15)
plt.tight_layout()
plt.show()# Import required libraries import pandas as pd import numpy as np import matplotlib.pyplot as plt from sklearn.datasets import load_diabetes from sklearn.model_selection import train_test_split from sklearn.impute import SimpleImputer from sklearn.preprocessing import StandardScaler from sklearn.pipeline import make_pipeline from sklearn.linear_model import LinearRegression, LogisticRegression from sklearn.metrics import r2_score, mean_squared_error, accuracy_score, roc_auc_score
# Load the built-in diabetes progression dataset with a continuous target
uci = load_diabetes(as_frame=True)
uci_data = uci.data.copy()
uci_data["Target"] = uci.target
print("Continuous-target diabetes dataset shape:", uci_data.shape)
uci_data.head()# Load the Pima Indians diabetes dataset
pima_data = pd.read_csv("diabetes.csv")
print("Pima dataset shape:", pima_data.shape)
pima_data.head()# Prepare the Pima dataset by treating impossible zero measurements as missing
pima_data = pima_data.copy()
zero_missing_columns = ["Glucose", "BloodPressure", "SkinThickness", "Insulin", "BMI"]
for column in zero_missing_columns:
if column in pima_data.columns:
pima_data[column] = pima_data[column].replace(0, np.nan)
print("Missing values in Pima dataset:")
print(pima_data.isnull().sum())# Calculate univariate statistics for numerical columns
def univariate_analysis(frame):
numeric = frame.select_dtypes(include=np.number)
summary = pd.DataFrame({
"Frequency": numeric.count(),
"Missing": numeric.isnull().sum(),
"Unique Values": numeric.nunique(),
"Mean": numeric.mean(),
"Median": numeric.median(),
"Mode": numeric.mode().iloc[0],
"Variance": numeric.var(),
"Standard Deviation": numeric.std(),
"Skewness": numeric.skew(),
"Kurtosis": numeric.kurtosis()
})
return summary.round(3)
print("Continuous-target Diabetes Univariate Analysis")
display(univariate_analysis(uci_data))
print("Pima Indians Diabetes Univariate Analysis")
display(univariate_analysis(pima_data))# Display target frequency counts for both datasets
print("Continuous-target diabetes summary:")
print(uci_data["Target"].describe())
print("\nPima outcome frequency:")
print(pima_data["Outcome"].value_counts().sort_index())
# Create a high/low progression label for logistic regression on the continuous UCI target
uci_median_target = uci_data["Target"].median()
uci_data["High_Progression"] = (uci_data["Target"] > uci_median_target).astype(int)
print("\nUCI high/low progression frequency:")
print(uci_data["High_Progression"].value_counts().sort_index())# Run bivariate linear regression for each feature
def bivariate_linear_results(frame, feature_columns, target_column):
results = []
for feature in feature_columns:
X = frame[[feature]]
y = frame[target_column]
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)
model = make_pipeline(SimpleImputer(strategy="median"), StandardScaler(), LinearRegression())
model.fit(X_train, y_train)
predictions = model.predict(X_test)
results.append({
"Feature": feature,
"R2": r2_score(y_test, predictions),
"RMSE": np.sqrt(mean_squared_error(y_test, predictions))
})
return pd.DataFrame(results).sort_values("R2", ascending=False)
uci_features = list(uci.data.columns)
pima_features = [column for column in pima_data.columns if column != "Outcome"]
print("Continuous-target bivariate linear regression results")
uci_bivariate_linear = bivariate_linear_results(uci_data, uci_features, "Target")
display(uci_bivariate_linear)
print("Pima bivariate linear regression results")
pima_bivariate_linear = bivariate_linear_results(pima_data, pima_features, "Outcome")
display(pima_bivariate_linear)# Run bivariate logistic regression for each feature
def bivariate_logistic_results(frame, feature_columns, target_column):
results = []
for feature in feature_columns:
X = frame[[feature]]
y = frame[target_column].astype(int)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42, stratify=y
)
model = make_pipeline(
SimpleImputer(strategy="median"),
StandardScaler(),
LogisticRegression(max_iter=2000)
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
probabilities = model.predict_proba(X_test)[:, 1]
results.append({
"Feature": feature,
"Accuracy": accuracy_score(y_test, predictions),
"ROC AUC": roc_auc_score(y_test, probabilities)
})
return pd.DataFrame(results).sort_values("ROC AUC", ascending=False)
print("Continuous-target logistic regression results (high vs low progression)")
uci_bivariate_logistic = bivariate_logistic_results(
uci_data, uci_features, "High_Progression"
)
display(uci_bivariate_logistic)
print("Pima bivariate logistic regression results (diabetes outcome)")
pima_bivariate_logistic = bivariate_logistic_results(
pima_data, pima_features, "Outcome"
)
display(pima_bivariate_logistic)# Perform multiple linear regression using all features
def multiple_linear_analysis(frame, feature_columns, target_column, dataset_name):
X = frame[feature_columns]
y = frame[target_column]
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42
)
model = make_pipeline(
SimpleImputer(strategy="median"),
StandardScaler(),
LinearRegression()
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
return {
"Dataset": dataset_name,
"R2": r2_score(y_test, predictions),
"RMSE": np.sqrt(mean_squared_error(y_test, predictions))
}, model
uci_multiple_linear, uci_linear_model = multiple_linear_analysis(
uci_data, uci_features, "Target", "Continuous-target Diabetes"
)
pima_multiple_linear, pima_linear_model = multiple_linear_analysis(
pima_data, pima_features, "Outcome", "Pima Indians"
)
multiple_linear_comparison = pd.DataFrame([uci_multiple_linear, pima_multiple_linear])
display(multiple_linear_comparison)# Perform multiple logistic regression using all features
def multiple_logistic_analysis(frame, feature_columns, target_column, dataset_name):
X = frame[feature_columns]
y = frame[target_column].astype(int)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.2, random_state=42, stratify=y
)
model = make_pipeline(
SimpleImputer(strategy="median"),
StandardScaler(),
LogisticRegression(max_iter=2000)
)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
probabilities = model.predict_proba(X_test)[:, 1]
return {
"Dataset": dataset_name,
"Accuracy": accuracy_score(y_test, predictions),
"ROC AUC": roc_auc_score(y_test, probabilities)
}, model
uci_multiple_logistic, uci_logistic_model = multiple_logistic_analysis(
uci_data, uci_features, "High_Progression", "Continuous-target Diabetes (high/low progression)"
)
pima_multiple_logistic, pima_logistic_model = multiple_logistic_analysis(
pima_data, pima_features, "Outcome", "Pima Indians (diabetes outcome)"
)
multiple_logistic_comparison = pd.DataFrame([uci_multiple_logistic, pima_multiple_logistic])
display(multiple_logistic_comparison)# Compare the bivariate and multiple regression results
linear_comparison = pd.DataFrame([
{"Dataset": "Continuous-target Diabetes", "Best Bivariate R2": uci_bivariate_linear["R2"].max(),
"Multiple R2": uci_multiple_linear["R2"], "Multiple RMSE": uci_multiple_linear["RMSE"]},
{"Dataset": "Pima Indians", "Best Bivariate R2": pima_bivariate_linear["R2"].max(),
"Multiple R2": pima_multiple_linear["R2"], "Multiple RMSE": pima_multiple_linear["RMSE"]}
])
logistic_comparison = pd.DataFrame([
{"Dataset": "Continuous-target Diabetes (high/low progression)",
"Best Bivariate Accuracy": uci_bivariate_logistic["Accuracy"].max(),
"Best Bivariate ROC AUC": uci_bivariate_logistic["ROC AUC"].max(),
"Multiple Accuracy": uci_multiple_logistic["Accuracy"],
"Multiple ROC AUC": uci_multiple_logistic["ROC AUC"]},
{"Dataset": "Pima Indians (diabetes outcome)",
"Best Bivariate Accuracy": pima_bivariate_logistic["Accuracy"].max(),
"Best Bivariate ROC AUC": pima_bivariate_logistic["ROC AUC"].max(),
"Multiple Accuracy": pima_multiple_logistic["Accuracy"],
"Multiple ROC AUC": pima_multiple_logistic["ROC AUC"]}
])
print("Linear regression comparison")
display(linear_comparison.round(3))
print("Logistic regression comparison")
display(logistic_comparison.round(3))# Plot multiple linear regression R2 scores and logistic regression ROC AUC scores
plt.figure(figsize=(8, 5))
plt.bar(multiple_linear_comparison["Dataset"], multiple_linear_comparison["R2"])
plt.ylabel("R2 Score")
plt.title("Multiple Linear Regression Comparison")
plt.xticks(rotation=15)
plt.tight_layout()
plt.show()
plt.figure(figsize=(9, 5))
plt.bar(multiple_logistic_comparison["Dataset"], multiple_logistic_comparison["ROC AUC"])
plt.ylabel("ROC AUC")
plt.title("Multiple Logistic Regression Comparison")
plt.xticks(rotation=15)
plt.tight_layout()
plt.show()