Hi All I’ve been stuck on this problem for a while. I can’t match the accuracy of the Nilearn Decoder in Sklearn no matter how hard I try. Is there something obvious I’m doing wrong? Nilean gives much higher accuracy scores.
Code below:
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
from sklearn.model_selection import LeaveOneOut
from sklearn.metrics import accuracy_score
from nilearn.maskers import NiftiMasker
masker = NiftiMasker(
mask_img=ffa_img,
runs=final_df['run_value'],
standardize="zscore_sample",
memory="nilearn_cache",
memory_level=1,
)
fmri_masked = masker.fit_transform(final_df['0_x'])
from sklearn.model_selection import cross_val_score
from sklearn.svm import SVC
svc = LinearSVC(penalty="l1", C=c, dual=False) # SVC with L1 regularization
fmri_masked.shape
# Step 1: Standardization (z-score standardization across samples)
# Masked fMRI data is already provided in `fmri_masked`, assumed to be of shape (n_samples, n_features)
X = fmri_masked # The masked fMRI data (n_samples, n_features)
y = final_df['0_y'].values # Classification labels (n_samples,)
# Step 2: Define the classifier (L1-penalized SVM)
svc = LinearSVC(penalty="l1", C=0.1, dual=False)
# Step 3: Define the cross-validation strategy (Leave-One-Out Cross-Validation)
cv = LeaveOneOut() # This aligns with the original step
# Step 4: Manual cross-validation loop
# Initialize an array to hold results
accuracy_scores = []
# Step 5: Leave-One-Out Cross-Validation
for train_idx, test_idx in cv.split(X):
# Standardize within the fold (on training data only)
scaler = StandardScaler()
# Extract training and test sets using the indices from the cross-validation split
X_train = scaler.fit_transform(X[train_idx]) # Fit the scaler on training data
X_test = scaler.transform(X[test_idx]) # Transform the test data using the same scaler
y_train = y[train_idx] # Extract the corresponding labels for training
y_test = y[test_idx] # Extract the corresponding labels for test
# Step 6: Train the model on training data
svc.fit(X_train, y_train)
# Step 7: Predict the test set and evaluate accuracy
y_pred = svc.predict(X_test)
accuracy = accuracy_score(y_test, y_pred)
# Append the accuracy score to the list
accuracy_scores.append(accuracy)
# Step 8: Calculate the mean accuracy across all folds
mean_accuracy = np.mean(accuracy_scores)
# Output the results
print(f"Mean accuracy using Leave-One-Out CV: {mean_accuracy:.3f}")