Summary of what happened:
I have used the xcpd to perform denoise for hcp minimal preprocessed fMRI data to obtain the nifti data. I used the xcpd 0.6.1rc1 to convert the hcp data to bids-style. And then, I used the xcpd to perform denoise using the bids-style data (–input-tyle fmriprep). The xcpd can running this process with no error, but it stuck in “denoise_bold_wf.regress_and_filter_bold” more than 12 hours. Is there anything wrong? Thank you very much for your help！
Command used (and if a helper script was used, a link to the helper script or the command generated):
step 1st: using xcpd 0.6.1rc1 to convert hcp data to bids style
#!/bin/bash
#SBATCH --job-name=xcpd
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=4
#SBATCH --mem-per-cpu 20G
#SBATCH -p q_fat_c
#SBATCH --qos=high_c
#SBATCH -o ./job.%j.out
#SBATCH -e ./job.%j.error.txt
module load singularity
subj=100610
Path=/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/xcpd0.3.2
hcpminiprep=/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/hcp_rest_1_miniprep
# Unzip rest-fMRI-1 data
for zipfile in /GPFS/PUB/ALL_Family_Subjects/Resting_State_fMRI_1_Preprocessed/${subj}*.zip
do
# Create a directory with the same name as the zip file (excluding the .zip extension)
# dirname=$(basename "$zipfile" .zip)
# mkdir -p "/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/hcp_unzip/$dirname"
# Create a common directory for all unzipped files (if it doesn't already exist)
mkdir -p $hcpminiprep
# Unzip the zip file to the corresponding directory without overwriting existing files
unzip -n "$zipfile" -d $hcpminiprep
done
# Unzip T1w data
for zipfile in /GPFS/PUB/ALL_Family_Subjects/Structural_Preprocessed/${subj}*.zip
do
# Unzip the zip file to the specified directory without overwriting existing files
unzip -n "$zipfile" -d $hcpminiprep
done
xcpd_op=${Path}/step_1st_hcp2bids
mkdir -p $xcpd_op
xcpd_wp=${Path}/step_1st_wd/wd_$subj
mkdir -p $xcpd_wp
fslic=/ibmgpfs/cuizaixu_lab/xulongzhou/tool/freesurfer
templateflow=/ibmgpfs/cuizaixu_lab/xulongzhou/tool/templateflow
#Run xcpd
echo ""
echo "Running xcpd on participant: sub-$subj"
echo ""
# convert hcp to bids
unset PYTHONPATH
export SINGULARITYENV_TEMPLATEFLOW_HOME=$templateflow
singularity run --cleanenv \
-B $hcpminiprep:/hcpminiprep \
-B $xcpd_op:/output \
-B $xcpd_wp:/work \
-B $fslic:/fslic \
-B $templateflow:$templateflow \
/ibmgpfs/cuizaixu_lab/xulongzhou/apps/singularity_update/xcpd-0.6.1rc1.simg \
/hcpminiprep /output participant --participant_label ${subj} \
-w /work --nthreads 4 --omp-nthreads 4 --mem_gb 80 \
--fs-license-file /fslic/license.txt \
--input-type hcp \
--nuisance-regressors none \
--disable-bandpass-filter \
--fd-thresh -1 \
--resource-monitor
step 2nd: constuct the fMRIprep data to complete xcpd needed
module load singularity
subj=$1
Path=/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/xcpd0.3.2
customCP=/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/custom_confounds_csf_global_2p
fslic=/ibmgpfs/cuizaixu_lab/xulongzhou/tool/freesurfer
templateflow=/ibmgpfs/cuizaixu_lab/xulongzhou/tool/templateflow
# construct bids-style file
bids=${Path}/bids
mkdir -p ${bids}/sub-${subj}
cp -r ${Path}/step_1st_wd/wd_$subj/dset_bids/derivatives/hcp/sub-${subj}/func ${Path}/bids/sub-${subj}/
cp -r ${Path}/step_1st_wd/wd_$subj/dset_bids/derivatives/hcp/sub-${subj}/anat ${Path}/bids/sub-${subj}/
# construct custom confounds file
module load MATLAB
filePath=${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-LR_run-1_desc-confounds_timeseries.tsv
savePath=${customCP}/sub-${subj}_task-rest_dir-LR_run-1_desc-confounds_timeseries.tsv
rm -rf ${savePath}
matlab -nodisplay -nosplash -nodesktop -r \
"extract_columns_by_title('${filePath}', '${savePath}', {'global_signal', 'csf'}); exit;"
filePath=${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-RL_run-1_desc-confounds_timeseries.tsv
savePath=${customCP}/sub-${subj}_task-rest_dir-RL_run-1_desc-confounds_timeseries.tsv
rm -rf ${savePath}
matlab -nodisplay -nosplash -nodesktop -r \
"extract_columns_by_title('${filePath}', '${savePath}', {'global_signal', 'csf'}); exit;"
# add the BIDS version in the json file
cp ${Path}/dataset_description.json ${bids}/dataset_description.json
# construct fmriprep-style file structure
cp ${bids}/sub-${subj}/anat/sub-${subj}_space-MNI152NLin6Asym_res-2_desc-preproc_T1w.nii.gz ${bids}/sub-${subj}/anat/sub-${subj}_desc-preproc_T1w.nii.gz
cp ${bids}/sub-${subj}/anat/sub-${subj}_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz ${bids}/sub-${subj}/anat/sub-${subj}_desc-brain_mask.nii.gz
cp ${bids}/sub-${subj}/anat/sub-${subj}_space-MNI152NLin6Asym_res-2_desc-aparcaseg_dseg.nii.gz ${bids}/sub-${subj}/anat/sub-${subj}_desc-aparcaseg_dseg.nii.gz
cp ${bids}/sub-${subj}/anat/sub-${subj}_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz ${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-RL_run-1_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz
cp ${bids}/sub-${subj}/anat/sub-${subj}_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz ${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-LR_run-1_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz
cp ${bids}/sub-${subj}/anat/sub-${subj}_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt ${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-LR_run-1_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt
cp ${bids}/sub-${subj}/anat/sub-${subj}_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt ${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-RL_run-1_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt
cp ${bids}/sub-${subj}/anat/sub-${subj}_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt ${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-LR_run-1_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt
cp ${bids}/sub-${subj}/anat/sub-${subj}_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt ${bids}/sub-${subj}/func/sub-${subj}_task-rest_dir-RL_run-1_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt
step 3rd: running xcpd using fmriprep style to obtain denoised nifti data
module load singularity
subj=100610
Path=/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/xcpd0.3.2
fslic=/ibmgpfs/cuizaixu_lab/xulongzhou/tool/freesurfer
templateflow=/ibmgpfs/cuizaixu_lab/xulongzhou/tool/templateflow
custom_confound=/ibmgpfs/cuizaixu_lab/xulongzhou/HCP_WMfMRI/custom_confounds_csf_global_2p
output=${Path}/step_2nd_24PcsfGlobal
mkdir -p ${output}
wd=${Path}/step_2nd_wd/sub-${subj}
mkdir -p ${wd}
# running xcpd
unset PYTHONPATH
export SINGULARITYENV_TEMPLATEFLOW_HOME=$templateflow
singularity run --cleanenv \
-B $Path/bids:/bids \
-B $output:/output \
-B $wd:/wd \
-B $custom_confound:/custom_confounds \
-B $fslic:/fslic \
-B $templateflow:$templateflow \
/ibmgpfs/cuizaixu_lab/xulongzhou/apps/singularity_update/xcpd-0.6.1rc1.simg \
/bids /output participant \
--participant_label ${subj} --task-id rest \
--input-type fmriprep \
--fs-license-file /fslic/license.txt \
-w /wd --nthreads 16 --omp-nthreads 8 --mem-gb 320 \
--nuisance-regressors 24P --despike -c /custom_confounds \
--lower-bpf=0.01 --upper-bpf=0.1 \
--dummy-scans auto \
--fd-thresh -1
Version:
singularity version 3.7.0+61-g920a6a4-dirty
xcp_d 0.6.1rc1
bids-style file
bids
├── dataset_description.json
└── sub-100610
├── anat
│ ├── sub-100610_desc-aparcaseg_dseg.nii.gz
│ ├── sub-100610_desc-brain_mask.nii.gz
│ ├── sub-100610_desc-preproc_T1w.nii.gz
│ ├── sub-100610_dseg.nii.gz
│ ├── sub-100610_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt
│ ├── sub-100610_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt
│ ├── sub-100610_space-fsLR_den-32k_hemi-L_pial.surf.gii
│ ├── sub-100610_space-fsLR_den-32k_hemi-L_smoothwm.surf.gii
│ ├── sub-100610_space-fsLR_den-32k_hemi-R_pial.surf.gii
│ ├── sub-100610_space-fsLR_den-32k_hemi-R_smoothwm.surf.gii
│ ├── sub-100610_space-fsLR_den-91k_curv.dscalar.nii
│ ├── sub-100610_space-fsLR_den-91k_desc-corrected_thickness.dscalar.nii
│ ├── sub-100610_space-fsLR_den-91k_desc-smoothed_myelinw.dscalar.nii
│ ├── sub-100610_space-fsLR_den-91k_myelinw.dscalar.nii
│ ├── sub-100610_space-fsLR_den-91k_sulc.dscalar.nii
│ ├── sub-100610_space-fsLR_den-91k_thickness.dscalar.nii
│ ├── sub-100610_space-MNI152NLin6Asym_res-2_desc-aparcaseg_dseg.nii.gz
│ ├── sub-100610_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz
│ ├── sub-100610_space-MNI152NLin6Asym_res-2_desc-preproc_T1w.nii.gz
│ ├── sub-100610_space-MNI152NLin6Asym_res-2_desc-ribbon_T1w.nii.gz
│ └── sub-100610_space-MNI152NLin6Asym_res-2_dseg.nii.gz
└── func
├── sub-100610_task-rest_dir-LR_run-1_desc-confounds_timeseries.json
├── sub-100610_task-rest_dir-LR_run-1_desc-confounds_timeseries.tsv
├── sub-100610_task-rest_dir-LR_run-1_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt
├── sub-100610_task-rest_dir-LR_run-1_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt
├── sub-100610_task-rest_dir-LR_run-1_space-fsLR_den-91k_bold.dtseries.json
├── sub-100610_task-rest_dir-LR_run-1_space-fsLR_den-91k_bold.dtseries.nii
├── sub-100610_task-rest_dir-LR_run-1_space-MNI152NLin6Asym_res-2_boldref.nii.gz
├── sub-100610_task-rest_dir-LR_run-1_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz
├── sub-100610_task-rest_dir-LR_run-1_space-MNI152NLin6Asym_res-2_desc-preproc_bold.json
├── sub-100610_task-rest_dir-LR_run-1_space-MNI152NLin6Asym_res-2_desc-preproc_bold.nii.gz
├── sub-100610_task-rest_dir-RL_run-1_desc-confounds_timeseries.json
├── sub-100610_task-rest_dir-RL_run-1_desc-confounds_timeseries.tsv
├── sub-100610_task-rest_dir-RL_run-1_from-MNI152NLin6Asym_to-T1w_mode-image_xfm.txt
├── sub-100610_task-rest_dir-RL_run-1_from-T1w_to-MNI152NLin6Asym_mode-image_xfm.txt
├── sub-100610_task-rest_dir-RL_run-1_space-fsLR_den-91k_bold.dtseries.json
├── sub-100610_task-rest_dir-RL_run-1_space-fsLR_den-91k_bold.dtseries.nii
├── sub-100610_task-rest_dir-RL_run-1_space-MNI152NLin6Asym_res-2_boldref.nii.gz
├── sub-100610_task-rest_dir-RL_run-1_space-MNI152NLin6Asym_res-2_desc-brain_mask.nii.gz
├── sub-100610_task-rest_dir-RL_run-1_space-MNI152NLin6Asym_res-2_desc-preproc_bold.json
└── sub-100610_task-rest_dir-RL_run-1_space-MNI152NLin6Asym_res-2_desc-preproc_bold.nii.gz
Relevant log outputs (up to 20 lines):
240408-19:59:51,377 nipype.workflow INFO:
[Node] Finished "despike3d", elapsed time 112.259767s.
240408-19:59:53,167 nipype.workflow INFO:
[Job 51] Completed (xcpd_wf.single_subject_100610_wf.nifti_postprocess_1_wf.despike_wf.despike3d).
240408-19:59:53,168 nipype.workflow INFO:
[MultiProc] Running 1 tasks, and 21 jobs ready. Free memory (GB): 304.00/320.00, Free processors: 8/16.
Currently running:
* xcpd_wf.single_subject_100610_wf.nifti_postprocess_0_wf.denoise_bold_wf.regress_and_filter_bold
240408-19:59:53,260 nipype.workflow INFO:
[Node] Setting-up "xcpd_wf.single_subject_100610_wf.nifti_postprocess_1_wf.prepare_confounds_wf.censor_report" in "/wd/xcpd_wf/single_subject_100610_wf/nifti_postprocess_1_wf/prepare_confounds_wf/censor_report".
240408-19:59:53,266 nipype.workflow INFO:
[Node] Executing "censor_report" <xcp_d.interfaces.plotting.CensoringPlot>
240408-19:59:53,378 nipype.workflow INFO:
[Node] Finished "censor_report", elapsed time 0.111797s.
240408-19:59:55,171 nipype.workflow INFO:
[Job 68] Completed (xcpd_wf.single_subject_100610_wf.nifti_postprocess_1_wf.prepare_confounds_wf.censor_report).
240408-19:59:55,172 nipype.workflow INFO:
[MultiProc] Running 1 tasks, and 21 jobs ready. Free memory (GB): 304.00/320.00, Free processors: 8/16.
Currently running:
* xcpd_wf.single_subject_100610_wf.nifti_postprocess_0_wf.denoise_bold_wf.regress_and_filter_bold
240408-19:59:55,263 nipype.workflow INFO:
[Node] Setting-up "xcpd_wf.single_subject_100610_wf.nifti_postprocess_1_wf.denoise_bold_wf.regress_and_filter_bold" in "/wd/xcpd_wf/single_subject_100610_wf/nifti_postprocess_1_wf/denoise_bold_wf/regress_and_filter_bold".
240408-19:59:55,268 nipype.workflow INFO:
[Node] Executing "regress_and_filter_bold" <xcp_d.interfaces.nilearn.DenoiseNifti>
240408-19:59:57,178 nipype.workflow INFO:
[MultiProc] Running 2 tasks, and 20 jobs ready. Free memory (GB): 288.00/320.00, Free processors: 0/16.
Currently running:
* xcpd_wf.single_subject_100610_wf.nifti_postprocess_1_wf.denoise_bold_wf.regress_and_filter_bold
* xcpd_wf.single_subject_100610_wf.nifti_postprocess_0_wf.denoise_bold_wf.regress_and_filter_bold
_____