I’ve been noticing in the log output that even when requesting larger amounts of virtual memory (250gb), I still run into memory allocation issues:
Fatal Python error: Segmentation fault
Thread 0x00002ae34d4ee700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae34d8f0700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae34d2ed700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae34d6ef700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Current thread 0x00002ae34def3700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/site-packages/numpy/linalg/linalg.py”, line 2010 in det
File “/usr/local/miniconda/lib/python3.6/site-packages/nibabel/nifti1.py”, line 1015 in set_qform
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/utils.py”, line 175 in _copyxform
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 29 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae34e0f4700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae34dcf2700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae34daf1700 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/selectors.py”, line 376 in select
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 1514 in _communicate
File “/usr/local/miniconda/lib/python3.6/subprocess.py”, line 843 in communicate
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 834 in run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 1030 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/niworkflows/interfaces/fixes.py”, line 24 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 262 in _applytfms
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 56 in run
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/thread.py”, line 69 in _worker
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 864 in run
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 916 in _bootstrap_inner
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 884 in _bootstrap
Thread 0x00002ae315603440 (most recent call first):
File “/usr/local/miniconda/lib/python3.6/threading.py”, line 295 in wait
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/_base.py”, line 427 in result
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/_base.py”, line 586 in result_iterator
File “/usr/local/miniconda/lib/python3.6/site-packages/fmriprep/interfaces/itk.py”, line 143 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/interfaces/base/core.py”, line 521 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/pipeline/engine/nodes.py”, line 644 in _run_command
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/pipeline/engine/nodes.py”, line 564 in _run_interface
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/pipeline/engine/nodes.py”, line 480 in run
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/pipeline/plugins/multiproc.py”, line 69 in run_node
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/process.py”, line 175 in _process_worker
File “/usr/local/miniconda/lib/python3.6/multiprocessing/process.py”, line 93 in run
File “/usr/local/miniconda/lib/python3.6/multiprocessing/process.py”, line 258 in _bootstrap
File “/usr/local/miniconda/lib/python3.6/multiprocessing/spawn.py”, line 118 in _main
File “/usr/local/miniconda/lib/python3.6/multiprocessing/forkserver.py”, line 231 in _serve_one
File “/usr/local/miniconda/lib/python3.6/multiprocessing/forkserver.py”, line 196 in main
File “”, line 1 in
exception calling callback for <Future at 0x2b7d6fb67ba8 state=finished raised BrokenProcessPool>
Traceback (most recent call last):
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/_base.py”, line 324, in _invoke_callbacks
callback(self)
File “/usr/local/miniconda/lib/python3.6/site-packages/nipype/pipeline/plugins/multiproc.py”, line 143, in _async_callback
result = args.result()
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/_base.py”, line 425, in result
return self.__get_result()
File “/usr/local/miniconda/lib/python3.6/concurrent/futures/_base.py”, line 384, in __get_result
raise self._exception
concurrent.futures.process.BrokenProcessPool: A process in the process pool was terminated abruptly while the future was running or pending.
Thanks again.