Summary of what happened:
I’m trying to understand how the -J/--jobs parameter of datalad save affects execution. There is a -J/--jobs parameter for git annex add, and I had expected that passing a value to datalad would cause it to be passed to git annex. But that’s not what I’m seeing in the output of pstree.
Command used (and if a helper script was used, a link to the helper script or the command generated):
# create dataset
root=/tmp
for bytes in 100000; do
echo "bytes: ${bytes}"
for n_files in 10000; do
echo "n_files: ${n_files}"
for cmd in annex; do
echo "cmd: ${cmd}"
for J in 1; do
ds=$root/dataset
mkdir "${ds}" || exit \
&& cd "${ds}" \
&& datalad create -d "${ds}"
for file in $(seq 1 $n_files); do
truncate -s "${bytes}" "${file}".gii
done
done
done
done
done
$ datalad -l critical save --jobs 10 &
# get process id (could assign to variable)
$ pgrep -a "datalad"
4045408 /home/ubuntu/mambaforge/envs/snapshot-v-test/bin/python /home/ubuntu/mambaforge/envs/snapshot-v-test/bin/datalad -l critical save --jobs 10
# report subprocesses.
# This is where I would have expected to see --jobs 10 with git annex add
$ pstree -atc 4045408
datalad /home/ubuntu/mambaforge/envs/test/bin/datalad -l critical save --jobs 10
├─git -c diff.ignoreSubmodules=none -c core.quotepath=false annex add --json --json-error-messages -c annex.dotfiles=true -- 1.gii 10.gii 100.gii 1000.gii 10000.gii 1001.gii 1002.gii 1003.gii 1004.gii ...
│ └─git-annex add --json --json-error-messages -c annex.dotfiles=true -- 1.gii 10.gii 100.gii 1000.gii 10000.gii 1001.gii 1002.gii 1003.gii 1004.gii 1005.gii 1006.gii10
│ ├─git --git-dir=.git --work-tree=. --literal-pathspecs -c annex.dotfiles=true cat-file --batch
│ ├─(git)
│ ├─(git)
│ ├─(git)
│ ├─(git)
│ ├─(git)
│ ├─(git)
│ ├─(git)
│ ├─(git)
│ ├─git --git-dir=.git --work-tree=. --literal-pathspecs -c annex.dotfiles=true check-attr -z --stdin annex.backend annex.largefiles annex.numcopies annex.mincopies --
│ ├─git --git-dir=.git --work-tree=. --literal-pathspecs -c annex.dotfiles=true cat-file --batch-check=%(objectname) %(objecttype) %(objectsize)
│ ├─git --git-dir=.git --work-tree=. --literal-pathspecs -c annex.dotfiles=true hash-object -w --no-filters --stdin-paths
│ ├─{ghc_ticker}
│ ├─{git-annex:w}
│ ├─{git-annex:w}
│ ├─{git-annex:w}
│ └─{git-annex:w}
├─{datalad}
├─{datalad}
├─{datalad}
└─{datalad}
Version:
$ datalad --version
datalad 0.19.3
$ git annex version
git-annex version: 10.20230626-g801c4b7
build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.7.9 persistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
operating system: linux x86_64
supported repository versions: 8 9 10
upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
Environment (Docker, Singularity, custom installation):
$ uname -a
Linux instance-20230117-1301 5.15.0-1040-oracle #46-Ubuntu SMP Fri Jul 14 21:47:13 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
mamba environment
channels:
- conda-forge
dependencies:
- _libgcc_mutex=0.1=conda_forge
- _openmp_mutex=4.5=2_gnu
- annexremote=1.6.0=py311h38be061_1
- attr=2.5.1=h166bdaf_1
- boto=2.49.0=py_0
- brotli-python=1.1.0=py311hb755f60_1
- bzip2=1.0.8=h7f98852_4
- c-ares=1.20.1=hd590300_0
- ca-certificates=2023.7.22=hbcca054_0
- certifi=2023.7.22=pyhd8ed1ab_0
- cffi=1.16.0=py311hb3a22ac_0
- chardet=5.2.0=py311h38be061_1
- charset-normalizer=3.3.0=pyhd8ed1ab_0
- colorama=0.4.6=pyhd8ed1ab_0
- cryptography=41.0.4=py311h63ff55d_0
- curl=8.3.0=hca28451_0
- datalad=0.19.3=py311h38be061_0
- dbus=1.13.6=h5008d03_3
- distro=1.8.0=pyhd8ed1ab_0
- exifread=3.0.0=pyhd8ed1ab_0
- expat=2.5.0=hcb278e6_1
- fasteners=0.17.3=pyhd8ed1ab_0
- freetype=2.12.1=h267a509_2
- future=0.18.3=pyhd8ed1ab_0
- gettext=0.21.1=h27087fc_0
- git=2.42.0=pl5321h86e50cf_0
- git-annex=10.20230626=alldep_h97b9560_100
- gmp=6.2.1=h58526e2_0
- gnupg=2.4.2=h1cac74d_1
- humanize=4.8.0=pyhd8ed1ab_0
- idna=3.4=pyhd8ed1ab_0
- importlib-metadata=6.8.0=pyha770c72_0
- importlib_metadata=6.8.0=hd8ed1ab_0
- iso8601=2.1.0=pyhd8ed1ab_0
- jaraco.classes=3.3.0=pyhd8ed1ab_0
- jeepney=0.8.0=pyhd8ed1ab_0
- keyring=24.2.0=py311h38be061_1
- keyrings.alt=4.2.0=pyhd8ed1ab_0
- keyutils=1.6.1=h166bdaf_0
- krb5=1.21.2=h659d440_0
- lcms2=2.15=hb7c19ff_3
- ld_impl_linux-64=2.40=h41732ed_0
- lerc=4.0.0=h27087fc_0
- libassuan=2.5.6=h59595ed_0
- libcap=2.69=h0f662aa_0
- libcbor=0.10.2=hcb278e6_0
- libcurl=8.3.0=hca28451_0
- libdeflate=1.19=hd590300_0
- libedit=3.1.20191231=he28a2e2_2
- libev=4.33=h516909a_1
- libexpat=2.5.0=hcb278e6_1
- libffi=3.4.2=h7f98852_5
- libfido2=1.13.0=h2e5b2a7_0
- libgcc-ng=13.2.0=h807b86a_2
- libgcrypt=1.10.1=h166bdaf_0
- libglib=2.78.0=hebfc3b9_0
- libgomp=13.2.0=h807b86a_2
- libgpg-error=1.47=h71f35ed_0
- libiconv=1.17=h166bdaf_0
- libjpeg-turbo=3.0.0=hd590300_1
- libksba=1.6.4=h59595ed_0
- libmagic=5.39=h753d276_1
- libnghttp2=1.52.0=h61bc06f_0
- libnsl=2.0.0=hd590300_1
- libpng=1.6.39=h753d276_0
- libsqlite=3.43.0=h2797004_0
- libssh2=1.11.0=h0841786_0
- libstdcxx-ng=13.2.0=h7e041cc_2
- libtiff=4.6.0=ha9c0a0a_2
- libudev1=254=h3f72095_0
- libuuid=2.38.1=h0b41bf4_0
- libwebp-base=1.3.2=hd590300_0
- libxcb=1.15=h0b41bf4_0
- libzlib=1.2.13=hd590300_5
- looseversion=1.3.0=pyhd8ed1ab_0
- lsof=4.89=h7f98852_1
- lz4-c=1.9.4=hcb278e6_0
- more-itertools=10.1.0=pyhd8ed1ab_0
- msgpack-python=1.0.6=py311h9547e67_0
- mutagen=1.47.0=pyhd8ed1ab_0
- ncurses=6.4=hcb278e6_0
- npth=1.6=h59595ed_1002
- ntbtls=0.3.1=hfc55251_1
- openjpeg=2.5.0=h488ebb8_3
- openssh=9.3p1=h2d3b35a_2
- openssl=3.1.3=hd590300_0
- p7zip=16.02=h9c3ff4c_1001
- patool=1.12=py311h38be061_1007
- pcre2=10.40=hc3806b6_0
- perl=5.32.1=4_hd590300_perl5
- pillow=10.0.1=py311ha6c5da5_2
- pip=23.2.1=pyhd8ed1ab_0
- platformdirs=3.11.0=pyhd8ed1ab_0
- popt=1.16=h0b475e3_2002
- psutil=5.9.5=py311h459d7ec_1
- pthread-stubs=0.4=h36c2ea0_1001
- pycparser=2.21=pyhd8ed1ab_0
- pyperclip=1.8.2=pyhd8ed1ab_2
- pysocks=1.7.1=pyha2e5f31_6
- python=3.11.6=hab00c5b_0_cpython
- python-dateutil=2.8.2=pyhd8ed1ab_0
- python-gitlab=3.15.0=pyhd8ed1ab_0
- python_abi=3.11=4_cp311
- readline=8.2=h8228510_1
- requests=2.31.0=pyhd8ed1ab_0
- requests-ftp=0.3.1=py_1
- requests-toolbelt=1.0.0=pyhd8ed1ab_0
- rsync=3.2.7=h70740c4_0
- secretstorage=3.3.3=py311h38be061_2
- setuptools=68.2.2=pyhd8ed1ab_0
- simplejson=3.19.2=py311h459d7ec_0
- six=1.16.0=pyh6c4a22f_0
- sqlite=3.43.0=h2c6b66d_0
- tk=8.6.13=h2797004_0
- tqdm=4.66.1=pyhd8ed1ab_0
- typing-extensions=4.8.0=hd8ed1ab_0
- typing_extensions=4.8.0=pyha770c72_0
- tzdata=2023c=h71feb2d_0
- urllib3=2.0.6=pyhd8ed1ab_0
- wheel=0.41.2=pyhd8ed1ab_0
- whoosh=2.7.4=py311h38be061_8
- xorg-libxau=1.0.11=hd590300_0
- xorg-libxdmcp=1.1.3=h7f98852_0
- xxhash=0.8.0=h7f98852_3
- xz=5.2.6=h166bdaf_0
- zipp=3.17.0=pyhd8ed1ab_0
- zlib=1.2.13=hd590300_5
- zstd=1.5.5=hfc55251_0
Data formatted according to a validatable standard? Please provide the output of the validator:
n/a
Relevant log outputs (up to 20 lines):
see above
Screenshots / relevant information:
thank you!