diff --git a/.hunspell.en.dic b/.hunspell.en.dic index 1ffeab742..846a6d6c8 100644 --- a/.hunspell.en.dic +++ b/.hunspell.en.dic @@ -1312,6 +1312,9 @@ profiler moduleTop struct load2 +load3 +ms +stopmod list2 lrem replaceFromList diff --git a/MIGRATING.rst b/MIGRATING.rst index a737971fe..03d4489d4 100644 --- a/MIGRATING.rst +++ b/MIGRATING.rst @@ -38,21 +38,44 @@ have been implemented: match the module root name specification, instead of comparing against all loaded modules. +* Lighter-weight resolution of an exact ``name/version`` specification: sites + with a modulepath directory holding hundreds or thousands of sibling + modulefiles reported slow ``module load`` commands even when the requested + version already exists. Correctly computing the implicit ``default`` and + ``latest`` symbolic versions of such a directory used to require validating + every sibling modulefile in it. The lookup now walks candidate versions from + the dictionary-highest to the dictionary-lowest and stops validating as soon + as this can be answered: either a higher valid version is found (the + requested one is then known not to be ``default`` or ``latest``), or the + requested version itself is reached, in which case + nothing higher exists. This bypasses the exhaustive scan without changing + the resolved result. + For this work, the :command:`mb` utility was extended with new benchmark tests -that loads 137 modulefiles, lists the loaded modules, and then purges them. -Running this benchmark highlights the performance improvements, as shown in -the table below: +that loads 137 modulefiles (``load2``), lists the loaded modules (``list2``), +and then purges them. Running this benchmark highlights the performance +improvements, as shown in the table below: +---------+-------------+--------------------+ | | v5.6.1 | v5.7.0 | +=========+=============+====================+ -| load | 1038 ms | 384 ms (-63%) | +| load2 | 1038 ms | 384 ms (-63%) | +---------+-------------+--------------------+ -| list | 222 ms | 29 ms (-87%) | +| list2 | 222 ms | 29 ms (-87%) | +---------+-------------+--------------------+ | purge | 568 ms | 303 ms (-46%) | +---------+-------------+--------------------+ +A ``load3`` benchmark test was also added, loading a low, non-default +version of a module out of 500 sibling versions in the same directory. It +specifically exercises the lighter-weight resolution described above: + ++---------+-------------+--------------------+ +| | v5.6.1 | v5.7.0 | ++=========+=============+====================+ +| load3 | 30 ms | 19 ms (-36%) | ++---------+-------------+--------------------+ + These measurements were obtained on a system where all modulefiles are stored on local flash storage. If your modulefiles reside on a shared network filesystem, using a `Module cache`_ is strongly recommended. diff --git a/NEWS.rst b/NEWS.rst index e9508bf06..c83cb2780 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -152,6 +152,13 @@ Modules 5.7.0 (not yet released) prepends and the entries it appends, so this entry is not silently absorbed by these commands' default de-duplication behavior nor relocated by the :mconfig:`path_entry_reorder` configuration option. +* Resolve an exact ``name/version`` specification without validating every + sibling modulefile found in its directory, by walking versions from the + dictionary-highest to the dictionary-lowest and stopping as soon as the + implicit ``default`` and ``latest`` symbolic versions are settled. A + substantial performance improvement is observed when loading an existing + modulefile out of a directory holding hundreds or thousands of sibling + versions. (fix issue #561) .. _5.6 release notes: diff --git a/script/mb b/script/mb index 49bbdb579..f917e7741 100755 --- a/script/mb +++ b/script/mb @@ -1,7 +1,7 @@ #!/usr/bin/env tclsh # # MB, make bench between modulecmd versions -# Copyright (C) 2019-2025 Xavier Delaruelle +# Copyright (C) 2019-2026 Xavier Delaruelle # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ Available tests: help, avail, avail_cache, avail2, avail2_cache, avail3, avail3_cache, avail4, avail4_cache, whatis, whatis_cache, whatis2, whatis2_cache, whatis3, whatis3_cache, apropos, load, load_cache, load2, load2_cache, - list, list2, purge, unload + load3, load3_cache, list, list2, purge, unload (all tests selected by default) Modes: @@ -196,7 +196,8 @@ foreach arg $argv { help - avail - avail_cache - avail2 - avail2_cache - avail3 -\ avail3_cache - avail4 - avail4_cache - whatis - whatis_cache -\ whatis2 - whatis2_cache - whatis3 - whatis3_cache - apropos - load -\ - load_cache - load2 - load2_cache - list - list2 - purge - unload { + load_cache - load2 - load2_cache - load3 - load3_cache -\ + list - list2 - purge - unload { lappend testlist $arg } --with-lmod=* { @@ -232,7 +233,7 @@ if {![info exists testlist]} { set testlist [list help avail avail_cache avail2 avail2_cache avail3\ avail3_cache avail4 avail4_cache whatis whatis_cache whatis2\ whatis2_cache whatis3 whatis3_cache apropos load load_cache load2\ - load2_cache list list2 purge unload] + load2_cache load3 load3_cache list list2 purge unload] } } elseif {[info exists lmod_cmd]} { foreach test $testlist { @@ -338,12 +339,70 @@ catch {unset env(__MODULES_SHARE_MODULEPATH)} # define bench tests and their arguments and environment set modpath $curdir/testsuite/modulefiles array set testsubcmdlist [list avail2 avail avail3 avail avail4 avail whatis2\ - whatis whatis3 whatis load2 load list2 list] + whatis whatis3 whatis load2 load load3 load list2 list] array set testarglist [list avail2 load whatis2 load load load/all unload\ - load/all load2 R-bundle-Bioconductor/3.18-foss-2023a-R-4.3.2] + load/all load2 R-bundle-Bioconductor/3.18-foss-2023a-R-4.3.2 load3\ + bigmod/1.1] array set testoptlist [list avail4_lmod --ignore_cache load2_lmod\ --ignore_cache] +# the load3 bench needs a module root holding several hundred sibling +# versions (mixing valid and invalid modulefiles, matching the shape +# reported by sites with large modulepath directories), generated on the +# fly rather than checked into the repository and removed once done +set bigmoddir $modpath.bigmod/bigmod +set need_bigmod [expr {{load3} in $testlist || {load3_cache} in $testlist}] +if {$need_bigmod} { + file mkdir $bigmoddir + set bigmod_content {#%Module + +proc ModulesHelp {} { + puts stderr {Lorem ipsum dolor sit amet, consectetur adipiscing elit.} + puts stderr {Sed non risus. Suspendisse lectus tortor, dignissim sit amet,} + puts stderr {adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam.} + puts stderr {Maecenas ligula massa, varius a, semper congue, euismod non, mi.} + puts stderr {Proin porttitor, orci nec nonummy molestie, enim est eleifend mi,} + puts stderr {non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa,} + puts stderr {scelerisque vitae, consequat in, } + puts stderr {Lorem ipsum dolor sit amet, consectetur adipiscing elit.} + puts stderr {Sed non risus. Suspendisse lectus tortor, dignissim sit amet,} + puts stderr {adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam.} + puts stderr {Maecenas ligula massa, varius a, semper congue, euismod non, mi.} + puts stderr {Proin porttitor, orci nec nonummy molestie, enim est eleifend mi,} + puts stderr {non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa,} + puts stderr {scelerisque vitae, consequat in, } + puts stderr {Lorem ipsum dolor sit amet, consectetur adipiscing elit.} + puts stderr {Sed non risus. Suspendisse lectus tortor, dignissim sit amet,} + puts stderr {adipiscing nec, ultricies sed, dolor. Cras elementum ultrices diam.} + puts stderr {Maecenas ligula massa, varius a, semper congue, euismod non, mi.} + puts stderr {Proin porttitor, orci nec nonummy molestie, enim est eleifend mi,} + puts stderr {non fermentum diam nisl sit amet erat. Duis semper. Duis arcu massa,} + puts stderr {scelerisque vitae, consequat in, } +} + +# comment code of the modulefile +module-whatis {returns all various module-info values} + +# comment code of the modulefile +setenv FOO value +append-path BAR /path/to/directory +setenv QUX {Lorem ipsum dolor sit amet, consectetur adipiscing elit.} +setenv FOO2 value +append-path BAR2 /path/to/directory +setenv QUX2 {Lorem ipsum dolor sit amet, consectetur adipiscing elit.} +} + for {set bigmodvers 1} {$bigmodvers <= 500} {incr bigmodvers} { + set fid [open $bigmoddir/1.$bigmodvers w] + puts $fid $bigmod_content + close $fid + } + foreach bigmodvers {50 150 250 350 450} { + set fid [open $bigmoddir/1.$bigmodvers.orig w] + puts $fid {not a modulefile} + close $fid + } +} + # load test environment recorded in external files foreach testenv_file [glob $curdir/testsuite/mb/testenv_*] { set test [string range [file tail $testenv_file] 8 end] @@ -368,7 +427,8 @@ set testenvlist(purge_lmod) $testenvlist(list2_lmod) # some tests have a minimum version requirement array set testcompatlist [list avail3 4.6.0 whatis3 4.6.0 avail_cache 5.3.0\ avail2_cache 5.3.0 avail3_cache 5.3.0 whatis_cache 5.3.0 whatis2_cache\ - 5.3.0 whatis3_cache 5.3.0 load_cache 5.3.0 load2_cache 5.3.0] + 5.3.0 whatis3_cache 5.3.0 load_cache 5.3.0 load2_cache 5.3.0 load3_cache\ + 5.3.0] # adapt output table to test mode if {$mode eq {profile}} { @@ -424,6 +484,11 @@ foreach tag $taglist { catch {file delete modulecmd.$tag lib/libtclenvmodules.so.$tag} } } + +# clean generated load3 bench modulefiles +if {$need_bigmod} { + file delete -force $modpath.bigmod +} catch {file delete tcl/mfinterp.tcl} # restore workspace if saved diff --git a/tcl/modfind.tcl.in b/tcl/modfind.tcl.in index 2311c3f64..30c921b7c 100644 --- a/tcl/modfind.tcl.in +++ b/tcl/modfind.tcl.in @@ -2984,10 +2984,112 @@ proc findModulesInMemCache {searchid} { return [list $match_searchid $mod_list] } +# when 'mod_name' is 'stop_mod''s own parent directory, narrow +# 'elt_list_name' (in the {path hid} pairs shape returned by +# getFilesInDirectory) down to only what is needed to tell whether stop_mod +# is the implicit 'default'/'latest' version of that directory: its +# dotfiles (always kept, the general consolidation phase in getModules +# deals with them normally) plus, among its plain elements, only the +# dictionary range from stop_mod up to either the first higher element that +# is itself valid and not hidden (stop_mod is then known not to be +# default/latest) or the highest element if none qualifies. Left untouched +# in any other case: this is not stop_mod's own directory, stop_mod does not +# exist in it, or it is not itself a valid modulefile, in which case there +# is nothing to optimize +proc trimElemListForStopmod {dir mod_name stop_mod elt_list_name} { + if {![string length $stop_mod] || $mod_name ne [file dirname $stop_mod]} { + return + } + + upvar $elt_list_name elt_list + array set elt_arr $elt_list + + if {![info exists elt_arr($dir/$stop_mod)] || [lindex [checkValidModule\ + $dir/$stop_mod] 0] ne {true}} { + return + } + + # first fill updated element list with rc files if any and remove them from + # reference array to work only on other items + array set up_elt_arr {} + foreach rc_tail {.modulerc .version} { + set elt $dir/$mod_name/$rc_tail + if {[info exists elt_arr($elt)]} { + set up_elt_arr($elt) $elt_arr($elt) + unset elt_arr($elt) + } + } + + set tail_list {} + foreach elt [array names elt_arr] { + lappend tail_list [file tail $elt] + } + + # source the modulepath root '.modulerc' then this directory's own + # '.modulerc' and '.version' right away (in that order) if not already + # done, as any of them may set a hiding rule needed below, to correctly + # tell if a sibling actually is a qualifying higher element (guarded the + # same way the general consolidation phase guards its own later, + # deferred sourcing of them) + global g_rcfilesSourced + set top_rc_file $dir/.modulerc + if {![info exists g_rcfilesSourced($top_rc_file)] && [file readable\ + $top_rc_file]} { + execute-modulerc $top_rc_file .modulerc .modulerc + set g_rcfilesSourced($top_rc_file) 1 + } + foreach rc_tail {.modulerc .version} { + set rc_elt $dir/$mod_name/$rc_tail + if {[info exists up_elt_arr($rc_elt)] && ![info exists\ + g_rcfilesSourced($rc_elt)]} { + set rc_mod $mod_name/$rc_tail + execute-modulerc $rc_elt $rc_mod $rc_mod + set g_rcfilesSourced($rc_elt) 1 + } + } + + # sort this directory's plain elements and look, starting from the + # highest one down to stop_mod, for the first element that is itself + # valid and not hidden: whether or not it turns out to be stop_mod + # itself, it is this directory's true implicit default/latest, needed + # as-is by the caller to correctly resolve it for every module in this + # directory, not just stop_mod. Only the elements from that point up to + # the highest one need to be kept to reflect it, plus stop_mod's own + # element if it lies below that point + set stop_tail [file tail $stop_mod] + set sorted_tails [lsort -dictionary $tail_list] + set stop_idx [lsearch -exact $sorted_tails $stop_tail] + set top_idx $stop_idx + for {set ti [expr {[llength $sorted_tails] - 1}]} {$ti > $stop_idx}\ + {incr ti -1} { + set fp_tail [lindex $sorted_tails $ti] + set fp_elt $dir/$mod_name/$fp_tail + if {[file isdirectory $fp_elt]} { + # a nested version directory: its own validity cannot be settled + # without recursing into it, give up trying to stop early, keep + # everything untouched like usual + return + } + if {[lindex [checkValidModule $fp_elt] 0] eq {true} &&\ + ![isModuleHidden $mod_name/$fp_tail $stop_mod 0 $fp_elt]} { + set top_idx $ti + break + } + } + + # only keep rc files, stop_mod and highest element the updated structure + set up_elt_arr($dir/$stop_mod) $elt_arr($dir/$stop_mod) + if {$top_idx > $stop_idx} { + set elt $dir/$mod_name/[lindex $sorted_tails $top_idx] + set up_elt_arr($elt) $elt_arr($elt) + } + set elt_list [array get up_elt_arr] +} + # Walk through provided list of directories and files to find modules proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ res_arrname {indir_arrname {}} {hidden_listname {}} {fknown_arrname {}}\ - {dknown_arrname {}}} { + {dknown_arrname {}} {stop_mod {}}} { # link to variables/arrays from upper context upvar $res_arrname mod_list if {$indir_arrname ne {}} { @@ -3024,6 +3126,12 @@ proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ set mod_list($modulename) [list accesserr [parseAccessIssue\ $element] $element] } else { + # narrow elt_list down to what is needed to correctly tell + # whether stop_mod is the implicit default/latest version of + # this directory, when this is stop_mod's own directory + # (left untouched in any other case) + trimElemListForStopmod $dir $modulename $stop_mod elt_list + # Add each element in the current directory to the list foreach {fpelt hid} $elt_list { lappend full_list $fpelt @@ -3082,13 +3190,15 @@ proc findModulesFromDirsAndFiles {dir full_list depthlvl fetch_mtime\ } } -# finds all module-related files matching mod in the module path dir -proc findModules {dir mod depthlvl fetch_mtime} { +# finds all module-related files matching mod in the module path dir. +# 'stop_mod' helps optimize the search when looking for an exact existing +# modulefile +proc findModules {dir mod depthlvl fetch_mtime {stop_mod {}}} { reportDebug "finding '$mod' in $dir (depthlvl=$depthlvl,\ fetch_mtime=$fetch_mtime)" # generated search id (for cache search/save) by compacting given args - set searchid $dir:$mod:$depthlvl:$fetch_mtime + set searchid $dir:$mod:$depthlvl:$fetch_mtime:$stop_mod # look at memory cache for a compatible result lassign [findModulesInMemCache $searchid] cache_searchid cache_list @@ -3127,7 +3237,8 @@ proc findModules {dir mod depthlvl fetch_mtime} { } # walk through list of dirs and files to find modules - findModulesFromDirsAndFiles $dir $full_list $depthlvl $fetch_mtime mod_list + findModulesFromDirsAndFiles $dir $full_list $depthlvl $fetch_mtime\ + mod_list {} {} {} {} $stop_mod reportDebug "found [array names mod_list]" @@ -3212,6 +3323,18 @@ proc getModules {dir {mod {}} {fetch_mtime 0} {search {}} {filter {}}} { set add_rc_defs 1 array set found_list {} } else { + # an exact 'name/version' query (no wildcard/contains, no result + # filtering), possibly along with a variant or an extra match search + # specifier, resolved in a single-module context (unlike a listing + # one, e.g. avail/whatis/search) lets findModules shortcut its + # directory search (see there); left empty in any other case + set stop_mod {} + set modnamevers [getModuleNameAndVersFromVersSpec $mod] + if {$resctx && $earlyfilter && $filter eq {} && [getCmpSpecFromVersSpec\ + $mod] eq {eq} && $querydepth == 1} { + set stop_mod $modnamevers + } + # find modules by searching mod root name in order to catch all module # related entries to correctly computed auto symbols afterward @@ -3233,7 +3356,8 @@ proc getModules {dir {mod {}} {fetch_mtime 0} {search {}} {filter {}}} { # unless EMS need to be performed (findModules should fetch everything) set depthlvl [expr {$indepth || $ems_required ? 0 : $querydepth + 1}] - array set found_list [findModules $dir $findmod $depthlvl $fetch_mtime] + array set found_list [findModules $dir $findmod $depthlvl\ + $fetch_mtime $stop_mod] } # Phase #1: consolidate every kind of entries (directory, modulefile, diff --git a/testsuite/mb/testenv_load3 b/testsuite/mb/testenv_load3 new file mode 100644 index 000000000..6852f9c17 --- /dev/null +++ b/testsuite/mb/testenv_load3 @@ -0,0 +1 @@ +MODULEPATH $modpath.bigmod diff --git a/testsuite/modulefiles.4/.modulerc b/testsuite/modulefiles.4/.modulerc index b51597ee2..bff9cf857 100644 --- a/testsuite/modulefiles.4/.modulerc +++ b/testsuite/modulefiles.4/.modulerc @@ -98,3 +98,11 @@ if {[info exists env(TESTSUITE_PROVIDE)]} { } } } + +if {[info exists env(TESTSUITE_FASTDEFVERS)]} { + switch -- $env(TESTSUITE_FASTDEFVERS) { + hide1 { + module-hide --hard loc_fdv4/2.0 + } + } +} diff --git a/testsuite/modulefiles.4/loc_fdv1/.modulerc b/testsuite/modulefiles.4/loc_fdv1/.modulerc new file mode 100644 index 000000000..5deac4da8 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/.modulerc @@ -0,0 +1,3 @@ +#%Module + +module-hide --hard loc_fdv1/2.9 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.0 b/testsuite/modulefiles.4/loc_fdv1/1.0 new file mode 100644 index 000000000..f5b474723 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.0 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.1 b/testsuite/modulefiles.4/loc_fdv1/1.1 new file mode 100644 index 000000000..8ca1cf47e --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.1 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.1 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.2 b/testsuite/modulefiles.4/loc_fdv1/1.2 new file mode 100644 index 000000000..27229f05f --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.2 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.2 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.3 b/testsuite/modulefiles.4/loc_fdv1/1.3 new file mode 100644 index 000000000..a6120ecf3 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.3 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.3 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.4 b/testsuite/modulefiles.4/loc_fdv1/1.4 new file mode 100644 index 000000000..fe702f6bc --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.4 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.4 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.5 b/testsuite/modulefiles.4/loc_fdv1/1.5 new file mode 100644 index 000000000..916633b64 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.5 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.5 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.6 b/testsuite/modulefiles.4/loc_fdv1/1.6 new file mode 100644 index 000000000..ff7b5e132 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.6 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.6 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.7 b/testsuite/modulefiles.4/loc_fdv1/1.7 new file mode 100644 index 000000000..d6f0e3f93 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.7 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.7 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.8 b/testsuite/modulefiles.4/loc_fdv1/1.8 new file mode 100644 index 000000000..372a60278 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.8 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.8 diff --git a/testsuite/modulefiles.4/loc_fdv1/1.9 b/testsuite/modulefiles.4/loc_fdv1/1.9 new file mode 100644 index 000000000..a0885d6ab --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/1.9 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/1.9 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.0 b/testsuite/modulefiles.4/loc_fdv1/2.0 new file mode 100644 index 000000000..bf09db769 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.0 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.1 b/testsuite/modulefiles.4/loc_fdv1/2.1 new file mode 100644 index 000000000..cd7085264 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.1 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.1 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.2 b/testsuite/modulefiles.4/loc_fdv1/2.2 new file mode 100644 index 000000000..8b756faf7 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.2 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.2 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.3 b/testsuite/modulefiles.4/loc_fdv1/2.3 new file mode 100644 index 000000000..5c065a96c --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.3 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.3 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.4 b/testsuite/modulefiles.4/loc_fdv1/2.4 new file mode 100644 index 000000000..5660d3924 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.4 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.4 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.5 b/testsuite/modulefiles.4/loc_fdv1/2.5 new file mode 100644 index 000000000..930b987fb --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.5 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.5 diff --git a/testsuite/modulefiles.4/loc_fdv1/2.9 b/testsuite/modulefiles.4/loc_fdv1/2.9 new file mode 100644 index 000000000..80d339450 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv1/2.9 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv1/2.9 diff --git a/testsuite/modulefiles.4/loc_fdv2/.version b/testsuite/modulefiles.4/loc_fdv2/.version new file mode 100644 index 000000000..f3f39e50f --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/.version @@ -0,0 +1,3 @@ +#%Module + +module-version ./1.2 default diff --git a/testsuite/modulefiles.4/loc_fdv2/1.0 b/testsuite/modulefiles.4/loc_fdv2/1.0 new file mode 100644 index 000000000..45f88a6d9 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/1.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/1.0 diff --git a/testsuite/modulefiles.4/loc_fdv2/1.1 b/testsuite/modulefiles.4/loc_fdv2/1.1 new file mode 100644 index 000000000..eaae4d117 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/1.1 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/1.1 diff --git a/testsuite/modulefiles.4/loc_fdv2/1.2 b/testsuite/modulefiles.4/loc_fdv2/1.2 new file mode 100644 index 000000000..712932b80 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/1.2 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/1.2 diff --git a/testsuite/modulefiles.4/loc_fdv2/1.3 b/testsuite/modulefiles.4/loc_fdv2/1.3 new file mode 100644 index 000000000..2b7f9e870 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/1.3 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/1.3 diff --git a/testsuite/modulefiles.4/loc_fdv2/1.4 b/testsuite/modulefiles.4/loc_fdv2/1.4 new file mode 100644 index 000000000..7e2938d98 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/1.4 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/1.4 diff --git a/testsuite/modulefiles.4/loc_fdv2/1.5 b/testsuite/modulefiles.4/loc_fdv2/1.5 new file mode 100644 index 000000000..44c5c86ee --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/1.5 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/1.5 diff --git a/testsuite/modulefiles.4/loc_fdv2/2.0 b/testsuite/modulefiles.4/loc_fdv2/2.0 new file mode 100644 index 000000000..d9a78461a --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/2.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/2.0 diff --git a/testsuite/modulefiles.4/loc_fdv2/2.1 b/testsuite/modulefiles.4/loc_fdv2/2.1 new file mode 100644 index 000000000..5fb061da4 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/2.1 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/2.1 diff --git a/testsuite/modulefiles.4/loc_fdv2/2.2 b/testsuite/modulefiles.4/loc_fdv2/2.2 new file mode 100644 index 000000000..3e3c824ee --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/2.2 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/2.2 diff --git a/testsuite/modulefiles.4/loc_fdv2/2.3 b/testsuite/modulefiles.4/loc_fdv2/2.3 new file mode 100644 index 000000000..bd0ae6fc7 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv2/2.3 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv2/2.3 diff --git a/testsuite/modulefiles.4/loc_fdv3/1.0 b/testsuite/modulefiles.4/loc_fdv3/1.0 new file mode 100644 index 000000000..7cf37a395 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv3/1.0 @@ -0,0 +1,3 @@ +#%Module +variant myvar val1 val2 +module-whatis loc_fdv3/1.0 diff --git a/testsuite/modulefiles.4/loc_fdv3/1.5 b/testsuite/modulefiles.4/loc_fdv3/1.5 new file mode 100644 index 000000000..ca1d578cb --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv3/1.5 @@ -0,0 +1,3 @@ +#%Module +variant myvar val1 val2 +module-whatis loc_fdv3/1.5 diff --git a/testsuite/modulefiles.4/loc_fdv3/2.0 b/testsuite/modulefiles.4/loc_fdv3/2.0 new file mode 100644 index 000000000..a7ad333cf --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv3/2.0 @@ -0,0 +1,3 @@ +#%Module +variant myvar val1 val2 +module-whatis loc_fdv3/2.0 diff --git a/testsuite/modulefiles.4/loc_fdv4/1.0 b/testsuite/modulefiles.4/loc_fdv4/1.0 new file mode 100644 index 000000000..f291eb664 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv4/1.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv4/1.0 diff --git a/testsuite/modulefiles.4/loc_fdv4/1.1 b/testsuite/modulefiles.4/loc_fdv4/1.1 new file mode 100644 index 000000000..37e2ebba8 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv4/1.1 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv4/1.1 diff --git a/testsuite/modulefiles.4/loc_fdv4/2.0 b/testsuite/modulefiles.4/loc_fdv4/2.0 new file mode 100644 index 000000000..6d0cd1618 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv4/2.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv4/2.0 diff --git a/testsuite/modulefiles.4/loc_fdv5/.modulerc b/testsuite/modulefiles.4/loc_fdv5/.modulerc new file mode 100644 index 000000000..71f10e209 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv5/.modulerc @@ -0,0 +1,3 @@ +#%Module + +module-hide --hard [file dirname $ModulesCurrentModulefile]/2.0 diff --git a/testsuite/modulefiles.4/loc_fdv5/1.0 b/testsuite/modulefiles.4/loc_fdv5/1.0 new file mode 100644 index 000000000..e478e5df0 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv5/1.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv5/1.0 diff --git a/testsuite/modulefiles.4/loc_fdv5/2.0 b/testsuite/modulefiles.4/loc_fdv5/2.0 new file mode 100644 index 000000000..cf1329052 --- /dev/null +++ b/testsuite/modulefiles.4/loc_fdv5/2.0 @@ -0,0 +1,2 @@ +#%Module +module-whatis loc_fdv5/2.0 diff --git a/testsuite/modules.20-locate/140-hide-full-path.exp b/testsuite/modules.20-locate/140-hide-full-path.exp index 17c063ab9..9d2f2bc14 100644 --- a/testsuite/modules.20-locate/140-hide-full-path.exp +++ b/testsuite/modules.20-locate/140-hide-full-path.exp @@ -131,6 +131,7 @@ set tserr "$modlin Aliases $modlin tag/1.1.al -> tag/1.0 $modlin Versions $modlin +loc_fdv2/default -> loc_fdv2/1.2 tag/1.1.sym -> tag/1.0" testouterr_cmd_re sh {aliases} OK $tserr @@ -167,6 +168,7 @@ set tserr "$modlin Aliases $modlin tag/1.1.al -> tag/1.0 $modlin Versions $modlin +loc_fdv2/default -> loc_fdv2/1.2 tag/1.1.sym -> tag/1.0" testouterr_cmd_re sh {aliases} OK $tserr diff --git a/testsuite/modules.20-locate/170-fastdefvers.exp b/testsuite/modules.20-locate/170-fastdefvers.exp new file mode 100644 index 000000000..79bf456c8 --- /dev/null +++ b/testsuite/modules.20-locate/170-fastdefvers.exp @@ -0,0 +1,244 @@ +############################################################################## +# Modules Revision 3.0 +# Providing a flexible user environment +# +# File: modules.20-locate/%M% +# Revision: %I% +# First Edition: 2026/07/26 +# Last Mod.: %U%, %G% +# +# Authors: Xavier Delaruelle, xavier.delaruelle@cea.fr +# +# Description: Testuite testsequence +# Command: load, display +# Modulefiles: loc_fdv1, loc_fdv2, loc_fdv3, loc_fdv4, loc_fdv5 +# Sub-Command: +# +# Comment: %C{ +# Check that the lighter-weight resolution attempted for an exact +# 'name/version' specification that matches an existing modulefile +# (see findModulesFromDirsAndFiles's 'stopmod' handling) resolves +# the implicit 'default' and 'latest' symbolic versions exactly +# like the exhaustive search does, on a directory holding many +# sibling versions: a hard hidden modulefile sorting above the +# actual highest usable version, an explicitly defined 'default' +# distinct from the implicit 'latest', a variant value specified +# along with the query, a hiding rule set in the modulepath's own +# top-level '.modulerc' and a hiding rule targeting a full path +# modulefile designation. +# }C% +# +############################################################################## + +# +# Variables for this test +# + +# use a dedicated modulepath so these new fixtures do not affect other tests +# that enumerate the whole content of the default modulepath +setenv_path_var MODULEPATH $modpath.4 + +set module_1 "loc_fdv1" +set module_1_low "$module_1/1.0" +set module_1_top "$module_1/2.5" +set module_1_hidden "$module_1/2.9" +set modulefile_1_low "$modpath.4/$module_1_low" +set modulefilere_1_low "$modpathre.4/$module_1_low" +set modulefile_1_top "$modpath.4/$module_1_top" +set modulefilere_1_top "$modpathre.4/$module_1_top" + +set module_2 "loc_fdv2" +set module_2_default "$module_2/1.2" +set module_2_top "$module_2/2.3" +set modulefile_2_default "$modpath.4/$module_2_default" +set modulefilere_2_default "$modpathre.4/$module_2_default" +set modulefile_2_top "$modpath.4/$module_2_top" +set modulefilere_2_top "$modpathre.4/$module_2_top" + +set module_3 "loc_fdv3" +set module_3_low "$module_3/1.0" +set module_3_top "$module_3/2.0" +set modulefile_3_low "$modpath.4/$module_3_low" +set modulefile_3_top "$modpath.4/$module_3_top" + +set module_4 "loc_fdv4" +set module_4_low "$module_4/1.0" +set module_4_top "$module_4/1.1" +set module_4_hidden "$module_4/2.0" +set modulefile_4_low "$modpath.4/$module_4_low" +set modulefile_4_top "$modpath.4/$module_4_top" + +set module_5 "loc_fdv5" +set module_5_top "$module_5/1.0" +set module_5_hidden "$module_5/2.0" +set modulefile_5_top "$modpath.4/$module_5_top" + +# +# loc_fdv1: bare name and low exact version both resolve correctly, skipping +# past an invalid modulefile (3.0) and a hard hidden one (2.9), neither of +# which should be considered a candidate for the implicit highest version +# + +set ans [list] +if {[is_conf_enabled advversspec implicitdefault]} { + lappend ans [list set __MODULES_LMALTNAME\ + $module_1_top&as|$module_1/default&as|$module_1/latest] +} +lappend ans [list set _LMFILES_ $modulefile_1_top] +lappend ans [list set LOADEDMODULES $module_1_top] + +test_cmd "sh" "load $module_1" [if_implicitdefault $ans ERR]\ + [if_implicitdefault 0 1] + +testouterr_cmd_re "sh" "display $module_1" [if_implicitdefault OK ERR]\ + [if_implicitdefault "$modlin\n$modulefilere_1_top:\n\nmodule-whatis\\s+$module_1_top\n$modlin"\ + "$err_nodefault'$module_1'"] + +# +# loading the low exact version directly must succeed and must not carry the +# 'default' or 'latest' alternate names, as a higher usable version exists +# + +set ans [list] +lappend ans [list set _LMFILES_ $modulefile_1_low] +lappend ans [list set LOADEDMODULES $module_1_low] + +test_cmd "sh" "load $module_1_low" $ans + +testouterr_cmd_re "sh" "display $module_1_low" "OK"\ + "$modlin\n$modulefilere_1_low:\n\nmodule-whatis\\s+$module_1_low\n$modlin" + +# +# loading the actual highest usable version directly must carry both the +# 'default' and 'latest' alternate names +# + +set ans [list] +if {[is_conf_enabled advversspec implicitdefault]} { + lappend ans [list set __MODULES_LMALTNAME\ + $module_1_top&as|$module_1/default&as|$module_1/latest] +} +lappend ans [list set _LMFILES_ $modulefile_1_top] +lappend ans [list set LOADEDMODULES $module_1_top] + +test_cmd "sh" "load $module_1_top" $ans + +# +# the hard hidden version stays unreachable even when queried by its exact +# name +# + +testouterr_cmd_re "sh" "display $module_1_hidden" "ERR"\ + "$err_path'$module_1_hidden'" + +skip_if_quick_mode + +# +# loc_fdv2: an explicitly defined 'default' distinct from the implicit +# 'latest' must each resolve independently and carry only their own +# alternate name, not the other one. Unlike loc_fdv1's implicit default, +# this one is set via a '.version' rule, so it resolves regardless of the +# 'implicit_default' configuration option +# + +set ans [list] +lappend ans [list set __MODULES_LMALTNAME\ + $module_2_default&$module_2/default&$module_2] +lappend ans [list set _LMFILES_ $modulefile_2_default] +lappend ans [list set LOADEDMODULES $module_2_default] + +test_cmd "sh" "load $module_2" $ans + +testouterr_cmd_re "sh" "display $module_2" "OK"\ + "$modlin\n$modulefilere_2_default:\n\nmodule-whatis\\s+$module_2_default\n$modlin" + +set ans [list] +if {[is_conf_enabled advversspec implicitdefault]} { + lappend ans [list set __MODULES_LMALTNAME\ + $module_2_top&as|$module_2/latest] +} +lappend ans [list set _LMFILES_ $modulefile_2_top] +lappend ans [list set LOADEDMODULES $module_2_top] + +test_cmd "sh" "load $module_2_top" $ans + +# +# loc_fdv3: a variant value specified along with an exact 'name/version' +# query must not prevent the fast path from being taken, nor affect its +# correctness +# + +if {[is_conf_enabled advversspec]} { + +set ans [list] +lappend ans [list set __MODULES_LMVARIANT $module_3_low&myvar|val2|0|0] +lappend ans [list set _LMFILES_ $modulefile_3_low] +lappend ans [list set LOADEDMODULES $module_3_low] + +test_cmd "sh" "load $module_3_low myvar=val2" $ans + +set ans [list] +lappend ans [list set __MODULES_LMVARIANT $module_3_top&myvar|val2|0|0] +if {[is_conf_enabled advversspec implicitdefault]} { + lappend ans [list set __MODULES_LMALTNAME\ + $module_3_top&as|$module_3/default&as|$module_3/latest] +} +lappend ans [list set _LMFILES_ $modulefile_3_top] +lappend ans [list set LOADEDMODULES $module_3_top] + +test_cmd "sh" "load $module_3_top myvar=val2" $ans + +} + +# +# loc_fdv4: a module-hide rule declared in the modulepath's own top-level +# '.modulerc' (as opposed to the module directory's own '.modulerc', already +# covered by loc_fdv1) must also be honored while resolving the implicit +# highest version +# + +setenv_var TESTSUITE_FASTDEFVERS hide1 + +set ans [list] +lappend ans [list set _LMFILES_ $modulefile_4_low] +lappend ans [list set LOADEDMODULES $module_4_low] + +test_cmd "sh" "load $module_4_low" $ans + +set ans [list] +if {[is_conf_enabled advversspec implicitdefault]} { + lappend ans [list set __MODULES_LMALTNAME\ + $module_4_top&as|$module_4/default&as|$module_4/latest] +} +lappend ans [list set _LMFILES_ $modulefile_4_top] +lappend ans [list set LOADEDMODULES $module_4_top] + +test_cmd "sh" "load $module_4_top" $ans + +testouterr_cmd_re "sh" "display $module_4_hidden" "ERR"\ + "$err_path'$module_4_hidden'" + +# +# loc_fdv5: a module-hide rule targeting a full path modulefile designation +# (as opposed to a relative 'name/version' one) must also be honored while +# resolving the implicit highest version +# + +set ans [list] +if {[is_conf_enabled advversspec implicitdefault]} { + lappend ans [list set __MODULES_LMALTNAME\ + $module_5_top&as|$module_5/default&as|$module_5/latest] +} +lappend ans [list set _LMFILES_ $modulefile_5_top] +lappend ans [list set LOADEDMODULES $module_5_top] + +test_cmd "sh" "load $module_5_top" $ans + +testouterr_cmd_re "sh" "display $module_5_hidden" "ERR"\ + "$err_path'$module_5_hidden'" + +# +# Cleanup +# + +reset_test_env diff --git a/testsuite/modules.51-scan/030-scan_eval-avail.exp b/testsuite/modules.51-scan/030-scan_eval-avail.exp index 65b992ae8..5398f3f88 100644 --- a/testsuite/modules.51-scan/030-scan_eval-avail.exp +++ b/testsuite/modules.51-scan/030-scan_eval-avail.exp @@ -257,7 +257,7 @@ testouterr_cmd_re sh {avail -T -t scan@1.1:2.1 foo=val4} OK $tserr set tserr ".* Get modules: {} matching 'scan@1.1:2.0 foo=val2' in '' .* -Get modules: {scan/2.0 scan/1.1} matching 'scan@1.1:2.0 foo=val2' in '$mpre' +Get modules: {scan/(2\.0|1\.1) scan/(2\.0|1\.1)} matching 'scan@1.1:2.0 foo=val2' in '$mpre' scan/1.1{foo=val1,val2,val3} scan/2.0{foo=val2,val3,val4}" testouterr_cmd_re sh {avail -T -t scan@1.1:2.0 foo=val2} OK $tserr @@ -332,7 +332,7 @@ testouterr_cmd_re sh {avail -T -t scan@1.1:2.0 foo=val4} OK $tserr set tserr ".* Get modules: {} matching 'scan@1.1:2.1 foo=val2' in '' .* -Get modules: {scan/2.0 scan/1.1 scan/2.1} matching 'scan@1.1:2.1 foo=val2' in '$mpre' +Get modules: {scan/(2\.0|1\.1|2\.1) scan/(2\.0|1\.1|2\.1) scan/(2\.0|1\.1|2\.1)} matching 'scan@1.1:2.1 foo=val2' in '$mpre' scan/1.1{foo=val1,val2,val3} scan/2.0{foo=val2,val3,val4} scan/2.1{foo=val2,val3,val4}" @@ -358,7 +358,7 @@ testouterr_cmd_re sh {avail -T -t scan@latest foo=val4} OK $tserr set tserr ".* Get modules: {} matching 'scan@1.1:2.0 foo=val2' in '' .* -Get modules: {scan/2.0 scan/1.1} matching 'scan@1.1:2.0 foo=val2' in '$mpre' +Get modules: {scan/(2\.0|1\.1) scan/(2\.0|1\.1)} matching 'scan@1.1:2.0 foo=val2' in '$mpre' scan/1.1{foo=val1,val2,val3} scan/2.0{foo=val2,val3,val4}" testouterr_cmd_re sh {avail -T -t scan@1.1:2.0 foo=val2} OK $tserr diff --git a/testsuite/modules.51-scan/033-scan_eval-spider.exp b/testsuite/modules.51-scan/033-scan_eval-spider.exp index 8eee908ae..71bf9075f 100644 --- a/testsuite/modules.51-scan/033-scan_eval-spider.exp +++ b/testsuite/modules.51-scan/033-scan_eval-spider.exp @@ -261,7 +261,7 @@ testouterr_cmd_re sh {spider -T -t scan@1.1:2.1 foo=val4} OK $tserr set tserr ".* Get modules: {} matching 'scan@1.1:2.0 foo=val2' in '' .* -Get modules: {scan/2.0 scan/1.1} matching 'scan@1.1:2.0 foo=val2' in '$mpre' +Get modules: {scan/(2\.0|1\.1) scan/(2\.0|1\.1)} matching 'scan@1.1:2.0 foo=val2' in '$mpre' scan/1.1{foo=val1,val2,val3} scan/2.0{foo=val2,val3,val4}" testouterr_cmd_re sh {spider -T -t scan@1.1:2.0 foo=val2} OK $tserr @@ -336,7 +336,7 @@ testouterr_cmd_re sh {spider -T -t scan@1.1:2.0 foo=val4} OK $tserr set tserr ".* Get modules: {} matching 'scan@1.1:2.1 foo=val2' in '' .* -Get modules: {scan/2.0 scan/1.1 scan/2.1} matching 'scan@1.1:2.1 foo=val2' in '$mpre' +Get modules: {scan/(2\.1|2\.0|1\.1) scan/(2\.1|2\.0|1\.1) scan/(2\.1|2\.0|1\.1)} matching 'scan@1.1:2.1 foo=val2' in '$mpre' scan/1.1{foo=val1,val2,val3} scan/2.0{foo=val2,val3,val4} scan/2.1{foo=val2,val3,val4}" @@ -362,7 +362,7 @@ testouterr_cmd_re sh {spider -T -t scan@latest foo=val4} OK $tserr set tserr ".* Get modules: {} matching 'scan@1.1:2.0 foo=val2' in '' .* -Get modules: {scan/2.0 scan/1.1} matching 'scan@1.1:2.0 foo=val2' in '$mpre' +Get modules: {scan/(2\.0|1\.1) scan/(2\.0|1\.1)} matching 'scan@1.1:2.0 foo=val2' in '$mpre' scan/1.1{foo=val1,val2,val3} scan/2.0{foo=val2,val3,val4}" testouterr_cmd_re sh {spider -T -t scan@1.1:2.0 foo=val2} OK $tserr