Skip to content

Commit c4ab0d4

Browse files
IanButterworthIanButterworth
authored
Increase build precompilation (#53682)
- Bake precompilepkgs into the sysimage - Enable compilation in stdlib JLLs @staticfloat I guess this is ok now we have pkgimages? - Add some hardcoded precompiles encountered locally - Disable a problematic llvm test, with a note With this there's no compilation loading Pkg in interactive mode ``` % ./julia --trace-compile=stderr --start=no -q julia> ] (@v1.12) pkg> ``` Before ``` % time julia +nightly --start=no --trace-compile=stderr -q -e "using Pkg" precompile(Tuple{typeof(Base.setindex!), Base.Dict{Symbol, Union{Nothing, Bool, Int64, String}}, Int64, Symbol}) precompile(Tuple{typeof(Base.haskey), Base.Dict{String, Any}, String}) precompile(Tuple{typeof(MbedTLS_jll.__init__)}) precompile(Tuple{typeof(LibSSH2_jll.__init__)}) precompile(Tuple{typeof(LibGit2_jll.__init__)}) precompile(Tuple{typeof(nghttp2_jll.__init__)}) precompile(Tuple{typeof(LibCURL_jll.__init__)}) precompile(Tuple{typeof(MozillaCACerts_jll.__init__)}) precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}}) precompile(Tuple{typeof(p7zip_jll.__init__)}) precompile(Tuple{typeof(p7zip_jll.init_p7zip_path)}) precompile(Tuple{typeof(Base.append!), Array{String, 1}, Array{String, 1}}) precompile(Tuple{typeof(Base.join), Array{String, 1}, Char}) julia +nightly --start=no --trace-compile=stderr -q -e "using Pkg" 1.48s user 0.40s system 206% cpu 0.910 total ``` PR ``` % time ./julia --start=no --trace-compile=stderr -q -e "using Pkg" ./julia --start=no --trace-compile=stderr -q -e "using Pkg" 1.62s user 0.13s system 402% cpu 0.433 total ```
1 parent b507330 commit c4ab0d4

File tree

26 files changed

+77
-58
lines changed

26 files changed

+77
-58
lines changed

‎base/precompilation.jl‎

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -354,12 +354,10 @@ function precompilepkgs(pkgs::Vector{String}=String[];
354354
_from_loading::Bool=false,
355355
configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()),
356356
io::IO=stderr,
357-
flags_cacheflags=nothing)
357+
# asking for timing disables fancy mode, as timing is shown in non-fancy mode
358+
fancyprint::Bool = can_fancyprint(io) && !timing
359+
)
358360

359-
if flags_cacheflags !== nothing
360-
# TODO: Remove `flags_cacheflags` once Pkg moves to the renamed kwarg `configs`
361-
configs = flags_cacheflags
362-
end
363361
configs = configs isa Config ? [configs] : configs
364362

365363
time_start = time_ns()
@@ -374,9 +372,6 @@ function precompilepkgs(pkgs::Vector{String}=String[];
374372
num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks)))
375373
parallel_limiter = Base.Semaphore(num_tasks)
376374

377-
# asking for timing disables fancy mode, as timing is shown in non-fancy mode
378-
fancyprint = can_fancyprint(io) && !timing
379-
380375
if _from_loading && !Sys.isinteractive() && Base.get_bool_env("JULIA_TESTS", false)
381376
# suppress passive loading printing in julia test suite. `JULIA_TESTS` is set in Base.runtests
382377
io = devnull
@@ -434,7 +429,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
434429
end
435430
end
436431

437-
@debug "precompile: deps collected" _group=:precompile
432+
@debug "precompile: deps collected"
438433
# this loop must be run after the full depsmap has been populated
439434
for (pkg, pkg_exts) in pkg_exts_map
440435
# find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s),
@@ -446,7 +441,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
446441
end
447442
end
448443
end
449-
@debug "precompile: extensions collected" _group=:precompile
444+
@debug "precompile: extensions collected"
450445

451446
# return early if no deps
452447
if isempty(depsmap)
@@ -474,7 +469,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
474469
was_recompiled[dep_config] = false
475470
end
476471
end
477-
@debug "precompile: signalling initialized" _group=:precompile
472+
@debug "precompile: signalling initialized"
478473

479474

480475
# find and guard against circular deps
@@ -515,7 +510,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
515510
if !isempty(circular_deps)
516511
@warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))"""
517512
end
518-
@debug "precompile: circular dep check done" _group=:precompile
513+
@debug "precompile: circular dep check done"
519514

520515
# if a list of packages is given, restrict to dependencies of given packages
521516
if !isempty(pkgs)
@@ -562,7 +557,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
562557
else
563558
target *= "..."
564559
end
565-
@debug "precompile: packages filtered" _group=:precompile
560+
@debug "precompile: packages filtered"
566561

567562
pkg_queue = PkgConfig[]
568563
failed_deps = Dict{PkgConfig, String}()
@@ -737,7 +732,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
737732
if !_from_loading
738733
Base.LOADING_CACHE[] = Base.LoadingCache()
739734
end
740-
@debug "precompile: starting precompilation loop" _group=:precompile
735+
@debug "precompile: starting precompilation loop" depsmap direct_deps
741736
## precompilation loop
742737
for (pkg, deps) in depsmap
743738
cachepaths = Base.find_all_in_cache_path(pkg)

‎contrib/generate_precompile.jl‎

Lines changed: 58 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,17 @@ precompile(Base.unsafe_string, (Ptr{Int8},))
3939
# loading.jl
4040
precompile(Base.__require_prelocked, (Base.PkgId, Nothing))
4141
precompile(Base._require, (Base.PkgId, Nothing))
42+
precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int))
43+
precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int))
44+
45+
# Pkg loading
46+
precompile(Tuple{typeof(Base.Filesystem.normpath), String, String, Vararg{String}})
47+
precompile(Tuple{typeof(Base.append!), Array{String, 1}, Array{String, 1}})
48+
precompile(Tuple{typeof(Base.join), Array{String, 1}, Char})
49+
precompile(Tuple{typeof(Base.getindex), Base.Dict{Any, Any}, Char})
50+
precompile(Tuple{typeof(Base.delete!), Base.Set{Any}, Char})
51+
precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Base.Dict{String, String}}}, Base.Dict{String, Any}})
52+
precompile(Tuple{typeof(Base.convert), Type{Base.Dict{String, Array{String, 1}}}, Base.Dict{String, Any}})
4253
4354
# REPL
4455
precompile(isequal, (String, String))
@@ -67,6 +78,10 @@ precompile(Tuple{typeof(haskey), Dict{Base.PkgId,Vector{Function}}, Base.PkgId})
6778
precompile(Tuple{typeof(delete!), Dict{Base.PkgId,Vector{Function}}, Base.PkgId})
6879
precompile(Tuple{typeof(push!), Vector{Function}, Function})
6980
81+
# preferences
82+
precompile(Base.get_preferences, (Base.UUID,))
83+
precompile(Base.record_compiletime_preference, (Base.UUID, String))
84+
7085
# miscellaneous
7186
precompile(Tuple{typeof(Base.exit)})
7287
precompile(Tuple{typeof(Base.require), Base.PkgId})
@@ -84,6 +99,23 @@ precompile(Base.CoreLogging.current_logger_for_env, (Base.CoreLogging.LogLevel,
8499
precompile(Base.CoreLogging.env_override_minlevel, (Symbol, Module))
85100
precompile(Base.StackTraces.lookup, (Ptr{Nothing},))
86101
precompile(Tuple{typeof(Base.run_module_init), Module, Int})
102+
103+
# precompilepkgs
104+
precompile(Tuple{typeof(Base.get), Type{Array{String, 1}}, Base.Dict{String, Any}, String})
105+
precompile(Tuple{typeof(Base.get), Type{Base.Dict{String, Any}}, Base.Dict{String, Any}, String})
106+
precompile(Tuple{typeof(Base.haskey), Base.Dict{String, Any}, String})
107+
precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Base.TTY, Bool}, Int, Int})
108+
precompile(Tuple{typeof(Base.indexed_iterate), Tuple{Base.TTY, Bool}, Int})
109+
precompile(Tuple{typeof(Base.open), Base.CmdRedirect, String, Base.TTY})
110+
precompile(Tuple{typeof(Base.Precompilation.precompilepkgs)})
111+
precompile(Tuple{typeof(Base.Precompilation.printpkgstyle), Base.TTY, Symbol, String})
112+
precompile(Tuple{typeof(Base.rawhandle), Base.TTY})
113+
precompile(Tuple{typeof(Base.setindex!), Base.Dict{String, Array{String, 1}}, Array{String, 1}, String})
114+
precompile(Tuple{typeof(Base.setindex!), GenericMemory{:not_atomic, Union{Base.Libc.RawFD, Base.SyncCloseFD, IO}, Core.AddrSpace{Core}(0x00)}, Base.TTY, Int})
115+
precompile(Tuple{typeof(Base.setup_stdio), Base.TTY, Bool})
116+
precompile(Tuple{typeof(Base.spawn_opts_inherit), Base.DevNull, Base.TTY, Base.TTY})
117+
precompile(Tuple{typeof(Core.kwcall), NamedTuple{(:context,), Tuple{Base.TTY}}, typeof(Base.sprint), Function})
118+
precompile(Tuple{Type{Base.UUID}, Base.UUID})
87119
"""
88120

89121
for T in (Float16, Float32, Float64), IO in (IOBuffer, IOContext{IOBuffer}, Base.TTY, IOContext{Base.TTY})
@@ -131,17 +163,17 @@ for match = Base._methods(+, (Int, Int), -1, Base.get_world_counter())
131163
push!(Expr[], Expr(:return, false))
132164
vcat(String[], String[])
133165
k, v = (:hello => nothing)
134-
precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int))
135-
precompile(Base.indexed_iterate, (Pair{Symbol, Union{Nothing, String}}, Int, Int))
166+
136167
# Preferences uses these
137-
precompile(Base.get_preferences, (Base.UUID,))
138-
precompile(Base.record_compiletime_preference, (Base.UUID, String))
139168
get(Dict{String,Any}(), "missing", nothing)
140169
delete!(Dict{String,Any}(), "missing")
141170
for (k, v) in Dict{String,Any}()
142171
println(k)
143172
end
144173
174+
# interactive statup uses this
175+
write(IOBuffer(), "")
176+
145177
break # only actually need to do this once
146178
end
147179
"""
@@ -223,10 +255,10 @@ ansi_disablecursor = "\e[?25l"
223255
blackhole = Sys.isunix() ? "/dev/null" : "nul"
224256
procenv = Dict{String,Any}(
225257
"JULIA_HISTORY" => blackhole,
226-
"JULIA_PROJECT" => nothing, # remove from environment
227-
"JULIA_LOAD_PATH" => "@stdlib",
258+
"JULIA_LOAD_PATH" => "@$(Sys.iswindows() ? ";" : ":")@stdlib",
228259
"JULIA_DEPOT_PATH" => Sys.iswindows() ? ";" : ":",
229260
"TERM" => "",
261+
# "JULIA_DEBUG" => "precompilation",
230262
"JULIA_FALLBACK_REPL" => "true")
231263

232264
generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printed
@@ -263,24 +295,32 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
263295
print_state("step1" => "R")
264296
# Also precompile a package here
265297
pkgname = "__PackagePrecompilationStatementModule"
266-
mkpath(joinpath(prec_path, pkgname, "src"))
267-
path = joinpath(prec_path, pkgname, "src", "$pkgname.jl")
268-
write(path,
269-
"""
270-
module $pkgname
271-
end
272-
""")
298+
pkguuid = "824efdaf-a0e9-431c-8ee7-3d356b2531c2"
299+
pkgpath = joinpath(prec_path, pkgname)
300+
mkpath(joinpath(pkgpath, "src"))
301+
write(joinpath(pkgpath, "src", "$pkgname.jl"),
302+
"""
303+
module $pkgname
304+
println("Precompiling $pkgname")
305+
end
306+
""")
307+
write(joinpath(pkgpath, "Project.toml"),
308+
"""
309+
name = "$pkgname"
310+
uuid = "$pkguuid"
311+
""")
312+
touch(joinpath(pkgpath, "Manifest.toml"))
273313
tmp_prec = tempname(prec_path)
274314
tmp_proc = tempname(prec_path)
275315
s = """
276-
pushfirst!(DEPOT_PATH, $(repr(prec_path)));
316+
pushfirst!(DEPOT_PATH, $(repr(joinpath(prec_path,"depot"))));
277317
Base.PRECOMPILE_TRACE_COMPILE[] = $(repr(tmp_prec));
278-
Base.compilecache(Base.PkgId($(repr(pkgname))), $(repr(path)))
318+
Base.Precompilation.precompilepkgs(;fancyprint=true);
279319
$precompile_script
280320
"""
281321
p = run(pipeline(addenv(`$(julia_exepath()) -O0 --trace-compile=$tmp_proc --sysimage $sysimg
282-
--cpu-target=native --startup-file=no --color=yes`, procenv),
283-
stdin=IOBuffer(s), stdout=debug_output))
322+
--cpu-target=native --startup-file=no --color=yes --project=$(pkgpath)`, procenv),
323+
stdin=IOBuffer(s), stderr=debug_output, stdout=debug_output))
284324
n_step1 = 0
285325
for f in (tmp_prec, tmp_proc)
286326
isfile(f) || continue
@@ -334,9 +374,8 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
334374
yield() # Make clock spinning
335375
print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : ""))
336376
catch ex
337-
@show backtrace()
338377
# See #28808
339-
@warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0
378+
@warn "Failed to precompile expression" form=statement exception=(ex,catch_backtrace()) _module=nothing _file=nothing _line=0
340379
end
341380
end
342381
wait(clock) # Stop asynchronous printing

‎stdlib/CompilerSupportLibraries_jll/src/CompilerSupportLibraries_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule CompilerSupportLibraries_jll
66
using Base, Libdl, Base.BinaryPlatforms
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
const PATH_list = String[]
109
const LIBPATH_list = String[]

‎stdlib/GMP_jll/src/GMP_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
## dummy stub for https://github.com/JuliaBinaryWrappers/GMP_jll.jl
44
baremodule GMP_jll
55
using Base, Libdl
6-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
76

87
const PATH_list = String[]
98
const LIBPATH_list = String[]

‎stdlib/LLD_jll/src/LLD_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule LLD_jll
66
using Base, Libdl
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
const PATH_list = String[]
109
const LIBPATH_list = String[]

‎stdlib/LLVMLibUnwind_jll/src/LLVMLibUnwind_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule LLVMLibUnwind_jll
66
using Base, Libdl
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
const PATH_list = String[]
109
const LIBPATH_list = String[]

‎stdlib/LibCURL_jll/src/LibCURL_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule LibCURL_jll
66
using Base, Libdl, nghttp2_jll
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
const PATH_list = String[]
109
const LIBPATH_list = String[]

‎stdlib/LibGit2_jll/src/LibGit2_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule LibGit2_jll
66
using Base, Libdl, MbedTLS_jll, LibSSH2_jll
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
const PATH_list = String[]
109
const LIBPATH_list = String[]

‎stdlib/LibSSH2_jll/src/LibSSH2_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule LibSSH2_jll
66
using Base, Libdl, MbedTLS_jll
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
const PATH_list = String[]
109
const LIBPATH_list = String[]

‎stdlib/LibUV_jll/src/LibUV_jll.jl‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44

55
baremodule LibUV_jll
66
using Base, Libdl
7-
Base.Experimental.@compiler_options compile=min optimize=0 infer=false
87

98
# NOTE: This file is currently empty, as we link libuv statically for now.
109

0 commit comments

Comments
 (0)