Make Compiler a stdlib

base/Base.jl

@@ -335,6 +335,7 @@ include(mapexpr::Function, mod::Module, _path::AbstractString) = _include(mapexp
 
 # Compatibility with when Compiler was in Core
 @eval Core const Compiler = Main.Base.Compiler
+@eval Compiler const fl_parse = Core.Main.Base.fl_parse
 
 # External libraries vendored into Base
 Core.println("JuliaSyntax/src/JuliaSyntax.jl")

base/Base_compiler.jl

@@ -247,15 +247,15 @@ include("namedtuple.jl")
 include("ordering.jl")
 using .Order
 
-include("compiler/compiler.jl")
+include("coreir.jl")
+include("osutils.jl")
+
+include("../stdlib/Compiler/src/Compiler.jl")
 
 const _return_type = Compiler.return_type
 
 # Enable compiler
-Core.eval(Compiler, quote
-include("compiler/bootstrap.jl")
-ccall(:jl_set_typeinf_func, Cvoid, (Any,), typeinf_ext_toplevel)
+Compiler.bootstrap!()
 
-include("compiler/parsing.jl")
+include("flparse.jl")
 Core._setparser!(fl_parse)
-end)

base/boot.jl

@@ -508,7 +508,6 @@ eval(Core, quote
     UpsilonNode(@nospecialize(val)) = $(Expr(:new, :UpsilonNode, :val))
     UpsilonNode() = $(Expr(:new, :UpsilonNode))
     Const(@nospecialize(v)) = $(Expr(:new, :Const, :v))
-    # NOTE the main constructor is defined within `Core.Compiler`
     _PartialStruct(@nospecialize(typ), fields::Array{Any, 1}) = $(Expr(:new, :PartialStruct, :typ, :fields))
     PartialOpaque(@nospecialize(typ), @nospecialize(env), parent::MethodInstance, source) = $(Expr(:new, :PartialOpaque, :typ, :env, :parent, :source))
     InterConditional(slot::Int, @nospecialize(thentype), @nospecialize(elsetype)) = $(Expr(:new, :InterConditional, :slot, :thentype, :elsetype))

base/coreir.jl

@@ -0,0 +1,54 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+Core.PhiNode() = Core.PhiNode(Int32[], Any[])
+
+"""
+    struct Const
+        val
+    end
+
+The type representing a constant value.
+"""
+Core.Const
+
+"""
+    struct PartialStruct
+        typ
+        fields::Vector{Any} # elements are other type lattice members
+    end
+
+This extended lattice element is introduced when we have information about an object's
+fields beyond what can be obtained from the object type. E.g. it represents a tuple where
+some elements are known to be constants or a struct whose `Any`-typed field is initialized
+with `Int` values.
+
+- `typ` indicates the type of the object
+- `fields` holds the lattice elements corresponding to each field of the object
+
+If `typ` is a struct, `fields` represents the fields of the struct that are guaranteed to be
+initialized. For instance, if the length of `fields` of `PartialStruct` representing a
+struct with 4 fields is 3, the 4th field may not be initialized. If the length is 4, all
+fields are guaranteed to be initialized.
+
+If `typ` is a tuple, the last element of `fields` may be `Vararg`. In this case, it is
+guaranteed that the number of elements in the tuple is at least `length(fields)-1`, but the
+exact number of elements is unknown.
+"""
+Core.PartialStruct
+
+"""
+    struct InterConditional
+        slot::Int
+        thentype
+        elsetype
+    end
+
+Similar to `Conditional`, but conveys inter-procedural constraints imposed on call arguments.
+This is separate from `Conditional` to catch logic errors: the lattice element name is `InterConditional`
+while processing a call, then `Conditional` everywhere else. Thus `InterConditional` does not appear in
+`CompilerTypes`—these type's usages are disjoint—though we define the lattice for `InterConditional`.
+"""
+Core.InterConditional
+
+InterConditional(var::SlotNumber, @nospecialize(thentype), @nospecialize(elsetype)) =
+    InterConditional(slot_id(var), thentype, elsetype)

base/loading.jl

@@ -2168,15 +2168,22 @@ const include_callbacks = Any[]
 const _concrete_dependencies = Pair{PkgId,UInt128}[] # these dependency versions are "set in stone", because they are explicitly loaded, and the process should try to avoid invalidating them
 const _require_dependencies = Any[] # a list of (mod, abspath, fsize, hash, mtime) tuples that are the file dependencies of the module currently being precompiled
 const _track_dependencies = Ref(false) # set this to true to track the list of file dependencies
-function _include_dependency(mod::Module, _path::AbstractString; track_content=true,
-                             path_may_be_dir=false)
+
+function _include_dependency(mod::Module, _path::AbstractString; track_content::Bool=true,
+                             path_may_be_dir::Bool=false)
+    _include_dependency!(_require_dependencies, _track_dependencies[], mod, _path, track_content, path_may_be_dir)
+end
+
+function _include_dependency!(dep_list::Vector{Any}, track_dependencies::Bool,
+                              mod::Module, _path::AbstractString,
+                              track_content::Bool, path_may_be_dir::Bool)
     prev = source_path(nothing)
     if prev === nothing
         path = abspath(_path)
     else
         path = normpath(joinpath(dirname(prev), _path))
     end
-    if !_track_dependencies[]
+    if !track_dependencies[]
         if !path_may_be_dir && !isfile(path)
             throw(SystemError("opening file $(repr(path))", Libc.ENOENT))
         elseif path_may_be_dir && !Filesystem.isreadable(path)
@@ -2187,9 +2194,9 @@ function _include_dependency(mod::Module, _path::AbstractString; track_content=t
             if track_content
                 hash = isdir(path) ? _crc32c(join(readdir(path))) : open(_crc32c, path, "r")
                 # use mtime=-1.0 here so that fsize==0 && mtime==0.0 corresponds to a missing include_dependency
-                push!(_require_dependencies, (mod, path, filesize(path), hash, -1.0))
+                push!(dep_list, (mod, path, filesize(path), hash, -1.0))
             else
-                push!(_require_dependencies, (mod, path, UInt64(0), UInt32(0), mtime(path)))
+                push!(dep_list, (mod, path, UInt64(0), UInt32(0), mtime(path)))
             end
         end
     end
@@ -2493,7 +2500,21 @@ end
 maybe_root_module(key::PkgId) = @lock require_lock get(loaded_modules, key, nothing)
 
 root_module_exists(key::PkgId) = @lock require_lock haskey(loaded_modules, key)
-loaded_modules_array() = @lock require_lock copy(loaded_modules_order)
+function loaded_modules_array()
+    @lock require_lock begin
+        ret = copy(loaded_modules_order)
+        if generating_output(false)
+            # If we're generating a non-incremental system image, also add the
+            # Compiler to the module list so that we generate native code for it.
+            # However, we do not add it to `loaded_modules_order` unless there
+            # is a Julia-level dependency on it. The (non-)loadedness of the compiler
+            # is an implementation detail and we do not want ordinary pkgimages to
+            # acquire a dependency on the Compiler if they do not use it internally.
+            push!(ret, Compiler)
+        end
+        return ret
+    end
+end
 
 # after unreference_module, a subsequent require call will try to load a new copy of it, if stale
 # reload(m) = (unreference_module(m); require(m))
@@ -3273,6 +3294,10 @@ mutable struct CacheHeaderIncludes
     const modpath::Vector{String}   # seemingly not needed in Base, but used by Revise
 end
 
+function CacheHeaderIncludes(dep_tuple::Tuple{Module, String, Int64, UInt32, Float64})
+    return CacheHeaderIncludes(PkgId(dep_tuple[1]), dep_tuple[2:end]..., String[])
+end
+
 function replace_depot_path(path::AbstractString, depots::Vector{String}=normalize_depots_for_relocation())
     for depot in depots
         if startswith(path, string(depot, Filesystem.pathsep())) || path == depot
@@ -3812,6 +3837,56 @@ function list_reasons(reasons::Dict{String,Int})
 end
 list_reasons(::Nothing) = ""
 
+function any_includes_stale(includes::Vector{CacheHeaderIncludes}, cachefile::String, reasons::Union{Dict{String,Int},Nothing}=nothing)
+    for chi in includes
+        f, fsize_req, hash_req, ftime_req = chi.filename, chi.fsize, chi.hash, chi.mtime
+        if startswith(f, string("@depot", Filesystem.pathsep()))
+            @debug("Rejecting stale cache file $cachefile because its depot could not be resolved")
+            record_reason(reasons, "nonresolveable depot")
+            return true
+        end
+        if !ispath(f)
+            _f = fixup_stdlib_path(f)
+            if _f != f && isfile(_f) && startswith(_f, Sys.STDLIB)
+                continue
+            end
+            @debug "Rejecting stale cache file $cachefile because file $f does not exist"
+            record_reason(reasons, "missing sourcefile")
+            return true
+        end
+        if ftime_req >= 0.0
+            # this is an include_dependency for which we only recorded the mtime
+            ftime = mtime(f)
+            is_stale = ( ftime != ftime_req ) &&
+                       ( ftime != floor(ftime_req) ) &&           # Issue #13606, PR #13613: compensate for Docker images rounding mtimes
+                       ( ftime != ceil(ftime_req) ) &&            # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching
+                       ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds
+                       ( ftime != 1.0 )  &&                       # PR #43090: provide compatibility with Nix mtime.
+                       !( 0 < (ftime_req - ftime) < 1e-6 )        # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond
+            if is_stale
+                @debug "Rejecting stale cache file $cachefile because mtime of include_dependency $f has changed (mtime $ftime, before $ftime_req)"
+                record_reason(reasons, "include_dependency mtime change")
+                return true
+            end
+        else
+            fstat = stat(f)
+            fsize = filesize(fstat)
+            if fsize != fsize_req
+                @debug "Rejecting stale cache file $cachefile because file size of $f has changed (file size $fsize, before $fsize_req)"
+                record_reason(reasons, "include_dependency fsize change")
+                return true
+            end
+            hash = isdir(fstat) ? _crc32c(join(readdir(f))) : open(_crc32c, f, "r")
+            if hash != hash_req
+                @debug "Rejecting stale cache file $cachefile because hash of $f has changed (hash $hash, before $hash_req)"
+                record_reason(reasons, "include_dependency fhash change")
+                return true
+            end
+        end
+    end
+    return false
+end
+
 # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey
 # otherwise returns the list of dependencies to also check
 @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false, requested_flags::CacheFlags=CacheFlags(), reasons=nothing)
@@ -3971,51 +4046,8 @@ end
                     return true
                 end
             end
-            for chi in includes
-                f, fsize_req, hash_req, ftime_req = chi.filename, chi.fsize, chi.hash, chi.mtime
-                if startswith(f, string("@depot", Filesystem.pathsep()))
-                    @debug("Rejecting stale cache file $cachefile because its depot could not be resolved")
-                    record_reason(reasons, "nonresolveable depot")
-                    return true
-                end
-                if !ispath(f)
-                    _f = fixup_stdlib_path(f)
-                    if _f != f && isfile(_f) && startswith(_f, Sys.STDLIB)
-                        continue
-                    end
-                    @debug "Rejecting stale cache file $cachefile because file $f does not exist"
-                    record_reason(reasons, "missing sourcefile")
-                    return true
-                end
-                if ftime_req >= 0.0
-                    # this is an include_dependency for which we only recorded the mtime
-                    ftime = mtime(f)
-                    is_stale = ( ftime != ftime_req ) &&
-                               ( ftime != floor(ftime_req) ) &&           # Issue #13606, PR #13613: compensate for Docker images rounding mtimes
-                               ( ftime != ceil(ftime_req) ) &&            # PR: #47433 Compensate for CirceCI's truncating of timestamps in its caching
-                               ( ftime != trunc(ftime_req, digits=6) ) && # Issue #20837, PR #20840: compensate for GlusterFS truncating mtimes to microseconds
-                               ( ftime != 1.0 )  &&                       # PR #43090: provide compatibility with Nix mtime.
-                               !( 0 < (ftime_req - ftime) < 1e-6 )        # PR #45552: Compensate for Windows tar giving mtimes that may be incorrect by up to one microsecond
-                    if is_stale
-                        @debug "Rejecting stale cache file $cachefile because mtime of include_dependency $f has changed (mtime $ftime, before $ftime_req)"
-                        record_reason(reasons, "include_dependency mtime change")
-                        return true
-                    end
-                else
-                    fstat = stat(f)
-                    fsize = filesize(fstat)
-                    if fsize != fsize_req
-                        @debug "Rejecting stale cache file $cachefile because file size of $f has changed (file size $fsize, before $fsize_req)"
-                        record_reason(reasons, "include_dependency fsize change")
-                        return true
-                    end
-                    hash = isdir(fstat) ? _crc32c(join(readdir(f))) : open(_crc32c, f, "r")
-                    if hash != hash_req
-                        @debug "Rejecting stale cache file $cachefile because hash of $f has changed (hash $hash, before $hash_req)"
-                        record_reason(reasons, "include_dependency fhash change")
-                        return true
-                    end
-                end
+            if any_includes_stale(includes, cachefile, reasons)
+                return true
             end
         end
 
@@ -4138,3 +4170,14 @@ precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{St
 precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) || @assert false
 precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), Cmd, IO, IO)) || @assert false
 precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), Cmd, IO, IO)) || @assert false
+
+is_core_or_base(mod::Module) = mod === Core || mod === Base
+function prepare_compiler_stub_image!()
+    ccall(:jl_add_to_module_init_list, Cvoid, (Any,), Compiler)
+    # Drop any dependencies on loaded modules. We know Base.Compiler only depends
+    # on very basic `Base` functionality, so we don't want to pick up dependencies
+    # on other stdlibs that may be loaded.
+    filter!(p->is_core_or_base(p.second), loaded_modules)
+    register_root_module(Compiler)
+    filter!(is_core_or_base, loaded_modules_order)
+end

base/show.jl

@@ -2826,7 +2826,7 @@ module IRShow
         Effects, ALWAYS_TRUE, ALWAYS_FALSE, DebugInfoStream, getdebugidx,
         VarState, InvalidIRError, argextype, widenconst, singleton_type,
         sptypes_from_meth_instance, EMPTY_SPTYPES
-    include("compiler/ssair/show.jl")
+    include("../stdlib/Compiler/src/ssair/show.jl")
 
     const __debuginfo = Dict{Symbol, Any}(
         # :full => src -> statementidx_lineinfo_printer(src), # and add variable slot information

contrib/generate_precompile.jl

@@ -359,6 +359,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe
             eval(PrecompileStagingArea, :(const $(Symbol(_mod)) = $_mod))
         end
     end
+    eval(PrecompileStagingArea, :(const Compiler = Base.Compiler))
 
     n_succeeded = 0
     # Make statements unique

src/module.c

@@ -1182,6 +1182,7 @@ JL_DLLEXPORT jl_uuid_t jl_module_uuid(jl_module_t* m) { return m->uuid; }
 
 // TODO: make this part of the module constructor and read-only?
 JL_DLLEXPORT void jl_set_module_uuid(jl_module_t *m, jl_uuid_t uuid) { m->uuid = uuid; }
+JL_DLLEXPORT void jl_set_module_parent(jl_module_t *m, jl_module_t *parent) { m->parent = parent; }
 
 int jl_is_submodule(jl_module_t *child, jl_module_t *parent) JL_NOTSAFEPOINT
 {
@@ -1214,15 +1215,20 @@ JL_DLLEXPORT void jl_clear_implicit_imports(jl_module_t *m)
     JL_UNLOCK(&m->lock);
 }
 
+JL_DLLEXPORT void jl_add_to_module_init_list(jl_value_t *mod)
+{
+    if (jl_module_init_order == NULL)
+        jl_module_init_order = jl_alloc_vec_any(0);
+    jl_array_ptr_1d_push(jl_module_init_order, mod);
+}
+
 JL_DLLEXPORT void jl_init_restored_module(jl_value_t *mod)
 {
     if (!jl_generating_output() || jl_options.incremental) {
         jl_module_run_initializer((jl_module_t*)mod);
     }
     else {
-        if (jl_module_init_order == NULL)
-            jl_module_init_order = jl_alloc_vec_any(0);
-        jl_array_ptr_1d_push(jl_module_init_order, mod);
+        jl_add_to_module_init_list(mod);
     }
 }