[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[gnumach] 01/05: Imported Upstream version 1.6+git20160311
|
From: |
Samuel Thibault |
|
Subject: |
[gnumach] 01/05: Imported Upstream version 1.6+git20160311 |
|
Date: |
Fri, 11 Mar 2016 23:22:34 +0000 |
This is an automated email from the git hooks/post-receive script.
sthibault pushed a commit to branch master
in repository gnumach.
commit f1e52f825e4dd5dceae317bf5b61b4251f835880
Author: Samuel Thibault <address@hidden>
Date: Fri Mar 11 22:03:15 2016 +0000
Imported Upstream version 1.6+git20160311
---
ChangeLog | 530 ++++++++++++++++++
Makefile.in | 85 ++-
Makefrag.am | 9 +-
build-aux/compile | 2 +-
build-aux/config.guess | 47 +-
build-aux/config.sub | 36 +-
build-aux/depcomp | 2 +-
build-aux/install-sh | 373 ++++++-------
build-aux/mdate-sh | 8 +-
build-aux/missing | 2 +-
build-aux/test-driver | 15 +-
configure | 20 +-
device/dev_lookup.c | 2 +-
device/dev_pager.c | 4 +-
device/ds_routines.c | 4 +-
device/net_io.c | 4 +-
doc/mach.info | 242 ++++----
doc/mach.info-1 | 18 +-
doc/mach.info-2 | 6 +-
doc/mach.texi | 5 +-
doc/stamp-vti | 8 +-
doc/version.texi | 8 +-
i386/Makefrag.am | 4 +
i386/i386/cpu.h | 110 ++++
i386/i386/db_trace.c | 4 +-
i386/i386/fpu.c | 51 +-
i386/i386/fpu.h | 1 +
i386/i386/locore.S | 14 +-
i386/i386/machine_task.c | 2 +-
i386/i386/pcb.c | 9 +-
i386/i386/pcb.h | 2 +-
i386/i386/thread.h | 1 +
i386/i386/vm_param.h | 53 +-
i386/i386at/biosmem.c | 910 +++++++++++++++++++++++++++++++
i386/i386at/biosmem.h | 88 +++
i386/i386at/elf.h | 61 +++
i386/i386at/kd_mouse.c | 2 +
i386/i386at/model_dep.c | 307 +----------
i386/i386at/model_dep.h | 9 +
i386/include/mach/i386/multiboot.h | 105 ++++
i386/include/mach/i386/vm_types.h | 9 +
i386/intel/pmap.c | 4 +-
include/mach/mach.defs | 41 ++
include/mach_debug/slab_info.h | 6 -
ipc/ipc_init.c | 8 +-
ipc/ipc_marequest.c | 2 +-
ipc/ipc_object.c | 3 +-
kern/act.c | 2 +-
kern/bootstrap.c | 16 +-
kern/cpu_number.h | 2 +
i386/i386at/model_dep.h => kern/exc.defs | 22 +-
kern/log2.h | 50 ++
kern/processor.c | 2 +-
kern/rdxtree.c | 2 +-
kern/sched_prim.c | 3 +
kern/slab.c | 354 ++++++------
kern/slab.h | 33 +-
kern/startup.c | 2 +-
kern/task.c | 2 +-
kern/thread.c | 27 +-
linux/dev/drivers/block/ahci.c | 47 +-
linux/dev/glue/glue.h | 4 +-
linux/dev/glue/kmem.c | 6 +-
linux/dev/include/linux/blk.h | 1 +
linux/dev/init/main.c | 143 +----
linux/pcmcia-cs/glue/ds.c | 6 -
linux/src/arch/i386/kernel/bios32.c | 2 +
linux/src/drivers/block/ide.c | 11 +-
linux/src/include/linux/compiler-gcc6.h | 67 +++
version.m4 | 2 +-
vm/memory_object_proxy.c | 2 +-
vm/pmap.h | 15 +-
vm/vm_external.c | 10 +-
vm/vm_external.h | 5 +
vm/vm_fault.c | 6 +-
vm/vm_init.c | 1 +
vm/vm_map.c | 64 +--
vm/vm_map.h | 3 -
vm/vm_object.c | 237 ++++----
vm/vm_object.h | 10 +-
vm/vm_page.c | 782 ++++++++++++++++++++++++++
vm/vm_page.h | 251 ++++++++-
vm/vm_pageout.c | 27 +-
vm/vm_resident.c | 593 ++++++--------------
vm/vm_user.c | 2 +-
85 files changed, 4248 insertions(+), 1802 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 3d8d038..b909aba 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,508 @@
+2016-03-11 Samuel Thibault <address@hidden>
+
+ Ship missing files
+ Makefrag.am (libkernel_a_SOURCES): Add kern/log2.h.
+ (EXTRA_DIST): Add kern/exc.defs.
+ i386/Makefrag.am (libkernel_a_SOURCES): Add i386/i386at/elf.h.
+
+2016-03-11 Richard Braun <address@hidden>
+
+ Merge remote-tracking branch 'remotes/origin/rbraun/vm_cache_policy'
+ Finally ;-).
+
+2016-03-09 Richard Braun <address@hidden>
+
+ Fix stack allocation on Xen
+ Stack allocation on Xen can fail because of fragmentation. This change
+ makes stack allocation use the slab allocator.
+
+ * kern/thread.c (thread_stack_cache): New global variable.
+ (stack_alloc): Use kmem_cache_alloc instead of vm_page_grab_contig.
+ (stack_collect): Use kmem_cache_free instead of vm_page_free_contig.
+ (kmem_cache_init): Initialize thread_stack_cache.
+
+2016-03-09 Richard Braun <address@hidden>
+
+ Relax slab allocation alignment constraint
+ * kern/slab.c (kmem_pagealloc_virtual): Pass alignment to function,
+ call kmem_alloc_aligned when greater than a page.
+ (kmem_pagealloc): Pass alignment to function.
+ (kmem_slab_create): Update call to kmem_pagealloc.
+ (kalloc): Likewise.
+ (kmem_cache_compute_properties): Fix handling of color with large slab
+ sizes.
+ (kmem_cache_init): Allow alignment greater than the page size.
+
+2016-03-06 Samuel Thibault <address@hidden>
+
+ Inherit fpu control word from parent to child
+ * i386/i386/thread.h (struct pcb): Add init_control field.
+ * i386/i386/fpu.h (fpinherit): New prototype.
+ * i386/i386/fpu.c (fpinit): Add thread parameter. When init_control
field is
+ set, use that value instead of a hardcoded one.
+ (fpinherit): New function.
+ (fp_load): Pass thread parameter to fpinit().
+ * kern/thread.c (thread_create): Pass parent task to pcb_init().
+ * i386/i386/pcb.c (pcb_init): Add parent_task parameter, call fpinherit
when
+ it is equal to current_task().
+
+2016-02-28 Justus Winter <address@hidden>
+
+ ipc: add missing kernel object type
+ * ipc/ipc_object.c (ikot_print_array): Add entry for IKOT_PAGER_PROXY.
+
+ doc: clarify memory object initialization
+ * doc/mach.texi: Mention another way how a memory manager can signal
+ that an object is ready.
+
+ i386: add parts of cpu.h from x15
+ * i386/Makefrag.am (libkernel_a_SOURCES): Add new file.
+ * i386/i386/cpu.h: New file.
+
+2016-02-26 Justus Winter <address@hidden>
+
+ include: avoid generating unused client stubs
+ * include/mach/mach.defs: Avoid generating unused client stubs, some
+ of which use an unreasonable amount of stack space and showed up in
+ compiler warnings.
+
+2016-02-26 Samuel Thibault <address@hidden>
+
+ Document thread_sleep about events woken from interrupt handlers
+ * kern/sched_prim.c (thread_sleep): Document case of events woken from
+ interrupt handlers.
+
+ Document why code is not racy
+ * i386/i386at/kd_mouse.c (kd_mouse_read): Document why the
+ assert_wait/thread_block pair is not racy.
+
+2016-02-23 Justus Winter <address@hidden>
+
+ Include the exception protocol in 'gnumach.msgids'
+ * Makefrag.am: Include the exception protocol in 'gnumach.msgids'.
+ * kern/exc.defs: New file.
+
+2016-02-22 Richard Braun <address@hidden>
+
+ Remove kmem cache flags from the debugging interface
+ * include/mach_debug/slab_info.h (CACHE_FLAGS_NO_CPU_POOL,
+ CACHE_FLAGS_SLAB_EXTERNAL, CACHE_FLAGS_NO_RECLAIM,
+ CACHE_FLAGS_VERIFY, CACHE_FLAGS_DIRECT): Remove macros.
+ * kern/slab.c (host_slab_info): Pass raw cache flags to caller.
+
+2016-02-22 Richard Braun <address@hidden>
+
+ Fix slab allocator option handling
+ The slab allocator has grown to use multiple ways to allocate slabs
+ as well as track them, which got a little messy. One consequence is
+ the breaking of the KMEM_CF_VERIFY option. In order to make the code
+ less confusing, this change expresses all options as explicit cache
+ flags and clearly defines their relationships.
+
+ The special kmem_slab and vm_map_entry caches are initialized
+ accordingly.
+
+ * kern/slab.c (KMEM_CF_DIRECTMAP): Rename to ...
+ (KMEM_CF_PHYSMEM): ... this new macro.
+ (KMEM_CF_DIRECT): Restore macro.
+ (KMEM_CF_USE_TREE, KMEM_CF_USE_PAGE): New macros.
+ (KMEM_CF_VERIFY): Update value.
+ (kmem_pagealloc_directmap): Rename to...
+ (kmem_pagealloc_physmem): ... this new function.
+ (kmem_pagefree_directmap): Rename to ...
+ (kmem_pagefree_physmem): ... this new function.
+ (kmem_pagealloc, kmem_pagefree): Update macro names.
+ (kmem_slab_use_tree): Remove function.
+ (kmem_slab_create, kmem_slab_destroy): Update according to the new
+ cache flags.
+ (kmem_cache_compute_sizes): Rename to ...
+ (kmem_cache_compute_properties): ... this new function, and update
+ to properly set cache flags.
+ (kmem_cache_init): Update call to kmem_cache_compute_properties.
+ (kmem_cache_alloc_from_slab): Check KMEM_CF_USE_TREE instead of
+ calling the defunct kmem_slab_use_tree function.
+ (kmem_cache_free_to_slab): Update according to the new cache flags.
+ kmem_cache_free_verify): Add assertion.
+ (slab_init): Update initialization of kmem_slab_cache.
+ * kern/slab.h (KMEM_CACHE_DIRECTMAP): Rename to ...
+ (KMEM_CACHE_PHYSMEM): ... this new macro.
+ * vm/vm_map.c (vm_map_init): Update initialization of
vm_map_entry_cache.
+
+2016-02-22 Richard Braun <address@hidden>
+
+ Optimize slab lookup on the free path
+ Caches that use external slab data but allocate slabs from the direct
+ physical mapping can look up slab data in constant time by associating
+ the slab data directly with the underlying page.
+
+ * kern/slab.c (kmem_slab_use_tree): Take KMEM_CF_DIRECTMAP into account.
+ (kmem_slab_create): Set page private data if relevant.
+ (kmem_slab_destroy): Clear page private data if relevant.
+ (kmem_cache_free_to_slab): Use page private data if relevant.
+ * vm/vm_page.c (vm_page_init_pa): Set `priv' member to NULL.
+ * vm/vm_page.h (vm_page_set_priv, vm_page_get_priv): New functions.
+
+2016-02-22 Richard Braun <address@hidden>
+
+ Fix unused variable warnings
+ * kern/slab.c (slab_init): Remove unused variables.
+
+2016-02-20 Richard Braun <address@hidden>
+
+ Avoid slab allocation failures caused by memory fragmentation
+ Since the slab allocator has been changed to sit directly on top of the
+ physical allocator, failures caused by fragmentation have been observed,
+ as one could expect. This change makes the slab allocator revert to
+ kernel virtual memory when allocating larger-than-page slabs. This
+ solution is motivated in part to avoid the complexity of other solutions
+ such as page mobility, and also because a microkernel cannot be extended
+ to new arbitrary uncontrolled usage patterns such as a monolithic kernel
+ with loadable modules. As such, large objects are rare, and their use
+ infrequent, which is compatible with the use of kernel virtual memory.
+
+ * kern/slab.c: Update module description.
+ (KMEM_CF_SLAB_EXTERNAL, KMEM_CF_VERIFY): Update values.
+ (KMEM_CF_DIRECT): Remove macro.
+ (KMEM_CF_DIRECTMAP): New macro.
+ (kmem_pagealloc_directmap, kmem_pagefree_directmap,
+ kmem_pagealloc_virtual, kmem_pagefree_virtual): New functions.
+ (kmem_pagealloc, kmem_pagefree, kmem_slab_create, kmem_slab_destroy,
+ kalloc, kfree): Update to use the new pagealloc functions.
+ (kmem_cache_compute_sizes): Update the algorithm used to determine slab
+ size and other cache properties.
+ (kmem_slab_use_tree, kmem_cache_free_to_slab, host_slab_info): Update to
+ correctly use the cache flags.
+ (slab_init): Add KMEM_CACHE_DIRECTMAP to the kmem_slab_cache init flags.
+ * kern/slab.h (KMEM_CACHE_VERIFY): Change value.
+ (KMEM_CACHE_DIRECTMAP): New macro.
+ * vm/vm_map.c (vm_map_init): Add KMEM_CACHE_DIRECTMAP to the
+ vm_map_entry_cache init flags.
+
+2016-02-16 Richard Braun <address@hidden>
+
+ Avoid panics on physical memory exhaustion
+ * vm/vm_resident (vm_page_grab): Return NULL instead of calling panic
+ on memory exhaustion.
+
+2016-02-14 Samuel Thibault <address@hidden>
+
+ Reduce VM_KERNEL_MAP_SIZE
+ Now that KMEM_MAP_SIZE (128MiB) has been removed.
+
+ * i386/i386/vm_param.h (VM_KERNEL_MAP_SIZE): Decrease by 128MiB.
+
+2016-02-07 Justus Winter <address@hidden>
+
+ vm: initialize external maps
+ * vm/vm_external.c (vm_external_create): Initialize allocated maps.
+
+2016-02-07 Richard Braun <address@hidden>
+
+ Fix page cache accounting
+ * vm/vm_object.c (vm_object_bootstrap): Set template object `cached'
+ member to FALSE.
+ (vm_object_cache_add, vm_object_cache_remove): New functions.
+ (vm_object_collect, vm_object_deallocate, vm_object_lookup,
+ vm_object_lookup_name, vm_object_destroy): Use new cache management
functions.
+ (vm_object_terminate, vm_object_collapse): Make sure object isn't
cached.
+ * vm/vm_object.h (struct vm_object): New `cached' member.
+
+2016-02-07 Justus Winter <address@hidden>
+
+ vm: allocate a large map for all objects larger than SMALL_SIZE
+ * vm/vm_external.c (vm_external_create): Allocate a large map for all
+ objects larger than SMALL_SIZE. 'vm_external_state_{g,s}et' can deal
+ with offsets larger than 'LARGE_SIZE', so currently objects larger
+ than 'LARGE_SIZE' are missing out on the optimization.
+
+ vm: remove unused field from struct vm_external
+ * vm/vm_external.h (struct vm_external): Remove unused field
+ 'existence_count'.
+
+2016-02-07 Richard Braun <address@hidden>
+
+ Remove kmem map
+ Now that the slab allocator doesn't use kernel virtual memory any more,
+ this map has become irrelevant.
+
+ * kern/slab.c (KMEM_MAP_SIZE): Remove macro.
+ (kmem_map_store, kmem_map): Remove variables.
+ (slab_init): Remove call kmem_submap.
+ * kern/slab.h (kmem_map): Remove extern declaration.
+
+2016-02-06 Richard Braun <address@hidden>
+
+ Change computation of slab size
+ Allocating directly out of the physical memory allocator makes the slab
+ allocator vulnerable to failures due to fragmentation. This change makes
+ the slab allocator use the lowest possible size for its slabs to reduce
+ the chance of contiguous allocation failures.
+
+ * kern/slab.c (KMEM_MIN_BUFS_PER_SLAB, KMEM_SLAB_SIZE_THRESHOLD): Remove
+ macros.
+ (kmem_cache_compute_sizes): Update the algorithm used to determine slab
+ size and other cache properties.
+
+2016-02-02 Richard Braun <address@hidden>
+
+ Fix various memory managment errors
+ A few errors were introduced in the latest changes.
+
+ o Add VM_PAGE_WAIT calls around physical allocation attempts in case of
+ memory exhaustion.
+ o Fix stack release.
+ o Fix memory exhaustion report.
+ o Fix free page accounting.
+
+ * kern/slab.c (kmem_pagealloc, kmem_pagefree): New functions
+ (kmem_slab_create, kmem_slab_destroy, kalloc, kfree): Use kmem_pagealloc
+ and kmem_pagefree instead of the raw page allocation functions.
+ (kmem_cache_compute_sizes): Don't store slab order.
+ * kern/slab.h (struct kmem_cache): Remove `slab_order' member.
+ * kern/thread.c (stack_alloc): Call VM_PAGE_WAIT in case of memory
+ exhaustion.
+ (stack_collect): Call vm_page_free_contig instead of kmem_free to
+ release pages.
+ * vm/vm_page.c (vm_page_seg_alloc): Fix memory exhaustion report.
+ (vm_page_setup): Don't update vm_page_free_count.
+ (vm_page_free_pa): Check page parameter.
+ (vm_page_mem_free): New function.
+ * vm/vm_page.h (vm_page_free_count): Remove extern declaration.
+ (vm_page_mem_free): New prototype.
+ * vm/vm_pageout.c: Update comments not to refer to vm_page_free_count.
+ (vm_pageout_scan, vm_pageout_continue, vm_pageout): Use vm_page_mem_free
+ instead of vm_page_free_count, update types accordingly.
+ * vm/vm_resident.c (vm_page_free_count, vm_page_free_count_minimum):
+ Remove variables.
+ (vm_page_free_avail): New variable.
+ (vm_page_bootstrap, vm_page_grab, vm_page_release, vm_page_grab_contig,
+ vm_page_free_contig, vm_page_wait): Use vm_page_mem_free instead of
vm_page_free_count,
+ update types accordingly, don't set vm_page_free_count_minimum.
+ * vm/vm_user.c (vm_statistics): Likewise.
+
+2016-02-02 Richard Braun <address@hidden>
+
+ Fix unused variable warnings
+ * i386/i386at/biosmem.c (biosmem_bootstrap): Remove unused variables.
+
+2016-02-02 Richard Braun <address@hidden>
+
+ Stack the slab allocator directly on top of the physical allocator
+ In order to increase the amount of memory available for kernel objects,
+ without reducing the amount of memory available for user processes,
+ a new allocation strategy is introduced in this change.
+
+ Instead of allocating kernel objects out of kernel virtual memory,
+ the slab allocator directly uses the direct mapping of physical
+ memory as its backend. This largely increases the kernel heap, and
+ removes the need for address translation updates.
+
+ In order to allow this strategy, an assumption made by the interrupt
+ code had to be removed. In addition, kernel stacks are now also
+ allocated directly from the physical allocator.
+
+ * i386/i386/db_trace.c: Include i386at/model_dep.h
+ (db_i386_reg_value): Update stack check.
+ * i386/i386/locore.S (trap_from_kernel, all_intrs,
+ int_from_intstack): Update interrupt handling.
+ * i386/i386at/model_dep.c: Include kern/macros.h.
+ (int_stack, int_stack_base): New variables.
+ (int_stack_high): Remove variable.
+ (i386at_init): Update interrupt stack initialization.
+ * i386/i386at/model_dep.h: Include i386/vm_param.h.
+ (int_stack_top, int_stack_base): New extern declarations.
+ (ON_INT_STACK): New macro.
+ * kern/slab.c: Include vm/vm_page.h
+ (KMEM_CF_NO_CPU_POOL, KMEM_CF_NO_RECLAIM): Remove macros.
+ (kmem_pagealloc, kmem_pagefree, kalloc_pagealloc, kalloc_pagefree):
Remove
+ functions.
+ (kmem_slab_create): Allocate slab pages directly from the physical
allocator.
+ (kmem_slab_destroy): Release slab pages directly to the physical
allocator.
+ (kmem_cache_compute_sizes): Update the slab size computation algorithm
to
+ return a power-of-two suitable for the physical allocator.
+ (kmem_cache_init): Remove custom allocation function pointers.
+ (kmem_cache_reap): Remove check on KMEM_CF_NO_RECLAIM.
+ (slab_init, kalloc_init): Update calls to kmem_cache_init.
+ (kalloc, kfree): Directly fall back on the physical allocator for big
+ allocation sizes.
+ (host_slab_info): Remove checks on defunct flags.
+ * kern/slab.h (kmem_slab_alloc_fn_t, kmem_slab_free_fn_t): Remove types.
+ (struct kmem_cache): Add `slab_order' member, remove `slab_alloc_fn' and
+ `slab_free_fn' members.
+ (KMEM_CACHE_NOCPUPOOL, KMEM_CACHE_NORECLAIM): Remove macros.
+ (kmem_cache_init): Update prototype, remove custom allocation functions.
+ * kern/thread.c (stack_alloc): Allocate stacks from the physical
allocator.
+ * vm/vm_map.c (vm_map_kentry_cache, kentry_data, kentry_data_size):
Remove
+ variables.
+ (kentry_pagealloc): Remove function.
+ (vm_map_init): Update calls to kmem_cache_init, remove initialization of
+ vm_map_kentry_cache.
+ (vm_map_create, _vm_map_entry_dispose, vm_map_copyout): Unconditionnally
+ use vm_map_entry_cache.
+ * vm/vm_map.h (kentry_data, kentry_data_size, kentry_count): Remove
extern
+ declarations.
+ * vm/vm_page.h (VM_PT_STACK): New page type.
+ * device/dev_lookup.c (dev_lookup_init): Update calls to
kmem_cache_init.
+ * device/dev_pager.c (dev_pager_hash_init, device_pager_init): Likewise.
+ * device/ds_routines.c (mach_device_init, mach_device_trap_init):
Likewise.
+ * device/net_io.c (net_io_init): Likewise.
+ * i386/i386/fpu.c (fpu_module_init): Likewise.
+ * i386/i386/machine_task.c (machine_task_module_init): Likewise.
+ * i386/i386/pcb.c (pcb_module_init): Likewise.
+ * i386/intel/pmap.c (pmap_init): Likewise.
+ * ipc/ipc_init.c (ipc_bootstrap): Likewise.
+ * ipc/ipc_marequest.c (ipc_marequest_init): Likewise.
+ * kern/act.c (global_act_init): Likewise.
+ * kern/processor.c (pset_sys_init): Likewise.
+ * kern/rdxtree.c (rdxtree_cache_init): Likewise.
+ * kern/task.c (task_init): Likewise.
+ * vm/memory_object_proxy.c (memory_object_proxy_init): Likewise.
+ * vm/vm_external.c (vm_external_module_initialize): Likewise.
+ * vm/vm_fault.c (vm_fault_init): Likewise.
+ * vm/vm_object.c (vm_object_bootstrap): Likewise.
+ * vm/vm_resident.c (vm_page_module_init): Likewise.
+ (vm_page_bootstrap): Remove initialization of kentry_data.
+
+2016-01-30 Richard Braun <address@hidden>
+
+ Fix early page allocation on Xen
+ The Xen target was completely ignored when porting the biosmem and
+ vm_page physical memory allocators. Let's fix this.
+
+ * i386/Makefrag.am (libkernel_a_SOURCES): Add i386/i386at/biosmem.{c,h}.
+ * i386/i386/vm_page.h (VM_PAGE_MAX_SEGS, VM_PAGE_DIRECTMAP_LIMIT,
+ VM_PAGE_HIGHMEM_LIMIT): Define for Xen.
+ * i386/i386at/biosmem.c: Include mach/xen.h.
+ (biosmem_panic_setup_msg): Comment out for Xen since it's unused.
+ (biosmem_map_build, biosmem_map_build_simple,
+ biosmem_save_cmdline_sizes, biosmem_find_boot_data_update,
+ biosmem_find_boot_data, biosmem_setup_allocator): Likewise.
+ (biosmem_bootstrap_common): New function.
+ (biosmem_xen_bootstrap): Likewise, for Xen.
+ (biosmem_bootalloc): Perform bottom-up allocations for Xen.
+ * i386/i386at/biosmem.h (biosmem_xen_bootstrap): New prototype, for Xen.
+ * i386/i386at/model_dep.c (i386at_init): Call biosmem_xen_bootstrap
instead
+ of biosmem_bootstrap on Xen.
+ * i386/include/mach/i386/vm_types.h (phys_addr_t): Define as an unsigned
+ 64-bits integer when PAE is enabled.
+
+2016-01-29 Samuel Thibault <address@hidden>
+
+ Disable probing legacy IDE when AHCI driver works
+ * linux/src/drivers/block/ide.c (default_io_base): Do not qualify const.
+ (ide_disable_base): New function.
+ * linux/dev/include/linux/blk.h (ide_disable_base): New declaration.
+ * linux/dev/drivers/block/ahci.c (ahci_probe_dev): Call ide_disable_base
+ with each I/O BAR of the AHCI PCI card.
+
+ Use PCI macros
+ * linux/dev/drivers/block/ahci.c (ahci_probe_dev): Use
+ PCI_BASE_ADDRESS_SPACE_IO and PCI_BASE_ADDRESS_MEM_MASK macros instead
of
+ hardcoded values.
+
+ ahci: print PCI bus/dev/fun in hexadecimal
+ * linux/dev/drivers/block/ahci.c: Print PCI bus/dev/fun number in
+ hexadecimal
+
+2016-01-29 Samuel Thibault <address@hidden>
+
+ Fallback on direct PCI access when no BIOS32 is available
+ Some hardware start shippping with no BIOS32 at all, and we'll have to
+ implement ACPI to get the address of the mmconfig table. In the
meanwhile,
+ we can hope that the direct probe works (it does on HP820 for instance).
+
+ * linux/src/arch/i386/kernel/bios32.c (pcibios_init): Also try
+ check_direct_pci() when bios32 probing failed.
+
+2016-01-23 Richard Braun <address@hidden>
+
+ Merge branch 'rbraun/vm_page'
+
+2016-01-23 Richard Braun <address@hidden>
+
+ Use vm_page as the physical memory allocator
+ This change replaces the historical page allocator with a buddy
allocator
+ implemented in vm/vm_page.c. This allocator allows easy contiguous
allocations
+ and also manages memory inside segments. In a future change, these
segments
+ will be used to service requests with special constraints, such as
"usable
+ for 16-bits DMA" or "must be part of the direct physical mapping".
+
+ * Makefrag.am (libkernel_a_SOURCES): Add vm/vm_page.c.
+ * i386/Makefrag.am (libkernel_a_SOURCES): Add i386/i386at/biosmem.{c,h}.
+ * i386/i386/vm_param.h: Include kern/macros.h.
+ (VM_PAGE_DMA_LIMIT, VM_PAGE_MAX_SEGS, VM_PAGE_DMA32_LIMIT,
+ VM_PAGE_DIRECTMAP_LIMIT, VM_PAGE_HIGHMEM_LIMIT, VM_PAGE_SEG_DMA,
+ VM_PAGE_SEG_DMA32, VM_PAGE_SEG_DIRECTMAP, VM_PAGE_SEG_HIGHMEM): New
macros.
+ * i386/i386at/model_dep.c: Include i386at/biosmem.h.
+ (avail_next, avail_remaining): Remove variables.
+ (mem_size_init): Remove function.
+ (i386at_init): Initialize and use the biosmem module for early physical
+ memory management.
+ (pmap_free_pages): Return phys_last_addr instead of avail_remaining.
+ (init_alloc_aligned): Turn into a wrapper for biosmem_bootalloc.
+ (pmap_grab_page): Directly call init_alloc_aligned instead of
pmap_next_page.
+ * i386/include/mach/i386/vm_types.h (phys_addr_t): New type.
+ * kern/bootstrap.c (free_bootstrap_pages): New function.
+ (bootstrap_create): Call free_bootstrap_pages instead of vm_page_create.
+ * kern/cpu_number.h (CPU_L1_SIZE): New macro.
+ * kern/slab.h: Include kern/cpu_number.h.
+ (CPU_L1_SIZE): Remove macro, moved to kern/cpu_number.h.
+ * kern/startup.c (setup_main): Change the value of
machine_info.memory_size.
+ * linux/dev/glue/glue.h (alloc_contig_mem, free_contig_mem): Update
prototypes.
+ * linux/dev/glue/kmem.c (linux_kmem_init): Don't use defunct page queue.
+ * linux/dev/init/main.c (linux_init): Don't free unused memory.
+ (alloc_contig_mem, free_contig_mem): Turn into wrappers for the vm_page
+ allocator.
+ * linux/pcmcia-cs/glue/ds.c (PAGE_SHIFT): Don't undefine.
+ * vm/pmap.h (pmap_startup, pmap_next_page): Remove prototypes.
+ * vm/vm_fault.c (vm_fault_page): Update calls to vm_page_convert.
+ * vm/vm_init.c (vm_mem_init): Call vm_page_info_all.
+ * vm/vm_object.c (vm_object_page_map): Update call to vm_page_init.
+ * vm/vm_page.h (vm_page_queue_free): Remove variable declaration.
+ (vm_page_create, vm_page_release_fictitious, vm_page_release): Remove
+ declarations.
+ (vm_page_convert, vm_page_init): Update prototypes.
+ (vm_page_grab_contig, vm_page_free_contig): New prototypes.
+ * vm/vm_resident.c (vm_page_template, vm_page_queue_free,
+ vm_page_big_pagenum): Remove variables.
+ (vm_page_bootstrap): Update and call vm_page_setup.
+ (pmap_steal_memory): Update and call vm_page_bootalloc.
+ (pmap_startup, vm_page_create, vm_page_grab_contiguous_pages): Remove
functions.
+ (vm_page_init_template, vm_page_grab_contig,
+ vm_page_free_contig): New functions.
+ (vm_page_init): Update and call vm_page_init_template.
+ (vm_page_release_fictitious): Make static.
+ (vm_page_more_fictitious): Update call to vm_page_init.
+ (vm_page_convert): Rewrite to comply with vm_page.
+ (vm_page_grab): Update and call vm_page_alloc_pa.
+ (vm_page_release): Update and call vm_page_free_pa.
+
+2016-01-23 Richard Braun <address@hidden>
+
+ Import the vm_page module from X15 and relicense to GPLv2+
+ * vm/vm_page.c: New File.
+ * vm/vm_page.h: Merge vm_page.h from X15.
+ (struct vm_page): New members: node, type, seg_index, order,
+ vm_page_header. Turn phys_addr into a phys_addr_t.
+
+ Import the biosmem module from X15 and relicense to GPLv2+
+ * i386/i386at/biosmem.c: New file.
+ * i386/i386at/biosmem.h: Likewise.
+
+ Import the multiboot module from X15 and relicense to GPLv2+
+ * i386/include/mach/i386/multiboot.h: Merge multiboot.h from X15.
+
+ Import the elf module from X15 and relicense to GPLv2+
+ * i386/i386at/elf.h: New file.
+
+ Import the log2 module from X15 and relicense to GPLv2+
+ * kern/log2.h: New file.
+
+2016-01-14 Samuel Thibault <address@hidden>
+
+ Fix build with gcc-6
+ * linux/src/include/linux/compiler-gcc6.h: New file.
+
2016-01-13 Samuel Thibault <address@hidden>
Increase kernel map size
@@ -4308,6 +4813,31 @@
Increase kernel map entry pool size
* vm/vm_map.h (KENTRY_DATA_SIZE): Set to 256 pages.
+2013-10-09 Richard Braun <address@hidden>
+
+ VM cache policy change
+ This patch lets the kernel unconditionnally cache non empty unreferenced
+ objects instead of using a fixed arbitrary limit. As the pageout daemon
+ evicts pages, it collects cached objects that have become empty. The
+ effective result is a graceful adjustment of the number of objects
+ related to memory management (virtual memory objects, their associated
+ ports, and potentially objects maintained in the external memory
+ managers). Physical memory can now be almost entirely filled up with
+ cached pages. In addition, these cached pages are not automatically
+ deactivated as objects can quickly be referenced again.
+
+ There are problems with this patch however. The first is that, on
+ machines with a large amount of physical memory (above 1 GiB but it also
+ depends on usage patterns), scalability issues are exposed. For example,
+ file systems which don't throttle their writeback requests can create
+ thread storms, strongly reducing system responsiveness. Other issues
+ such as linear scans of memory objects also add visible CPU overhead.
+
+ The second is that, as most memory is used, it increases the chances of
+ swapping deadlocks. Applications that map large objects and quickly
+ cause lots of page faults can still easily bring the system to its
+ knees.
+
2013-09-28 Samuel Thibault <address@hidden>
Add files missing in distrib tarball
diff --git a/Makefile.in b/Makefile.in
index 90b4555..23a8556 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -953,8 +953,11 @@ noinst_PROGRAMS = gnumach.o$(EXEEXT)
@address@hidden = \
@HOST_ix86_TRUE@ i386/i386at/autoconf.c \
@HOST_ix86_TRUE@ i386/i386at/autoconf.h \
address@hidden@ i386/i386at/biosmem.c \
address@hidden@ i386/i386at/biosmem.h \
@HOST_ix86_TRUE@ i386/i386at/conf.c \
@HOST_ix86_TRUE@ i386/i386at/cons_conf.c \
address@hidden@ i386/i386at/elf.h \
@HOST_ix86_TRUE@ i386/i386at/idt.h \
@HOST_ix86_TRUE@ i386/i386at/model_dep.c \
@HOST_ix86_TRUE@ i386/i386at/model_dep.h \
@@ -1020,6 +1023,7 @@ noinst_PROGRAMS = gnumach.o$(EXEEXT)
@HOST_ix86_TRUE@ i386/i386/ast.h \
@HOST_ix86_TRUE@ i386/i386/ast_check.c \
@HOST_ix86_TRUE@ i386/i386/ast_types.h \
address@hidden@ i386/i386/cpu.h \
@HOST_ix86_TRUE@ i386/i386/cpu_number.h \
@HOST_ix86_TRUE@ i386/i386/cswitch.S \
@HOST_ix86_TRUE@ i386/i386/db_disasm.c \
@@ -1210,6 +1214,7 @@ nodist_lib_dep_tr_for_defs_a_OBJECTS =
vm/lib_dep_tr_for_defs_a-memory_object_us
kern/lib_dep_tr_for_defs_a-mach_debug.server.defs.$(OBJEXT) \
kern/lib_dep_tr_for_defs_a-mach_host.server.defs.$(OBJEXT) \
ipc/lib_dep_tr_for_defs_a-notify.none.defs.$(OBJEXT) \
+ kern/lib_dep_tr_for_defs_a-exc.none.defs.$(OBJEXT) \
$(am__objects_1)
lib_dep_tr_for_defs_a_OBJECTS = \
$(nodist_lib_dep_tr_for_defs_a_OBJECTS)
@@ -1249,7 +1254,7 @@ am__libkernel_a_SOURCES_DIST = ddb/db_access.c
ddb/db_access.h \
kern/ipc_kobject.h kern/ipc_mig.c kern/ipc_mig.h \
kern/ipc_sched.c kern/ipc_sched.h kern/ipc_tt.c kern/ipc_tt.h \
kern/kalloc.h kern/kern_types.h kern/list.h kern/lock.c \
- kern/lock.h kern/lock_mon.c kern/mach_clock.c \
+ kern/lock.h kern/lock_mon.c kern/log2.h kern/mach_clock.c \
kern/mach_clock.h kern/mach_factor.c kern/mach_factor.h \
kern/machine.c kern/machine.h kern/macros.h kern/pc_sample.c \
kern/pc_sample.h kern/printf.c kern/printf.h kern/priority.c \
@@ -1271,11 +1276,11 @@ am__libkernel_a_SOURCES_DIST = ddb/db_access.c
ddb/db_access.h \
vm/vm_external.c vm/vm_external.h vm/vm_fault.c vm/vm_fault.h \
vm/vm_init.c vm/vm_init.h vm/vm_kern.c vm/vm_kern.h \
vm/vm_map.c vm/vm_map.h vm/vm_object.c vm/vm_object.h \
- vm/vm_page.h vm/vm_pageout.c vm/vm_pageout.h vm/vm_print.h \
- vm/vm_resident.c vm/vm_resident.h vm/vm_types.h vm/vm_user.c \
- vm/vm_user.h device/blkio.c device/blkio.h device/buf.h \
- device/chario.c device/chario.h device/cirbuf.h device/conf.h \
- device/cons.c device/cons.h device/device_emul.h \
+ vm/vm_page.c vm/vm_page.h vm/vm_pageout.c vm/vm_pageout.h \
+ vm/vm_print.h vm/vm_resident.c vm/vm_resident.h vm/vm_types.h \
+ vm/vm_user.c vm/vm_user.h device/blkio.c device/blkio.h \
+ device/buf.h device/chario.c device/chario.h device/cirbuf.h \
+ device/conf.h device/cons.c device/cons.h device/device_emul.h \
device/dev_hdr.h device/dev_lookup.c device/dev_master.h \
device/dev_name.c device/dev_pager.c device/dev_pager.h \
device/device_init.c device/device_init.h device/device_port.h \
@@ -1306,7 +1311,8 @@ am__libkernel_a_SOURCES_DIST = ddb/db_access.c
ddb/db_access.h \
xen/net.c xen/net.h xen/ring.c xen/ring.h xen/store.c \
xen/store.h xen/time.c xen/time.h xen/xen.c xen/xen.h \
i386/i386at/autoconf.c i386/i386at/autoconf.h \
- i386/i386at/conf.c i386/i386at/cons_conf.c i386/i386at/idt.h \
+ i386/i386at/biosmem.c i386/i386at/biosmem.h i386/i386at/conf.c \
+ i386/i386at/cons_conf.c i386/i386at/elf.h i386/i386at/idt.h \
i386/i386at/model_dep.c i386/i386at/model_dep.h \
i386/include/mach/sa/stdarg.h i386/i386at/boothdr.S \
i386/i386at/com.c i386/i386at/com.h i386/i386at/comreg.h \
@@ -1326,7 +1332,7 @@ am__libkernel_a_SOURCES_DIST = ddb/db_access.c
ddb/db_access.h \
i386/grub/misc.h i386/grub/types.h i386/grub/time.h \
i386/grub/i18n.h i386/grub/compiler.h i386/grub/glue.h \
i386/i386at/lpr.c i386/i386at/lpr.h i386/i386/ast.h \
- i386/i386/ast_check.c i386/i386/ast_types.h \
+ i386/i386/ast_check.c i386/i386/ast_types.h i386/i386/cpu.h \
i386/i386/cpu_number.h i386/i386/cswitch.S \
i386/i386/db_disasm.c i386/i386/db_interface.c \
i386/i386/db_interface.h i386/i386/db_machdep.h \
@@ -1382,6 +1388,7 @@ am__libkernel_a_SOURCES_DIST = ddb/db_access.c
ddb/db_access.h \
@PLATFORM_xen_TRUE@ xen/ring.$(OBJEXT) xen/store.$(OBJEXT) \
@PLATFORM_xen_TRUE@ xen/time.$(OBJEXT) xen/xen.$(OBJEXT)
@address@hidden = i386/i386at/autoconf.$(OBJEXT) \
address@hidden@ i386/i386at/biosmem.$(OBJEXT) \
@HOST_ix86_TRUE@ i386/i386at/conf.$(OBJEXT) \
@HOST_ix86_TRUE@ i386/i386at/cons_conf.$(OBJEXT) \
@HOST_ix86_TRUE@ i386/i386at/model_dep.$(OBJEXT)
@@ -1477,17 +1484,17 @@ am_libkernel_a_OBJECTS = $(am__objects_2)
ipc/ipc_entry.$(OBJEXT) \
vm/vm_debug.$(OBJEXT) vm/vm_external.$(OBJEXT) \
vm/vm_fault.$(OBJEXT) vm/vm_init.$(OBJEXT) \
vm/vm_kern.$(OBJEXT) vm/vm_map.$(OBJEXT) \
- vm/vm_object.$(OBJEXT) vm/vm_pageout.$(OBJEXT) \
- vm/vm_resident.$(OBJEXT) vm/vm_user.$(OBJEXT) \
- device/blkio.$(OBJEXT) device/chario.$(OBJEXT) \
- device/cons.$(OBJEXT) device/dev_lookup.$(OBJEXT) \
- device/dev_name.$(OBJEXT) device/dev_pager.$(OBJEXT) \
- device/device_init.$(OBJEXT) device/ds_routines.$(OBJEXT) \
- device/net_io.$(OBJEXT) device/subrs.$(OBJEXT) \
- $(am__objects_3) $(am__objects_4) $(am__objects_5) \
- $(am__objects_6) $(am__objects_7) $(am__objects_8) \
- $(am__objects_9) $(am__objects_10) $(am__objects_11) \
- $(am__objects_12)
+ vm/vm_object.$(OBJEXT) vm/vm_page.$(OBJEXT) \
+ vm/vm_pageout.$(OBJEXT) vm/vm_resident.$(OBJEXT) \
+ vm/vm_user.$(OBJEXT) device/blkio.$(OBJEXT) \
+ device/chario.$(OBJEXT) device/cons.$(OBJEXT) \
+ device/dev_lookup.$(OBJEXT) device/dev_name.$(OBJEXT) \
+ device/dev_pager.$(OBJEXT) device/device_init.$(OBJEXT) \
+ device/ds_routines.$(OBJEXT) device/net_io.$(OBJEXT) \
+ device/subrs.$(OBJEXT) $(am__objects_3) $(am__objects_4) \
+ $(am__objects_5) $(am__objects_6) $(am__objects_7) \
+ $(am__objects_8) $(am__objects_9) $(am__objects_10) \
+ $(am__objects_11) $(am__objects_12)
@address@hidden = i386/i386/mach_i386.server.$(OBJEXT)
nodist_libkernel_a_OBJECTS = version.$(OBJEXT) \
vm/memory_object_user.user.$(OBJEXT) \
@@ -2452,7 +2459,7 @@ DIST_SUBDIRS =
# Building a distribution.
#
EXTRA_DIST = gensym.awk ipc/mach_port.srv ipc/notify.defs \
- kern/mach.srv kern/mach4.srv kern/gnumach.srv \
+ kern/exc.defs kern/mach.srv kern/mach4.srv kern/gnumach.srv \
kern/mach_debug.srv kern/mach_host.srv kern/task_notify.cli \
vm/memory_object_default.cli vm/memory_object_user.cli \
device/device.srv device/device_pager.srv \
@@ -2578,7 +2585,7 @@ libkernel_a_SOURCES = $(am__append_2) ipc/ipc_entry.c
ipc/ipc_entry.h \
kern/ipc_kobject.h kern/ipc_mig.c kern/ipc_mig.h \
kern/ipc_sched.c kern/ipc_sched.h kern/ipc_tt.c kern/ipc_tt.h \
kern/kalloc.h kern/kern_types.h kern/list.h kern/lock.c \
- kern/lock.h kern/lock_mon.c kern/mach_clock.c \
+ kern/lock.h kern/lock_mon.c kern/log2.h kern/mach_clock.c \
kern/mach_clock.h kern/mach_factor.c kern/mach_factor.h \
kern/machine.c kern/machine.h kern/macros.h kern/pc_sample.c \
kern/pc_sample.h kern/printf.c kern/printf.h kern/priority.c \
@@ -2600,11 +2607,11 @@ libkernel_a_SOURCES = $(am__append_2) ipc/ipc_entry.c
ipc/ipc_entry.h \
vm/vm_external.c vm/vm_external.h vm/vm_fault.c vm/vm_fault.h \
vm/vm_init.c vm/vm_init.h vm/vm_kern.c vm/vm_kern.h \
vm/vm_map.c vm/vm_map.h vm/vm_object.c vm/vm_object.h \
- vm/vm_page.h vm/vm_pageout.c vm/vm_pageout.h vm/vm_print.h \
- vm/vm_resident.c vm/vm_resident.h vm/vm_types.h vm/vm_user.c \
- vm/vm_user.h device/blkio.c device/blkio.h device/buf.h \
- device/chario.c device/chario.h device/cirbuf.h device/conf.h \
- device/cons.c device/cons.h device/device_emul.h \
+ vm/vm_page.c vm/vm_page.h vm/vm_pageout.c vm/vm_pageout.h \
+ vm/vm_print.h vm/vm_resident.c vm/vm_resident.h vm/vm_types.h \
+ vm/vm_user.c vm/vm_user.h device/blkio.c device/blkio.h \
+ device/buf.h device/chario.c device/chario.h device/cirbuf.h \
+ device/conf.h device/cons.c device/cons.h device/device_emul.h \
device/dev_hdr.h device/dev_lookup.c device/dev_master.h \
device/dev_name.c device/dev_pager.c device/dev_pager.h \
device/device_init.c device/device_init.h device/device_port.h \
@@ -2642,7 +2649,7 @@ nodist_libkernel_a_SOURCES = version.c
vm/memory_object_user.user.h \
kern/mach_debug.server.c kern/mach_debug.server.msgids \
kern/mach_host.server.h kern/mach_host.server.c \
kern/mach_host.server.msgids ipc/notify.none.msgids \
- $(am__append_122)
+ kern/exc.none.msgids $(am__append_122)
gnumach_o_LDADD = libkernel.a $(am__append_6) $(am__append_100)
gnumach_SOURCES =
gnumach_LINKFLAGS = $(am__append_123) $(am__append_126)
@@ -2681,7 +2688,7 @@ nodist_lib_dep_tr_for_defs_a_SOURCES = \
kern/mach.server.defs.c kern/mach4.server.defs.c \
kern/gnumach.server.defs.c kern/mach_debug.server.defs.c \
kern/mach_host.server.defs.c ipc/notify.none.defs.c \
- $(am__append_121)
+ kern/exc.none.defs.c $(am__append_121)
# Preprocess only.
lib_dep_tr_for_defs_a_CPPFLAGS = $(AM_CPPFLAGS) \
-E
@@ -3065,6 +3072,8 @@
kern/lib_dep_tr_for_defs_a-mach_host.server.defs.$(OBJEXT): \
kern/$(am__dirstamp) kern/$(DEPDIR)/$(am__dirstamp)
ipc/lib_dep_tr_for_defs_a-notify.none.defs.$(OBJEXT): \
ipc/$(am__dirstamp) ipc/$(DEPDIR)/$(am__dirstamp)
+kern/lib_dep_tr_for_defs_a-exc.none.defs.$(OBJEXT): \
+ kern/$(am__dirstamp) kern/$(DEPDIR)/$(am__dirstamp)
i386/i386/$(am__dirstamp):
@$(MKDIR_P) i386/i386
@: > i386/i386/$(am__dirstamp)
@@ -3270,6 +3279,7 @@ vm/vm_kern.$(OBJEXT): vm/$(am__dirstamp)
vm/$(DEPDIR)/$(am__dirstamp)
vm/vm_map.$(OBJEXT): vm/$(am__dirstamp) vm/$(DEPDIR)/$(am__dirstamp)
vm/vm_object.$(OBJEXT): vm/$(am__dirstamp) \
vm/$(DEPDIR)/$(am__dirstamp)
+vm/vm_page.$(OBJEXT): vm/$(am__dirstamp) vm/$(DEPDIR)/$(am__dirstamp)
vm/vm_pageout.$(OBJEXT): vm/$(am__dirstamp) \
vm/$(DEPDIR)/$(am__dirstamp)
vm/vm_resident.$(OBJEXT): vm/$(am__dirstamp) \
@@ -3321,6 +3331,8 @@ i386/i386at/$(DEPDIR)/$(am__dirstamp):
@: > i386/i386at/$(DEPDIR)/$(am__dirstamp)
i386/i386at/autoconf.$(OBJEXT): i386/i386at/$(am__dirstamp) \
i386/i386at/$(DEPDIR)/$(am__dirstamp)
+i386/i386at/biosmem.$(OBJEXT): i386/i386at/$(am__dirstamp) \
+ i386/i386at/$(DEPDIR)/$(am__dirstamp)
i386/i386at/conf.$(OBJEXT): i386/i386at/$(am__dirstamp) \
i386/i386at/$(DEPDIR)/$(am__dirstamp)
i386/i386at/cons_conf.$(OBJEXT): i386/i386at/$(am__dirstamp) \
@@ -4232,6 +4244,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
address@hidden@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/i386at/$(DEPDIR)/address@hidden@
@@ -4292,6 +4305,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
address@hidden@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@@ -4490,6 +4504,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
address@hidden@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@AMDEP_TRUE@@am__include@ @address@hidden/$(DEPDIR)/address@hidden@
@@ -4745,6 +4760,20 @@ ipc/lib_dep_tr_for_defs_a-notify.none.defs.obj:
ipc/notify.none.defs.c
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE)
$(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(address@hidden@)$(CC) $(DEFS) $(DEFAULT_INCLUDES)
$(INCLUDES) $(lib_dep_tr_for_defs_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS)
$(CFLAGS) -c -o ipc/lib_dep_tr_for_defs_a-notify.none.defs.obj `if test -f
'ipc/notify.none.defs.c'; then $(CYGPATH_W) 'ipc/notify.none.defs.c'; else
$(CYGPATH_W) '$(srcdir)/ipc/notify.none.defs.c'; fi`
+kern/lib_dep_tr_for_defs_a-exc.none.defs.o: kern/exc.none.defs.c
address@hidden@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES)
$(lib_dep_tr_for_defs_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT
kern/lib_dep_tr_for_defs_a-exc.none.defs.o -MD -MP -MF
kern/$(DEPDIR)/lib_dep_tr_for_defs_a-exc.none.defs.Tpo -c -o
kern/lib_dep_tr_for_defs_a-exc.none.defs.o `test -f 'kern/exc.none.defs.c' ||
echo '$(srcdir)/'`kern/exc.none.defs.c
address@hidden@ $(AM_V_at)$(am__mv)
kern/$(DEPDIR)/lib_dep_tr_for_defs_a-exc.none.defs.Tpo
kern/$(DEPDIR)/lib_dep_tr_for_defs_a-exc.none.defs.Po
address@hidden@@am__fastdepCC_FALSE@ $(AM_V_CC)source='kern/exc.none.defs.c'
object='kern/lib_dep_tr_for_defs_a-exc.none.defs.o' libtool=no @AMDEPBACKSLASH@
address@hidden@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE)
$(depcomp) @AMDEPBACKSLASH@
address@hidden@ $(address@hidden@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES)
$(lib_dep_tr_for_defs_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o
kern/lib_dep_tr_for_defs_a-exc.none.defs.o `test -f 'kern/exc.none.defs.c' ||
echo '$(srcdir)/'`kern/exc.none.defs.c
+
+kern/lib_dep_tr_for_defs_a-exc.none.defs.obj: kern/exc.none.defs.c
address@hidden@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES)
$(lib_dep_tr_for_defs_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT
kern/lib_dep_tr_for_defs_a-exc.none.defs.obj -MD -MP -MF
kern/$(DEPDIR)/lib_dep_tr_for_defs_a-exc.none.defs.Tpo -c -o
kern/lib_dep_tr_for_defs_a-exc.none.defs.obj `if test -f
'kern/exc.none.defs.c'; then $(CYGPATH_W) 'kern/exc.none.defs.c'; else
$(CYGPATH_W) '$(srcdir)/kern/exc.none.defs.c'; fi`
address@hidden@ $(AM_V_at)$(am__mv)
kern/$(DEPDIR)/lib_dep_tr_for_defs_a-exc.none.defs.Tpo
kern/$(DEPDIR)/lib_dep_tr_for_defs_a-exc.none.defs.Po
address@hidden@@am__fastdepCC_FALSE@ $(AM_V_CC)source='kern/exc.none.defs.c'
object='kern/lib_dep_tr_for_defs_a-exc.none.defs.obj' libtool=no
@AMDEPBACKSLASH@
address@hidden@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE)
$(depcomp) @AMDEPBACKSLASH@
address@hidden@ $(address@hidden@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES)
$(lib_dep_tr_for_defs_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o
kern/lib_dep_tr_for_defs_a-exc.none.defs.obj `if test -f
'kern/exc.none.defs.c'; then $(CYGPATH_W) 'kern/exc.none.defs.c'; else
$(CYGPATH_W) '$(srcdir)/kern/exc.none.defs.c'; fi`
+
i386/i386/lib_dep_tr_for_defs_a-mach_i386.server.defs.o:
i386/i386/mach_i386.server.defs.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES)
$(lib_dep_tr_for_defs_a_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT
i386/i386/lib_dep_tr_for_defs_a-mach_i386.server.defs.o -MD -MP -MF
i386/i386/$(DEPDIR)/lib_dep_tr_for_defs_a-mach_i386.server.defs.Tpo -c -o
i386/i386/lib_dep_tr_for_defs_a-mach_i386.server.defs.o `test -f
'i386/i386/mach_i386.server.defs.c' || echo
'$(srcdir)/'`i386/i386/mach_i386.server.defs.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv)
i386/i386/$(DEPDIR)/lib_dep_tr_for_defs_a-mach_i386.server.defs.Tpo
i386/i386/$(DEPDIR)/lib_dep_tr_for_defs_a-mach_i386.server.defs.Po
diff --git a/Makefrag.am b/Makefrag.am
index 823ece5..6ffc8cc 100644
--- a/Makefrag.am
+++ b/Makefrag.am
@@ -162,6 +162,7 @@ libkernel_a_SOURCES += \
kern/lock.c \
kern/lock.h \
kern/lock_mon.c \
+ kern/log2.h \
kern/mach_clock.c \
kern/mach_clock.h \
kern/mach_factor.c \
@@ -217,6 +218,7 @@ libkernel_a_SOURCES += \
kern/elf-load.c \
kern/boot_script.c
EXTRA_DIST += \
+ kern/exc.defs \
kern/mach.srv \
kern/mach4.srv \
kern/gnumach.srv \
@@ -259,6 +261,7 @@ libkernel_a_SOURCES += \
vm/vm_map.h \
vm/vm_object.c \
vm/vm_object.h \
+ vm/vm_page.c \
vm/vm_page.h \
vm/vm_pageout.c \
vm/vm_pageout.h \
@@ -548,9 +551,11 @@ nodist_libkernel_a_SOURCES += \
# Stand-alone rule to generate the list of message ids when neither
# the client nor the server stubs are required.
nodist_lib_dep_tr_for_defs_a_SOURCES += \
- ipc/notify.none.defs.c
+ ipc/notify.none.defs.c \
+ kern/exc.none.defs.c
nodist_libkernel_a_SOURCES += \
- ipc/notify.none.msgids
+ ipc/notify.none.msgids \
+ kern/exc.none.msgids
# ipc/notify.none.defs
# rpctrace can make use of that.
diff --git a/build-aux/compile b/build-aux/compile
index 531136b..a85b723 100755
--- a/build-aux/compile
+++ b/build-aux/compile
@@ -3,7 +3,7 @@
scriptversion=2012-10-14.11; # UTC
-# Copyright (C) 1999-2013 Free Software Foundation, Inc.
+# Copyright (C) 1999-2014 Free Software Foundation, Inc.
# Written by Tom Tromey <address@hidden>.
#
# This program is free software; you can redistribute it and/or modify
diff --git a/build-aux/config.guess b/build-aux/config.guess
index 1f5c50c..1659250 100755
--- a/build-aux/config.guess
+++ b/build-aux/config.guess
@@ -1,8 +1,8 @@
#! /bin/sh
# Attempt to guess a canonical system name.
-# Copyright 1992-2014 Free Software Foundation, Inc.
+# Copyright 1992-2015 Free Software Foundation, Inc.
-timestamp='2014-03-23'
+timestamp='2015-08-20'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@@ -24,12 +24,12 @@ timestamp='2014-03-23'
# program. This Exception is an additional permission under section 7
# of the GNU General Public License, version 3 ("GPLv3").
#
-# Originally written by Per Bothner.
+# Originally written by Per Bothner; maintained since 2000 by Ben Elliston.
#
# You can get the latest version of this script from:
#
http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD
#
-# Please send patches with a ChangeLog entry to address@hidden
+# Please send patches to <address@hidden>.
me=`echo "$0" | sed -e 's,.*/,,'`
@@ -50,7 +50,7 @@ version="\
GNU config.guess ($timestamp)
Originally written by Per Bothner.
-Copyright 1992-2014 Free Software Foundation, Inc.
+Copyright 1992-2015 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -168,20 +168,27 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
# Note: NetBSD doesn't particularly care about the vendor
# portion of the name. We always set it to "unknown".
sysctl="sysctl -n hw.machine_arch"
- UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \
- /usr/sbin/$sysctl 2>/dev/null || echo unknown)`
+ UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \
+ /sbin/$sysctl 2>/dev/null || \
+ /usr/sbin/$sysctl 2>/dev/null || \
+ echo unknown)`
case "${UNAME_MACHINE_ARCH}" in
armeb) machine=armeb-unknown ;;
arm*) machine=arm-unknown ;;
sh3el) machine=shl-unknown ;;
sh3eb) machine=sh-unknown ;;
sh5el) machine=sh5le-unknown ;;
+ earmv*)
+ arch=`echo ${UNAME_MACHINE_ARCH} | sed -e
's,^e\(armv[0-9]\).*$,\1,'`
+ endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne
's,^.*\(eb\)$,\1,p'`
+ machine=${arch}${endian}-unknown
+ ;;
*) machine=${UNAME_MACHINE_ARCH}-unknown ;;
esac
# The Operating System including object format, if it has switched
# to ELF recently, or will in the future.
case "${UNAME_MACHINE_ARCH}" in
- arm*|i386|m68k|ns32k|sh3*|sparc|vax)
+ arm*|earm*|i386|m68k|ns32k|sh3*|sparc|vax)
eval $set_cc_for_build
if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \
| grep -q __ELF__
@@ -197,6 +204,13 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
os=netbsd
;;
esac
+ # Determine ABI tags.
+ case "${UNAME_MACHINE_ARCH}" in
+ earm*)
+ expr='s/^earmv[0-9]/-eabi/;s/eb$//'
+ abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"`
+ ;;
+ esac
# The OS release
# Debian GNU/NetBSD machines have a different userland, and
# thus, need a distinct triplet. However, they do not need
@@ -207,13 +221,13 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
release='-gnu'
;;
*)
- release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'`
+ release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d.
-f1,2`
;;
esac
# Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM:
# contains redundant information, the shorter form:
# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used.
- echo "${machine}-${os}${release}"
+ echo "${machine}-${os}${release}${abi}"
exit ;;
*:Bitrig:*:*)
UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'`
@@ -235,6 +249,9 @@ case
"${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in
*:MirBSD:*:*)
echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE}
exit ;;
+ *:Sortix:*:*)
+ echo ${UNAME_MACHINE}-unknown-sortix
+ exit ;;
alpha:OSF1:*:*)
case $UNAME_RELEASE in
*4.0)
@@ -579,8 +596,9 @@ EOF
else
IBM_ARCH=powerpc
fi
- if [ -x /usr/bin/oslevel ] ; then
- IBM_REV=`/usr/bin/oslevel`
+ if [ -x /usr/bin/lslpp ] ; then
+ IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc |
+ awk -F: '{ print $3 }' | sed s/[0-9]*$/0/`
else
IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE}
fi
@@ -932,6 +950,9 @@ EOF
crisv32:Linux:*:*)
echo ${UNAME_MACHINE}-axis-linux-${LIBC}
exit ;;
+ e2k:Linux:*:*)
+ echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+ exit ;;
frv:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
exit ;;
@@ -1020,7 +1041,7 @@ EOF
echo ${UNAME_MACHINE}-dec-linux-${LIBC}
exit ;;
x86_64:Linux:*:*)
- echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
+ echo ${UNAME_MACHINE}-pc-linux-${LIBC}
exit ;;
xtensa*:Linux:*:*)
echo ${UNAME_MACHINE}-unknown-linux-${LIBC}
diff --git a/build-aux/config.sub b/build-aux/config.sub
index bba4efb..1acc966 100755
--- a/build-aux/config.sub
+++ b/build-aux/config.sub
@@ -1,8 +1,8 @@
#! /bin/sh
# Configuration validation subroutine script.
-# Copyright 1992-2014 Free Software Foundation, Inc.
+# Copyright 1992-2015 Free Software Foundation, Inc.
-timestamp='2014-09-11'
+timestamp='2015-08-20'
# This file is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
@@ -25,7 +25,7 @@ timestamp='2014-09-11'
# of the GNU General Public License, version 3 ("GPLv3").
-# Please send patches with a ChangeLog entry to address@hidden
+# Please send patches to <address@hidden>.
#
# Configuration subroutine to validate and canonicalize a configuration type.
# Supply the specified configuration type as an argument.
@@ -68,7 +68,7 @@ Report bugs and patches to <address@hidden>."
version="\
GNU config.sub ($timestamp)
-Copyright 1992-2014 Free Software Foundation, Inc.
+Copyright 1992-2015 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE."
@@ -117,7 +117,7 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'`
case $maybe_os in
nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \
linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* |
kfreebsd*-gnu* | \
- knetbsd*-gnu* | netbsd*-gnu* | \
+ knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
kopensolaris*-gnu* | \
storm-chaos* | os2-emx* | rtmk-nova*)
os=-$maybe_os
@@ -255,12 +255,13 @@ case $basic_machine in
| arc | arceb \
| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
| avr | avr32 \
+ | ba \
| be32 | be64 \
| bfin \
| c4x | c8051 | clipper \
| d10v | d30v | dlx | dsp16xx \
- | epiphany \
- | fido | fr30 | frv \
+ | e2k | epiphany \
+ | fido | fr30 | frv | ft32 \
| h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \
| hexagon \
| i370 | i860 | i960 | ia64 \
@@ -305,7 +306,7 @@ case $basic_machine in
| riscv32 | riscv64 \
| rl78 | rx \
| score \
- | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb |
shbe | shle | sh[1234]le | sh3ele \
+ | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb |
shbe | shle | sh[1234]le | sh3ele \
| sh64 | sh64le \
| sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet |
sparclite \
| sparcv8 | sparcv9 | sparcv9b | sparcv9v \
@@ -313,6 +314,7 @@ case $basic_machine in
| tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \
| ubicom32 \
| v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \
+ | visium \
| we32k \
| x86 | xc16x | xstormy16 | xtensa \
| z8k | z80)
@@ -327,6 +329,9 @@ case $basic_machine in
c6x)
basic_machine=tic6x-unknown
;;
+ leon|leon[3-9])
+ basic_machine=sparc-$basic_machine
+ ;;
m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip)
basic_machine=$basic_machine-unknown
os=-none
@@ -372,12 +377,13 @@ case $basic_machine in
| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
| arm-* | armbe-* | armle-* | armeb-* | armv*-* \
| avr-* | avr32-* \
+ | ba-* \
| be32-* | be64-* \
| bfin-* | bs2000-* \
| c[123]* | c30-* | [cjt]90-* | c4x-* \
| c8051-* | clipper-* | craynv-* | cydra-* \
| d10v-* | d30v-* | dlx-* \
- | elxsi-* \
+ | e2k-* | elxsi-* \
| f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \
| h8300-* | h8500-* \
| hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \
@@ -424,12 +430,13 @@ case $basic_machine in
| pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \
| powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \
| pyramid-* \
+ | riscv32-* | riscv64-* \
| rl78-* | romp-* | rs6000-* | rx-* \
| sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-*
| sheb-* | shbe-* \
| shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \
| sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* |
sparclet-* \
| sparclite-* \
- | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \
+ | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \
| tahoe-* \
| tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \
| tile*-* \
@@ -437,6 +444,7 @@ case $basic_machine in
| ubicom32-* \
| v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \
| vax-* \
+ | visium-* \
| we32k-* \
| x86-* | x86_64-* | xc16x-* | xps100-* \
| xstormy16-* | xtensa*-* \
@@ -513,6 +521,9 @@ case $basic_machine in
basic_machine=i386-pc
os=-aros
;;
+ asmjs)
+ basic_machine=asmjs-unknown
+ ;;
aux)
basic_machine=m68k-apple
os=-aux
@@ -774,6 +785,9 @@ case $basic_machine in
basic_machine=m68k-isi
os=-sysv
;;
+ leon-*|leon[3-9]-*)
+ basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'`
+ ;;
m68knommu)
basic_machine=m68k-unknown
os=-linux
@@ -1365,7 +1379,7 @@ case $os in
| -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* |
-solaris* \
| -sym* | -kopensolaris* | -plan9* \
| -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \
- | -aos* | -aros* \
+ | -aos* | -aros* | -cloudabi* | -sortix* \
| -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \
| -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \
| -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \
diff --git a/build-aux/depcomp b/build-aux/depcomp
index 4ebd5b3..fc98710 100755
--- a/build-aux/depcomp
+++ b/build-aux/depcomp
@@ -3,7 +3,7 @@
scriptversion=2013-05-30.07; # UTC
-# Copyright (C) 1999-2013 Free Software Foundation, Inc.
+# Copyright (C) 1999-2014 Free Software Foundation, Inc.
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
diff --git a/build-aux/install-sh b/build-aux/install-sh
index 377bb86..59990a1 100755
--- a/build-aux/install-sh
+++ b/build-aux/install-sh
@@ -1,7 +1,7 @@
#!/bin/sh
# install - install a program, script, or datafile
-scriptversion=2011-11-20.07; # UTC
+scriptversion=2014-09-12.12; # UTC
# This originates from X11R5 (mit/util/scripts/install.sh), which was
# later released in X11R6 (xc/config/util/install.sh) with the
@@ -41,19 +41,15 @@ scriptversion=2011-11-20.07; # UTC
# This script is compatible with the BSD install script, but was written
# from scratch.
+tab=' '
nl='
'
-IFS=" "" $nl"
+IFS=" $tab$nl"
-# set DOITPROG to echo to test this script
+# Set DOITPROG to "echo" to test this script.
-# Don't use :- since 4.3BSD and earlier shells don't like it.
doit=${DOITPROG-}
-if test -z "$doit"; then
- doit_exec=exec
-else
- doit_exec=$doit
-fi
+doit_exec=${doit:-exec}
# Put in absolute file names if you don't have them in your path;
# or use environment vars.
@@ -68,17 +64,6 @@ mvprog=${MVPROG-mv}
rmprog=${RMPROG-rm}
stripprog=${STRIPPROG-strip}
-posix_glob='?'
-initialize_posix_glob='
- test "$posix_glob" != "?" || {
- if (set -f) 2>/dev/null; then
- posix_glob=
- else
- posix_glob=:
- fi
- }
-'
-
posix_mkdir=
# Desired mode of installed file.
@@ -97,7 +82,7 @@ dir_arg=
dst_arg=
copy_on_change=false
-no_target_directory=
+is_target_a_directory=possibly
usage="\
Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
@@ -137,46 +122,57 @@ while test $# -ne 0; do
-d) dir_arg=true;;
-g) chgrpcmd="$chgrpprog $2"
- shift;;
+ shift;;
--help) echo "$usage"; exit $?;;
-m) mode=$2
- case $mode in
- *' '* | *' '* | *'
-'* | *'*'* | *'?'* | *'['*)
- echo "$0: invalid mode: $mode" >&2
- exit 1;;
- esac
- shift;;
+ case $mode in
+ *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*)
+ echo "$0: invalid mode: $mode" >&2
+ exit 1;;
+ esac
+ shift;;
-o) chowncmd="$chownprog $2"
- shift;;
+ shift;;
-s) stripcmd=$stripprog;;
- -t) dst_arg=$2
- # Protect names problematic for 'test' and other utilities.
- case $dst_arg in
- -* | [=\(\)!]) dst_arg=./$dst_arg;;
- esac
- shift;;
+ -t)
+ is_target_a_directory=always
+ dst_arg=$2
+ # Protect names problematic for 'test' and other utilities.
+ case $dst_arg in
+ -* | [=\(\)!]) dst_arg=./$dst_arg;;
+ esac
+ shift;;
- -T) no_target_directory=true;;
+ -T) is_target_a_directory=never;;
--version) echo "$0 $scriptversion"; exit $?;;
- --) shift
- break;;
+ --) shift
+ break;;
- -*) echo "$0: invalid option: $1" >&2
- exit 1;;
+ -*) echo "$0: invalid option: $1" >&2
+ exit 1;;
*) break;;
esac
shift
done
+# We allow the use of options -d and -T together, by making -d
+# take the precedence; this is for compatibility with GNU install.
+
+if test -n "$dir_arg"; then
+ if test -n "$dst_arg"; then
+ echo "$0: target directory not allowed when installing a directory." >&2
+ exit 1
+ fi
+fi
+
if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
# When -d is used, all remaining arguments are directories to create.
# When -t is used, the destination is already specified.
@@ -208,6 +204,15 @@ if test $# -eq 0; then
fi
if test -z "$dir_arg"; then
+ if test $# -gt 1 || test "$is_target_a_directory" = always; then
+ if test ! -d "$dst_arg"; then
+ echo "$0: $dst_arg: Is not a directory." >&2
+ exit 1
+ fi
+ fi
+fi
+
+if test -z "$dir_arg"; then
do_exit='(exit $ret); exit $ret'
trap "ret=129; $do_exit" 1
trap "ret=130; $do_exit" 2
@@ -223,16 +228,16 @@ if test -z "$dir_arg"; then
*[0-7])
if test -z "$stripcmd"; then
- u_plus_rw=
+ u_plus_rw=
else
- u_plus_rw='% 200'
+ u_plus_rw='% 200'
fi
cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
*)
if test -z "$stripcmd"; then
- u_plus_rw=
+ u_plus_rw=
else
- u_plus_rw=,u+rw
+ u_plus_rw=,u+rw
fi
cp_umask=$mode$u_plus_rw;;
esac
@@ -269,41 +274,15 @@ do
# If destination is a directory, append the input filename; won't work
# if double slashes aren't ignored.
if test -d "$dst"; then
- if test -n "$no_target_directory"; then
- echo "$0: $dst_arg: Is a directory" >&2
- exit 1
+ if test "$is_target_a_directory" = never; then
+ echo "$0: $dst_arg: Is a directory" >&2
+ exit 1
fi
dstdir=$dst
dst=$dstdir/`basename "$src"`
dstdir_status=0
else
- # Prefer dirname, but fall back on a substitute if dirname fails.
- dstdir=`
- (dirname "$dst") 2>/dev/null ||
- expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
- X"$dst" : 'X\(//\)[^/]' \| \
- X"$dst" : 'X\(//\)$' \| \
- X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
- echo X"$dst" |
- sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
- s//\1/
- q
- }
- /^X\(\/\/\)[^/].*/{
- s//\1/
- q
- }
- /^X\(\/\/\)$/{
- s//\1/
- q
- }
- /^X\(\/\).*/{
- s//\1/
- q
- }
- s/.*/./; q'
- `
-
+ dstdir=`dirname "$dst"`
test -d "$dstdir"
dstdir_status=$?
fi
@@ -314,74 +293,81 @@ do
if test $dstdir_status != 0; then
case $posix_mkdir in
'')
- # Create intermediate dirs using mode 755 as modified by the umask.
- # This is like FreeBSD 'install' as of 1997-10-28.
- umask=`umask`
- case $stripcmd.$umask in
- # Optimize common cases.
- *[2367][2367]) mkdir_umask=$umask;;
- .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
-
- *[0-7])
- mkdir_umask=`expr $umask + 22 \
- - $umask % 100 % 40 + $umask % 20 \
- - $umask % 10 % 4 + $umask % 2
- `;;
- *) mkdir_umask=$umask,go-w;;
- esac
-
- # With -d, create the new directory with the user-specified mode.
- # Otherwise, rely on $mkdir_umask.
- if test -n "$dir_arg"; then
- mkdir_mode=-m$mode
- else
- mkdir_mode=
- fi
-
- posix_mkdir=false
- case $umask in
- *[123567][0-7][0-7])
- # POSIX mkdir -p sets u+wx bits regardless of umask, which
- # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
- ;;
- *)
- tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
- trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
-
- if (umask $mkdir_umask &&
- exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
- then
- if test -z "$dir_arg" || {
- # Check for POSIX incompatibilities with -m.
- # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
- # other-writable bit of parent directory when it shouldn't.
- # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
- ls_ld_tmpdir=`ls -ld "$tmpdir"`
- case $ls_ld_tmpdir in
- d????-?r-*) different_mode=700;;
- d????-?--*) different_mode=755;;
- *) false;;
- esac &&
- $mkdirprog -m$different_mode -p -- "$tmpdir" && {
- ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
- test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
- }
- }
- then posix_mkdir=:
- fi
- rmdir "$tmpdir/d" "$tmpdir"
- else
- # Remove any dirs left behind by ancient mkdir implementations.
- rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
- fi
- trap '' 0;;
- esac;;
+ # Create intermediate dirs using mode 755 as modified by the umask.
+ # This is like FreeBSD 'install' as of 1997-10-28.
+ umask=`umask`
+ case $stripcmd.$umask in
+ # Optimize common cases.
+ *[2367][2367]) mkdir_umask=$umask;;
+ .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
+
+ *[0-7])
+ mkdir_umask=`expr $umask + 22 \
+ - $umask % 100 % 40 + $umask % 20 \
+ - $umask % 10 % 4 + $umask % 2
+ `;;
+ *) mkdir_umask=$umask,go-w;;
+ esac
+
+ # With -d, create the new directory with the user-specified mode.
+ # Otherwise, rely on $mkdir_umask.
+ if test -n "$dir_arg"; then
+ mkdir_mode=-m$mode
+ else
+ mkdir_mode=
+ fi
+
+ posix_mkdir=false
+ case $umask in
+ *[123567][0-7][0-7])
+ # POSIX mkdir -p sets u+wx bits regardless of umask, which
+ # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
+ ;;
+ *)
+ # $RANDOM is not portable (e.g. dash); use it when possible to
+ # lower collision chance
+ tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
+ trap 'ret=$?; rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir"
2>/dev/null; exit $ret' 0
+
+ # As "mkdir -p" follows symlinks and we work in /tmp possibly; so
+ # create the $tmpdir first (and fail if unsuccessful) to make sure
+ # that nobody tries to guess the $tmpdir name.
+ if (umask $mkdir_umask &&
+ $mkdirprog $mkdir_mode "$tmpdir" &&
+ exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null
2>&1
+ then
+ if test -z "$dir_arg" || {
+ # Check for POSIX incompatibilities with -m.
+ # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
+ # other-writable bit of parent directory when it shouldn't.
+ # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
+ test_tmpdir="$tmpdir/a"
+ ls_ld_tmpdir=`ls -ld "$test_tmpdir"`
+ case $ls_ld_tmpdir in
+ d????-?r-*) different_mode=700;;
+ d????-?--*) different_mode=755;;
+ *) false;;
+ esac &&
+ $mkdirprog -m$different_mode -p -- "$test_tmpdir" && {
+ ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"`
+ test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
+ }
+ }
+ then posix_mkdir=:
+ fi
+ rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir"
+ else
+ # Remove any dirs left behind by ancient mkdir implementations.
+ rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null
+ fi
+ trap '' 0;;
+ esac;;
esac
if
$posix_mkdir && (
- umask $mkdir_umask &&
- $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
+ umask $mkdir_umask &&
+ $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
)
then :
else
@@ -391,53 +377,51 @@ do
# directory the slow way, step by step, checking for races as we go.
case $dstdir in
- /*) prefix='/';;
- [-=\(\)!]*) prefix='./';;
- *) prefix='';;
+ /*) prefix='/';;
+ [-=\(\)!]*) prefix='./';;
+ *) prefix='';;
esac
- eval "$initialize_posix_glob"
-
oIFS=$IFS
IFS=/
- $posix_glob set -f
+ set -f
set fnord $dstdir
shift
- $posix_glob set +f
+ set +f
IFS=$oIFS
prefixes=
for d
do
- test X"$d" = X && continue
-
- prefix=$prefix$d
- if test -d "$prefix"; then
- prefixes=
- else
- if $posix_mkdir; then
- (umask=$mkdir_umask &&
- $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
- # Don't fail if two instances are running concurrently.
- test -d "$prefix" || exit 1
- else
- case $prefix in
- *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
- *) qprefix=$prefix;;
- esac
- prefixes="$prefixes '$qprefix'"
- fi
- fi
- prefix=$prefix/
+ test X"$d" = X && continue
+
+ prefix=$prefix$d
+ if test -d "$prefix"; then
+ prefixes=
+ else
+ if $posix_mkdir; then
+ (umask=$mkdir_umask &&
+ $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
+ # Don't fail if two instances are running concurrently.
+ test -d "$prefix" || exit 1
+ else
+ case $prefix in
+ *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
+ *) qprefix=$prefix;;
+ esac
+ prefixes="$prefixes '$qprefix'"
+ fi
+ fi
+ prefix=$prefix/
done
if test -n "$prefixes"; then
- # Don't fail if two instances are running concurrently.
- (umask $mkdir_umask &&
- eval "\$doit_exec \$mkdirprog $prefixes") ||
- test -d "$dstdir" || exit 1
- obsolete_mkdir_used=true
+ # Don't fail if two instances are running concurrently.
+ (umask $mkdir_umask &&
+ eval "\$doit_exec \$mkdirprog $prefixes") ||
+ test -d "$dstdir" || exit 1
+ obsolete_mkdir_used=true
fi
fi
fi
@@ -472,15 +456,12 @@ do
# If -C, don't bother to copy if it wouldn't change the file.
if $copy_on_change &&
- old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` &&
- new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` &&
-
- eval "$initialize_posix_glob" &&
- $posix_glob set -f &&
+ old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` &&
+ new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` &&
+ set -f &&
set X $old && old=:$2:$4:$5:$6 &&
set X $new && new=:$2:$4:$5:$6 &&
- $posix_glob set +f &&
-
+ set +f &&
test "$old" = "$new" &&
$cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
then
@@ -493,24 +474,24 @@ do
# to itself, or perhaps because mv is so ancient that it does not
# support -f.
{
- # Now remove or move aside any old file at destination location.
- # We try this two ways since rm can't unlink itself on some
- # systems and the destination file might be busy for other
- # reasons. In this case, the final cleanup might fail but the new
- # file should still install successfully.
- {
- test ! -f "$dst" ||
- $doit $rmcmd -f "$dst" 2>/dev/null ||
- { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
- { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
- } ||
- { echo "$0: cannot unlink or rename $dst" >&2
- (exit 1); exit 1
- }
- } &&
-
- # Now rename the file to the real destination.
- $doit $mvcmd "$dsttmp" "$dst"
+ # Now remove or move aside any old file at destination location.
+ # We try this two ways since rm can't unlink itself on some
+ # systems and the destination file might be busy for other
+ # reasons. In this case, the final cleanup might fail but the new
+ # file should still install successfully.
+ {
+ test ! -f "$dst" ||
+ $doit $rmcmd -f "$dst" 2>/dev/null ||
+ { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
+ { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
+ } ||
+ { echo "$0: cannot unlink or rename $dst" >&2
+ (exit 1); exit 1
+ }
+ } &&
+
+ # Now rename the file to the real destination.
+ $doit $mvcmd "$dsttmp" "$dst"
}
fi || exit 1
diff --git a/build-aux/mdate-sh b/build-aux/mdate-sh
index b3719cf..9e2c0c9 100755
--- a/build-aux/mdate-sh
+++ b/build-aux/mdate-sh
@@ -1,9 +1,9 @@
#!/bin/sh
# Get modification time of a file or directory and pretty-print it.
-scriptversion=2010-08-21.06; # UTC
+scriptversion=2015-04-09.19; # UTC
-# Copyright (C) 1995-2013 Free Software Foundation, Inc.
+# Copyright (C) 1995-2014 Free Software Foundation, Inc.
# written by Ulrich Drepper <address@hidden>, June 1995
#
# This program is free software; you can redistribute it and/or modify
@@ -74,6 +74,10 @@ export LC_ALL
LC_TIME=C
export LC_TIME
+# Use UTC to get reproducible result
+TZ=UTC
+export TZ
+
# GNU ls changes its time format in response to the TIME_STYLE
# variable. Since we cannot assume 'unset' works, revert this
# variable to its documented default.
diff --git a/build-aux/missing b/build-aux/missing
index db98974..f62bbae 100755
--- a/build-aux/missing
+++ b/build-aux/missing
@@ -3,7 +3,7 @@
scriptversion=2013-10-28.13; # UTC
-# Copyright (C) 1996-2013 Free Software Foundation, Inc.
+# Copyright (C) 1996-2014 Free Software Foundation, Inc.
# Originally written by Fran,cois Pinard <address@hidden>, 1996.
# This program is free software; you can redistribute it and/or modify
diff --git a/build-aux/test-driver b/build-aux/test-driver
index d306056..8e575b0 100755
--- a/build-aux/test-driver
+++ b/build-aux/test-driver
@@ -3,7 +3,7 @@
scriptversion=2013-07-13.22; # UTC
-# Copyright (C) 2011-2013 Free Software Foundation, Inc.
+# Copyright (C) 2011-2014 Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -106,11 +106,14 @@ trap "st=143; $do_exit" 15
# Test script is run here.
"$@" >$log_file 2>&1
estatus=$?
+
if test $enable_hard_errors = no && test $estatus -eq 99; then
- estatus=1
+ tweaked_estatus=1
+else
+ tweaked_estatus=$estatus
fi
-case $estatus:$expect_failure in
+case $tweaked_estatus:$expect_failure in
0:yes) col=$red res=XPASS recheck=yes gcopy=yes;;
0:*) col=$grn res=PASS recheck=no gcopy=no;;
77:*) col=$blu res=SKIP recheck=no gcopy=yes;;
@@ -119,6 +122,12 @@ case $estatus:$expect_failure in
*:*) col=$red res=FAIL recheck=yes gcopy=yes;;
esac
+# Report the test outcome and exit status in the logs, so that one can
+# know whether the test passed or failed simply by looking at the '.log'
+# file, without the need of also peaking into the corresponding '.trs'
+# file (automake bug#11814).
+echo "$res $test_name (exit status: $estatus)" >>$log_file
+
# Report outcome to console.
echo "${col}${res}${std}: $test_name"
diff --git a/configure b/configure
index 5f29c10..f4e3f76 100755
--- a/configure
+++ b/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for GNU Mach 1.6+git20160114.
+# Generated by GNU Autoconf 2.69 for GNU Mach 1.6+git20160311.
#
# Report bugs to <address@hidden>.
#
@@ -579,8 +579,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='GNU Mach'
PACKAGE_TARNAME='gnumach'
-PACKAGE_VERSION='1.6+git20160114'
-PACKAGE_STRING='GNU Mach 1.6+git20160114'
+PACKAGE_VERSION='1.6+git20160311'
+PACKAGE_STRING='GNU Mach 1.6+git20160311'
PACKAGE_BUGREPORT='address@hidden'
PACKAGE_URL=''
@@ -1595,7 +1595,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures GNU Mach 1.6+git20160114 to adapt to many kinds of
systems.
+\`configure' configures GNU Mach 1.6+git20160311 to adapt to many kinds of
systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1666,7 +1666,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of GNU Mach 1.6+git20160114:";;
+ short | recursive ) echo "Configuration of GNU Mach 1.6+git20160311:";;
esac
cat <<\_ACEOF
@@ -2014,7 +2014,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-GNU Mach configure 1.6+git20160114
+GNU Mach configure 1.6+git20160311
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2106,7 +2106,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by GNU Mach $as_me 1.6+git20160114, which was
+It was created by GNU Mach $as_me 1.6+git20160311, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@@ -2972,7 +2972,7 @@ fi
# Define the identity of the package.
PACKAGE='gnumach'
- VERSION='1.6+git20160114'
+ VERSION='1.6+git20160311'
# Some tools Automake needs.
@@ -12128,7 +12128,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by GNU Mach $as_me 1.6+git20160114, which was
+This file was extended by GNU Mach $as_me 1.6+git20160311, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -12199,7 +12199,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //;
s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
-GNU Mach config.status 1.6+git20160114
+GNU Mach config.status 1.6+git20160311
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"
diff --git a/device/dev_lookup.c b/device/dev_lookup.c
index a80830c..9af7508 100644
--- a/device/dev_lookup.c
+++ b/device/dev_lookup.c
@@ -366,5 +366,5 @@ dev_lookup_init(void)
queue_init(&dev_number_hash_table[i]);
kmem_cache_init(&dev_hdr_cache, "mach_device",
- sizeof(struct mach_device), 0, NULL, NULL, NULL, 0);
+ sizeof(struct mach_device), 0, NULL, 0);
}
diff --git a/device/dev_pager.c b/device/dev_pager.c
index 815473a..4033170 100644
--- a/device/dev_pager.c
+++ b/device/dev_pager.c
@@ -173,7 +173,7 @@ void dev_pager_hash_init(void)
size = sizeof(struct dev_pager_entry);
kmem_cache_init(&dev_pager_hash_cache, "dev_pager_entry", size, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
for (i = 0; i < DEV_PAGER_HASH_COUNT; i++)
queue_init(&dev_pager_hashtable[i]);
simple_lock_init(&dev_pager_hash_lock);
@@ -705,7 +705,7 @@ void device_pager_init(void)
*/
size = sizeof(struct dev_pager);
kmem_cache_init(&dev_pager_cache, "dev_pager", size, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
/*
* Initialize the name port hashing stuff.
diff --git a/device/ds_routines.c b/device/ds_routines.c
index 43ed5b5..dbff7f8 100644
--- a/device/ds_routines.c
+++ b/device/ds_routines.c
@@ -1554,7 +1554,7 @@ void mach_device_init(void)
device_io_map->wait_for_space = TRUE;
kmem_cache_init(&io_inband_cache, "io_buf_ptr_inband",
- sizeof(io_buf_ptr_inband_t), 0, NULL, NULL, NULL, 0);
+ sizeof(io_buf_ptr_inband_t), 0, NULL, 0);
mach_device_trap_init();
}
@@ -1598,7 +1598,7 @@ static void
mach_device_trap_init(void)
{
kmem_cache_init(&io_trap_cache, "io_req", IOTRAP_REQSIZE, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
}
/*
diff --git a/device/net_io.c b/device/net_io.c
index 47ef2ea..99af0b2 100644
--- a/device/net_io.c
+++ b/device/net_io.c
@@ -1495,11 +1495,11 @@ net_io_init(void)
size = sizeof(struct net_rcv_port);
kmem_cache_init(&net_rcv_cache, "net_rcv_port", size, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
size = sizeof(struct net_hash_entry);
kmem_cache_init(&net_hash_entry_cache, "net_hash_entry", size, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
size = ikm_plus_overhead(sizeof(struct net_rcv_msg));
net_kmsg_size = round_page(size);
diff --git a/doc/mach.info b/doc/mach.info
index 995b99a..463a76b 100644
--- a/doc/mach.info
+++ b/doc/mach.info
@@ -1,9 +1,9 @@
-This is mach.info, produced by makeinfo version 6.0 from mach.texi.
+This is mach.info, produced by makeinfo version 6.1 from mach.texi.
This file documents the GNU Mach microkernel.
- This is edition 0.4, last updated on 12 September 2015, of 'The GNU
-Mach Reference Manual', for version 1.6+git20160114.
+ This is edition 0.4, last updated on 10 March 2016, of 'The GNU Mach
+Reference Manual', for version 1.6+git20160311.
Copyright (C) 2001, 2002, 2006, 2007, 2008 Free Software Foundation,
Inc.
@@ -39,126 +39,126 @@ END-INFO-DIR-ENTRY
Indirect:
-mach.info-1: 1643
-mach.info-2: 302570
+mach.info-1: 1639
+mach.info-2: 302600
Tag Table:
(Indirect)
-Node: Top1643
-Node: Introduction11284
-Node: Audience12115
-Node: Features13150
-Node: Overview14977
-Node: History16170
-Node: Installing16315
-Node: Binary Distributions17540
-Node: Compilation18348
-Node: Configuration19581
-Node: Cross-Compilation35992
-Node: Bootstrap36773
-Ref: Bootstrap-Footnote-137216
-Node: Bootloader37453
-Ref: Bootloader-Footnote-138733
-Node: Modules38819
-Node: Inter Process Communication39646
-Node: Major Concepts40269
-Node: Messaging Interface44074
-Node: Mach Message Call44804
-Node: Message Format48119
-Node: Exchanging Port Rights59311
-Ref: Exchanging Port Rights-Footnote-164873
-Node: Memory65045
-Ref: Memory-Footnote-168139
-Node: Message Send68481
-Ref: Message Send-Footnote-175503
-Node: Message Receive75786
-Ref: Message Receive-Footnote-185438
-Node: Atomicity85719
-Node: Port Manipulation Interface88493
-Node: Port Creation90048
-Node: Port Destruction94837
-Node: Port Names97980
-Node: Port Rights102227
-Node: Ports and other Tasks106016
-Node: Receive Rights110109
-Node: Port Sets117040
-Node: Request Notifications119443
-Node: Inherited Ports124247
-Node: Virtual Memory Interface127931
-Node: Memory Allocation129184
-Node: Memory Deallocation131709
-Node: Data Transfer133173
-Node: Memory Attributes136699
-Node: Mapping Memory Objects146138
-Node: Memory Statistics149447
-Node: External Memory Management151021
-Node: Memory Object Server151726
-Node: Memory Object Creation154435
-Node: Memory Object Termination158441
-Node: Memory Objects and Data161380
-Node: Memory Object Locking178526
-Node: Memory Object Attributes184421
-Node: Default Memory Manager190258
-Node: Threads and Tasks195980
-Node: Thread Interface196317
-Node: Thread Creation197313
-Node: Thread Termination198430
-Node: Thread Information198901
-Node: Thread Settings205000
-Node: Thread Execution206234
-Node: Scheduling213527
-Node: Thread Priority213882
-Node: Hand-Off Scheduling216516
-Node: Scheduling Policy221641
-Node: Thread Special Ports222973
-Node: Exceptions225419
-Node: Task Interface226289
-Node: Task Creation227301
-Node: Task Termination228636
-Node: Task Information229238
-Node: Task Execution236140
-Node: Task Special Ports240553
-Node: Syscall Emulation244407
-Node: Profiling245638
-Node: Host Interface249401
-Node: Host Ports250386
-Node: Host Information252459
-Node: Host Time257842
-Node: Host Reboot260509
-Node: Processors and Processor Sets261061
-Node: Processor Set Interface262039
-Node: Processor Set Ports262806
-Node: Processor Set Access263636
-Node: Processor Set Creation265896
-Node: Processor Set Destruction266923
-Node: Tasks and Threads on Sets267844
-Node: Processor Set Priority273011
-Node: Processor Set Policy274301
-Node: Processor Set Info275915
-Node: Processor Interface279728
-Node: Hosted Processors280453
-Node: Processor Control281444
-Node: Processors and Sets282910
-Node: Processor Info284788
-Node: Device Interface287530
-Node: Device Reply Server289145
-Node: Device Open290437
-Node: Device Close292560
-Node: Device Read293139
-Node: Device Write296058
-Node: Device Map298863
-Node: Device Status299754
-Node: Device Filter302570
-Node: Kernel Debugger308317
-Node: Operation309044
-Node: Commands312021
-Node: Variables325806
-Node: Expressions327194
-Node: Copying328543
-Node: Documentation License347772
-Node: GNU Free Documentation License348361
-Node: CMU License370760
-Node: Concept Index371995
-Node: Function and Data Index375841
+Node: Top1639
+Node: Introduction11276
+Node: Audience12107
+Node: Features13142
+Node: Overview14969
+Node: History16162
+Node: Installing16307
+Node: Binary Distributions17532
+Node: Compilation18340
+Node: Configuration19573
+Node: Cross-Compilation35984
+Node: Bootstrap36765
+Ref: Bootstrap-Footnote-137208
+Node: Bootloader37445
+Ref: Bootloader-Footnote-138725
+Node: Modules38811
+Node: Inter Process Communication39638
+Node: Major Concepts40261
+Node: Messaging Interface44066
+Node: Mach Message Call44796
+Node: Message Format48111
+Node: Exchanging Port Rights59303
+Ref: Exchanging Port Rights-Footnote-164865
+Node: Memory65037
+Ref: Memory-Footnote-168131
+Node: Message Send68473
+Ref: Message Send-Footnote-175495
+Node: Message Receive75778
+Ref: Message Receive-Footnote-185430
+Node: Atomicity85711
+Node: Port Manipulation Interface88485
+Node: Port Creation90040
+Node: Port Destruction94829
+Node: Port Names97972
+Node: Port Rights102219
+Node: Ports and other Tasks106008
+Node: Receive Rights110101
+Node: Port Sets117032
+Node: Request Notifications119435
+Node: Inherited Ports124239
+Node: Virtual Memory Interface127923
+Node: Memory Allocation129176
+Node: Memory Deallocation131701
+Node: Data Transfer133165
+Node: Memory Attributes136691
+Node: Mapping Memory Objects146130
+Node: Memory Statistics149439
+Node: External Memory Management151013
+Node: Memory Object Server151718
+Node: Memory Object Creation154427
+Node: Memory Object Termination158475
+Node: Memory Objects and Data161414
+Node: Memory Object Locking178560
+Node: Memory Object Attributes184455
+Node: Default Memory Manager190292
+Node: Threads and Tasks196014
+Node: Thread Interface196351
+Node: Thread Creation197347
+Node: Thread Termination198464
+Node: Thread Information198935
+Node: Thread Settings205034
+Node: Thread Execution206268
+Node: Scheduling213561
+Node: Thread Priority213916
+Node: Hand-Off Scheduling216550
+Node: Scheduling Policy221675
+Node: Thread Special Ports223007
+Node: Exceptions225453
+Node: Task Interface226323
+Node: Task Creation227335
+Node: Task Termination228670
+Node: Task Information229272
+Node: Task Execution236174
+Node: Task Special Ports240587
+Node: Syscall Emulation244441
+Node: Profiling245672
+Node: Host Interface249435
+Node: Host Ports250420
+Node: Host Information252493
+Node: Host Time257876
+Node: Host Reboot260543
+Node: Processors and Processor Sets261095
+Node: Processor Set Interface262073
+Node: Processor Set Ports262840
+Node: Processor Set Access263670
+Node: Processor Set Creation265930
+Node: Processor Set Destruction266957
+Node: Tasks and Threads on Sets267878
+Node: Processor Set Priority273045
+Node: Processor Set Policy274335
+Node: Processor Set Info275949
+Node: Processor Interface279762
+Node: Hosted Processors280487
+Node: Processor Control281478
+Node: Processors and Sets282944
+Node: Processor Info284822
+Node: Device Interface287564
+Node: Device Reply Server289179
+Node: Device Open290471
+Node: Device Close292594
+Node: Device Read293173
+Node: Device Write296092
+Node: Device Map298897
+Node: Device Status299788
+Node: Device Filter302600
+Node: Kernel Debugger308347
+Node: Operation309074
+Node: Commands312051
+Node: Variables325836
+Node: Expressions327224
+Node: Copying328573
+Node: Documentation License347802
+Node: GNU Free Documentation License348391
+Node: CMU License370790
+Node: Concept Index372025
+Node: Function and Data Index375871
End Tag Table
diff --git a/doc/mach.info-1 b/doc/mach.info-1
index b95106f..41ec161 100644
--- a/doc/mach.info-1
+++ b/doc/mach.info-1
@@ -1,9 +1,9 @@
-This is mach.info, produced by makeinfo version 6.0 from mach.texi.
+This is mach.info, produced by makeinfo version 6.1 from mach.texi.
This file documents the GNU Mach microkernel.
- This is edition 0.4, last updated on 12 September 2015, of 'The GNU
-Mach Reference Manual', for version 1.6+git20160114.
+ This is edition 0.4, last updated on 10 March 2016, of 'The GNU Mach
+Reference Manual', for version 1.6+git20160311.
Copyright (C) 2001, 2002, 2006, 2007, 2008 Free Software Foundation,
Inc.
@@ -45,8 +45,8 @@ Main Menu
This file documents the GNU Mach microkernel.
- This is edition 0.4, last updated on 12 September 2015, of 'The GNU
-Mach Reference Manual', for version 1.6+git20160114.
+ This is edition 0.4, last updated on 10 March 2016, of 'The GNU Mach
+Reference Manual', for version 1.6+git20160311.
Copyright (C) 2001, 2002, 2006, 2007, 2008 Free Software Foundation,
Inc.
@@ -3524,10 +3524,10 @@ File: mach.info, Node: Memory Object Creation, Next:
Memory Object Termination
the calling kernel is included for planning purposes.
When the memory manager is prepared to accept requests for data for
- this object, it must call 'memory_object_ready' with the attribute.
- Otherwise the kernel will not process requests on this object. To
- reject all mappings of this object, the memory manager may use
- 'memory_object_destroy'.
+ this object, it must call 'memory_object_ready', or set the ready
+ flag using 'memory_object_set_attributes'. Otherwise the kernel
+ will not process requests on this object. To reject all mappings
+ of this object, the memory manager may use 'memory_object_destroy'.
The argument MEMORY_OBJECT is the port that represents the memory
object data, as supplied to the kernel in a 'vm_map' call.
diff --git a/doc/mach.info-2 b/doc/mach.info-2
index 4e8ee4a..39fe14a 100644
--- a/doc/mach.info-2
+++ b/doc/mach.info-2
@@ -1,9 +1,9 @@
-This is mach.info, produced by makeinfo version 6.0 from mach.texi.
+This is mach.info, produced by makeinfo version 6.1 from mach.texi.
This file documents the GNU Mach microkernel.
- This is edition 0.4, last updated on 12 September 2015, of 'The GNU
-Mach Reference Manual', for version 1.6+git20160114.
+ This is edition 0.4, last updated on 10 March 2016, of 'The GNU Mach
+Reference Manual', for version 1.6+git20160311.
Copyright (C) 2001, 2002, 2006, 2007, 2008 Free Software Foundation,
Inc.
diff --git a/doc/mach.texi b/doc/mach.texi
index 6fc79f7..0aeed76 100644
--- a/doc/mach.texi
+++ b/doc/mach.texi
@@ -3526,8 +3526,9 @@ each will perform a @code{memory_object_init} call with
new request and
name ports. The virtual page size that is used by the calling kernel is
included for planning purposes.
-When the memory manager is prepared to accept requests for data for this
-object, it must call @code{memory_object_ready} with the attribute.
+When the memory manager is prepared to accept requests for data for
+this object, it must call @code{memory_object_ready}, or set the ready
+flag using @code{memory_object_set_attributes}.
Otherwise the kernel will not process requests on this object. To
reject all mappings of this object, the memory manager may use
@code{memory_object_destroy}.
diff --git a/doc/stamp-vti b/doc/stamp-vti
index 69dbe4d..fd5f466 100644
--- a/doc/stamp-vti
+++ b/doc/stamp-vti
@@ -1,4 +1,4 @@
address@hidden UPDATED 12 September 2015
address@hidden UPDATED-MONTH September 2015
address@hidden EDITION 1.6+git20160114
address@hidden VERSION 1.6+git20160114
address@hidden UPDATED 10 March 2016
address@hidden UPDATED-MONTH March 2016
address@hidden EDITION 1.6+git20160311
address@hidden VERSION 1.6+git20160311
diff --git a/doc/version.texi b/doc/version.texi
index 69dbe4d..fd5f466 100644
--- a/doc/version.texi
+++ b/doc/version.texi
@@ -1,4 +1,4 @@
address@hidden UPDATED 12 September 2015
address@hidden UPDATED-MONTH September 2015
address@hidden EDITION 1.6+git20160114
address@hidden VERSION 1.6+git20160114
address@hidden UPDATED 10 March 2016
address@hidden UPDATED-MONTH March 2016
address@hidden EDITION 1.6+git20160311
address@hidden VERSION 1.6+git20160311
diff --git a/i386/Makefrag.am b/i386/Makefrag.am
index ef393d5..8b0ef7f 100644
--- a/i386/Makefrag.am
+++ b/i386/Makefrag.am
@@ -20,8 +20,11 @@
libkernel_a_SOURCES += \
i386/i386at/autoconf.c \
i386/i386at/autoconf.h \
+ i386/i386at/biosmem.c \
+ i386/i386at/biosmem.h \
i386/i386at/conf.c \
i386/i386at/cons_conf.c \
+ i386/i386at/elf.h \
i386/i386at/idt.h \
i386/i386at/model_dep.c \
i386/i386at/model_dep.h \
@@ -95,6 +98,7 @@ libkernel_a_SOURCES += \
i386/i386/ast.h \
i386/i386/ast_check.c \
i386/i386/ast_types.h \
+ i386/i386/cpu.h \
i386/i386/cpu_number.h \
i386/i386/cswitch.S \
i386/i386/db_disasm.c \
diff --git a/i386/i386/cpu.h b/i386/i386/cpu.h
new file mode 100644
index 0000000..1bf40dc
--- /dev/null
+++ b/i386/i386/cpu.h
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2010-2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _X86_CPU_H
+#define _X86_CPU_H
+
+#include <kern/macros.h>
+
+/*
+ * EFLAGS register flags.
+ */
+#define CPU_EFL_ONE 0x00000002
+#define CPU_EFL_IF 0x00000200
+
+/*
+ * Return the content of the EFLAGS register.
+ *
+ * Implies a compiler barrier.
+ */
+static __always_inline unsigned long
+cpu_get_eflags(void)
+{
+ unsigned long eflags;
+
+ asm volatile("pushf\n"
+ "pop %0\n"
+ : "=r" (eflags)
+ : : "memory");
+
+ return eflags;
+}
+
+/*
+ * Enable local interrupts.
+ *
+ * Implies a compiler barrier.
+ */
+static __always_inline void
+cpu_intr_enable(void)
+{
+ asm volatile("sti" : : : "memory");
+}
+
+/*
+ * Disable local interrupts.
+ *
+ * Implies a compiler barrier.
+ */
+static __always_inline void
+cpu_intr_disable(void)
+{
+ asm volatile("cli" : : : "memory");
+}
+
+/*
+ * Restore the content of the EFLAGS register, possibly enabling interrupts.
+ *
+ * Implies a compiler barrier.
+ */
+static __always_inline void
+cpu_intr_restore(unsigned long flags)
+{
+ asm volatile("push %0\n"
+ "popf\n"
+ : : "r" (flags)
+ : "memory");
+}
+
+/*
+ * Disable local interrupts, returning the previous content of the EFLAGS
+ * register.
+ *
+ * Implies a compiler barrier.
+ */
+static __always_inline void
+cpu_intr_save(unsigned long *flags)
+{
+ *flags = cpu_get_eflags();
+ cpu_intr_disable();
+}
+
+/*
+ * Return true if interrupts are enabled.
+ *
+ * Implies a compiler barrier.
+ */
+static __always_inline int
+cpu_intr_enabled(void)
+{
+ unsigned long eflags;
+
+ eflags = cpu_get_eflags();
+ return (eflags & CPU_EFL_IF) ? 1 : 0;
+}
+
+#endif /* _X86_CPU_H */
diff --git a/i386/i386/db_trace.c b/i386/i386/db_trace.c
index ec33859..c8789e7 100644
--- a/i386/i386/db_trace.c
+++ b/i386/i386/db_trace.c
@@ -37,6 +37,7 @@
#include <machine/machspl.h>
#include <machine/db_interface.h>
#include <machine/db_trace.h>
+#include <i386at/model_dep.h>
#include <ddb/db_access.h>
#include <ddb/db_command.h>
@@ -129,7 +130,6 @@ db_i386_reg_value(
long *dp = 0;
db_expr_t null_reg = 0;
thread_t thread = ap->thread;
- extern unsigned int_stack_high;
if (db_option(ap->modif, 'u')) {
if (thread == THREAD_NULL) {
@@ -139,7 +139,7 @@ db_i386_reg_value(
if (thread == current_thread()) {
if (ddb_regs.cs & 0x3)
dp = vp->valuep;
- else if (ddb_regs.ebp < int_stack_high)
+ else if (ON_INT_STACK(ddb_regs.ebp))
db_error("cannot get/set user registers in nested
interrupt\n");
}
} else {
diff --git a/i386/i386/fpu.c b/i386/i386/fpu.c
index 0f34833..6ee2015 100644
--- a/i386/i386/fpu.c
+++ b/i386/i386/fpu.c
@@ -189,7 +189,7 @@ fpu_module_init(void)
{
kmem_cache_init(&ifps_cache, "i386_fpsave_state",
sizeof(struct i386_fpsave_state), 16,
- NULL, NULL, NULL, 0);
+ NULL, 0);
}
/*
@@ -494,27 +494,52 @@ ASSERT_IPL(SPL0);
*
* Use 53-bit precision.
*/
-void fpinit(void)
+static void fpinit(thread_t thread)
{
unsigned short control;
ASSERT_IPL(SPL0);
clear_ts();
fninit();
- fnstcw(&control);
- control &= ~(FPC_PC|FPC_RC); /* Clear precision & rounding control */
- control |= (FPC_PC_53 | /* Set precision */
- FPC_RC_RN | /* round-to-nearest */
- FPC_ZE | /* Suppress zero-divide */
- FPC_OE | /* and overflow */
- FPC_UE | /* underflow */
- FPC_IE | /* Allow NaNQs and +-INF */
- FPC_DE | /* Allow denorms as operands */
- FPC_PE); /* No trap for precision loss */
+ if (thread->pcb->init_control) {
+ control = thread->pcb->init_control;
+ }
+ else
+ {
+ fnstcw(&control);
+ control &= ~(FPC_PC|FPC_RC); /* Clear precision & rounding
control */
+ control |= (FPC_PC_53 | /* Set precision */
+ FPC_RC_RN | /* round-to-nearest */
+ FPC_ZE | /* Suppress zero-divide */
+ FPC_OE | /* and overflow */
+ FPC_UE | /* underflow */
+ FPC_IE | /* Allow NaNQs and +-INF */
+ FPC_DE | /* Allow denorms as operands */
+ FPC_PE); /* No trap for precision loss */
+ }
fldcw(control);
}
/*
+ * Inherit FPU state from a parent to a child, if any
+ */
+void fpinherit(thread_t parent_thread, thread_t thread)
+{
+ pcb_t pcb = parent_thread->pcb;
+ struct i386_fpsave_state *ifps;
+
+ ifps = pcb->ims.ifps;
+ if (ifps) {
+ /* Parent does have a state, inherit it */
+ if (ifps->fp_valid == TRUE)
+ thread->pcb->init_control =
ifps->fp_save_state.fp_control;
+ else
+ /* State is in the FPU, fetch from there */
+ fnstcw(&thread->pcb->init_control);
+ }
+}
+
+/*
* Coprocessor not present.
*/
void
@@ -778,7 +803,7 @@ ASSERT_IPL(SPL0);
ifps = (struct i386_fpsave_state *) kmem_cache_alloc(&ifps_cache);
memset(ifps, 0, sizeof *ifps);
pcb->ims.ifps = ifps;
- fpinit();
+ fpinit(thread);
#if 1
/*
* I'm not sure this is needed. Does the fpu regenerate the interrupt in
diff --git a/i386/i386/fpu.h b/i386/i386/fpu.h
index 7bc6438..caade5d 100644
--- a/i386/i386/fpu.h
+++ b/i386/i386/fpu.h
@@ -125,5 +125,6 @@ extern void fpexterrflt(void);
extern void fpastintr(void);
extern void init_fpu(void);
extern void fpintr(int unit);
+extern void fpinherit(thread_t parent_thread, thread_t thread);
#endif /* _I386_FPU_H_ */
diff --git a/i386/i386/locore.S b/i386/i386/locore.S
index 8cefbcc..c715d95 100644
--- a/i386/i386/locore.S
+++ b/i386/i386/locore.S
@@ -542,8 +542,10 @@ trap_from_kernel:
#if MACH_KDB || MACH_TTD
movl %esp,%ebx /* save current stack */
- cmpl EXT(int_stack_high),%esp /* on an interrupt stack? */
- jb 1f /* OK if so */
+ movl %esp,%edx /* on an interrupt stack? */
+ and $(~(KERNEL_STACK_SIZE-1)),%edx
+ cmpl EXT(int_stack_base),%edx
+ je 1f /* OK if so */
CPU_NUMBER(%edx) /* get CPU number */
cmpl CX(EXT(kernel_stack),%edx),%esp
@@ -647,8 +649,10 @@ ENTRY(all_intrs)
pushl %edx
cld /* clear direction flag */
- cmpl %ss:EXT(int_stack_high),%esp /* on an interrupt stack? */
- jb int_from_intstack /* if not: */
+ movl %esp,%edx /* on an interrupt stack? */
+ and $(~(KERNEL_STACK_SIZE-1)),%edx
+ cmpl %ss:EXT(int_stack_base),%edx
+ je int_from_intstack /* if not: */
pushl %ds /* save segment registers */
pushl %es
@@ -707,7 +711,7 @@ LEXT(return_to_iret) /* ( label for
kdb_kintr and hardclock) */
iret /* return to caller */
int_from_intstack:
- cmpl $EXT(_intstack),%esp /* seemingly looping? */
+ cmpl EXT(int_stack_base),%esp /* seemingly looping? */
jb stack_overflowed /* if not: */
call EXT(interrupt) /* call interrupt routine */
_return_to_iret_i: /* ( label for kdb_kintr) */
diff --git a/i386/i386/machine_task.c b/i386/i386/machine_task.c
index 490b102..d592838 100644
--- a/i386/i386/machine_task.c
+++ b/i386/i386/machine_task.c
@@ -38,7 +38,7 @@ void
machine_task_module_init (void)
{
kmem_cache_init (&machine_task_iopb_cache, "i386_task_iopb", IOPB_BYTES, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
}
diff --git a/i386/i386/pcb.c b/i386/i386/pcb.c
index 3a0eba0..dd2042d 100644
--- a/i386/i386/pcb.c
+++ b/i386/i386/pcb.c
@@ -371,12 +371,12 @@ thread_t switch_context(
void pcb_module_init(void)
{
kmem_cache_init(&pcb_cache, "pcb", sizeof(struct pcb), 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
fpu_module_init();
}
-void pcb_init(thread_t thread)
+void pcb_init(task_t parent_task, thread_t thread)
{
pcb_t pcb;
@@ -406,6 +406,11 @@ void pcb_init(thread_t thread)
pcb->iss.efl = EFL_USER_SET;
thread->pcb = pcb;
+
+ /* This is a new thread for the current task, make it inherit our FPU
+ state. */
+ if (parent_task == current_task())
+ fpinherit(current_thread(), thread);
}
void pcb_terminate(thread_t thread)
diff --git a/i386/i386/pcb.h b/i386/i386/pcb.h
index 708f30d..cf47694 100644
--- a/i386/i386/pcb.h
+++ b/i386/i386/pcb.h
@@ -31,7 +31,7 @@
#include <mach/thread_status.h>
#include <machine/thread.h>
-extern void pcb_init (thread_t thread);
+extern void pcb_init (task_t parent_task, thread_t thread);
extern void pcb_terminate (thread_t thread);
diff --git a/i386/i386/thread.h b/i386/i386/thread.h
index 35a2802..9bda11f 100644
--- a/i386/i386/thread.h
+++ b/i386/i386/thread.h
@@ -177,6 +177,7 @@ typedef struct pcb {
struct i386_saved_state iss;
struct i386_machine_state ims;
decl_simple_lock_data(, lock)
+ unsigned short init_control; /* Initial FPU control to set */
#ifdef LINUX_DEV
void *data;
#endif /* LINUX_DEV */
diff --git a/i386/i386/vm_param.h b/i386/i386/vm_param.h
index 6292ca2..2635c2c 100644
--- a/i386/i386/vm_param.h
+++ b/i386/i386/vm_param.h
@@ -23,6 +23,8 @@
#ifndef _I386_KERNEL_I386_VM_PARAM_
#define _I386_KERNEL_I386_VM_PARAM_
+#include <kern/macros.h>
+
/* XXX use xu/vm_param.h */
#include <mach/vm_param.h>
#ifdef MACH_PV_PAGETABLES
@@ -56,9 +58,9 @@
/* Reserve mapping room for kmem. */
#ifdef MACH_XEN
-#define VM_KERNEL_MAP_SIZE (256 * 1024 * 1024)
+#define VM_KERNEL_MAP_SIZE (128 * 1024 * 1024)
#else
-#define VM_KERNEL_MAP_SIZE (224 * 1024 * 1024)
+#define VM_KERNEL_MAP_SIZE (96 * 1024 * 1024)
#endif
/* The kernel virtual address space is actually located
@@ -101,4 +103,51 @@
#define kvtolin(a) ((vm_offset_t)(a) - VM_MIN_KERNEL_ADDRESS +
LINEAR_MIN_KERNEL_ADDRESS)
#define lintokv(a) ((vm_offset_t)(a) - LINEAR_MIN_KERNEL_ADDRESS +
VM_MIN_KERNEL_ADDRESS)
+/*
+ * Physical memory properties.
+ */
+#define VM_PAGE_DMA_LIMIT DECL_CONST(0x1000000, UL)
+
+#ifdef MACH_XEN
+/* TODO Completely check Xen physical/virtual layout */
+#define VM_PAGE_MAX_SEGS 3
+#define VM_PAGE_DIRECTMAP_LIMIT (VM_MAX_KERNEL_ADDRESS \
+ - VM_MIN_KERNEL_ADDRESS \
+ - VM_KERNEL_MAP_SIZE)
+#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, ULL)
+#else /* MACH_XEN */
+#ifdef __LP64__
+#define VM_PAGE_MAX_SEGS 4
+#define VM_PAGE_DMA32_LIMIT DECL_CONST(0x100000000, UL)
+#define VM_PAGE_DIRECTMAP_LIMIT DECL_CONST(0x400000000000, UL)
+#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, UL)
+#else /* __LP64__ */
+#define VM_PAGE_DIRECTMAP_LIMIT (VM_MAX_KERNEL_ADDRESS \
+ - VM_MIN_KERNEL_ADDRESS \
+ - VM_KERNEL_MAP_SIZE + 1)
+#ifdef PAE
+#define VM_PAGE_MAX_SEGS 3
+#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0x10000000000000, ULL)
+#else /* PAE */
+#define VM_PAGE_MAX_SEGS 3
+#define VM_PAGE_HIGHMEM_LIMIT DECL_CONST(0xfffff000, UL)
+#endif /* PAE */
+#endif /* __LP64__ */
+#endif /* MACH_XEN */
+
+/*
+ * Physical segment indexes.
+ */
+#define VM_PAGE_SEG_DMA 0
+
+#ifdef __LP64__
+#define VM_PAGE_SEG_DMA32 1
+#define VM_PAGE_SEG_DIRECTMAP 2
+#define VM_PAGE_SEG_HIGHMEM 3
+#else /* __LP64__ */
+#define VM_PAGE_SEG_DMA32 1 /* Alias for the DIRECTMAP segment */
+#define VM_PAGE_SEG_DIRECTMAP 1
+#define VM_PAGE_SEG_HIGHMEM 2
+#endif /* __LP64__ */
+
#endif /* _I386_KERNEL_I386_VM_PARAM_ */
diff --git a/i386/i386at/biosmem.c b/i386/i386at/biosmem.c
new file mode 100644
index 0000000..a7a440e
--- /dev/null
+++ b/i386/i386at/biosmem.c
@@ -0,0 +1,910 @@
+/*
+ * Copyright (c) 2010-2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <string.h>
+#include <i386/model_dep.h>
+#include <i386at/biosmem.h>
+#include <i386at/elf.h>
+#include <kern/assert.h>
+#include <kern/debug.h>
+#include <kern/macros.h>
+#include <kern/printf.h>
+#include <mach/vm_param.h>
+#include <mach/xen.h>
+#include <mach/machine/multiboot.h>
+#include <sys/types.h>
+#include <vm/vm_page.h>
+
+#define __boot
+#define __bootdata
+#define __init
+
+#define boot_memmove memmove
+#define boot_panic panic
+#define boot_strlen strlen
+
+#define BOOT_CGAMEM phystokv(0xb8000)
+#define BOOT_CGACHARS (80 * 25)
+#define BOOT_CGACOLOR 0x7
+
+extern char _start, _end;
+
+/*
+ * Maximum number of entries in the BIOS memory map.
+ *
+ * Because of adjustments of overlapping ranges, the memory map can grow
+ * to twice this size.
+ */
+#define BIOSMEM_MAX_MAP_SIZE 128
+
+/*
+ * Memory range types.
+ */
+#define BIOSMEM_TYPE_AVAILABLE 1
+#define BIOSMEM_TYPE_RESERVED 2
+#define BIOSMEM_TYPE_ACPI 3
+#define BIOSMEM_TYPE_NVS 4
+#define BIOSMEM_TYPE_UNUSABLE 5
+#define BIOSMEM_TYPE_DISABLED 6
+
+/*
+ * Memory map entry.
+ */
+struct biosmem_map_entry {
+ uint64_t base_addr;
+ uint64_t length;
+ unsigned int type;
+};
+
+/*
+ * Contiguous block of physical memory.
+ *
+ * Tha "available" range records what has been passed to the VM system as
+ * available inside the segment.
+ */
+struct biosmem_segment {
+ phys_addr_t start;
+ phys_addr_t end;
+ phys_addr_t avail_start;
+ phys_addr_t avail_end;
+};
+
+/*
+ * Memory map built from the information passed by the boot loader.
+ *
+ * If the boot loader didn't pass a valid memory map, a simple map is built
+ * based on the mem_lower and mem_upper multiboot fields.
+ */
+static struct biosmem_map_entry biosmem_map[BIOSMEM_MAX_MAP_SIZE * 2]
+ __bootdata;
+static unsigned int biosmem_map_size __bootdata;
+
+/*
+ * Physical segment boundaries.
+ */
+static struct biosmem_segment biosmem_segments[VM_PAGE_MAX_SEGS] __bootdata;
+
+/*
+ * Boundaries of the simple bootstrap heap.
+ *
+ * This heap is located above BIOS memory.
+ */
+static uint32_t biosmem_heap_start __bootdata;
+static uint32_t biosmem_heap_cur __bootdata;
+static uint32_t biosmem_heap_end __bootdata;
+
+static char biosmem_panic_toobig_msg[] __bootdata
+ = "biosmem: too many memory map entries";
+#ifndef MACH_HYP
+static char biosmem_panic_setup_msg[] __bootdata
+ = "biosmem: unable to set up the early memory allocator";
+#endif /* MACH_HYP */
+static char biosmem_panic_noseg_msg[] __bootdata
+ = "biosmem: unable to find any memory segment";
+static char biosmem_panic_inval_msg[] __bootdata
+ = "biosmem: attempt to allocate 0 page";
+static char biosmem_panic_nomem_msg[] __bootdata
+ = "biosmem: unable to allocate memory";
+
+#ifndef MACH_HYP
+
+static void __boot
+biosmem_map_build(const struct multiboot_raw_info *mbi)
+{
+ struct multiboot_raw_mmap_entry *mb_entry, *mb_end;
+ struct biosmem_map_entry *start, *entry, *end;
+ unsigned long addr;
+
+ addr = phystokv(mbi->mmap_addr);
+ mb_entry = (struct multiboot_raw_mmap_entry *)addr;
+ mb_end = (struct multiboot_raw_mmap_entry *)(addr + mbi->mmap_length);
+ start = biosmem_map;
+ entry = start;
+ end = entry + BIOSMEM_MAX_MAP_SIZE;
+
+ while ((mb_entry < mb_end) && (entry < end)) {
+ entry->base_addr = mb_entry->base_addr;
+ entry->length = mb_entry->length;
+ entry->type = mb_entry->type;
+
+ mb_entry = (void *)mb_entry + sizeof(mb_entry->size) + mb_entry->size;
+ entry++;
+ }
+
+ biosmem_map_size = entry - start;
+}
+
+static void __boot
+biosmem_map_build_simple(const struct multiboot_raw_info *mbi)
+{
+ struct biosmem_map_entry *entry;
+
+ entry = biosmem_map;
+ entry->base_addr = 0;
+ entry->length = mbi->mem_lower << 10;
+ entry->type = BIOSMEM_TYPE_AVAILABLE;
+
+ entry++;
+ entry->base_addr = BIOSMEM_END;
+ entry->length = mbi->mem_upper << 10;
+ entry->type = BIOSMEM_TYPE_AVAILABLE;
+
+ biosmem_map_size = 2;
+}
+
+#endif /* MACH_HYP */
+
+static int __boot
+biosmem_map_entry_is_invalid(const struct biosmem_map_entry *entry)
+{
+ return (entry->base_addr + entry->length) <= entry->base_addr;
+}
+
+static void __boot
+biosmem_map_filter(void)
+{
+ struct biosmem_map_entry *entry;
+ unsigned int i;
+
+ i = 0;
+
+ while (i < biosmem_map_size) {
+ entry = &biosmem_map[i];
+
+ if (biosmem_map_entry_is_invalid(entry)) {
+ biosmem_map_size--;
+ boot_memmove(entry, entry + 1,
+ (biosmem_map_size - i) * sizeof(*entry));
+ continue;
+ }
+
+ i++;
+ }
+}
+
+static void __boot
+biosmem_map_sort(void)
+{
+ struct biosmem_map_entry tmp;
+ unsigned int i, j;
+
+ /*
+ * Simple insertion sort.
+ */
+ for (i = 1; i < biosmem_map_size; i++) {
+ tmp = biosmem_map[i];
+
+ for (j = i - 1; j < i; j--) {
+ if (biosmem_map[j].base_addr < tmp.base_addr)
+ break;
+
+ biosmem_map[j + 1] = biosmem_map[j];
+ }
+
+ biosmem_map[j + 1] = tmp;
+ }
+}
+
+static void __boot
+biosmem_map_adjust(void)
+{
+ struct biosmem_map_entry tmp, *a, *b, *first, *second;
+ uint64_t a_end, b_end, last_end;
+ unsigned int i, j, last_type;
+
+ biosmem_map_filter();
+
+ /*
+ * Resolve overlapping areas, giving priority to most restrictive
+ * (i.e. numerically higher) types.
+ */
+ for (i = 0; i < biosmem_map_size; i++) {
+ a = &biosmem_map[i];
+ a_end = a->base_addr + a->length;
+
+ j = i + 1;
+
+ while (j < biosmem_map_size) {
+ b = &biosmem_map[j];
+ b_end = b->base_addr + b->length;
+
+ if ((a->base_addr >= b_end) || (a_end <= b->base_addr)) {
+ j++;
+ continue;
+ }
+
+ if (a->base_addr < b->base_addr) {
+ first = a;
+ second = b;
+ } else {
+ first = b;
+ second = a;
+ }
+
+ if (a_end > b_end) {
+ last_end = a_end;
+ last_type = a->type;
+ } else {
+ last_end = b_end;
+ last_type = b->type;
+ }
+
+ tmp.base_addr = second->base_addr;
+ tmp.length = MIN(a_end, b_end) - tmp.base_addr;
+ tmp.type = MAX(a->type, b->type);
+ first->length = tmp.base_addr - first->base_addr;
+ second->base_addr += tmp.length;
+ second->length = last_end - second->base_addr;
+ second->type = last_type;
+
+ /*
+ * Filter out invalid entries.
+ */
+ if (biosmem_map_entry_is_invalid(a)
+ && biosmem_map_entry_is_invalid(b)) {
+ *a = tmp;
+ biosmem_map_size--;
+ memmove(b, b + 1, (biosmem_map_size - j) * sizeof(*b));
+ continue;
+ } else if (biosmem_map_entry_is_invalid(a)) {
+ *a = tmp;
+ j++;
+ continue;
+ } else if (biosmem_map_entry_is_invalid(b)) {
+ *b = tmp;
+ j++;
+ continue;
+ }
+
+ if (tmp.type == a->type)
+ first = a;
+ else if (tmp.type == b->type)
+ first = b;
+ else {
+
+ /*
+ * If the overlapping area can't be merged with one of its
+ * neighbors, it must be added as a new entry.
+ */
+
+ if (biosmem_map_size >= ARRAY_SIZE(biosmem_map))
+ boot_panic(biosmem_panic_toobig_msg);
+
+ biosmem_map[biosmem_map_size] = tmp;
+ biosmem_map_size++;
+ j++;
+ continue;
+ }
+
+ if (first->base_addr > tmp.base_addr)
+ first->base_addr = tmp.base_addr;
+
+ first->length += tmp.length;
+ j++;
+ }
+ }
+
+ biosmem_map_sort();
+}
+
+static int __boot
+biosmem_map_find_avail(phys_addr_t *phys_start, phys_addr_t *phys_end)
+{
+ const struct biosmem_map_entry *entry, *map_end;
+ phys_addr_t seg_start, seg_end;
+ uint64_t start, end;
+
+ seg_start = (phys_addr_t)-1;
+ seg_end = (phys_addr_t)-1;
+ map_end = biosmem_map + biosmem_map_size;
+
+ for (entry = biosmem_map; entry < map_end; entry++) {
+ if (entry->type != BIOSMEM_TYPE_AVAILABLE)
+ continue;
+
+ start = vm_page_round(entry->base_addr);
+
+ if (start >= *phys_end)
+ break;
+
+ end = vm_page_trunc(entry->base_addr + entry->length);
+
+ if ((start < end) && (start < *phys_end) && (end > *phys_start)) {
+ if (seg_start == (phys_addr_t)-1)
+ seg_start = start;
+
+ seg_end = end;
+ }
+ }
+
+ if ((seg_start == (phys_addr_t)-1) || (seg_end == (phys_addr_t)-1))
+ return -1;
+
+ if (seg_start > *phys_start)
+ *phys_start = seg_start;
+
+ if (seg_end < *phys_end)
+ *phys_end = seg_end;
+
+ return 0;
+}
+
+static void __boot
+biosmem_set_segment(unsigned int seg_index, phys_addr_t start, phys_addr_t end)
+{
+ biosmem_segments[seg_index].start = start;
+ biosmem_segments[seg_index].end = end;
+}
+
+static phys_addr_t __boot
+biosmem_segment_end(unsigned int seg_index)
+{
+ return biosmem_segments[seg_index].end;
+}
+
+static phys_addr_t __boot
+biosmem_segment_size(unsigned int seg_index)
+{
+ return biosmem_segments[seg_index].end - biosmem_segments[seg_index].start;
+}
+
+#ifndef MACH_HYP
+
+static void __boot
+biosmem_save_cmdline_sizes(struct multiboot_raw_info *mbi)
+{
+ struct multiboot_raw_module *mod;
+ uint32_t i, va;
+
+ if (mbi->flags & MULTIBOOT_LOADER_CMDLINE) {
+ va = phystokv(mbi->cmdline);
+ mbi->unused0 = boot_strlen((char *)va) + 1;
+ }
+
+ if (mbi->flags & MULTIBOOT_LOADER_MODULES) {
+ unsigned long addr;
+
+ addr = phystokv(mbi->mods_addr);
+
+ for (i = 0; i < mbi->mods_count; i++) {
+ mod = (struct multiboot_raw_module *)addr + i;
+ va = phystokv(mod->string);
+ mod->reserved = boot_strlen((char *)va) + 1;
+ }
+ }
+}
+
+static void __boot
+biosmem_find_boot_data_update(uint32_t min, uint32_t *start, uint32_t *end,
+ uint32_t data_start, uint32_t data_end)
+{
+ if ((min <= data_start) && (data_start < *start)) {
+ *start = data_start;
+ *end = data_end;
+ }
+}
+
+/*
+ * Find the first boot data in the given range, and return their containing
+ * area (start address is returned directly, end address is returned in end).
+ * The following are considered boot data :
+ * - the kernel
+ * - the kernel command line
+ * - the module table
+ * - the modules
+ * - the modules command lines
+ * - the ELF section header table
+ * - the ELF .shstrtab, .symtab and .strtab sections
+ *
+ * If no boot data was found, 0 is returned, and the end address isn't set.
+ */
+static uint32_t __boot
+biosmem_find_boot_data(const struct multiboot_raw_info *mbi, uint32_t min,
+ uint32_t max, uint32_t *endp)
+{
+ struct multiboot_raw_module *mod;
+ struct elf_shdr *shdr;
+ uint32_t i, start, end = end;
+ unsigned long tmp;
+
+ start = max;
+
+ biosmem_find_boot_data_update(min, &start, &end, _kvtophys(&_start),
+ _kvtophys(&_end));
+
+ if ((mbi->flags & MULTIBOOT_LOADER_CMDLINE) && (mbi->cmdline != 0))
+ biosmem_find_boot_data_update(min, &start, &end, mbi->cmdline,
+ mbi->cmdline + mbi->unused0);
+
+ if (mbi->flags & MULTIBOOT_LOADER_MODULES) {
+ i = mbi->mods_count * sizeof(struct multiboot_raw_module);
+ biosmem_find_boot_data_update(min, &start, &end, mbi->mods_addr,
+ mbi->mods_addr + i);
+ tmp = phystokv(mbi->mods_addr);
+
+ for (i = 0; i < mbi->mods_count; i++) {
+ mod = (struct multiboot_raw_module *)tmp + i;
+ biosmem_find_boot_data_update(min, &start, &end, mod->mod_start,
+ mod->mod_end);
+
+ if (mod->string != 0)
+ biosmem_find_boot_data_update(min, &start, &end, mod->string,
+ mod->string + mod->reserved);
+ }
+ }
+
+ if (mbi->flags & MULTIBOOT_LOADER_SHDR) {
+ tmp = mbi->shdr_num * mbi->shdr_size;
+ biosmem_find_boot_data_update(min, &start, &end, mbi->shdr_addr,
+ mbi->shdr_addr + tmp);
+ tmp = phystokv(mbi->shdr_addr);
+
+ for (i = 0; i < mbi->shdr_num; i++) {
+ shdr = (struct elf_shdr *)(tmp + (i * mbi->shdr_size));
+
+ if ((shdr->type != ELF_SHT_SYMTAB)
+ && (shdr->type != ELF_SHT_STRTAB))
+ continue;
+
+ biosmem_find_boot_data_update(min, &start, &end, shdr->addr,
+ shdr->addr + shdr->size);
+ }
+ }
+
+ if (start == max)
+ return 0;
+
+ *endp = end;
+ return start;
+}
+
+static void __boot
+biosmem_setup_allocator(struct multiboot_raw_info *mbi)
+{
+ uint32_t heap_start, heap_end, max_heap_start, max_heap_end;
+ uint32_t mem_end, next;
+
+ /*
+ * Find some memory for the heap. Look for the largest unused area in
+ * upper memory, carefully avoiding all boot data.
+ */
+ mem_end = vm_page_trunc((mbi->mem_upper + 1024) << 10);
+
+#ifndef __LP64__
+ if (mem_end > VM_PAGE_DIRECTMAP_LIMIT)
+ mem_end = VM_PAGE_DIRECTMAP_LIMIT;
+#endif /* __LP64__ */
+
+ max_heap_start = 0;
+ max_heap_end = 0;
+ next = BIOSMEM_END;
+
+ do {
+ heap_start = next;
+ heap_end = biosmem_find_boot_data(mbi, heap_start, mem_end, &next);
+
+ if (heap_end == 0) {
+ heap_end = mem_end;
+ next = 0;
+ }
+
+ if ((heap_end - heap_start) > (max_heap_end - max_heap_start)) {
+ max_heap_start = heap_start;
+ max_heap_end = heap_end;
+ }
+ } while (next != 0);
+
+ max_heap_start = vm_page_round(max_heap_start);
+ max_heap_end = vm_page_trunc(max_heap_end);
+
+ if (max_heap_start >= max_heap_end)
+ boot_panic(biosmem_panic_setup_msg);
+
+ biosmem_heap_start = max_heap_start;
+ biosmem_heap_end = max_heap_end;
+ biosmem_heap_cur = biosmem_heap_end;
+}
+
+#endif /* MACH_HYP */
+
+static void __boot
+biosmem_bootstrap_common(void)
+{
+ phys_addr_t phys_start, phys_end, last_addr;
+ int error;
+
+ biosmem_map_adjust();
+
+ phys_start = BIOSMEM_BASE;
+ phys_end = VM_PAGE_DMA_LIMIT;
+ error = biosmem_map_find_avail(&phys_start, &phys_end);
+
+ if (error)
+ boot_panic(biosmem_panic_noseg_msg);
+
+ biosmem_set_segment(VM_PAGE_SEG_DMA, phys_start, phys_end);
+ last_addr = phys_end;
+
+ phys_start = VM_PAGE_DMA_LIMIT;
+#ifdef VM_PAGE_DMA32_LIMIT
+ phys_end = VM_PAGE_DMA32_LIMIT;
+ error = biosmem_map_find_avail(&phys_start, &phys_end);
+
+ if (error)
+ goto out;
+
+ biosmem_set_segment(VM_PAGE_SEG_DMA32, phys_start, phys_end);
+ last_addr = phys_end;
+
+ phys_start = VM_PAGE_DMA32_LIMIT;
+#endif /* VM_PAGE_DMA32_LIMIT */
+ phys_end = VM_PAGE_DIRECTMAP_LIMIT;
+ error = biosmem_map_find_avail(&phys_start, &phys_end);
+
+ if (error)
+ goto out;
+
+ biosmem_set_segment(VM_PAGE_SEG_DIRECTMAP, phys_start, phys_end);
+ last_addr = phys_end;
+
+ phys_start = VM_PAGE_DIRECTMAP_LIMIT;
+ phys_end = VM_PAGE_HIGHMEM_LIMIT;
+ error = biosmem_map_find_avail(&phys_start, &phys_end);
+
+ if (error)
+ goto out;
+
+ biosmem_set_segment(VM_PAGE_SEG_HIGHMEM, phys_start, phys_end);
+
+out:
+ /* XXX phys_last_addr must be part of the direct physical mapping */
+ phys_last_addr = last_addr;
+}
+
+#ifdef MACH_HYP
+
+void
+biosmem_xen_bootstrap(void)
+{
+ struct biosmem_map_entry *entry;
+
+ entry = biosmem_map;
+ entry->base_addr = 0;
+ entry->length = boot_info.nr_pages << PAGE_SHIFT;
+ entry->type = BIOSMEM_TYPE_AVAILABLE;
+
+ biosmem_map_size = 1;
+
+ biosmem_bootstrap_common();
+
+ biosmem_heap_start = _kvtophys(boot_info.pt_base)
+ + (boot_info.nr_pt_frames + 3) * 0x1000;
+ biosmem_heap_end = boot_info.nr_pages << PAGE_SHIFT;
+
+#ifndef __LP64__
+ /* TODO Check that this actually makes sense */
+ if (biosmem_heap_end > VM_PAGE_DIRECTMAP_LIMIT)
+ biosmem_heap_end = VM_PAGE_DIRECTMAP_LIMIT;
+#endif /* __LP64__ */
+
+ /*
+ * XXX Allocation on Xen must be bottom-up :
+ * At the "start of day", only 512k are available after the boot
+ * data. The pmap module then creates a 4g mapping so all physical
+ * memory is available, but it uses this allocator to do so.
+ * Therefore, it must return pages from this small 512k regions
+ * first.
+ */
+ biosmem_heap_cur = biosmem_heap_start;
+}
+
+#else /* MACH_HYP */
+
+void __boot
+biosmem_bootstrap(struct multiboot_raw_info *mbi)
+{
+ if (mbi->flags & MULTIBOOT_LOADER_MMAP)
+ biosmem_map_build(mbi);
+ else
+ biosmem_map_build_simple(mbi);
+
+ biosmem_bootstrap_common();
+
+ /*
+ * The kernel and modules command lines will be memory mapped later
+ * during initialization. Their respective sizes must be saved.
+ */
+ biosmem_save_cmdline_sizes(mbi);
+ biosmem_setup_allocator(mbi);
+}
+
+#endif /* MACH_HYP */
+
+unsigned long __boot
+biosmem_bootalloc(unsigned int nr_pages)
+{
+ unsigned long addr, size;
+
+ assert(!vm_page_ready());
+
+ size = vm_page_ptoa(nr_pages);
+
+ if (size == 0)
+ boot_panic(biosmem_panic_inval_msg);
+
+#ifdef MACH_HYP
+ addr = biosmem_heap_cur;
+#else /* MACH_HYP */
+ /* Top-down allocation to avoid unnecessarily filling DMA segments */
+ addr = biosmem_heap_cur - size;
+#endif /* MACH_HYP */
+
+ if ((addr < biosmem_heap_start) || (addr > biosmem_heap_cur))
+ boot_panic(biosmem_panic_nomem_msg);
+
+#ifdef MACH_HYP
+ biosmem_heap_cur += size;
+#else /* MACH_HYP */
+ biosmem_heap_cur = addr;
+#endif /* MACH_HYP */
+
+ return addr;
+}
+
+phys_addr_t __boot
+biosmem_directmap_size(void)
+{
+ if (biosmem_segment_size(VM_PAGE_SEG_DIRECTMAP) != 0)
+ return biosmem_segment_end(VM_PAGE_SEG_DIRECTMAP);
+ else if (biosmem_segment_size(VM_PAGE_SEG_DMA32) != 0)
+ return biosmem_segment_end(VM_PAGE_SEG_DMA32);
+ else
+ return biosmem_segment_end(VM_PAGE_SEG_DMA);
+}
+
+static const char * __init
+biosmem_type_desc(unsigned int type)
+{
+ switch (type) {
+ case BIOSMEM_TYPE_AVAILABLE:
+ return "available";
+ case BIOSMEM_TYPE_RESERVED:
+ return "reserved";
+ case BIOSMEM_TYPE_ACPI:
+ return "ACPI";
+ case BIOSMEM_TYPE_NVS:
+ return "ACPI NVS";
+ case BIOSMEM_TYPE_UNUSABLE:
+ return "unusable";
+ default:
+ return "unknown (reserved)";
+ }
+}
+
+static void __init
+biosmem_map_show(void)
+{
+ const struct biosmem_map_entry *entry, *end;
+
+ printf("biosmem: physical memory map:\n");
+
+ for (entry = biosmem_map, end = entry + biosmem_map_size;
+ entry < end;
+ entry++)
+ printf("biosmem: %018llx:%018llx, %s\n", entry->base_addr,
+ entry->base_addr + entry->length,
+ biosmem_type_desc(entry->type));
+
+ printf("biosmem: heap: %x-%x\n", biosmem_heap_start, biosmem_heap_end);
+}
+
+static void __init
+biosmem_load_segment(struct biosmem_segment *seg, uint64_t max_phys_end,
+ phys_addr_t phys_start, phys_addr_t phys_end,
+ phys_addr_t avail_start, phys_addr_t avail_end)
+{
+ unsigned int seg_index;
+
+ seg_index = seg - biosmem_segments;
+
+ if (phys_end > max_phys_end) {
+ if (max_phys_end <= phys_start) {
+ printf("biosmem: warning: segment %s physically unreachable, "
+ "not loaded\n", vm_page_seg_name(seg_index));
+ return;
+ }
+
+ printf("biosmem: warning: segment %s truncated to %#llx\n",
+ vm_page_seg_name(seg_index), max_phys_end);
+ phys_end = max_phys_end;
+ }
+
+ if ((avail_start < phys_start) || (avail_start >= phys_end))
+ avail_start = phys_start;
+
+ if ((avail_end <= phys_start) || (avail_end > phys_end))
+ avail_end = phys_end;
+
+ seg->avail_start = avail_start;
+ seg->avail_end = avail_end;
+ vm_page_load(seg_index, phys_start, phys_end, avail_start, avail_end);
+}
+
+void __init
+biosmem_setup(void)
+{
+ struct biosmem_segment *seg;
+ unsigned int i;
+
+ biosmem_map_show();
+
+ for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
+ if (biosmem_segment_size(i) == 0)
+ break;
+
+ seg = &biosmem_segments[i];
+ biosmem_load_segment(seg, VM_PAGE_HIGHMEM_LIMIT, seg->start, seg->end,
+ biosmem_heap_start, biosmem_heap_cur);
+ }
+}
+
+static void __init
+biosmem_free_usable_range(phys_addr_t start, phys_addr_t end)
+{
+ struct vm_page *page;
+
+ printf("biosmem: release to vm_page: %llx-%llx (%lluk)\n",
+ (unsigned long long)start, (unsigned long long)end,
+ (unsigned long long)((end - start) >> 10));
+
+ while (start < end) {
+ page = vm_page_lookup_pa(start);
+ assert(page != NULL);
+ vm_page_manage(page);
+ start += PAGE_SIZE;
+ }
+}
+
+static void __init
+biosmem_free_usable_update_start(phys_addr_t *start, phys_addr_t res_start,
+ phys_addr_t res_end)
+{
+ if ((*start >= res_start) && (*start < res_end))
+ *start = res_end;
+}
+
+static phys_addr_t __init
+biosmem_free_usable_start(phys_addr_t start)
+{
+ const struct biosmem_segment *seg;
+ unsigned int i;
+
+ biosmem_free_usable_update_start(&start, _kvtophys(&_start),
+ _kvtophys(&_end));
+ biosmem_free_usable_update_start(&start, biosmem_heap_start,
+ biosmem_heap_end);
+
+ for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
+ seg = &biosmem_segments[i];
+ biosmem_free_usable_update_start(&start, seg->avail_start,
+ seg->avail_end);
+ }
+
+ return start;
+}
+
+static int __init
+biosmem_free_usable_reserved(phys_addr_t addr)
+{
+ const struct biosmem_segment *seg;
+ unsigned int i;
+
+ if ((addr >= _kvtophys(&_start))
+ && (addr < _kvtophys(&_end)))
+ return 1;
+
+ if ((addr >= biosmem_heap_start) && (addr < biosmem_heap_end))
+ return 1;
+
+ for (i = 0; i < ARRAY_SIZE(biosmem_segments); i++) {
+ seg = &biosmem_segments[i];
+
+ if ((addr >= seg->avail_start) && (addr < seg->avail_end))
+ return 1;
+ }
+
+ return 0;
+}
+
+static phys_addr_t __init
+biosmem_free_usable_end(phys_addr_t start, phys_addr_t entry_end)
+{
+ while (start < entry_end) {
+ if (biosmem_free_usable_reserved(start))
+ break;
+
+ start += PAGE_SIZE;
+ }
+
+ return start;
+}
+
+static void __init
+biosmem_free_usable_entry(phys_addr_t start, phys_addr_t end)
+{
+ phys_addr_t entry_end;
+
+ entry_end = end;
+
+ for (;;) {
+ start = biosmem_free_usable_start(start);
+
+ if (start >= entry_end)
+ return;
+
+ end = biosmem_free_usable_end(start, entry_end);
+ biosmem_free_usable_range(start, end);
+ start = end;
+ }
+}
+
+void __init
+biosmem_free_usable(void)
+{
+ struct biosmem_map_entry *entry;
+ uint64_t start, end;
+ unsigned int i;
+
+ for (i = 0; i < biosmem_map_size; i++) {
+ entry = &biosmem_map[i];
+
+ if (entry->type != BIOSMEM_TYPE_AVAILABLE)
+ continue;
+
+ start = vm_page_round(entry->base_addr);
+
+ if (start >= VM_PAGE_HIGHMEM_LIMIT)
+ break;
+
+ end = vm_page_trunc(entry->base_addr + entry->length);
+
+ if (start < BIOSMEM_BASE)
+ start = BIOSMEM_BASE;
+
+ biosmem_free_usable_entry(start, end);
+ }
+}
diff --git a/i386/i386at/biosmem.h b/i386/i386at/biosmem.h
new file mode 100644
index 0000000..1db63f9
--- /dev/null
+++ b/i386/i386at/biosmem.h
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2010-2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _X86_BIOSMEM_H
+#define _X86_BIOSMEM_H
+
+#include <mach/machine/vm_types.h>
+#include <mach/machine/multiboot.h>
+
+/*
+ * Address where the address of the Extended BIOS Data Area segment can be
+ * found.
+ */
+#define BIOSMEM_EBDA_PTR 0x40e
+
+/*
+ * Significant low memory addresses.
+ *
+ * The first 64 KiB are reserved for various reasons (e.g. to preserve BIOS
+ * data and to work around data corruption on some hardware).
+ */
+#define BIOSMEM_BASE 0x010000
+#define BIOSMEM_BASE_END 0x0a0000
+#define BIOSMEM_EXT_ROM 0x0e0000
+#define BIOSMEM_ROM 0x0f0000
+#define BIOSMEM_END 0x100000
+
+/*
+ * Early initialization of the biosmem module.
+ *
+ * This function processes the given multiboot data for BIOS-provided
+ * memory information, and sets up a bootstrap physical page allocator.
+ *
+ * It is called before paging is enabled.
+ */
+#ifdef MACH_HYP
+void biosmem_xen_bootstrap(void);
+#else /* MACH_HYP */
+void biosmem_bootstrap(struct multiboot_raw_info *mbi);
+#endif /* MACH_HYP */
+
+/*
+ * Allocate contiguous physical pages during bootstrap.
+ *
+ * This function is called before paging is enabled. It should only be used
+ * to allocate initial page table pages. Those pages are later loaded into
+ * the VM system (as reserved pages) which means they can be freed like other
+ * regular pages. Users should fix up the type of those pages once the VM
+ * system is initialized.
+ */
+unsigned long biosmem_bootalloc(unsigned int nr_pages);
+
+/*
+ * Return the amount of physical memory that can be directly mapped.
+ *
+ * This includes the size of both the DMA/DMA32 and DIRECTMAP segments.
+ */
+phys_addr_t biosmem_directmap_size(void);
+
+/*
+ * Set up physical memory based on the information obtained during bootstrap
+ * and load it in the VM system.
+ */
+void biosmem_setup(void);
+
+/*
+ * Free all usable memory.
+ *
+ * This includes ranges that weren't part of the bootstrap allocator initial
+ * heap, e.g. because they contained boot data.
+ */
+void biosmem_free_usable(void);
+
+#endif /* _X86_BIOSMEM_H */
diff --git a/i386/i386at/elf.h b/i386/i386at/elf.h
new file mode 100644
index 0000000..26f4d87
--- /dev/null
+++ b/i386/i386at/elf.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2013 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _X86_ELF_H
+#define _X86_ELF_H
+
+#define ELF_SHT_SYMTAB 2
+#define ELF_SHT_STRTAB 3
+
+struct elf_shdr {
+ unsigned int name;
+ unsigned int type;
+ unsigned int flags;
+ unsigned long addr;
+ unsigned long offset;
+ unsigned int size;
+ unsigned int link;
+ unsigned int info;
+ unsigned int addralign;
+ unsigned int entsize;
+};
+
+#ifdef __LP64__
+
+struct elf_sym {
+ unsigned int name;
+ unsigned char info;
+ unsigned char other;
+ unsigned short shndx;
+ unsigned long value;
+ unsigned long size;
+};
+
+#else /* __LP64__ */
+
+struct elf_sym {
+ unsigned int name;
+ unsigned long value;
+ unsigned long size;
+ unsigned char info;
+ unsigned char other;
+ unsigned short shndx;
+};
+
+#endif /* __LP64__ */
+
+#endif /* _X86_ELF_H */
diff --git a/i386/i386at/kd_mouse.c b/i386/i386at/kd_mouse.c
index 0f1881f..ece13ef 100644
--- a/i386/i386at/kd_mouse.c
+++ b/i386/i386at/kd_mouse.c
@@ -630,6 +630,8 @@ int kd_mouse_read(void)
while (mousebufindex <= mouse_char_index) {
mouse_char_wanted = TRUE;
assert_wait((event_t) &mousebuf, FALSE);
+ /* We are at tty SPL level, interrupts can not happen between
+ * assert_wait and thread_block. */
thread_block((void (*)()) 0);
}
diff --git a/i386/i386at/model_dep.c b/i386/i386at/model_dep.c
index 04cf695..62763ae 100644
--- a/i386/i386at/model_dep.c
+++ b/i386/i386at/model_dep.c
@@ -47,6 +47,7 @@
#include <kern/cpu_number.h>
#include <kern/debug.h>
#include <kern/mach_clock.h>
+#include <kern/macros.h>
#include <kern/printf.h>
#include <kern/startup.h>
#include <sys/time.h>
@@ -64,6 +65,7 @@
#include <i386/locore.h>
#include <i386/model_dep.h>
#include <i386at/autoconf.h>
+#include <i386at/biosmem.h>
#include <i386at/idt.h>
#include <i386at/int_init.h>
#include <i386at/kd.h>
@@ -127,27 +129,14 @@ struct multiboot_info boot_info;
/* Command line supplied to kernel. */
char *kernel_cmdline = "";
-/* This is used for memory initialization:
- it gets bumped up through physical memory
- that exists and is not occupied by boot gunk.
- It is not necessarily page-aligned. */
-static vm_offset_t avail_next
-#ifndef MACH_HYP
- = RESERVED_BIOS /* XX end of BIOS data area */
-#endif /* MACH_HYP */
- ;
-
-/* Possibly overestimated amount of available memory
- still remaining to be handed to the VM system. */
-static vm_size_t avail_remaining;
-
extern char version[];
/* If set, reboot the system on ctrl-alt-delete. */
boolean_t rebootflag = FALSE; /* exported to kdintr */
-/* XX interrupt stack pointer and highwater mark, for locore.S. */
-vm_offset_t int_stack_top, int_stack_high;
+/* Interrupt stack. */
+static char int_stack[KERNEL_STACK_SIZE] __aligned(KERNEL_STACK_SIZE);
+vm_offset_t int_stack_top, int_stack_base;
#ifdef LINUX_DEV
extern void linux_init(void);
@@ -275,91 +264,6 @@ void db_reset_cpu(void)
halt_all_cpus(1);
}
-
-/*
- * Compute physical memory size and other parameters.
- */
-void
-mem_size_init(void)
-{
- vm_offset_t max_phys_size;
-
- /* Physical memory on all PCs starts at physical address 0.
- XX make it a constant. */
- phys_first_addr = 0;
-
-#ifdef MACH_HYP
- if (boot_info.nr_pages >= 0x100000) {
- printf("Truncating memory size to 4GiB\n");
- phys_last_addr = 0xffffffffU;
- } else
- phys_last_addr = boot_info.nr_pages * 0x1000;
-#else /* MACH_HYP */
- vm_size_t phys_last_kb;
-
- if (boot_info.flags & MULTIBOOT_MEM_MAP) {
- struct multiboot_mmap *map, *map_end;
-
- map = (void*) phystokv(boot_info.mmap_addr);
- map_end = (void*) map + boot_info.mmap_count;
-
- while (map + 1 <= map_end) {
- if (map->Type == MB_ARD_MEMORY) {
- unsigned long long start = map->BaseAddr, end =
map->BaseAddr + map->Length;;
-
- if (start >= 0x100000000ULL) {
- printf("Ignoring %luMiB RAM region
above 4GiB\n", (unsigned long) (map->Length >> 20));
- } else {
- if (end >= 0x100000000ULL) {
- printf("Truncating memory
region to 4GiB\n");
- end = 0x0ffffffffU;
- }
- if (end > phys_last_addr)
- phys_last_addr = end;
-
- printf("AT386 boot: physical memory map
from 0x%lx to 0x%lx\n",
- (unsigned long) start,
- (unsigned long) end);
- }
- }
- map = (void*) map + map->size + sizeof(map->size);
- }
- } else {
- phys_last_kb = 0x400 + boot_info.mem_upper;
- /* Avoid 4GiB overflow. */
- if (phys_last_kb < 0x400 || phys_last_kb >= 0x400000) {
- printf("Truncating memory size to 4GiB\n");
- phys_last_addr = 0xffffffffU;
- } else
- phys_last_addr = phys_last_kb * 0x400;
- }
-#endif /* MACH_HYP */
-
- printf("AT386 boot: physical memory from 0x%lx to 0x%lx\n",
- phys_first_addr, phys_last_addr);
-
- /* Reserve room for virtual mappings.
- * Yes, this loses memory. Blame i386. */
- max_phys_size = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS -
VM_KERNEL_MAP_SIZE;
- if (phys_last_addr - phys_first_addr > max_phys_size) {
- phys_last_addr = phys_first_addr + max_phys_size;
- printf("Truncating memory to %luMiB\n", (phys_last_addr -
phys_first_addr) / (1024 * 1024));
- /* TODO Xen: be nice, free lost memory */
- }
-
- phys_first_addr = round_page(phys_first_addr);
- phys_last_addr = trunc_page(phys_last_addr);
-
-#ifdef MACH_HYP
- /* Memory is just contiguous */
- avail_remaining = phys_last_addr;
-#else /* MACH_HYP */
- avail_remaining
- = phys_last_addr - (0x100000 - (boot_info.mem_lower * 0x400)
- - RESERVED_BIOS);
-#endif /* MACH_HYP */
-}
-
/*
* Basic PC VM initialization.
* Turns on paging and changes the kernel segments to use high linear
addresses.
@@ -382,9 +286,13 @@ i386at_init(void)
#endif /* MACH_HYP */
/*
- * Find memory size parameters.
+ * Read memory map and load it into the physical page allocator.
*/
- mem_size_init();
+#ifdef MACH_HYP
+ biosmem_xen_bootstrap();
+#else /* MACH_HYP */
+ biosmem_bootstrap((struct multiboot_raw_info *) &boot_info);
+#endif /* MACH_HYP */
#ifdef MACH_XEN
kernel_cmdline = (char*) boot_info.cmd_line;
@@ -432,6 +340,13 @@ i386at_init(void)
pmap_bootstrap();
/*
+ * Load physical segments into the VM system.
+ * The early allocation functions become unusable after
+ * this point.
+ */
+ biosmem_setup();
+
+ /*
* We'll have to temporarily install a direct mapping
* between physical memory and low linear memory,
* until we start using our new kernel segment descriptors.
@@ -485,11 +400,6 @@ i386at_init(void)
pmap_clear_bootstrap_pagetable((void *)boot_info.pt_base);
#endif /* MACH_PV_PAGETABLES */
- /* Interrupt stacks are allocated in physical memory,
- while kernel stacks are allocated in kernel virtual memory,
- so phys_last_addr serves as a convenient dividing point. */
- int_stack_high = phystokv(phys_last_addr);
-
/*
* Initialize and activate the real i386 protected-mode structures.
*/
@@ -535,10 +445,8 @@ i386at_init(void)
hyp_p2m_init();
#endif /* MACH_XEN */
- /* XXX We'll just use the initialization stack we're already running on
- as the interrupt stack for now. Later this will have to change,
- because the init stack will get freed after bootup. */
- asm("movl %%esp,%0" : "=m" (int_stack_top));
+ int_stack_base = (vm_offset_t)&int_stack;
+ int_stack_top = int_stack_base + KERNEL_STACK_SIZE - 4;
}
/*
@@ -706,187 +614,20 @@ resettodr(void)
unsigned int pmap_free_pages(void)
{
- return atop(avail_remaining);
+ return vm_page_atop(phys_last_addr); /* XXX */
}
-/* Always returns page-aligned regions. */
boolean_t
init_alloc_aligned(vm_size_t size, vm_offset_t *addrp)
{
- vm_offset_t addr;
-
-#ifdef MACH_HYP
- /* There is none */
- if (!avail_next)
- avail_next = _kvtophys(boot_info.pt_base) +
(boot_info.nr_pt_frames + 3) * 0x1000;
-#else /* MACH_HYP */
- extern char start[], end[];
- int i;
- static int wrapped = 0;
-
- /* Memory regions to skip. */
- vm_offset_t cmdline_start_pa = boot_info.flags & MULTIBOOT_CMDLINE
- ? boot_info.cmdline : 0;
- vm_offset_t cmdline_end_pa = cmdline_start_pa
- ? cmdline_start_pa+strlen((char*)phystokv(cmdline_start_pa))+1
- : 0;
- vm_offset_t mods_start_pa = boot_info.flags & MULTIBOOT_MODS
- ? boot_info.mods_addr : 0;
- vm_offset_t mods_end_pa = mods_start_pa
- ? mods_start_pa
- + boot_info.mods_count * sizeof(struct multiboot_module)
- : 0;
-
- retry:
-#endif /* MACH_HYP */
-
- /* Page-align the start address. */
- avail_next = round_page(avail_next);
+ *addrp = biosmem_bootalloc(vm_page_atop(vm_page_round(size)));
-#ifndef MACH_HYP
- /* Start with memory above 16MB, reserving the low memory for later. */
- /* Don't care on Xen */
- if (!wrapped && phys_last_addr > 16 * 1024*1024)
- {
- if (avail_next < 16 * 1024*1024)
- avail_next = 16 * 1024*1024;
- else if (avail_next == phys_last_addr)
- {
- /* We have used all the memory above 16MB, so now start on
- the low memory. This will wind up at the end of the list
- of free pages, so it should not have been allocated to any
- other use in early initialization before the Linux driver
- glue initialization needs to allocate low memory. */
- avail_next = RESERVED_BIOS;
- wrapped = 1;
- }
- }
-#endif /* MACH_HYP */
-
- /* Check if we have reached the end of memory. */
- if (avail_next ==
- (
-#ifndef MACH_HYP
- wrapped ? 16 * 1024*1024 :
-#endif /* MACH_HYP */
- phys_last_addr))
+ if (*addrp == 0)
return FALSE;
- /* Tentatively assign the current location to the caller. */
- addr = avail_next;
-
- /* Bump the pointer past the newly allocated region
- and see where that puts us. */
- avail_next += size;
-
-#ifndef MACH_HYP
- /* Skip past the I/O and ROM area. */
- if (boot_info.flags & MULTIBOOT_MEM_MAP)
- {
- struct multiboot_mmap *map, *map_end, *current = NULL, *next =
NULL;
- unsigned long long minimum_next = ~0ULL;
-
- map = (void*) phystokv(boot_info.mmap_addr);
- map_end = (void*) map + boot_info.mmap_count;
-
- /* Find both our current map, and the next one */
- while (map + 1 <= map_end)
- {
- if (map->Type == MB_ARD_MEMORY)
- {
- unsigned long long start = map->BaseAddr;
- unsigned long long end = start + map->Length;;
-
- if (start <= addr && avail_next <= end)
- {
- /* Ok, fits in the current map */
- current = map;
- break;
- }
- else if (avail_next <= start && start <
minimum_next)
- {
- /* This map is not far from avail_next
*/
- next = map;
- minimum_next = start;
- }
- }
- map = (void*) map + map->size + sizeof(map->size);
- }
-
- if (!current) {
- /* Area does not fit in the current map, switch to next
- * map if any */
- if (!next || next->BaseAddr >= phys_last_addr)
- {
- /* No further reachable map, we have reached
- * the end of memory, but possibly wrap around
- * 16MiB. */
- avail_next = phys_last_addr;
- goto retry;
- }
-
- /* Start from next map */
- avail_next = next->BaseAddr;
- goto retry;
- }
- }
- else if ((avail_next > (boot_info.mem_lower * 0x400)) && (addr <
0x100000))
- {
- avail_next = 0x100000;
- goto retry;
- }
-
- /* Skip our own kernel code, data, and bss. */
- if ((phystokv(avail_next) > (vm_offset_t)start) && (phystokv(addr) <
(vm_offset_t)end))
- {
- avail_next = _kvtophys(end);
- goto retry;
- }
-
- /* Skip any areas occupied by valuable boot_info data. */
- if ((avail_next > cmdline_start_pa) && (addr < cmdline_end_pa))
- {
- avail_next = cmdline_end_pa;
- goto retry;
- }
- if ((avail_next > mods_start_pa) && (addr < mods_end_pa))
- {
- avail_next = mods_end_pa;
- goto retry;
- }
- if ((phystokv(avail_next) > kern_sym_start) && (phystokv(addr) <
kern_sym_end))
- {
- avail_next = _kvtophys(kern_sym_end);
- goto retry;
- }
- if (boot_info.flags & MULTIBOOT_MODS)
- {
- struct multiboot_module *m = (struct multiboot_module *)
- phystokv(boot_info.mods_addr);
- for (i = 0; i < boot_info.mods_count; i++)
- {
- if ((avail_next > m[i].mod_start)
- && (addr < m[i].mod_end))
- {
- avail_next = m[i].mod_end;
- goto retry;
- }
- /* XXX string */
- }
- }
-#endif /* MACH_HYP */
-
- avail_remaining -= size;
-
- *addrp = addr;
return TRUE;
}
-boolean_t pmap_next_page(vm_offset_t *addrp)
-{
- return init_alloc_aligned(PAGE_SIZE, addrp);
-}
-
/* Grab a physical page:
the standard memory allocation mechanism
during system initialization. */
@@ -894,7 +635,7 @@ vm_offset_t
pmap_grab_page(void)
{
vm_offset_t addr;
- if (!pmap_next_page(&addr))
+ if (!init_alloc_aligned(PAGE_SIZE, &addr))
panic("Not enough memory to initialize Mach");
return addr;
}
diff --git a/i386/i386at/model_dep.h b/i386/i386at/model_dep.h
index aa24032..47551b8 100644
--- a/i386/i386at/model_dep.h
+++ b/i386/i386at/model_dep.h
@@ -19,8 +19,17 @@
#ifndef _MODEL_DEP_H_
#define _MODEL_DEP_H_
+#include <i386/vm_param.h>
#include <mach/vm_prot.h>
+/*
+ * Interrupt stack.
+ */
+extern vm_offset_t int_stack_top, int_stack_base;
+
+/* Check whether P points to the interrupt stack. */
+#define ON_INT_STACK(P) (((P) & ~(KERNEL_STACK_SIZE-1)) ==
int_stack_base)
+
extern int timemmap(dev_t dev, vm_offset_t off, vm_prot_t prot);
void inittodr(void);
diff --git a/i386/include/mach/i386/multiboot.h
b/i386/include/mach/i386/multiboot.h
index 8f1c47b..c66ca03 100644
--- a/i386/include/mach/i386/multiboot.h
+++ b/i386/include/mach/i386/multiboot.h
@@ -188,5 +188,110 @@ struct multiboot_mmap
/* usable memory "Type", all others are reserved. */
#define MB_ARD_MEMORY 1
+/*
+ * Copyright (c) 2010, 2012 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/*
+ * Versions used by the biosmem module.
+ */
+
+#include <kern/macros.h>
+
+/*
+ * Magic number provided by the OS to the boot loader.
+ */
+#define MULTIBOOT_OS_MAGIC 0x1badb002
+
+/*
+ * Multiboot flags requesting services from the boot loader.
+ */
+#define MULTIBOOT_OS_MEMORY_INFO 0x2
+
+#define MULTIBOOT_OS_FLAGS MULTIBOOT_OS_MEMORY_INFO
+
+/*
+ * Magic number to identify a multiboot compliant boot loader.
+ */
+#define MULTIBOOT_LOADER_MAGIC 0x2badb002
+
+/*
+ * Multiboot flags set by the boot loader.
+ */
+#define MULTIBOOT_LOADER_MEMORY 0x01
+#define MULTIBOOT_LOADER_CMDLINE 0x04
+#define MULTIBOOT_LOADER_MODULES 0x08
+#define MULTIBOOT_LOADER_SHDR 0x20
+#define MULTIBOOT_LOADER_MMAP 0x40
+
+/*
+ * A multiboot module.
+ */
+struct multiboot_raw_module {
+ uint32_t mod_start;
+ uint32_t mod_end;
+ uint32_t string;
+ uint32_t reserved;
+} __packed;
+
+/*
+ * Memory map entry.
+ */
+struct multiboot_raw_mmap_entry {
+ uint32_t size;
+ uint64_t base_addr;
+ uint64_t length;
+ uint32_t type;
+} __packed;
+
+/*
+ * Multiboot information structure as passed by the boot loader.
+ */
+struct multiboot_raw_info {
+ uint32_t flags;
+ uint32_t mem_lower;
+ uint32_t mem_upper;
+ uint32_t unused0;
+ uint32_t cmdline;
+ uint32_t mods_count;
+ uint32_t mods_addr;
+ uint32_t shdr_num;
+ uint32_t shdr_size;
+ uint32_t shdr_addr;
+ uint32_t shdr_strndx;
+ uint32_t mmap_length;
+ uint32_t mmap_addr;
+ uint32_t unused1[9];
+} __packed;
+
+/*
+ * Versions of the multiboot structures suitable for use with 64-bit pointers.
+ */
+
+struct multiboot_os_module {
+ void *mod_start;
+ void *mod_end;
+ char *string;
+};
+
+struct multiboot_os_info {
+ uint32_t flags;
+ char *cmdline;
+ struct multiboot_module *mods_addr;
+ uint32_t mods_count;
+};
#endif /* _MACH_I386_MULTIBOOT_H_ */
diff --git a/i386/include/mach/i386/vm_types.h
b/i386/include/mach/i386/vm_types.h
index 1439940..4a58b1c 100644
--- a/i386/include/mach/i386/vm_types.h
+++ b/i386/include/mach/i386/vm_types.h
@@ -77,6 +77,15 @@ typedef unsigned long vm_offset_t;
typedef vm_offset_t * vm_offset_array_t;
/*
+ * A type for physical addresses.
+ */
+#ifdef PAE
+typedef unsigned long long phys_addr_t;
+#else /* PAE */
+typedef unsigned long phys_addr_t;
+#endif /* PAE */
+
+/*
* A vm_size_t is the proper type for e.g.
* expressing the difference between two
* vm_offset_t entities.
diff --git a/i386/intel/pmap.c b/i386/intel/pmap.c
index 22e20c9..0771a08 100644
--- a/i386/intel/pmap.c
+++ b/i386/intel/pmap.c
@@ -978,9 +978,9 @@ void pmap_init(void)
* and of the physical-to-virtual entries.
*/
s = (vm_size_t) sizeof(struct pmap);
- kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, NULL, NULL, 0);
+ kmem_cache_init(&pmap_cache, "pmap", s, 0, NULL, 0);
s = (vm_size_t) sizeof(struct pv_entry);
- kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, NULL, NULL, 0);
+ kmem_cache_init(&pv_list_cache, "pv_entry", s, 0, NULL, 0);
#if NCPUS > 1
/*
diff --git a/include/mach/mach.defs b/include/mach/mach.defs
index 3786f65..20dc863 100644
--- a/include/mach/mach.defs
+++ b/include/mach/mach.defs
@@ -58,6 +58,9 @@ skip; /* old port_select */
skip; /* old port_set_backlog */
skip; /* old port_status */
+/* We use only a handful of RPCs as client. Skip the rest. */
+#if ! KERNEL_USER
+
/*
* Create a new task with an empty set of IPC rights,
* and having an address space constructed from the
@@ -310,6 +313,18 @@ skip; /* old u*x_pid */
skip; /* old netipc_listen */
skip; /* old netipc_ignore */
+#else /* ! KERNEL_USER */
+
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip;
+
+#endif /* ! KERNEL_USER */
+
/*
* Provide the data contents of a range of the given memory
* object, with the access restriction specified. [Only
@@ -351,6 +366,8 @@ routine memory_object_get_attributes(
out may_cache : boolean_t;
out copy_strategy : memory_object_copy_strategy_t);
+#if ! KERNEL_USER
+
/*
* Sets the default memory manager, the port to which
* newly-created temporary memory objects are delivered.
@@ -361,6 +378,12 @@ routine vm_set_default_memory_manager(
host_priv : host_priv_t;
inout default_manager : mach_port_make_send_t);
+#else /* ! KERNEL_USER */
+
+skip;
+
+#endif /* ! KERNEL_USER */
+
skip; /* old pager_flush_request */
/*
@@ -413,6 +436,8 @@ skip; /* old netport_enter */
skip; /* old netport_remove */
skip; /* old thread_set_priority */
+#if ! KERNEL_USER
+
/*
* Increment the suspend count for the target task.
* No threads within a task may run when the suspend
@@ -613,6 +638,18 @@ routine vm_map(
inheritance : vm_inherit_t);
#endif /* EMULATOR */
+#else /* ! KERNEL_USER */
+
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip; skip;
+skip; skip; skip; skip;
+
+#endif /* ! KERNEL_USER */
+
/*
* Indicate that a range of the specified memory object cannot
* be provided at this time. [Threads waiting for memory pages
@@ -684,6 +721,8 @@ simpleroutine memory_object_change_attributes(
reply_to : mach_port_t =
MACH_MSG_TYPE_MAKE_SEND_ONCE|polymorphic);
+#if ! KERNEL_USER
+
skip; /* old host_callout_statistics_reset */
skip; /* old port_set_select */
skip; /* old port_set_backup */
@@ -702,6 +741,8 @@ routine vm_machine_attribute(
skip; /* old host_fpa_counters_reset */
+#endif /* ! KERNEL_USER */
+
/*
* There is no more room in this interface for additional calls.
*/
diff --git a/include/mach_debug/slab_info.h b/include/mach_debug/slab_info.h
index 37dcb8c..7d12cc1 100644
--- a/include/mach_debug/slab_info.h
+++ b/include/mach_debug/slab_info.h
@@ -36,12 +36,6 @@
#define CACHE_NAME_MAX_LEN 32
-#define CACHE_FLAGS_NO_CPU_POOL 0x01
-#define CACHE_FLAGS_SLAB_EXTERNAL 0x02
-#define CACHE_FLAGS_NO_RECLAIM 0x04
-#define CACHE_FLAGS_VERIFY 0x08
-#define CACHE_FLAGS_DIRECT 0x10
-
typedef struct cache_info {
int flags;
size_t cpu_pool_size;
diff --git a/ipc/ipc_init.c b/ipc/ipc_init.c
index 2c58a6e..5ed800f 100644
--- a/ipc/ipc_init.c
+++ b/ipc/ipc_init.c
@@ -73,16 +73,16 @@ ipc_bootstrap(void)
ipc_port_timestamp_data = 0;
kmem_cache_init(&ipc_space_cache, "ipc_space",
- sizeof(struct ipc_space), 0, NULL, NULL, NULL, 0);
+ sizeof(struct ipc_space), 0, NULL, 0);
kmem_cache_init(&ipc_entry_cache, "ipc_entry",
- sizeof(struct ipc_entry), 0, NULL, NULL, NULL, 0);
+ sizeof(struct ipc_entry), 0, NULL, 0);
kmem_cache_init(&ipc_object_caches[IOT_PORT], "ipc_port",
- sizeof(struct ipc_port), 0, NULL, NULL, NULL, 0);
+ sizeof(struct ipc_port), 0, NULL, 0);
kmem_cache_init(&ipc_object_caches[IOT_PORT_SET], "ipc_pset",
- sizeof(struct ipc_pset), 0, NULL, NULL, NULL, 0);
+ sizeof(struct ipc_pset), 0, NULL, 0);
/* create special spaces */
diff --git a/ipc/ipc_marequest.c b/ipc/ipc_marequest.c
index ded1711..736db83 100644
--- a/ipc/ipc_marequest.c
+++ b/ipc/ipc_marequest.c
@@ -137,7 +137,7 @@ ipc_marequest_init(void)
}
kmem_cache_init(&ipc_marequest_cache, "ipc_marequest",
- sizeof(struct ipc_marequest), 0, NULL, NULL, NULL, 0);
+ sizeof(struct ipc_marequest), 0, NULL, 0);
}
/*
diff --git a/ipc/ipc_object.c b/ipc/ipc_object.c
index 320fbcb..a6457c3 100644
--- a/ipc/ipc_object.c
+++ b/ipc/ipc_object.c
@@ -998,7 +998,8 @@ char *ikot_print_array[IKOT_MAX_TYPE] = {
"(SEMAPHORE) ",
"(LOCK_SET) ",
"(CLOCK) ",
- "(CLOCK_CTRL) ", /* 26 */
+ "(CLOCK_CTRL) ",
+ "(PAGER_PROXY) ", /* 27 */
/* << new entries here */
"(UNKNOWN) " /* magic catchall */
}; /* Please keep in sync with kern/ipc_kobject.h */
diff --git a/kern/act.c b/kern/act.c
index 3186f7e..3819ef3 100644
--- a/kern/act.c
+++ b/kern/act.c
@@ -68,7 +68,7 @@ global_act_init(void)
{
#ifndef ACT_STATIC_KLUDGE
kmem_cache_init(&act_cache, "Act", sizeof(struct Act), 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
#else
int i;
diff --git a/kern/bootstrap.c b/kern/bootstrap.c
index 249c605..0836276 100644
--- a/kern/bootstrap.c
+++ b/kern/bootstrap.c
@@ -107,6 +107,20 @@ task_insert_send_right(
return name;
}
+static void
+free_bootstrap_pages(phys_addr_t start, phys_addr_t end)
+{
+ struct vm_page *page;
+
+ while (start < end)
+ {
+ page = vm_page_lookup_pa(start);
+ assert(page != NULL);
+ vm_page_manage(page);
+ start += PAGE_SIZE;
+ }
+}
+
void bootstrap_create(void)
{
int compat;
@@ -265,7 +279,7 @@ void bootstrap_create(void)
/* XXX we could free the memory used
by the boot loader's descriptors and such. */
for (n = 0; n < boot_info.mods_count; n++)
- vm_page_create(bmods[n].mod_start, bmods[n].mod_end);
+ free_bootstrap_pages(bmods[n].mod_start, bmods[n].mod_end);
}
static void
diff --git a/kern/cpu_number.h b/kern/cpu_number.h
index 44bbd64..650f404 100644
--- a/kern/cpu_number.h
+++ b/kern/cpu_number.h
@@ -37,5 +37,7 @@ int master_cpu; /* 'master' processor - keeps time */
/* cpu number is always 0 on a single processor system */
#define cpu_number() (0)
+#define CPU_L1_SIZE (1 << CPU_L1_SHIFT)
+
#endif /* NCPUS == 1 */
#endif /* _KERN_CPU_NUMBER_H_ */
diff --git a/i386/i386at/model_dep.h b/kern/exc.defs
similarity index 56%
copy from i386/i386at/model_dep.h
copy to kern/exc.defs
index aa24032..e614fff 100644
--- a/i386/i386at/model_dep.h
+++ b/kern/exc.defs
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2013 Free Software Foundation.
+ * Copyright (c) 2016 Free Software Foundation.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -11,20 +11,12 @@
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
-#ifndef _MODEL_DEP_H_
-#define _MODEL_DEP_H_
+/* We use custom functions to send exceptions. These functions can
+ be found in `exception.c'. We use this file merely to produce the
+ list of message ids. */
-#include <mach/vm_prot.h>
-
-extern int timemmap(dev_t dev, vm_offset_t off, vm_prot_t prot);
-
-void inittodr(void);
-
-boolean_t init_alloc_aligned(vm_size_t size, vm_offset_t *addrp);
-
-#endif /* _MODEL_DEP_H_ */
+#include <mach/exc.defs>
diff --git a/kern/log2.h b/kern/log2.h
new file mode 100644
index 0000000..0e67701
--- /dev/null
+++ b/kern/log2.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Integer base 2 logarithm operations.
+ */
+
+#ifndef _KERN_LOG2_H
+#define _KERN_LOG2_H
+
+#include <kern/assert.h>
+
+#ifdef __LP64__
+#define LONG_BIT 64
+#else /* __LP64__ */
+#define LONG_BIT 32
+#endif /* __LP64__ */
+
+static inline unsigned int
+ilog2(unsigned long x)
+{
+ assert(x != 0);
+ return LONG_BIT - __builtin_clzl(x) - 1;
+}
+
+static inline unsigned int
+iorder2(unsigned long size)
+{
+ assert(size != 0);
+
+ if (size == 1)
+ return 0;
+
+ return ilog2(size - 1) + 1;
+}
+
+#endif /* _KERN_LOG2_H */
diff --git a/kern/processor.c b/kern/processor.c
index 48e9273..0a88469 100644
--- a/kern/processor.c
+++ b/kern/processor.c
@@ -109,7 +109,7 @@ void pset_sys_init(void)
* Allocate the cache for processor sets.
*/
kmem_cache_init(&pset_cache, "processor_set",
- sizeof(struct processor_set), 0, NULL, NULL, NULL, 0);
+ sizeof(struct processor_set), 0, NULL, 0);
/*
* Give each processor a control port.
diff --git a/kern/rdxtree.c b/kern/rdxtree.c
index 78868b1..a23d6e7 100644
--- a/kern/rdxtree.c
+++ b/kern/rdxtree.c
@@ -124,7 +124,7 @@ void
rdxtree_cache_init(void)
{
kmem_cache_init(&rdxtree_node_cache, "rdxtree_node",
- sizeof(struct rdxtree_node), 0, NULL, NULL, NULL, 0);
+ sizeof(struct rdxtree_node), 0, NULL, 0);
}
#ifdef RDXTREE_ENABLE_NODE_CREATION_FAILURES
diff --git a/kern/sched_prim.c b/kern/sched_prim.c
index 580ca43..0cef160 100644
--- a/kern/sched_prim.c
+++ b/kern/sched_prim.c
@@ -446,6 +446,9 @@ void thread_wakeup_prim(
* occurs. The specified lock is unlocked before releasing
* the cpu. (This is a convenient way to sleep without manually
* calling assert_wait).
+ *
+ * Note: if the event may be woken from an interrupt handler, this must be
+ * called at an spl level that prevents such interrupts.
*/
void thread_sleep(
event_t event,
diff --git a/kern/slab.c b/kern/slab.c
index 8bc1b06..eeb94f8 100644
--- a/kern/slab.c
+++ b/kern/slab.c
@@ -58,15 +58,10 @@
* over a hash table. Unlike a hash table, a BST provides a "lookup nearest"
* operation, so obtaining the slab data (whether it is embedded in the slab or
* off slab) from a buffer address simply consists of a "lookup nearest towards
- * 0" tree search. Storing slabs instead of buffers also considerably reduces
- * the number of elements to retain. Finally, a self-balancing tree is a true
- * self-scaling data structure, whereas a hash table requires periodic
- * maintenance and complete resizing, which is expensive. The only drawback is
- * that releasing a buffer to the slab layer takes logarithmic time instead of
- * constant time. But as the data set size is kept reasonable (because slabs
- * are stored instead of buffers) and because the CPU pool layer services most
- * requests, avoiding many accesses to the slab layer, it is considered an
- * acceptable tradeoff.
+ * 0" tree search. Finally, a self-balancing tree is a true self-scaling data
+ * structure, whereas a hash table requires periodic maintenance and complete
+ * resizing, which is expensive. The only drawback is that releasing a buffer
+ * to the slab layer takes logarithmic time instead of constant time.
*
* This implementation uses per-cpu pools of objects, which service most
* allocation requests. These pools act as caches (but are named differently
@@ -87,6 +82,7 @@
#include <mach/vm_param.h>
#include <mach/machine/vm_types.h>
#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
#include <vm/vm_types.h>
#include <sys/types.h>
@@ -111,19 +107,6 @@
#define KMEM_ALIGN_MIN 8
/*
- * Minimum number of buffers per slab.
- *
- * This value is ignored when the slab size exceeds a threshold.
- */
-#define KMEM_MIN_BUFS_PER_SLAB 8
-
-/*
- * Special slab size beyond which the minimum number of buffers per slab is
- * ignored when computing the slab size of a cache.
- */
-#define KMEM_SLAB_SIZE_THRESHOLD (8 * PAGE_SIZE)
-
-/*
* Special buffer size under which slab data is unconditionnally allocated
* from its associated slab.
*/
@@ -163,11 +146,6 @@
#define KMEM_REDZONE_BYTE 0xbb
/*
- * Size of the VM submap from which default backend functions allocate.
- */
-#define KMEM_MAP_SIZE (128 * 1024 * 1024)
-
-/*
* Shift for the first kalloc cache size.
*/
#define KALLOC_FIRST_SHIFT 5
@@ -217,11 +195,17 @@
*
* The flags don't change once set and can be tested without locking.
*/
-#define KMEM_CF_NO_CPU_POOL 0x01 /* CPU pool layer disabled */
-#define KMEM_CF_SLAB_EXTERNAL 0x02 /* Slab data is off slab */
-#define KMEM_CF_NO_RECLAIM 0x04 /* Slabs are not reclaimable */
-#define KMEM_CF_VERIFY 0x08 /* Debugging facilities enabled */
-#define KMEM_CF_DIRECT 0x10 /* No buf-to-slab tree lookup */
+#define KMEM_CF_SLAB_EXTERNAL 0x01 /* Slab data is off slab */
+#define KMEM_CF_PHYSMEM 0x02 /* Allocate from physical memory */
+#define KMEM_CF_DIRECT 0x04 /* Direct buf-to-slab translation
+ (implies !KMEM_CF_SLAB_EXTERNAL) */
+#define KMEM_CF_USE_TREE 0x08 /* Use red-black tree to track slab
+ data */
+#define KMEM_CF_USE_PAGE 0x10 /* Use page private data to track slab
+ data (implies KMEM_CF_SLAB_EXTERNAL
+ and KMEM_CF_PHYSMEM) */
+#define KMEM_CF_VERIFY 0x20 /* Debugging facilities enabled
+ (implies KMEM_CF_USE_TREE) */
/*
* Options for kmem_cache_alloc_verify().
@@ -278,12 +262,6 @@ static unsigned int kmem_nr_caches;
static simple_lock_data_t __attribute__((used)) kmem_cache_list_lock;
/*
- * VM submap for slab caches.
- */
-static struct vm_map kmem_map_store;
-vm_map_t kmem_map = &kmem_map_store;
-
-/*
* Time of the last memory reclaim, in clock ticks.
*/
static unsigned long kmem_gc_last_tick;
@@ -384,12 +362,49 @@ static inline void * kmem_bufctl_to_buf(union kmem_bufctl
*bufctl,
return (void *)bufctl - cache->bufctl_dist;
}
-static vm_offset_t kmem_pagealloc(vm_size_t size)
+static vm_offset_t
+kmem_pagealloc_physmem(vm_size_t size)
+{
+ struct vm_page *page;
+
+ assert(size == PAGE_SIZE);
+
+ for (;;) {
+ page = vm_page_grab_contig(size, VM_PAGE_SEL_DIRECTMAP);
+
+ if (page != NULL)
+ break;
+
+ VM_PAGE_WAIT(NULL);
+ }
+
+ return phystokv(vm_page_to_pa(page));
+}
+
+static void
+kmem_pagefree_physmem(vm_offset_t addr, vm_size_t size)
+{
+ struct vm_page *page;
+
+ assert(size == PAGE_SIZE);
+ page = vm_page_lookup_pa(kvtophys(addr));
+ assert(page != NULL);
+ vm_page_free_contig(page, size);
+}
+
+static vm_offset_t
+kmem_pagealloc_virtual(vm_size_t size, vm_size_t align)
{
vm_offset_t addr;
kern_return_t kr;
- kr = kmem_alloc_wired(kmem_map, &addr, size);
+ assert(size > PAGE_SIZE);
+ size = vm_page_round(size);
+
+ if (align <= PAGE_SIZE)
+ kr = kmem_alloc_wired(kernel_map, &addr, size);
+ else
+ kr = kmem_alloc_aligned(kernel_map, &addr, size);
if (kr != KERN_SUCCESS)
return 0;
@@ -397,9 +412,29 @@ static vm_offset_t kmem_pagealloc(vm_size_t size)
return addr;
}
-static void kmem_pagefree(vm_offset_t ptr, vm_size_t size)
+static void
+kmem_pagefree_virtual(vm_offset_t addr, vm_size_t size)
{
- kmem_free(kmem_map, ptr, size);
+ assert(size > PAGE_SIZE);
+ size = vm_page_round(size);
+ kmem_free(kernel_map, addr, size);
+}
+
+static vm_offset_t
+kmem_pagealloc(vm_size_t size, vm_size_t align, int flags)
+{
+ assert(align <= size);
+ return (flags & KMEM_CF_PHYSMEM)
+ ? kmem_pagealloc_physmem(size)
+ : kmem_pagealloc_virtual(size, align);
+}
+
+static void
+kmem_pagefree(vm_offset_t addr, vm_size_t size, int flags)
+{
+ return (flags & KMEM_CF_PHYSMEM)
+ ? kmem_pagefree_physmem(addr, size)
+ : kmem_pagefree_virtual(addr, size);
}
static void kmem_slab_create_verify(struct kmem_slab *slab,
@@ -434,28 +469,28 @@ static struct kmem_slab * kmem_slab_create(struct
kmem_cache *cache,
union kmem_bufctl *bufctl;
size_t buf_size;
unsigned long buffers;
- void *slab_buf;
+ vm_offset_t slab_buf;
- if (cache->slab_alloc_fn == NULL)
- slab_buf = (void *)kmem_pagealloc(cache->slab_size);
- else
- slab_buf = (void *)cache->slab_alloc_fn(cache->slab_size);
+ slab_buf = kmem_pagealloc(cache->slab_size, cache->align, cache->flags);
- if (slab_buf == NULL)
+ if (slab_buf == 0)
return NULL;
if (cache->flags & KMEM_CF_SLAB_EXTERNAL) {
- assert(!(cache->flags & KMEM_CF_NO_RECLAIM));
slab = (struct kmem_slab *)kmem_cache_alloc(&kmem_slab_cache);
if (slab == NULL) {
- if (cache->slab_free_fn == NULL)
- kmem_pagefree((vm_offset_t)slab_buf, cache->slab_size);
- else
- cache->slab_free_fn((vm_offset_t)slab_buf, cache->slab_size);
-
+ kmem_pagefree(slab_buf, cache->slab_size, cache->flags);
return NULL;
}
+
+ if (cache->flags & KMEM_CF_USE_PAGE) {
+ struct vm_page *page;
+
+ page = vm_page_lookup_pa(kvtophys(slab_buf));
+ assert(page != NULL);
+ vm_page_set_priv(page, slab);
+ }
} else {
slab = (struct kmem_slab *)(slab_buf + cache->slab_size) - 1;
}
@@ -464,7 +499,7 @@ static struct kmem_slab * kmem_slab_create(struct
kmem_cache *cache,
rbtree_node_init(&slab->tree_node);
slab->nr_refs = 0;
slab->first_free = NULL;
- slab->addr = slab_buf + color;
+ slab->addr = (void *)(slab_buf + color);
buf_size = cache->buf_size;
bufctl = kmem_buf_to_bufctl(slab->addr, cache);
@@ -518,25 +553,26 @@ static void kmem_slab_destroy(struct kmem_slab *slab,
struct kmem_cache *cache)
assert(slab->nr_refs == 0);
assert(slab->first_free != NULL);
- assert(!(cache->flags & KMEM_CF_NO_RECLAIM));
if (cache->flags & KMEM_CF_VERIFY)
kmem_slab_destroy_verify(slab, cache);
slab_buf = (vm_offset_t)P2ALIGN((unsigned long)slab->addr, PAGE_SIZE);
- if (cache->slab_free_fn == NULL)
- kmem_pagefree(slab_buf, cache->slab_size);
- else
- cache->slab_free_fn(slab_buf, cache->slab_size);
+ if (cache->flags & KMEM_CF_SLAB_EXTERNAL) {
+ if (cache->flags & KMEM_CF_USE_PAGE) {
+ struct vm_page *page;
+
+ /* Not strictly needed, but let's increase safety */
+ page = vm_page_lookup_pa(kvtophys(slab_buf));
+ assert(page != NULL);
+ vm_page_set_priv(page, NULL);
+ }
- if (cache->flags & KMEM_CF_SLAB_EXTERNAL)
kmem_cache_free(&kmem_slab_cache, (vm_offset_t)slab);
-}
+ }
-static inline int kmem_slab_use_tree(int flags)
-{
- return !(flags & KMEM_CF_DIRECT) || (flags & KMEM_CF_VERIFY);
+ kmem_pagefree(slab_buf, cache->slab_size, cache->flags);
}
static inline int kmem_slab_cmp_lookup(const void *addr,
@@ -694,82 +730,81 @@ static void kmem_cache_error(struct kmem_cache *cache,
void *buf, int error,
}
/*
- * Compute an appropriate slab size for the given cache.
+ * Compute properties such as slab size for the given cache.
*
* Once the slab size is known, this function sets the related properties
- * (buffers per slab and maximum color). It can also set the KMEM_CF_DIRECT
- * and/or KMEM_CF_SLAB_EXTERNAL flags depending on the resulting layout.
+ * (buffers per slab and maximum color). It can also set some KMEM_CF_xxx
+ * flags depending on the resulting layout.
*/
-static void kmem_cache_compute_sizes(struct kmem_cache *cache, int flags)
+static void kmem_cache_compute_properties(struct kmem_cache *cache, int flags)
{
- size_t i, buffers, buf_size, slab_size, free_slab_size, optimal_size = 0;
- size_t waste, waste_min;
- int embed, optimal_embed = 0;
-
- buf_size = cache->buf_size;
+ size_t size, waste;
+ int embed;
- if (buf_size < KMEM_BUF_SIZE_THRESHOLD)
+ if (cache->buf_size < KMEM_BUF_SIZE_THRESHOLD)
flags |= KMEM_CACHE_NOOFFSLAB;
- i = 0;
- waste_min = (size_t)-1;
-
- do {
- i++;
- slab_size = P2ROUND(i * buf_size, PAGE_SIZE);
- free_slab_size = slab_size;
-
- if (flags & KMEM_CACHE_NOOFFSLAB)
- free_slab_size -= sizeof(struct kmem_slab);
-
- buffers = free_slab_size / buf_size;
- waste = free_slab_size % buf_size;
-
- if (buffers > i)
- i = buffers;
+ cache->slab_size = PAGE_SIZE;
+ for (;;) {
if (flags & KMEM_CACHE_NOOFFSLAB)
embed = 1;
- else if (sizeof(struct kmem_slab) <= waste) {
- embed = 1;
- waste -= sizeof(struct kmem_slab);
- } else {
- embed = 0;
+ else {
+ waste = cache->slab_size % cache->buf_size;
+ embed = (sizeof(struct kmem_slab) <= waste);
}
- if (waste <= waste_min) {
- waste_min = waste;
- optimal_size = slab_size;
- optimal_embed = embed;
- }
- } while ((buffers < KMEM_MIN_BUFS_PER_SLAB)
- && (slab_size < KMEM_SLAB_SIZE_THRESHOLD));
+ size = cache->slab_size;
- assert(optimal_size > 0);
- assert(!(flags & KMEM_CACHE_NOOFFSLAB) || optimal_embed);
+ if (embed)
+ size -= sizeof(struct kmem_slab);
- cache->slab_size = optimal_size;
- slab_size = cache->slab_size - (optimal_embed
- ? sizeof(struct kmem_slab)
- : 0);
- cache->bufs_per_slab = slab_size / buf_size;
- cache->color_max = slab_size % buf_size;
+ if (size >= cache->buf_size)
+ break;
+
+ cache->slab_size += PAGE_SIZE;
+ }
+
+ cache->bufs_per_slab = size / cache->buf_size;
+ cache->color_max = size % cache->buf_size;
if (cache->color_max >= PAGE_SIZE)
- cache->color_max = PAGE_SIZE - 1;
+ cache->color_max = 0;
+
+ if (!embed)
+ cache->flags |= KMEM_CF_SLAB_EXTERNAL;
+
+ if ((flags & KMEM_CACHE_PHYSMEM) || (cache->slab_size == PAGE_SIZE)) {
+ cache->flags |= KMEM_CF_PHYSMEM;
- if (optimal_embed) {
+ /*
+ * Avoid using larger-than-page slabs backed by the direct physical
+ * mapping to completely prevent physical memory fragmentation from
+ * making slab allocations fail.
+ */
+ if (cache->slab_size != PAGE_SIZE)
+ panic("slab: invalid cache parameters");
+ }
+
+ if (cache->flags & KMEM_CF_VERIFY)
+ cache->flags |= KMEM_CF_USE_TREE;
+
+ if (cache->flags & KMEM_CF_SLAB_EXTERNAL) {
+ if (cache->flags & KMEM_CF_PHYSMEM)
+ cache->flags |= KMEM_CF_USE_PAGE;
+ else
+ cache->flags |= KMEM_CF_USE_TREE;
+ } else {
if (cache->slab_size == PAGE_SIZE)
cache->flags |= KMEM_CF_DIRECT;
- } else {
- cache->flags |= KMEM_CF_SLAB_EXTERNAL;
+ else
+ cache->flags |= KMEM_CF_USE_TREE;
}
}
void kmem_cache_init(struct kmem_cache *cache, const char *name,
- size_t obj_size, size_t align, kmem_cache_ctor_t ctor,
- kmem_slab_alloc_fn_t slab_alloc_fn,
- kmem_slab_free_fn_t slab_free_fn, int flags)
+ size_t obj_size, size_t align,
+ kmem_cache_ctor_t ctor, int flags)
{
#if SLAB_USE_CPU_POOLS
struct kmem_cpu_pool_type *cpu_pool_type;
@@ -783,15 +818,6 @@ void kmem_cache_init(struct kmem_cache *cache, const char
*name,
cache->flags = 0;
#endif /* SLAB_VERIFY */
- if (flags & KMEM_CACHE_NOCPUPOOL)
- cache->flags |= KMEM_CF_NO_CPU_POOL;
-
- if (flags & KMEM_CACHE_NORECLAIM) {
- assert(slab_free_fn == NULL);
- flags |= KMEM_CACHE_NOOFFSLAB;
- cache->flags |= KMEM_CF_NO_RECLAIM;
- }
-
if (flags & KMEM_CACHE_VERIFY)
cache->flags |= KMEM_CF_VERIFY;
@@ -800,7 +826,6 @@ void kmem_cache_init(struct kmem_cache *cache, const char
*name,
assert(obj_size > 0);
assert(ISP2(align));
- assert(align < PAGE_SIZE);
buf_size = P2ROUND(obj_size, align);
@@ -819,8 +844,6 @@ void kmem_cache_init(struct kmem_cache *cache, const char
*name,
cache->nr_slabs = 0;
cache->nr_free_slabs = 0;
cache->ctor = ctor;
- cache->slab_alloc_fn = slab_alloc_fn;
- cache->slab_free_fn = slab_free_fn;
strncpy(cache->name, name, sizeof(cache->name));
cache->name[sizeof(cache->name) - 1] = '\0';
cache->buftag_dist = 0;
@@ -835,7 +858,7 @@ void kmem_cache_init(struct kmem_cache *cache, const char
*name,
cache->buf_size = buf_size;
}
- kmem_cache_compute_sizes(cache, flags);
+ kmem_cache_compute_properties(cache, flags);
#if SLAB_USE_CPU_POOLS
for (cpu_pool_type = kmem_cpu_pool_types;
@@ -908,9 +931,6 @@ static void kmem_cache_reap(struct kmem_cache *cache)
struct list dead_slabs;
unsigned long nr_free_slabs;
- if (cache->flags & KMEM_CF_NO_RECLAIM)
- return;
-
simple_lock(&cache->lock);
list_set_head(&dead_slabs, &cache->free_slabs);
list_init(&cache->free_slabs);
@@ -971,7 +991,7 @@ static void * kmem_cache_alloc_from_slab(struct kmem_cache
*cache)
cache->nr_free_slabs--;
}
- if ((slab->nr_refs == 1) && kmem_slab_use_tree(cache->flags))
+ if ((slab->nr_refs == 1) && (cache->flags & KMEM_CF_USE_TREE))
rbtree_insert(&cache->active_slabs, &slab->tree_node,
kmem_slab_cmp_insert);
@@ -992,17 +1012,26 @@ static void kmem_cache_free_to_slab(struct kmem_cache
*cache, void *buf)
assert(cache->slab_size == PAGE_SIZE);
slab = (struct kmem_slab *)P2END((unsigned long)buf, cache->slab_size)
- 1;
+ } else if (cache->flags & KMEM_CF_USE_PAGE) {
+ struct vm_page *page;
+
+ page = vm_page_lookup_pa(kvtophys((vm_offset_t)buf));
+ assert(page != NULL);
+ slab = vm_page_get_priv(page);
} else {
struct rbtree_node *node;
+ assert(cache->flags & KMEM_CF_USE_TREE);
node = rbtree_lookup_nearest(&cache->active_slabs, buf,
kmem_slab_cmp_lookup, RBTREE_LEFT);
assert(node != NULL);
slab = rbtree_entry(node, struct kmem_slab, tree_node);
- assert((unsigned long)buf < (P2ALIGN((unsigned long)slab->addr
- + cache->slab_size, PAGE_SIZE)));
}
+ assert((unsigned long)buf >= (unsigned long)slab->addr);
+ assert(((unsigned long)buf + cache->buf_size)
+ <= vm_page_trunc((unsigned long)slab->addr + cache->slab_size));
+
assert(slab->nr_refs >= 1);
assert(slab->nr_refs <= cache->bufs_per_slab);
bufctl = kmem_buf_to_bufctl(buf, cache);
@@ -1014,7 +1043,7 @@ static void kmem_cache_free_to_slab(struct kmem_cache
*cache, void *buf)
if (slab->nr_refs == 0) {
/* The slab has become free */
- if (kmem_slab_use_tree(cache->flags))
+ if (cache->flags & KMEM_CF_USE_TREE)
rbtree_remove(&cache->active_slabs, &slab->tree_node);
if (cache->bufs_per_slab > 1)
@@ -1135,6 +1164,8 @@ static void kmem_cache_free_verify(struct kmem_cache
*cache, void *buf)
unsigned char *redzone_byte;
unsigned long slabend;
+ assert(cache->flags & KMEM_CF_USE_TREE);
+
simple_lock(&cache->lock);
node = rbtree_lookup_nearest(&cache->active_slabs, buf,
kmem_slab_cmp_lookup, RBTREE_LEFT);
@@ -1280,16 +1311,12 @@ void slab_bootstrap(void)
void slab_init(void)
{
- vm_offset_t min, max;
-
#if SLAB_USE_CPU_POOLS
struct kmem_cpu_pool_type *cpu_pool_type;
char name[KMEM_CACHE_NAME_SIZE];
size_t i, size;
#endif /* SLAB_USE_CPU_POOLS */
- kmem_submap(kmem_map, kernel_map, &min, &max, KMEM_MAP_SIZE, FALSE);
-
#if SLAB_USE_CPU_POOLS
for (i = 0; i < ARRAY_SIZE(kmem_cpu_pool_types); i++) {
cpu_pool_type = &kmem_cpu_pool_types[i];
@@ -1297,7 +1324,7 @@ void slab_init(void)
sprintf(name, "kmem_cpu_array_%d", cpu_pool_type->array_size);
size = sizeof(void *) * cpu_pool_type->array_size;
kmem_cache_init(cpu_pool_type->array_cache, name, size,
- cpu_pool_type->array_align, NULL, NULL, NULL, 0);
+ cpu_pool_type->array_align, NULL, 0);
}
#endif /* SLAB_USE_CPU_POOLS */
@@ -1305,25 +1332,7 @@ void slab_init(void)
* Prevent off slab data for the slab cache to avoid infinite recursion.
*/
kmem_cache_init(&kmem_slab_cache, "kmem_slab", sizeof(struct kmem_slab),
- 0, NULL, NULL, NULL, KMEM_CACHE_NOOFFSLAB);
-}
-
-static vm_offset_t kalloc_pagealloc(vm_size_t size)
-{
- vm_offset_t addr;
- kern_return_t kr;
-
- kr = kmem_alloc_wired(kmem_map, &addr, size);
-
- if (kr != KERN_SUCCESS)
- return 0;
-
- return addr;
-}
-
-static void kalloc_pagefree(vm_offset_t ptr, vm_size_t size)
-{
- kmem_free(kmem_map, ptr, size);
+ 0, NULL, KMEM_CACHE_NOOFFSLAB);
}
void kalloc_init(void)
@@ -1335,8 +1344,7 @@ void kalloc_init(void)
for (i = 0; i < ARRAY_SIZE(kalloc_caches); i++) {
sprintf(name, "kalloc_%lu", size);
- kmem_cache_init(&kalloc_caches[i], name, size, 0, NULL,
- kalloc_pagealloc, kalloc_pagefree, 0);
+ kmem_cache_init(&kalloc_caches[i], name, size, 0, NULL, 0);
size <<= 1;
}
}
@@ -1387,8 +1395,9 @@ vm_offset_t kalloc(vm_size_t size)
if ((buf != 0) && (cache->flags & KMEM_CF_VERIFY))
kalloc_verify(cache, buf, size);
- } else
- buf = (void *)kalloc_pagealloc(size);
+ } else {
+ buf = (void *)kmem_pagealloc_virtual(size, 0);
+ }
return (vm_offset_t)buf;
}
@@ -1429,7 +1438,7 @@ void kfree(vm_offset_t data, vm_size_t size)
kmem_cache_free(cache, data);
} else {
- kalloc_pagefree(data, size);
+ kmem_pagefree_virtual(data, size);
}
}
@@ -1529,16 +1538,7 @@ kern_return_t host_slab_info(host_t host,
cache_info_array_t *infop,
list_for_each_entry(&kmem_cache_list, cache, node) {
simple_lock(&cache->lock);
- info[i].flags = ((cache->flags & KMEM_CF_NO_CPU_POOL)
- ? CACHE_FLAGS_NO_CPU_POOL : 0)
- | ((cache->flags & KMEM_CF_SLAB_EXTERNAL)
- ? CACHE_FLAGS_SLAB_EXTERNAL : 0)
- | ((cache->flags & KMEM_CF_NO_RECLAIM)
- ? CACHE_FLAGS_NO_RECLAIM : 0)
- | ((cache->flags & KMEM_CF_VERIFY)
- ? CACHE_FLAGS_VERIFY : 0)
- | ((cache->flags & KMEM_CF_DIRECT)
- ? CACHE_FLAGS_DIRECT : 0);
+ info[i].flags = cache->flags;
#if SLAB_USE_CPU_POOLS
info[i].cpu_pool_size = cache->cpu_pool_type->array_size;
#else /* SLAB_USE_CPU_POOLS */
diff --git a/kern/slab.h b/kern/slab.h
index 77db7c1..8527c9d 100644
--- a/kern/slab.h
+++ b/kern/slab.h
@@ -48,6 +48,7 @@
#define _KERN_SLAB_H
#include <cache.h>
+#include <kern/cpu_number.h>
#include <kern/lock.h>
#include <kern/list.h>
#include <kern/rbtree.h>
@@ -56,10 +57,6 @@
#include <vm/vm_types.h>
#if SLAB_USE_CPU_POOLS
-/*
- * L1 cache line size.
- */
-#define CPU_L1_SIZE (1 << CPU_L1_SHIFT)
/*
* Per-processor cache of pre-constructed objects.
@@ -140,14 +137,6 @@ struct kmem_slab {
typedef void (*kmem_cache_ctor_t)(void *obj);
/*
- * Types for slab allocation/free functions.
- *
- * All addresses and sizes must be page-aligned.
- */
-typedef vm_offset_t (*kmem_slab_alloc_fn_t)(vm_size_t);
-typedef void (*kmem_slab_free_fn_t)(vm_offset_t, vm_size_t);
-
-/*
* Cache name buffer size. The size is chosen so that struct
* kmem_cache fits into two cache lines. The size of a cache line on
* a typical CPU is 64 bytes.
@@ -192,8 +181,6 @@ struct kmem_cache {
size_t color_max;
unsigned long nr_bufs; /* Total number of buffers */
unsigned long nr_slabs;
- kmem_slab_alloc_fn_t slab_alloc_fn;
- kmem_slab_free_fn_t slab_free_fn;
char name[KMEM_CACHE_NAME_SIZE];
size_t buftag_dist; /* Distance from buffer to buftag */
size_t redzone_pad; /* Bytes from end of object to redzone word */
@@ -206,26 +193,18 @@ typedef struct kmem_cache *kmem_cache_t;
#define KMEM_CACHE_NULL ((kmem_cache_t) 0)
/*
- * VM submap for slab allocations.
- */
-extern vm_map_t kmem_map;
-
-/*
* Cache initialization flags.
*/
-#define KMEM_CACHE_NOCPUPOOL 0x1 /* Don't use the per-cpu pools */
-#define KMEM_CACHE_NOOFFSLAB 0x2 /* Don't allocate external slab data */
-#define KMEM_CACHE_NORECLAIM 0x4 /* Never give slabs back to their source,
- implies KMEM_CACHE_NOOFFSLAB */
-#define KMEM_CACHE_VERIFY 0x8 /* Use debugging facilities */
+#define KMEM_CACHE_NOOFFSLAB 0x1 /* Don't allocate external slab data */
+#define KMEM_CACHE_PHYSMEM 0x2 /* Allocate from physical memory */
+#define KMEM_CACHE_VERIFY 0x4 /* Use debugging facilities */
/*
* Initialize a cache.
*/
void kmem_cache_init(struct kmem_cache *cache, const char *name,
- size_t obj_size, size_t align, kmem_cache_ctor_t ctor,
- kmem_slab_alloc_fn_t slab_alloc_fn,
- kmem_slab_free_fn_t slab_free_fn, int flags);
+ size_t obj_size, size_t align,
+ kmem_cache_ctor_t ctor, int flags);
/*
* Allocate an object from a cache.
diff --git a/kern/startup.c b/kern/startup.c
index 30cff5c..bd29694 100644
--- a/kern/startup.c
+++ b/kern/startup.c
@@ -136,7 +136,7 @@ void setup_main(void)
mapable_time_init();
machine_info.max_cpus = NCPUS;
- machine_info.memory_size = phys_last_addr - phys_first_addr; /* XXX
mem_size */
+ machine_info.memory_size = vm_page_mem_size(); /* XXX phys_addr_t ->
vm_size_t */
machine_info.avail_cpus = 0;
machine_info.major_version = KERNEL_MAJOR_VERSION;
machine_info.minor_version = KERNEL_MINOR_VERSION;
diff --git a/kern/task.c b/kern/task.c
index e9e6ba2..0f24e44 100644
--- a/kern/task.c
+++ b/kern/task.c
@@ -63,7 +63,7 @@ ipc_port_t new_task_notification = NULL;
void task_init(void)
{
kmem_cache_init(&task_cache, "task", sizeof(struct task), 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
eml_init();
machine_task_module_init ();
diff --git a/kern/thread.c b/kern/thread.c
index 3e90079..ce44ed1 100644
--- a/kern/thread.c
+++ b/kern/thread.c
@@ -70,6 +70,7 @@ thread_t active_threads[NCPUS];
vm_offset_t active_stacks[NCPUS];
struct kmem_cache thread_cache;
+struct kmem_cache thread_stack_cache;
queue_head_t reaper_queue;
decl_simple_lock_data(, reaper_lock)
@@ -195,16 +196,8 @@ kern_return_t stack_alloc(
(void) splx(s);
if (stack == 0) {
- kern_return_t kr;
- /*
- * Kernel stacks should be naturally aligned,
- * so that it is easy to find the starting/ending
- * addresses of a stack given an address in the middle.
- */
- kr = kmem_alloc_aligned(kmem_map, &stack, KERNEL_STACK_SIZE);
- if (kr != KERN_SUCCESS)
- return kr;
-
+ stack = kmem_cache_alloc(&thread_stack_cache);
+ assert(stack != 0);
#if MACH_DEBUG
stack_init(stack);
#endif /* MACH_DEBUG */
@@ -265,7 +258,7 @@ void stack_collect(void)
#if MACH_DEBUG
stack_finalize(stack);
#endif /* MACH_DEBUG */
- kmem_free(kmem_map, stack, KERNEL_STACK_SIZE);
+ kmem_cache_free(&thread_stack_cache, stack);
s = splsched();
stack_lock();
@@ -298,7 +291,15 @@ void stack_privilege(
void thread_init(void)
{
kmem_cache_init(&thread_cache, "thread", sizeof(struct thread), 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
+ /*
+ * Kernel stacks should be naturally aligned,
+ * so that it is easy to find the starting/ending
+ * addresses of a stack given an address in the middle.
+ */
+ kmem_cache_init(&thread_stack_cache, "thread_stack",
+ KERNEL_STACK_SIZE, KERNEL_STACK_SIZE,
+ NULL, 0);
/*
* Fill in a template thread for fast initialization.
@@ -430,7 +431,7 @@ kern_return_t thread_create(
* Create a pcb. The kernel stack is created later,
* when the thread is swapped-in.
*/
- pcb_init(new_thread);
+ pcb_init(parent_task, new_thread);
ipc_thread_init(new_thread);
diff --git a/linux/dev/drivers/block/ahci.c b/linux/dev/drivers/block/ahci.c
index c243982..b60f1a1 100644
--- a/linux/dev/drivers/block/ahci.c
+++ b/linux/dev/drivers/block/ahci.c
@@ -826,41 +826,56 @@ static void ahci_probe_dev(unsigned char bus, unsigned
char device)
/* Get configuration */
if (pcibios_read_config_byte(bus, device, PCI_HEADER_TYPE, &hdrtype) !=
PCIBIOS_SUCCESSFUL) {
- printk("ahci: %02u:%02u.%u: Can not read configuration", bus,
dev, fun);
+ printk("ahci: %02x:%02x.%x: Can not read configuration", bus,
dev, fun);
return;
}
if (hdrtype != 0) {
- printk("ahci: %02u:%02u.%u: Unknown hdrtype %d\n", bus, dev,
fun, hdrtype);
+ printk("ahci: %02x:%02x.%x: Unknown hdrtype %d\n", bus, dev,
fun, hdrtype);
return;
}
if (pcibios_read_config_dword(bus, device, PCI_BASE_ADDRESS_5, &bar) !=
PCIBIOS_SUCCESSFUL) {
- printk("ahci: %02u:%02u.%u: Can not read BAR 5", bus, dev, fun);
+ printk("ahci: %02x:%02x.%x: Can not read BAR 5", bus, dev, fun);
return;
}
- if (bar & 0x01) {
- printk("ahci: %02u:%02u.%u: BAR 5 is I/O?!", bus, dev, fun);
+ if (bar & PCI_BASE_ADDRESS_SPACE_IO) {
+ printk("ahci: %02x:%02x.%x: BAR 5 is I/O?!", bus, dev, fun);
return;
}
- bar &= ~0x0f;
+ bar &= PCI_BASE_ADDRESS_MEM_MASK;
if (pcibios_read_config_byte(bus, device, PCI_INTERRUPT_LINE, &irq) !=
PCIBIOS_SUCCESSFUL) {
- printk("ahci: %02u:%02u.%u: Can not read IRQ", bus, dev, fun);
+ printk("ahci: %02x:%02x.%x: Can not read IRQ", bus, dev, fun);
return;
}
- printk("AHCI SATA %02u:%02u.%u BAR 0x%x IRQ %u\n", bus, dev, fun, bar,
irq);
+ printk("AHCI SATA %02x:%02x.%x BAR 0x%x IRQ %u\n", bus, dev, fun, bar,
irq);
/* Map mmio */
ahci_host = vremap(bar, 0x2000);
/* Request IRQ */
if (request_irq(irq, &ahci_interrupt, SA_SHIRQ, "ahci", (void*)
ahci_host)) {
- printk("ahci: %02u:%02u.%u: Can not get irq %u\n", bus, dev,
fun, irq);
+ printk("ahci: %02x:%02x.%x: Can not get irq %u\n", bus, dev,
fun, irq);
return;
}
+#ifdef CONFIG_BLK_DEV_IDE
+ /* OK, we will handle it. Disable probing on legacy IDE ports it may
have. */
+ for (i = 0; i < 6; i++)
+ {
+ unsigned mybar;
+ if (pcibios_read_config_dword(bus, device, PCI_BASE_ADDRESS_0 +
i*4, &mybar) == PCIBIOS_SUCCESSFUL) {
+ if (!(bar & PCI_BASE_ADDRESS_SPACE_IO))
+ /* Memory, don't care */
+ continue;
+ /* printk("ahci: %02x:%02x.%x: BAR %d is %x\n", bus,
dev, fun, i, mybar); */
+ ide_disable_base(bar & PCI_BASE_ADDRESS_IO_MASK);
+ }
+ }
+#endif
+
nports = (readl(&ahci_host->cap) & 0x1f) + 1;
port_map = readl(&ahci_host->pi);
@@ -869,7 +884,7 @@ static void ahci_probe_dev(unsigned char bus, unsigned char
device)
n++;
if (nports != n) {
- printk("ahci: %02u:%02u.%u: Odd number of ports %u, assuming %u
is correct\n", bus, dev, fun, n, nports);
+ printk("ahci: %02x:%02x.%x: Odd number of ports %u, assuming %u
is correct\n", bus, dev, fun, n, nports);
port_map = 0;
}
if (!port_map) {
@@ -893,16 +908,16 @@ static void ahci_probe_dev(unsigned char bus, unsigned
char device)
/* Device not present */
continue;
case 0x1:
- printk("ahci: %02u:%02u.%u: Port %u
communication not established. TODO: power on device\n", bus, dev, fun, i);
+ printk("ahci: %02x:%02x.%x: Port %u
communication not established. TODO: power on device\n", bus, dev, fun, i);
continue;
case 0x3:
/* Present and communication established */
break;
case 0x4:
- printk("ahci: %02u:%02u.%u: Port %u phy
offline?!\n", bus, dev, fun, i);
+ printk("ahci: %02x:%02x.%x: Port %u phy
offline?!\n", bus, dev, fun, i);
continue;
default:
- printk("ahci: %02u:%02u.%u: Unknown port %u SPD
%x\n", bus, dev, fun, i, spd);
+ printk("ahci: %02x:%02x.%x: Unknown port %u SPD
%x\n", bus, dev, fun, i, spd);
continue;
}
@@ -916,13 +931,13 @@ static void ahci_probe_dev(unsigned char bus, unsigned
char device)
/* Active */
break;
case 0x2:
- printk("ahci: %02u:%02u.%u: Port %u in Partial
power management. TODO: power on device\n", bus, dev, fun, i);
+ printk("ahci: %02x:%02x.%x: Port %u in Partial
power management. TODO: power on device\n", bus, dev, fun, i);
continue;
case 0x6:
- printk("ahci: %02u:%02u.%u: Port %u in Slumber
power management. TODO: power on device\n", bus, dev, fun, i);
+ printk("ahci: %02x:%02x.%x: Port %u in Slumber
power management. TODO: power on device\n", bus, dev, fun, i);
continue;
default:
- printk("ahci: %02u:%02u.%u: Unknown port %u IPM
%x\n", bus, dev, fun, i, ipm);
+ printk("ahci: %02x:%02x.%x: Unknown port %u IPM
%x\n", bus, dev, fun, i, ipm);
continue;
}
diff --git a/linux/dev/glue/glue.h b/linux/dev/glue/glue.h
index 5d4f6d8..8cb118c 100644
--- a/linux/dev/glue/glue.h
+++ b/linux/dev/glue/glue.h
@@ -25,8 +25,8 @@
extern int linux_auto_config;
extern int linux_intr_pri;
-extern void *alloc_contig_mem (unsigned, unsigned, unsigned, vm_page_t *);
-extern void free_contig_mem (vm_page_t);
+extern unsigned long alloc_contig_mem (unsigned, unsigned, unsigned, vm_page_t
*);
+extern void free_contig_mem (vm_page_t, unsigned);
extern void init_IRQ (void);
extern void restore_IRQ (void);
extern void linux_kmem_init (void);
diff --git a/linux/dev/glue/kmem.c b/linux/dev/glue/kmem.c
index ff052ff..ed57610 100644
--- a/linux/dev/glue/kmem.c
+++ b/linux/dev/glue/kmem.c
@@ -111,10 +111,8 @@ linux_kmem_init ()
for (p = pages, j = 0; j < MEM_CHUNK_SIZE - PAGE_SIZE; j += PAGE_SIZE)
{
assert (p->phys_addr < MEM_DMA_LIMIT);
- assert (p->phys_addr + PAGE_SIZE
- == ((vm_page_t) p->pageq.next)->phys_addr);
-
- p = (vm_page_t) p->pageq.next;
+ assert (p->phys_addr + PAGE_SIZE == (p + 1)->phys_addr);
+ p++;
}
pages_free[i].end = pages_free[i].start + MEM_CHUNK_SIZE;
diff --git a/linux/dev/include/linux/blk.h b/linux/dev/include/linux/blk.h
index 156d91c..b924a14 100644
--- a/linux/dev/include/linux/blk.h
+++ b/linux/dev/include/linux/blk.h
@@ -78,6 +78,7 @@ extern int hd_init(void);
#endif
#ifdef CONFIG_BLK_DEV_IDE
extern int ide_init(void);
+extern void ide_disable_base(unsigned base);
#endif
#ifdef CONFIG_BLK_DEV_XD
extern int xd_init(void);
diff --git a/linux/dev/init/main.c b/linux/dev/init/main.c
index 8737b62..d69b3fc 100644
--- a/linux/dev/init/main.c
+++ b/linux/dev/init/main.c
@@ -98,7 +98,7 @@ void
linux_init (void)
{
int addr;
- unsigned memory_start, memory_end;
+ unsigned long memory_start, memory_end;
vm_page_t pages;
/*
@@ -131,9 +131,7 @@ linux_init (void)
/*
* Allocate contiguous memory below 16 MB.
*/
- memory_start = (unsigned long) alloc_contig_mem (CONTIG_ALLOC,
- 16 * 1024 * 1024,
- 0, &pages);
+ memory_start = alloc_contig_mem (CONTIG_ALLOC, 16 * 1024 * 1024, 0, &pages);
if (memory_start == 0)
panic ("linux_init: alloc_contig_mem failed");
memory_end = memory_start + CONTIG_ALLOC;
@@ -147,14 +145,6 @@ linux_init (void)
panic ("linux_init: ran out memory");
/*
- * Free unused memory.
- */
- while (pages && phystokv(pages->phys_addr) < round_page (memory_start))
- pages = (vm_page_t) pages->pageq.next;
- if (pages)
- free_contig_mem (pages);
-
- /*
* Initialize devices.
*/
#ifdef CONFIG_INET
@@ -182,140 +172,31 @@ linux_init (void)
/*
* Allocate contiguous memory with the given constraints.
- * This routine is horribly inefficient but it is presently
- * only used during initialization so it's not that bad.
*/
-void *
+unsigned long
alloc_contig_mem (unsigned size, unsigned limit,
unsigned mask, vm_page_t * pages)
{
- int i, j, bits_len;
- unsigned *bits, len;
- void *m;
- vm_page_t p, page_list, tail, prev;
- vm_offset_t addr = 0, max_addr;
-
- if (size == 0)
- return (NULL);
- size = round_page (size);
- if ((size >> PAGE_SHIFT) > vm_page_free_count)
- return (NULL);
-
- /* Allocate bit array. */
- max_addr = phys_last_addr;
- if (max_addr > limit)
- max_addr = limit;
- bits_len = ((((max_addr >> PAGE_SHIFT) + NBPW - 1) / NBPW)
- * sizeof (unsigned));
- bits = (unsigned *) kalloc (bits_len);
- if (!bits)
- return (NULL);
- memset (bits, 0, bits_len);
+ vm_page_t p;
- /*
- * Walk the page free list and set a bit for every usable page.
- */
- simple_lock (&vm_page_queue_free_lock);
- p = vm_page_queue_free;
- while (p)
- {
- if (p->phys_addr < limit)
- (bits[(p->phys_addr >> PAGE_SHIFT) / NBPW]
- |= 1 << ((p->phys_addr >> PAGE_SHIFT) % NBPW));
- p = (vm_page_t) p->pageq.next;
- }
+ p = vm_page_grab_contig(size, VM_PAGE_SEL_DMA);
- /*
- * Scan bit array for contiguous pages.
- */
- len = 0;
- m = NULL;
- for (i = 0; len < size && i < bits_len / sizeof (unsigned); i++)
- for (j = 0; len < size && j < NBPW; j++)
- if (!(bits[i] & (1 << j)))
- {
- len = 0;
- m = NULL;
- }
- else
- {
- if (len == 0)
- {
- addr = ((vm_offset_t) (i * NBPW + j)
- << PAGE_SHIFT);
- if ((addr & mask) == 0)
- {
- len += PAGE_SIZE;
- m = (void *) addr;
- }
- }
- else
- len += PAGE_SIZE;
- }
-
- if (len != size)
- {
- simple_unlock (&vm_page_queue_free_lock);
- kfree ((vm_offset_t) bits, bits_len);
- return (NULL);
- }
-
- /*
- * Remove pages from free list
- * and construct list to return to caller.
- */
- page_list = NULL;
- for (len = 0; len < size; len += PAGE_SIZE, addr += PAGE_SIZE)
- {
- prev = NULL;
- for (p = vm_page_queue_free; p; p = (vm_page_t) p->pageq.next)
- {
- if (p->phys_addr == addr)
- break;
- prev = p;
- }
- if (!p)
- panic ("alloc_contig_mem: page not on free list");
- if (prev)
- prev->pageq.next = p->pageq.next;
- else
- vm_page_queue_free = (vm_page_t) p->pageq.next;
- p->free = FALSE;
- p->pageq.next = NULL;
- if (!page_list)
- page_list = tail = p;
- else
- {
- tail->pageq.next = (queue_entry_t) p;
- tail = p;
- }
- vm_page_free_count--;
- }
+ if (p == NULL)
+ return 0;
- simple_unlock (&vm_page_queue_free_lock);
- kfree ((vm_offset_t) bits, bits_len);
if (pages)
- *pages = page_list;
- return (void *) phystokv(m);
+ *pages = p;
+
+ return phystokv(vm_page_to_pa(p));
}
/*
* Free memory allocated by alloc_contig_mem.
*/
void
-free_contig_mem (vm_page_t pages)
+free_contig_mem (vm_page_t pages, unsigned size)
{
- int i;
- vm_page_t p;
-
- for (p = pages, i = 0; p->pageq.next; p = (vm_page_t) p->pageq.next, i++)
- p->free = TRUE;
- p->free = TRUE;
- simple_lock (&vm_page_queue_free_lock);
- vm_page_free_count += i + 1;
- p->pageq.next = (queue_entry_t) vm_page_queue_free;
- vm_page_queue_free = pages;
- simple_unlock (&vm_page_queue_free_lock);
+ vm_page_free_contig(pages, size);
}
/* This is the number of bits of precision for the loops_per_second. Each
diff --git a/linux/pcmcia-cs/glue/ds.c b/linux/pcmcia-cs/glue/ds.c
index 8f88b55..cc4b92b 100644
--- a/linux/pcmcia-cs/glue/ds.c
+++ b/linux/pcmcia-cs/glue/ds.c
@@ -24,12 +24,6 @@
/* This file is included from linux/pcmcia-cs/modules/ds.c. */
/*
- * Prepare the namespace for inclusion of Mach header files.
- */
-
-#undef PAGE_SHIFT
-
-/*
* This is really ugly. But this is glue code, so... It's about the `kfree'
* symbols in <linux/malloc.h> and <kern/kalloc.h>.
*/
diff --git a/linux/src/arch/i386/kernel/bios32.c
b/linux/src/arch/i386/kernel/bios32.c
index b069ce4..c10cc0c 100644
--- a/linux/src/arch/i386/kernel/bios32.c
+++ b/linux/src/arch/i386/kernel/bios32.c
@@ -909,6 +909,8 @@ unsigned long pcibios_init(unsigned long memory_start,
unsigned long memory_end)
}
if (bios32_entry && check_pcibios())
access_pci = &pci_bios_access;
+ else
+ access_pci = check_direct_pci();
#endif
return memory_start;
}
diff --git a/linux/src/drivers/block/ide.c b/linux/src/drivers/block/ide.c
index 41a2601..dc20fcb 100644
--- a/linux/src/drivers/block/ide.c
+++ b/linux/src/drivers/block/ide.c
@@ -325,7 +325,7 @@
#endif /* CONFIG_BLK_DEV_PROMISE */
static const byte ide_hwif_to_major[MAX_HWIFS] = {IDE0_MAJOR, IDE1_MAJOR,
IDE2_MAJOR, IDE3_MAJOR};
-static const unsigned short default_io_base[MAX_HWIFS] = {0x1f0, 0x170, 0x1e8,
0x168};
+static unsigned short default_io_base[MAX_HWIFS] = {0x1f0, 0x170, 0x1e8,
0x168};
static const byte default_irqs[MAX_HWIFS] = {14, 15, 11, 10};
static int idebus_parameter; /* holds the "idebus=" parameter */
static int system_bus_speed; /* holds what we think is VESA/PCI bus speed
*/
@@ -367,6 +367,15 @@ static void set_recovery_timer (ide_hwif_t *hwif)
#endif /* DISK_RECOVERY_TIME */
+/* Called by other drivers to disable the legacy IDE driver on a given IDE
base. */
+void ide_disable_base(unsigned base)
+{
+ unsigned i;
+ for (i = 0; i < MAX_HWIFS; i++)
+ if (default_io_base[i] == base)
+ default_io_base[i] = 0;
+}
+
/*
* Do not even *think* about calling this!
diff --git a/linux/src/include/linux/compiler-gcc6.h
b/linux/src/include/linux/compiler-gcc6.h
new file mode 100644
index 0000000..cc2e86a
--- /dev/null
+++ b/linux/src/include/linux/compiler-gcc6.h
@@ -0,0 +1,67 @@
+#ifndef __LINUX_COMPILER_H
+#error "Please don't include <linux/compiler-gcc6.h> directly, include
<linux/compiler.h> instead."
+#endif
+
+#define __used __attribute__((__used__))
+#define __must_check __attribute__((warn_unused_result))
+#define __compiler_offsetof(a, b) __builtin_offsetof(a, b)
+
+/* Mark functions as cold. gcc will assume any path leading to a call
+ to them will be unlikely. This means a lot of manual unlikely()s
+ are unnecessary now for any paths leading to the usual suspects
+ like BUG(), printk(), panic() etc. [but let's keep them for now for
+ older compilers]
+
+ Early snapshots of gcc 4.3 don't support this and we can't detect this
+ in the preprocessor, but we can live with this because they're unreleased.
+ Maketime probing would be overkill here.
+
+ gcc also has a __attribute__((__hot__)) to move hot functions into
+ a special section, but I don't see any sense in this right now in
+ the kernel context */
+#define __cold __attribute__((__cold__))
+
+#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__)
+
+#ifndef __CHECKER__
+# define __compiletime_warning(message) __attribute__((warning(message)))
+# define __compiletime_error(message) __attribute__((error(message)))
+#endif /* __CHECKER__ */
+
+/*
+ * Mark a position in code as unreachable. This can be used to
+ * suppress control flow warnings after asm blocks that transfer
+ * control elsewhere.
+ *
+ * Early snapshots of gcc 4.5 don't support this and we can't detect
+ * this in the preprocessor, but we can live with this because they're
+ * unreleased. Really, we need to have autoconf for the kernel.
+ */
+#define unreachable() __builtin_unreachable()
+
+/* Mark a function definition as prohibited from being cloned. */
+#define __noclone __attribute__((__noclone__))
+
+/*
+ * Tell the optimizer that something else uses this function or variable.
+ */
+#define __visible __attribute__((externally_visible))
+
+/*
+ * GCC 'asm goto' miscompiles certain code sequences:
+ *
+ * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670
+ *
+ * Work it around via a compiler barrier quirk suggested by Jakub Jelinek.
+ *
+ * (asm goto is automatically volatile - the naming reflects this.)
+ */
+#define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0)
+
+#ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP
+#define __HAVE_BUILTIN_BSWAP32__
+#define __HAVE_BUILTIN_BSWAP64__
+#define __HAVE_BUILTIN_BSWAP16__
+#endif /* CONFIG_ARCH_USE_BUILTIN_BSWAP */
+
+#define KASAN_ABI_VERSION 4
diff --git a/version.m4 b/version.m4
index 5d14353..029631f 100644
--- a/version.m4
+++ b/version.m4
@@ -1,4 +1,4 @@
m4_define([AC_PACKAGE_NAME],[GNU Mach])
-m4_define([AC_PACKAGE_VERSION],[1.6+git20160114])
+m4_define([AC_PACKAGE_VERSION],[1.6+git20160311])
m4_define([AC_PACKAGE_BUGREPORT],address@hidden)
m4_define([AC_PACKAGE_TARNAME],[gnumach])
diff --git a/vm/memory_object_proxy.c b/vm/memory_object_proxy.c
index a64bfcc..01bce2a 100644
--- a/vm/memory_object_proxy.c
+++ b/vm/memory_object_proxy.c
@@ -64,7 +64,7 @@ void
memory_object_proxy_init (void)
{
kmem_cache_init (&memory_object_proxy_cache, "memory_object_proxy",
- sizeof (struct memory_object_proxy), 0, NULL, NULL, NULL, 0);
+ sizeof (struct memory_object_proxy), 0, NULL, 0);
}
/* Lookup a proxy memory object by its port. */
diff --git a/vm/pmap.h b/vm/pmap.h
index 134f9c6..9bbcdc3 100644
--- a/vm/pmap.h
+++ b/vm/pmap.h
@@ -67,9 +67,6 @@
extern vm_offset_t pmap_steal_memory(vm_size_t);
/* During VM initialization, report remaining unused physical pages. */
extern unsigned int pmap_free_pages(void);
-/* During VM initialization, use remaining physical pages to allocate page
- * frames. */
-extern void pmap_startup(vm_offset_t *, vm_offset_t *);
/* Initialization, after kernel runs in virtual memory. */
extern void pmap_init(void);
@@ -80,18 +77,14 @@ extern void pmap_init(void);
* Otherwise, it must implement
* pmap_free_pages
* pmap_virtual_space
- * pmap_next_page
* pmap_init
- * and vm/vm_resident.c implements pmap_steal_memory and pmap_startup
- * using pmap_free_pages, pmap_next_page, pmap_virtual_space,
- * and pmap_enter. pmap_free_pages may over-estimate the number
- * of unused physical pages, and pmap_next_page may return FALSE
- * to indicate that there are no more unused pages to return.
+ * and vm/vm_resident.c implements pmap_steal_memory using
+ * pmap_free_pages, pmap_virtual_space, and pmap_enter.
+ *
+ * pmap_free_pages may over-estimate the number of unused physical pages.
* However, for best performance pmap_free_pages should be accurate.
*/
-/* During VM initialization, return the next unused physical page. */
-extern boolean_t pmap_next_page(vm_offset_t *);
/* During VM initialization, report virtual space available for the kernel. */
extern void pmap_virtual_space(vm_offset_t *, vm_offset_t *);
#endif /* MACHINE_PAGES */
diff --git a/vm/vm_external.c b/vm/vm_external.c
index 2e2593b..3b1a287 100644
--- a/vm/vm_external.c
+++ b/vm/vm_external.c
@@ -35,6 +35,7 @@
#include <vm/vm_external.h>
#include <mach/vm_param.h>
#include <kern/assert.h>
+#include <string.h>
@@ -69,11 +70,12 @@ vm_external_t vm_external_create(vm_offset_t size)
result->existence_map =
(char *)
kmem_cache_alloc(&vm_object_small_existence_map_cache);
result->existence_size = SMALL_SIZE;
- } else if (bytes <= LARGE_SIZE) {
+ } else {
result->existence_map =
(char *)
kmem_cache_alloc(&vm_object_large_existence_map_cache);
result->existence_size = LARGE_SIZE;
}
+ memset (result->existence_map, 0, result->existence_size);
return(result);
}
@@ -138,13 +140,13 @@ void vm_external_module_initialize(void)
vm_size_t size = (vm_size_t) sizeof(struct vm_external);
kmem_cache_init(&vm_external_cache, "vm_external", size, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
kmem_cache_init(&vm_object_small_existence_map_cache,
"small_existence_map", SMALL_SIZE, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
kmem_cache_init(&vm_object_large_existence_map_cache,
"large_existence_map", LARGE_SIZE, 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
}
diff --git a/vm/vm_external.h b/vm/vm_external.h
index 55c9e48..4e44ddf 100644
--- a/vm/vm_external.h
+++ b/vm/vm_external.h
@@ -46,9 +46,14 @@ typedef struct vm_external {
* been written to backing
* storage.
*/
+#if 0
+ /* XXX: Currently, existence_count is not used. I guess it
+ could be useful to get rid of the map if the count drops to
+ zero. */
int existence_count;/* Number of bits turned on in
* existence_map.
*/
+#endif
} *vm_external_t;
#define VM_EXTERNAL_NULL ((vm_external_t) 0)
diff --git a/vm/vm_fault.c b/vm/vm_fault.c
index 46779f6..09e2c54 100644
--- a/vm/vm_fault.c
+++ b/vm/vm_fault.c
@@ -105,7 +105,7 @@ extern struct db_watchpoint *db_watchpoint_list;
void vm_fault_init(void)
{
kmem_cache_init(&vm_fault_state_cache, "vm_fault_state",
- sizeof(vm_fault_state_t), 0, NULL, NULL, NULL, 0);
+ sizeof(vm_fault_state_t), 0, NULL, 0);
}
/*
@@ -607,7 +607,7 @@ vm_fault_return_t vm_fault_page(
* won't block for pages.
*/
- if (m->fictitious && !vm_page_convert(m,
FALSE)) {
+ if (m->fictitious && !vm_page_convert(&m,
FALSE)) {
VM_PAGE_FREE(m);
vm_fault_cleanup(object, first_m);
return(VM_FAULT_MEMORY_SHORTAGE);
@@ -725,7 +725,7 @@ vm_fault_return_t vm_fault_page(
assert(m->object == object);
first_m = VM_PAGE_NULL;
- if (m->fictitious && !vm_page_convert(m,
!object->internal)) {
+ if (m->fictitious && !vm_page_convert(&m,
!object->internal)) {
VM_PAGE_FREE(m);
vm_fault_cleanup(object, VM_PAGE_NULL);
return(VM_FAULT_MEMORY_SHORTAGE);
diff --git a/vm/vm_init.c b/vm/vm_init.c
index 3d1081c..23d5d46 100644
--- a/vm/vm_init.c
+++ b/vm/vm_init.c
@@ -83,4 +83,5 @@ void vm_mem_init(void)
{
vm_object_init();
memory_object_proxy_init();
+ vm_page_info_all();
}
diff --git a/vm/vm_map.c b/vm/vm_map.c
index 3a231de..89a2b38 100644
--- a/vm/vm_map.c
+++ b/vm/vm_map.c
@@ -126,7 +126,6 @@ MACRO_END
struct kmem_cache vm_map_cache; /* cache for vm_map structures
*/
struct kmem_cache vm_map_entry_cache; /* cache for vm_map_entry
structures */
-struct kmem_cache vm_map_kentry_cache; /* cache for kernel entry
structures */
struct kmem_cache vm_map_copy_cache; /* cache for vm_map_copy
structures */
/*
@@ -147,47 +146,24 @@ vm_object_t vm_submap_object =
&vm_submap_object_store;
* Map and entry structures are allocated from caches -- we must
* initialize those caches.
*
- * There are three caches of interest:
+ * There are two caches of interest:
*
* vm_map_cache: used to allocate maps.
* vm_map_entry_cache: used to allocate map entries.
- * vm_map_kentry_cache: used to allocate map entries for the kernel.
*
- * Kernel map entries are allocated from a special cache, using a custom
- * page allocation function to avoid recursion. It would be difficult
- * (perhaps impossible) for the kernel to allocate more memory to an entry
- * cache when it became empty since the very act of allocating memory
- * implies the creation of a new entry.
+ * We make sure the map entry cache allocates memory directly from the
+ * physical allocator to avoid recursion with this module.
*/
-vm_offset_t kentry_data;
-vm_size_t kentry_data_size = KENTRY_DATA_SIZE;
-
-static vm_offset_t kentry_pagealloc(vm_size_t size)
-{
- vm_offset_t result;
-
- if (size > kentry_data_size)
- panic("vm_map: kentry memory exhausted");
-
- result = kentry_data;
- kentry_data += size;
- kentry_data_size -= size;
- return result;
-}
-
void vm_map_init(void)
{
kmem_cache_init(&vm_map_cache, "vm_map", sizeof(struct vm_map), 0,
- NULL, NULL, NULL, 0);
+ NULL, 0);
kmem_cache_init(&vm_map_entry_cache, "vm_map_entry",
- sizeof(struct vm_map_entry), 0, NULL, NULL, NULL, 0);
- kmem_cache_init(&vm_map_kentry_cache, "vm_map_kentry",
- sizeof(struct vm_map_entry), 0, NULL, kentry_pagealloc,
- NULL, KMEM_CACHE_NOCPUPOOL | KMEM_CACHE_NOOFFSLAB
- | KMEM_CACHE_NORECLAIM);
+ sizeof(struct vm_map_entry), 0, NULL,
+ KMEM_CACHE_NOOFFSLAB | KMEM_CACHE_PHYSMEM);
kmem_cache_init(&vm_map_copy_cache, "vm_map_copy",
- sizeof(struct vm_map_copy), 0, NULL, NULL, NULL, 0);
+ sizeof(struct vm_map_copy), 0, NULL, 0);
/*
* Submap object is initialized by vm_object_init.
@@ -261,15 +237,9 @@ vm_map_t vm_map_create(
vm_map_entry_t _vm_map_entry_create(map_header)
const struct vm_map_header *map_header;
{
- kmem_cache_t cache;
vm_map_entry_t entry;
- if (map_header->entries_pageable)
- cache = &vm_map_entry_cache;
- else
- cache = &vm_map_kentry_cache;
-
- entry = (vm_map_entry_t) kmem_cache_alloc(cache);
+ entry = (vm_map_entry_t) kmem_cache_alloc(&vm_map_entry_cache);
if (entry == VM_MAP_ENTRY_NULL)
panic("vm_map_entry_create");
@@ -291,14 +261,9 @@ void _vm_map_entry_dispose(map_header, entry)
const struct vm_map_header *map_header;
vm_map_entry_t entry;
{
- kmem_cache_t cache;
-
- if (map_header->entries_pageable)
- cache = &vm_map_entry_cache;
- else
- cache = &vm_map_kentry_cache;
+ (void)map_header;
- kmem_cache_free(cache, (vm_offset_t) entry);
+ kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry);
}
/*
@@ -2539,15 +2504,8 @@ kern_return_t vm_map_copyout(
* Mismatches occur when dealing with the default
* pager.
*/
- kmem_cache_t old_cache;
vm_map_entry_t next, new;
- /*
- * Find the cache that the copies were allocated from
- */
- old_cache = (copy->cpy_hdr.entries_pageable)
- ? &vm_map_entry_cache
- : &vm_map_kentry_cache;
entry = vm_map_copy_first_entry(copy);
/*
@@ -2571,7 +2529,7 @@ kern_return_t vm_map_copyout(
vm_map_copy_last_entry(copy),
new);
next = entry->vme_next;
- kmem_cache_free(old_cache, (vm_offset_t) entry);
+ kmem_cache_free(&vm_map_entry_cache, (vm_offset_t) entry);
entry = next;
}
}
diff --git a/vm/vm_map.h b/vm/vm_map.h
index 9b31f90..b4ba7c7 100644
--- a/vm/vm_map.h
+++ b/vm/vm_map.h
@@ -363,9 +363,6 @@ MACRO_END
* Exported procedures that operate on vm_map_t.
*/
-extern vm_offset_t kentry_data;
-extern vm_size_t kentry_data_size;
-extern int kentry_count;
/* Initialize the module */
extern void vm_map_init(void);
diff --git a/vm/vm_object.c b/vm/vm_object.c
index 6666fcb..bc30128 100644
--- a/vm/vm_object.c
+++ b/vm/vm_object.c
@@ -59,6 +59,11 @@
#include <ddb/db_output.h>
#endif /* MACH_KDB */
+void memory_object_release(
+ ipc_port_t pager,
+ pager_request_t pager_request,
+ ipc_port_t pager_name); /* forward */
+
/*
* Virtual memory objects maintain the actual data
* associated with allocated virtual memory. A given
@@ -159,8 +164,9 @@ vm_object_t kernel_object = &kernel_object_store;
*
* The kernel may choose to terminate objects from this
* queue in order to reclaim storage. The current policy
- * is to permit a fixed maximum number of unreferenced
- * objects (vm_object_cached_max).
+ * is to let memory pressure dynamically adjust the number
+ * of unreferenced objects. The pageout daemon attempts to
+ * collect objects after removing pages from them.
*
* A simple lock (accessed by routines
* vm_object_cache_{lock,lock_try,unlock}) governs the
@@ -176,7 +182,6 @@ vm_object_t kernel_object = &kernel_object_store;
*/
queue_head_t vm_object_cached_list;
int vm_object_cached_count;
-int vm_object_cached_max = 4000; /* may be patched*/
decl_simple_lock_data(,vm_object_cached_lock_data)
@@ -262,7 +267,7 @@ vm_object_t vm_object_allocate(
void vm_object_bootstrap(void)
{
kmem_cache_init(&vm_object_cache, "vm_object",
- sizeof(struct vm_object), 0, NULL, NULL, NULL, 0);
+ sizeof(struct vm_object), 0, NULL, 0);
queue_init(&vm_object_cached_list);
simple_lock_init(&vm_object_cached_lock_data);
@@ -298,6 +303,7 @@ void vm_object_bootstrap(void)
vm_object_template.paging_in_progress = 0;
vm_object_template.can_persist = FALSE;
+ vm_object_template.cached = FALSE;
vm_object_template.internal = TRUE;
vm_object_template.temporary = TRUE;
vm_object_template.alive = TRUE;
@@ -344,6 +350,60 @@ void vm_object_init(void)
}
/*
+ * Object cache management functions.
+ *
+ * Both the cache and the object must be locked
+ * before calling these functions.
+ */
+
+static void vm_object_cache_add(
+ vm_object_t object)
+{
+ assert(!object->cached);
+ queue_enter(&vm_object_cached_list, object, vm_object_t, cached_list);
+ vm_object_cached_count++;
+ vm_object_cached_pages_update(object->resident_page_count);
+ object->cached = TRUE;
+}
+
+static void vm_object_cache_remove(
+ vm_object_t object)
+{
+ assert(object->cached);
+ queue_remove(&vm_object_cached_list, object, vm_object_t, cached_list);
+ vm_object_cached_count--;
+ vm_object_cached_pages_update(-object->resident_page_count);
+ object->cached = FALSE;
+}
+
+void vm_object_collect(
+ register vm_object_t object)
+{
+ vm_object_unlock(object);
+
+ /*
+ * The cache lock must be acquired in the proper order.
+ */
+
+ vm_object_cache_lock();
+ vm_object_lock(object);
+
+ /*
+ * If the object was referenced while the lock was
+ * dropped, cancel the termination.
+ */
+
+ if (!vm_object_collectable(object)) {
+ vm_object_unlock(object);
+ vm_object_cache_unlock();
+ return;
+ }
+
+ vm_object_cache_remove(object);
+ vm_object_terminate(object);
+}
+
+/*
* vm_object_reference:
*
* Gets another reference to the given object.
@@ -403,103 +463,31 @@ void vm_object_deallocate(
/*
* See whether this object can persist. If so, enter
- * it in the cache, then deactivate all of its
- * pages.
+ * it in the cache.
*/
- if (object->can_persist) {
- boolean_t overflow;
-
- /*
- * Enter the object onto the queue
- * of "cached" objects. Remember whether
- * we've caused the queue to overflow,
- * as a hint.
- */
-
- queue_enter(&vm_object_cached_list, object,
- vm_object_t, cached_list);
- overflow = (++vm_object_cached_count >
vm_object_cached_max);
-
vm_object_cached_pages_update(object->resident_page_count);
+ if (object->can_persist && (object->resident_page_count > 0)) {
+ vm_object_cache_add(object);
vm_object_cache_unlock();
-
- vm_object_deactivate_pages(object);
vm_object_unlock(object);
+ return;
+ }
- /*
- * If we didn't overflow, or if the queue has
- * been reduced back to below the specified
- * minimum, then quit.
- */
- if (!overflow)
- return;
-
- while (TRUE) {
- vm_object_cache_lock();
- if (vm_object_cached_count <=
- vm_object_cached_max) {
- vm_object_cache_unlock();
- return;
- }
-
- /*
- * If we must trim down the queue, take
- * the first object, and proceed to
- * terminate it instead of the original
- * object. Have to wait for pager init.
- * if it's in progress.
- */
- object= (vm_object_t)
- queue_first(&vm_object_cached_list);
- vm_object_lock(object);
-
- if (!(object->pager_created &&
- !object->pager_initialized)) {
-
- /*
- * Ok to terminate, hang on to lock.
- */
- break;
- }
-
- vm_object_assert_wait(object,
- VM_OBJECT_EVENT_INITIALIZED, FALSE);
- vm_object_unlock(object);
- vm_object_cache_unlock();
- thread_block((void (*)()) 0);
-
- /*
- * Continue loop to check if cache still
- * needs to be trimmed.
- */
- }
+ if (object->pager_created &&
+ !object->pager_initialized) {
/*
- * Actually remove object from cache.
+ * Have to wait for initialization.
+ * Put reference back and retry
+ * when it's initialized.
*/
- queue_remove(&vm_object_cached_list, object,
- vm_object_t, cached_list);
- vm_object_cached_count--;
-
- assert(object->ref_count == 0);
- }
- else {
- if (object->pager_created &&
- !object->pager_initialized) {
-
- /*
- * Have to wait for initialization.
- * Put reference back and retry
- * when it's initialized.
- */
- object->ref_count++;
- vm_object_assert_wait(object,
- VM_OBJECT_EVENT_INITIALIZED, FALSE);
- vm_object_unlock(object);
- vm_object_cache_unlock();
- thread_block((void (*)()) 0);
- continue;
- }
+ object->ref_count++;
+ vm_object_assert_wait(object,
+ VM_OBJECT_EVENT_INITIALIZED, FALSE);
+ vm_object_unlock(object);
+ vm_object_cache_unlock();
+ thread_block((void (*)()) 0);
+ continue;
}
/*
@@ -630,6 +618,7 @@ void vm_object_terminate(
assert(object->ref_count == 0);
assert(object->paging_in_progress == 0);
+ assert(!object->cached);
/*
* Throw away port rights... note that they may
@@ -862,28 +851,6 @@ kern_return_t memory_object_destroy(
}
/*
- * vm_object_deactivate_pages
- *
- * Deactivate all pages in the specified object. (Keep its pages
- * in memory even though it is no longer referenced.)
- *
- * The object must be locked.
- */
-void vm_object_deactivate_pages(
- vm_object_t object)
-{
- vm_page_t p;
-
- queue_iterate(&object->memq, p, vm_page_t, listq) {
- vm_page_lock_queues();
- if (!p->busy)
- vm_page_deactivate(p);
- vm_page_unlock_queues();
- }
-}
-
-
-/*
* Routine: vm_object_pmap_protect
*
* Purpose:
@@ -1837,12 +1804,8 @@ vm_object_t vm_object_lookup(
assert(object->alive);
- if (object->ref_count == 0) {
- queue_remove(&vm_object_cached_list, object,
- vm_object_t, cached_list);
- vm_object_cached_count--;
-
vm_object_cached_pages_update(-object->resident_page_count);
- }
+ if (object->ref_count == 0)
+ vm_object_cache_remove(object);
object->ref_count++;
vm_object_unlock(object);
@@ -1869,12 +1832,8 @@ vm_object_t vm_object_lookup_name(
assert(object->alive);
- if (object->ref_count == 0) {
- queue_remove(&vm_object_cached_list, object,
- vm_object_t, cached_list);
- vm_object_cached_count--;
-
vm_object_cached_pages_update(-object->resident_page_count);
- }
+ if (object->ref_count == 0)
+ vm_object_cache_remove(object);
object->ref_count++;
vm_object_unlock(object);
@@ -1906,12 +1865,8 @@ void vm_object_destroy(
object = (vm_object_t) pager->ip_kobject;
vm_object_lock(object);
- if (object->ref_count == 0) {
- queue_remove(&vm_object_cached_list, object,
- vm_object_t, cached_list);
- vm_object_cached_count--;
- vm_object_cached_pages_update(-object->resident_page_count);
- }
+ if (object->ref_count == 0)
+ vm_object_cache_remove(object);
object->ref_count++;
object->can_persist = FALSE;
@@ -2059,12 +2014,8 @@ restart:
if ((object != VM_OBJECT_NULL) && !must_init) {
vm_object_lock(object);
- if (object->ref_count == 0) {
- queue_remove(&vm_object_cached_list, object,
- vm_object_t, cached_list);
- vm_object_cached_count--;
-
vm_object_cached_pages_update(-object->resident_page_count);
- }
+ if (object->ref_count == 0)
+ vm_object_cache_remove(object);
object->ref_count++;
vm_object_unlock(object);
@@ -2573,6 +2524,7 @@ void vm_object_collapse(
);
assert(backing_object->alive);
+ assert(!backing_object->cached);
backing_object->alive = FALSE;
vm_object_unlock(backing_object);
@@ -2701,7 +2653,7 @@ void vm_object_page_remove(
* It balances vm_object_lookup vs iteration.
*/
- if (atop(end - start) < (unsigned)object->resident_page_count/16) {
+ if (atop(end - start) < object->resident_page_count/16) {
vm_object_page_remove_lookup++;
for (; start < end; start += PAGE_SIZE) {
@@ -2891,7 +2843,8 @@ vm_object_page_map(
VM_PAGE_FREE(old_page);
}
- vm_page_init(m, addr);
+ vm_page_init(m);
+ m->phys_addr = addr;
m->private = TRUE; /* don`t free page */
m->wire_count = 1;
vm_page_lock_queues();
@@ -2927,7 +2880,7 @@ void vm_object_print(
(vm_offset_t) object, (vm_offset_t) object->size,
object->ref_count);
printf("\n");
- iprintf("%d resident pages,", object->resident_page_count);
+ iprintf("%lu resident pages,", object->resident_page_count);
printf(" %d absent pages,", object->absent_count);
printf(" %d paging ops\n", object->paging_in_progress);
indent += 1;
diff --git a/vm/vm_object.h b/vm/vm_object.h
index 71c8545..eb8a0c2 100644
--- a/vm/vm_object.h
+++ b/vm/vm_object.h
@@ -72,7 +72,7 @@ struct vm_object {
*/
int ref_count; /* Number of references */
- int resident_page_count;
+ unsigned long resident_page_count;
/* number of resident pages */
struct vm_object *copy; /* Object that should receive
@@ -148,8 +148,9 @@ struct vm_object {
*/
/* boolean_t */ use_shared_copy : 1,/* Use shared (i.e.,
* delayed) copy on write */
- /* boolean_t */ shadowed: 1; /* Shadow may exist */
+ /* boolean_t */ shadowed: 1, /* Shadow may exist */
+ /* boolean_t */ cached: 1; /* Object is cached */
queue_chain_t cached_list; /* Attachment point for the list
* of objects cached as a result
* of their can_persist value
@@ -169,6 +170,7 @@ vm_object_t kernel_object; /* the single kernel
object */
extern void vm_object_bootstrap(void);
extern void vm_object_init(void);
+extern void vm_object_collect(vm_object_t);
extern void vm_object_terminate(vm_object_t);
extern vm_object_t vm_object_allocate(vm_size_t);
extern void vm_object_reference(vm_object_t);
@@ -290,6 +292,10 @@ vm_object_t vm_object_copy_delayed(
* Routines implemented as macros
*/
+#define vm_object_collectable(object) \
+ (((object)->ref_count == 0) \
+ && ((object)->resident_page_count == 0))
+
#define vm_object_paging_begin(object)
\
((object)->paging_in_progress++)
diff --git a/vm/vm_page.c b/vm/vm_page.c
new file mode 100644
index 0000000..a868fce
--- /dev/null
+++ b/vm/vm_page.c
@@ -0,0 +1,782 @@
+/*
+ * Copyright (c) 2010-2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * This implementation uses the binary buddy system to manage its heap.
+ * Descriptions of the buddy system can be found in the following works :
+ * - "UNIX Internals: The New Frontiers", by Uresh Vahalia.
+ * - "Dynamic Storage Allocation: A Survey and Critical Review",
+ * by Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles.
+ *
+ * In addition, this allocator uses per-CPU pools of pages for order 0
+ * (i.e. single page) allocations. These pools act as caches (but are named
+ * differently to avoid confusion with CPU caches) that reduce contention on
+ * multiprocessor systems. When a pool is empty and cannot provide a page,
+ * it is filled by transferring multiple pages from the backend buddy system.
+ * The symmetric case is handled likewise.
+ */
+
+#include <string.h>
+#include <kern/assert.h>
+#include <kern/cpu_number.h>
+#include <kern/debug.h>
+#include <kern/list.h>
+#include <kern/lock.h>
+#include <kern/macros.h>
+#include <kern/printf.h>
+#include <kern/thread.h>
+#include <mach/vm_param.h>
+#include <machine/pmap.h>
+#include <sys/types.h>
+#include <vm/vm_page.h>
+
+#define __init
+#define __initdata
+#define __read_mostly
+
+#define thread_pin()
+#define thread_unpin()
+
+/*
+ * Number of free block lists per segment.
+ */
+#define VM_PAGE_NR_FREE_LISTS 11
+
+/*
+ * The size of a CPU pool is computed by dividing the number of pages in its
+ * containing segment by this value.
+ */
+#define VM_PAGE_CPU_POOL_RATIO 1024
+
+/*
+ * Maximum number of pages in a CPU pool.
+ */
+#define VM_PAGE_CPU_POOL_MAX_SIZE 128
+
+/*
+ * The transfer size of a CPU pool is computed by dividing the pool size by
+ * this value.
+ */
+#define VM_PAGE_CPU_POOL_TRANSFER_RATIO 2
+
+/*
+ * Per-processor cache of pages.
+ */
+struct vm_page_cpu_pool {
+ simple_lock_data_t lock;
+ int size;
+ int transfer_size;
+ int nr_pages;
+ struct list pages;
+} __aligned(CPU_L1_SIZE);
+
+/*
+ * Special order value for pages that aren't in a free list. Such pages are
+ * either allocated, or part of a free block of pages but not the head page.
+ */
+#define VM_PAGE_ORDER_UNLISTED ((unsigned short)-1)
+
+/*
+ * Doubly-linked list of free blocks.
+ */
+struct vm_page_free_list {
+ unsigned long size;
+ struct list blocks;
+};
+
+/*
+ * Segment name buffer size.
+ */
+#define VM_PAGE_NAME_SIZE 16
+
+/*
+ * Segment of contiguous memory.
+ */
+struct vm_page_seg {
+ struct vm_page_cpu_pool cpu_pools[NCPUS];
+
+ phys_addr_t start;
+ phys_addr_t end;
+ struct vm_page *pages;
+ struct vm_page *pages_end;
+ simple_lock_data_t lock;
+ struct vm_page_free_list free_lists[VM_PAGE_NR_FREE_LISTS];
+ unsigned long nr_free_pages;
+};
+
+/*
+ * Bootstrap information about a segment.
+ */
+struct vm_page_boot_seg {
+ phys_addr_t start;
+ phys_addr_t end;
+ phys_addr_t avail_start;
+ phys_addr_t avail_end;
+};
+
+static int vm_page_is_ready __read_mostly;
+
+/*
+ * Segment table.
+ *
+ * The system supports a maximum of 4 segments :
+ * - DMA: suitable for DMA
+ * - DMA32: suitable for DMA when devices support 32-bits addressing
+ * - DIRECTMAP: direct physical mapping, allows direct access from
+ * the kernel with a simple offset translation
+ * - HIGHMEM: must be mapped before it can be accessed
+ *
+ * Segments are ordered by priority, 0 being the lowest priority. Their
+ * relative priorities are DMA < DMA32 < DIRECTMAP < HIGHMEM. Some segments
+ * may actually be aliases for others, e.g. if DMA is always possible from
+ * the direct physical mapping, DMA and DMA32 are aliases for DIRECTMAP,
+ * in which case the segment table contains DIRECTMAP and HIGHMEM only.
+ */
+static struct vm_page_seg vm_page_segs[VM_PAGE_MAX_SEGS];
+
+/*
+ * Bootstrap segment table.
+ */
+static struct vm_page_boot_seg vm_page_boot_segs[VM_PAGE_MAX_SEGS] __initdata;
+
+/*
+ * Number of loaded segments.
+ */
+static unsigned int vm_page_segs_size __read_mostly;
+
+static void __init
+vm_page_init_pa(struct vm_page *page, unsigned short seg_index, phys_addr_t pa)
+{
+ memset(page, 0, sizeof(*page));
+ vm_page_init(page); /* vm_resident members */
+ page->type = VM_PT_RESERVED;
+ page->seg_index = seg_index;
+ page->order = VM_PAGE_ORDER_UNLISTED;
+ page->priv = NULL;
+ page->phys_addr = pa;
+}
+
+void
+vm_page_set_type(struct vm_page *page, unsigned int order, unsigned short type)
+{
+ unsigned int i, nr_pages;
+
+ nr_pages = 1 << order;
+
+ for (i = 0; i < nr_pages; i++)
+ page[i].type = type;
+}
+
+static void __init
+vm_page_free_list_init(struct vm_page_free_list *free_list)
+{
+ free_list->size = 0;
+ list_init(&free_list->blocks);
+}
+
+static inline void
+vm_page_free_list_insert(struct vm_page_free_list *free_list,
+ struct vm_page *page)
+{
+ assert(page->order == VM_PAGE_ORDER_UNLISTED);
+
+ free_list->size++;
+ list_insert_head(&free_list->blocks, &page->node);
+}
+
+static inline void
+vm_page_free_list_remove(struct vm_page_free_list *free_list,
+ struct vm_page *page)
+{
+ assert(page->order != VM_PAGE_ORDER_UNLISTED);
+
+ free_list->size--;
+ list_remove(&page->node);
+}
+
+static struct vm_page *
+vm_page_seg_alloc_from_buddy(struct vm_page_seg *seg, unsigned int order)
+{
+ struct vm_page_free_list *free_list = free_list;
+ struct vm_page *page, *buddy;
+ unsigned int i;
+
+ assert(order < VM_PAGE_NR_FREE_LISTS);
+
+ for (i = order; i < VM_PAGE_NR_FREE_LISTS; i++) {
+ free_list = &seg->free_lists[i];
+
+ if (free_list->size != 0)
+ break;
+ }
+
+ if (i == VM_PAGE_NR_FREE_LISTS)
+ return NULL;
+
+ page = list_first_entry(&free_list->blocks, struct vm_page, node);
+ vm_page_free_list_remove(free_list, page);
+ page->order = VM_PAGE_ORDER_UNLISTED;
+
+ while (i > order) {
+ i--;
+ buddy = &page[1 << i];
+ vm_page_free_list_insert(&seg->free_lists[i], buddy);
+ buddy->order = i;
+ }
+
+ seg->nr_free_pages -= (1 << order);
+ return page;
+}
+
+static void
+vm_page_seg_free_to_buddy(struct vm_page_seg *seg, struct vm_page *page,
+ unsigned int order)
+{
+ struct vm_page *buddy;
+ phys_addr_t pa, buddy_pa;
+ unsigned int nr_pages;
+
+ assert(page >= seg->pages);
+ assert(page < seg->pages_end);
+ assert(page->order == VM_PAGE_ORDER_UNLISTED);
+ assert(order < VM_PAGE_NR_FREE_LISTS);
+
+ nr_pages = (1 << order);
+ pa = page->phys_addr;
+
+ while (order < (VM_PAGE_NR_FREE_LISTS - 1)) {
+ buddy_pa = pa ^ vm_page_ptoa(1 << order);
+
+ if ((buddy_pa < seg->start) || (buddy_pa >= seg->end))
+ break;
+
+ buddy = &seg->pages[vm_page_atop(buddy_pa - seg->start)];
+
+ if (buddy->order != order)
+ break;
+
+ vm_page_free_list_remove(&seg->free_lists[order], buddy);
+ buddy->order = VM_PAGE_ORDER_UNLISTED;
+ order++;
+ pa &= -vm_page_ptoa(1 << order);
+ page = &seg->pages[vm_page_atop(pa - seg->start)];
+ }
+
+ vm_page_free_list_insert(&seg->free_lists[order], page);
+ page->order = order;
+ seg->nr_free_pages += nr_pages;
+}
+
+static void __init
+vm_page_cpu_pool_init(struct vm_page_cpu_pool *cpu_pool, int size)
+{
+ simple_lock_init(&cpu_pool->lock);
+ cpu_pool->size = size;
+ cpu_pool->transfer_size = (size + VM_PAGE_CPU_POOL_TRANSFER_RATIO - 1)
+ / VM_PAGE_CPU_POOL_TRANSFER_RATIO;
+ cpu_pool->nr_pages = 0;
+ list_init(&cpu_pool->pages);
+}
+
+static inline struct vm_page_cpu_pool *
+vm_page_cpu_pool_get(struct vm_page_seg *seg)
+{
+ return &seg->cpu_pools[cpu_number()];
+}
+
+static inline struct vm_page *
+vm_page_cpu_pool_pop(struct vm_page_cpu_pool *cpu_pool)
+{
+ struct vm_page *page;
+
+ assert(cpu_pool->nr_pages != 0);
+ cpu_pool->nr_pages--;
+ page = list_first_entry(&cpu_pool->pages, struct vm_page, node);
+ list_remove(&page->node);
+ return page;
+}
+
+static inline void
+vm_page_cpu_pool_push(struct vm_page_cpu_pool *cpu_pool, struct vm_page *page)
+{
+ assert(cpu_pool->nr_pages < cpu_pool->size);
+ cpu_pool->nr_pages++;
+ list_insert_head(&cpu_pool->pages, &page->node);
+}
+
+static int
+vm_page_cpu_pool_fill(struct vm_page_cpu_pool *cpu_pool,
+ struct vm_page_seg *seg)
+{
+ struct vm_page *page;
+ int i;
+
+ assert(cpu_pool->nr_pages == 0);
+
+ simple_lock(&seg->lock);
+
+ for (i = 0; i < cpu_pool->transfer_size; i++) {
+ page = vm_page_seg_alloc_from_buddy(seg, 0);
+
+ if (page == NULL)
+ break;
+
+ vm_page_cpu_pool_push(cpu_pool, page);
+ }
+
+ simple_unlock(&seg->lock);
+
+ return i;
+}
+
+static void
+vm_page_cpu_pool_drain(struct vm_page_cpu_pool *cpu_pool,
+ struct vm_page_seg *seg)
+{
+ struct vm_page *page;
+ int i;
+
+ assert(cpu_pool->nr_pages == cpu_pool->size);
+
+ simple_lock(&seg->lock);
+
+ for (i = cpu_pool->transfer_size; i > 0; i--) {
+ page = vm_page_cpu_pool_pop(cpu_pool);
+ vm_page_seg_free_to_buddy(seg, page, 0);
+ }
+
+ simple_unlock(&seg->lock);
+}
+
+static phys_addr_t __init
+vm_page_seg_size(struct vm_page_seg *seg)
+{
+ return seg->end - seg->start;
+}
+
+static int __init
+vm_page_seg_compute_pool_size(struct vm_page_seg *seg)
+{
+ phys_addr_t size;
+
+ size = vm_page_atop(vm_page_seg_size(seg)) / VM_PAGE_CPU_POOL_RATIO;
+
+ if (size == 0)
+ size = 1;
+ else if (size > VM_PAGE_CPU_POOL_MAX_SIZE)
+ size = VM_PAGE_CPU_POOL_MAX_SIZE;
+
+ return size;
+}
+
+static void __init
+vm_page_seg_init(struct vm_page_seg *seg, phys_addr_t start, phys_addr_t end,
+ struct vm_page *pages)
+{
+ phys_addr_t pa;
+ int pool_size;
+ unsigned int i;
+
+ seg->start = start;
+ seg->end = end;
+ pool_size = vm_page_seg_compute_pool_size(seg);
+
+ for (i = 0; i < ARRAY_SIZE(seg->cpu_pools); i++)
+ vm_page_cpu_pool_init(&seg->cpu_pools[i], pool_size);
+
+ seg->pages = pages;
+ seg->pages_end = pages + vm_page_atop(vm_page_seg_size(seg));
+ simple_lock_init(&seg->lock);
+
+ for (i = 0; i < ARRAY_SIZE(seg->free_lists); i++)
+ vm_page_free_list_init(&seg->free_lists[i]);
+
+ seg->nr_free_pages = 0;
+ i = seg - vm_page_segs;
+
+ for (pa = seg->start; pa < seg->end; pa += PAGE_SIZE)
+ vm_page_init_pa(&pages[vm_page_atop(pa - seg->start)], i, pa);
+}
+
+static struct vm_page *
+vm_page_seg_alloc(struct vm_page_seg *seg, unsigned int order,
+ unsigned short type)
+{
+ struct vm_page_cpu_pool *cpu_pool;
+ struct vm_page *page;
+ int filled;
+
+ assert(order < VM_PAGE_NR_FREE_LISTS);
+
+ if (order == 0) {
+ thread_pin();
+ cpu_pool = vm_page_cpu_pool_get(seg);
+ simple_lock(&cpu_pool->lock);
+
+ if (cpu_pool->nr_pages == 0) {
+ filled = vm_page_cpu_pool_fill(cpu_pool, seg);
+
+ if (!filled) {
+ simple_unlock(&cpu_pool->lock);
+ thread_unpin();
+ return NULL;
+ }
+ }
+
+ page = vm_page_cpu_pool_pop(cpu_pool);
+ simple_unlock(&cpu_pool->lock);
+ thread_unpin();
+ } else {
+ simple_lock(&seg->lock);
+ page = vm_page_seg_alloc_from_buddy(seg, order);
+ simple_unlock(&seg->lock);
+
+ if (page == NULL)
+ return NULL;
+ }
+
+ assert(page->type == VM_PT_FREE);
+ vm_page_set_type(page, order, type);
+ return page;
+}
+
+static void
+vm_page_seg_free(struct vm_page_seg *seg, struct vm_page *page,
+ unsigned int order)
+{
+ struct vm_page_cpu_pool *cpu_pool;
+
+ assert(page->type != VM_PT_FREE);
+ assert(order < VM_PAGE_NR_FREE_LISTS);
+
+ vm_page_set_type(page, order, VM_PT_FREE);
+
+ if (order == 0) {
+ thread_pin();
+ cpu_pool = vm_page_cpu_pool_get(seg);
+ simple_lock(&cpu_pool->lock);
+
+ if (cpu_pool->nr_pages == cpu_pool->size)
+ vm_page_cpu_pool_drain(cpu_pool, seg);
+
+ vm_page_cpu_pool_push(cpu_pool, page);
+ simple_unlock(&cpu_pool->lock);
+ thread_unpin();
+ } else {
+ simple_lock(&seg->lock);
+ vm_page_seg_free_to_buddy(seg, page, order);
+ simple_unlock(&seg->lock);
+ }
+}
+
+void __init
+vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end,
+ phys_addr_t avail_start, phys_addr_t avail_end)
+{
+ struct vm_page_boot_seg *seg;
+
+ assert(seg_index < ARRAY_SIZE(vm_page_boot_segs));
+ assert(vm_page_aligned(start));
+ assert(vm_page_aligned(end));
+ assert(vm_page_aligned(avail_start));
+ assert(vm_page_aligned(avail_end));
+ assert(start < end);
+ assert(start <= avail_start);
+ assert(avail_end <= end);
+ assert(vm_page_segs_size < ARRAY_SIZE(vm_page_boot_segs));
+
+ seg = &vm_page_boot_segs[seg_index];
+ seg->start = start;
+ seg->end = end;
+ seg->avail_start = avail_start;
+ seg->avail_end = avail_end;
+ vm_page_segs_size++;
+}
+
+int
+vm_page_ready(void)
+{
+ return vm_page_is_ready;
+}
+
+static unsigned int
+vm_page_select_alloc_seg(unsigned int selector)
+{
+ unsigned int seg_index;
+
+ switch (selector) {
+ case VM_PAGE_SEL_DMA:
+ seg_index = VM_PAGE_SEG_DMA;
+ break;
+ case VM_PAGE_SEL_DMA32:
+ seg_index = VM_PAGE_SEG_DMA32;
+ break;
+ case VM_PAGE_SEL_DIRECTMAP:
+ seg_index = VM_PAGE_SEG_DIRECTMAP;
+ break;
+ case VM_PAGE_SEL_HIGHMEM:
+ seg_index = VM_PAGE_SEG_HIGHMEM;
+ break;
+ default:
+ panic("vm_page: invalid selector");
+ }
+
+ return MIN(vm_page_segs_size - 1, seg_index);
+}
+
+static int __init
+vm_page_boot_seg_loaded(const struct vm_page_boot_seg *seg)
+{
+ return (seg->end != 0);
+}
+
+static void __init
+vm_page_check_boot_segs(void)
+{
+ unsigned int i;
+ int expect_loaded;
+
+ if (vm_page_segs_size == 0)
+ panic("vm_page: no physical memory loaded");
+
+ for (i = 0; i < ARRAY_SIZE(vm_page_boot_segs); i++) {
+ expect_loaded = (i < vm_page_segs_size);
+
+ if (vm_page_boot_seg_loaded(&vm_page_boot_segs[i]) == expect_loaded)
+ continue;
+
+ panic("vm_page: invalid boot segment table");
+ }
+}
+
+static phys_addr_t __init
+vm_page_boot_seg_size(struct vm_page_boot_seg *seg)
+{
+ return seg->end - seg->start;
+}
+
+static phys_addr_t __init
+vm_page_boot_seg_avail_size(struct vm_page_boot_seg *seg)
+{
+ return seg->avail_end - seg->avail_start;
+}
+
+unsigned long __init
+vm_page_bootalloc(size_t size)
+{
+ struct vm_page_boot_seg *seg;
+ phys_addr_t pa;
+ unsigned int i;
+
+ for (i = vm_page_select_alloc_seg(VM_PAGE_SEL_DIRECTMAP);
+ i < vm_page_segs_size;
+ i--) {
+ seg = &vm_page_boot_segs[i];
+
+ if (size <= vm_page_boot_seg_avail_size(seg)) {
+ pa = seg->avail_start;
+ seg->avail_start += vm_page_round(size);
+ return pa;
+ }
+ }
+
+ panic("vm_page: no physical memory available");
+}
+
+void __init
+vm_page_setup(void)
+{
+ struct vm_page_boot_seg *boot_seg;
+ struct vm_page_seg *seg;
+ struct vm_page *table, *page, *end;
+ size_t nr_pages, table_size;
+ unsigned long va;
+ unsigned int i;
+ phys_addr_t pa;
+
+ vm_page_check_boot_segs();
+
+ /*
+ * Compute the page table size.
+ */
+ nr_pages = 0;
+
+ for (i = 0; i < vm_page_segs_size; i++)
+ nr_pages += vm_page_atop(vm_page_boot_seg_size(&vm_page_boot_segs[i]));
+
+ table_size = vm_page_round(nr_pages * sizeof(struct vm_page));
+ printf("vm_page: page table size: %lu entries (%luk)\n", nr_pages,
+ table_size >> 10);
+ table = (struct vm_page *)pmap_steal_memory(table_size);
+ va = (unsigned long)table;
+
+ /*
+ * Initialize the segments, associating them to the page table. When
+ * the segments are initialized, all their pages are set allocated.
+ * Pages are then released, which populates the free lists.
+ */
+ for (i = 0; i < vm_page_segs_size; i++) {
+ seg = &vm_page_segs[i];
+ boot_seg = &vm_page_boot_segs[i];
+ vm_page_seg_init(seg, boot_seg->start, boot_seg->end, table);
+ page = seg->pages + vm_page_atop(boot_seg->avail_start
+ - boot_seg->start);
+ end = seg->pages + vm_page_atop(boot_seg->avail_end
+ - boot_seg->start);
+
+ while (page < end) {
+ page->type = VM_PT_FREE;
+ vm_page_seg_free_to_buddy(seg, page, 0);
+ page++;
+ }
+
+ table += vm_page_atop(vm_page_seg_size(seg));
+ }
+
+ while (va < (unsigned long)table) {
+ pa = pmap_extract(kernel_pmap, va);
+ page = vm_page_lookup_pa(pa);
+ assert((page != NULL) && (page->type == VM_PT_RESERVED));
+ page->type = VM_PT_TABLE;
+ va += PAGE_SIZE;
+ }
+
+ vm_page_is_ready = 1;
+}
+
+void __init
+vm_page_manage(struct vm_page *page)
+{
+ assert(page->seg_index < ARRAY_SIZE(vm_page_segs));
+ assert(page->type == VM_PT_RESERVED);
+
+ vm_page_set_type(page, 0, VM_PT_FREE);
+ vm_page_seg_free_to_buddy(&vm_page_segs[page->seg_index], page, 0);
+}
+
+struct vm_page *
+vm_page_lookup_pa(phys_addr_t pa)
+{
+ struct vm_page_seg *seg;
+ unsigned int i;
+
+ for (i = 0; i < vm_page_segs_size; i++) {
+ seg = &vm_page_segs[i];
+
+ if ((pa >= seg->start) && (pa < seg->end))
+ return &seg->pages[vm_page_atop(pa - seg->start)];
+ }
+
+ return NULL;
+}
+
+struct vm_page *
+vm_page_alloc_pa(unsigned int order, unsigned int selector, unsigned short
type)
+{
+ struct vm_page *page;
+ unsigned int i;
+
+ for (i = vm_page_select_alloc_seg(selector); i < vm_page_segs_size; i--) {
+ page = vm_page_seg_alloc(&vm_page_segs[i], order, type);
+
+ if (page != NULL)
+ return page;
+ }
+
+ if (type == VM_PT_PMAP)
+ panic("vm_page: unable to allocate pmap page");
+
+ return NULL;
+}
+
+void
+vm_page_free_pa(struct vm_page *page, unsigned int order)
+{
+ assert(page != NULL);
+ assert(page->seg_index < ARRAY_SIZE(vm_page_segs));
+
+ vm_page_seg_free(&vm_page_segs[page->seg_index], page, order);
+}
+
+const char *
+vm_page_seg_name(unsigned int seg_index)
+{
+ /* Don't use a switch statement since segments can be aliased */
+ if (seg_index == VM_PAGE_SEG_HIGHMEM)
+ return "HIGHMEM";
+ else if (seg_index == VM_PAGE_SEG_DIRECTMAP)
+ return "DIRECTMAP";
+ else if (seg_index == VM_PAGE_SEG_DMA32)
+ return "DMA32";
+ else if (seg_index == VM_PAGE_SEG_DMA)
+ return "DMA";
+ else
+ panic("vm_page: invalid segment index");
+}
+
+void
+vm_page_info_all(void)
+{
+ struct vm_page_seg *seg;
+ unsigned long pages;
+ unsigned int i;
+
+ for (i = 0; i < vm_page_segs_size; i++) {
+ seg = &vm_page_segs[i];
+ pages = (unsigned long)(seg->pages_end - seg->pages);
+ printf("vm_page: %s: pages: %lu (%luM), free: %lu (%luM)\n",
+ vm_page_seg_name(i), pages, pages >> (20 - PAGE_SHIFT),
+ seg->nr_free_pages, seg->nr_free_pages >> (20 - PAGE_SHIFT));
+ }
+}
+
+phys_addr_t
+vm_page_mem_size(void)
+{
+ phys_addr_t total;
+ unsigned int i;
+
+ total = 0;
+
+ for (i = 0; i < vm_page_segs_size; i++) {
+ /* XXX */
+ if (i > VM_PAGE_SEG_DIRECTMAP)
+ continue;
+
+ total += vm_page_seg_size(&vm_page_segs[i]);
+ }
+
+ return total;
+}
+
+unsigned long
+vm_page_mem_free(void)
+{
+ unsigned long total;
+ unsigned int i;
+
+ total = 0;
+
+ for (i = 0; i < vm_page_segs_size; i++) {
+ /* XXX */
+ if (i > VM_PAGE_SEG_DIRECTMAP)
+ continue;
+
+ total += vm_page_segs[i].nr_free_pages;
+ }
+
+ return total;
+}
diff --git a/vm/vm_page.h b/vm/vm_page.h
index e6a8c49..f2e20a7 100644
--- a/vm/vm_page.h
+++ b/vm/vm_page.h
@@ -36,11 +36,12 @@
#include <mach/boolean.h>
#include <mach/vm_prot.h>
-#include <mach/vm_param.h>
+#include <machine/vm_param.h>
#include <vm/vm_object.h>
#include <vm/vm_types.h>
#include <kern/queue.h>
#include <kern/lock.h>
+#include <kern/log2.h>
#include <kern/macros.h>
#include <kern/sched_prim.h> /* definitions of wait/wakeup */
@@ -76,6 +77,23 @@
*/
struct vm_page {
+ /* Members used in the vm_page module only */
+ struct list node;
+ unsigned short type;
+ unsigned short seg_index;
+ unsigned short order;
+ void *priv;
+
+ /*
+ * This member is used throughout the code and may only change for
+ * fictitious pages.
+ */
+ phys_addr_t phys_addr;
+
+ /* We use an empty struct as the delimiter. */
+ struct {} vm_page_header;
+#define VM_PAGE_HEADER_SIZE offsetof(struct vm_page, vm_page_header)
+
queue_chain_t pageq; /* queue info for FIFO
* queue or free list (P) */
queue_chain_t listq; /* all pages in same object (O) */
@@ -110,8 +128,6 @@ struct vm_page {
* without having data. (O)
* [See vm_object_overwrite] */
- vm_offset_t phys_addr; /* Physical address of page, passed
- * to pmap_enter (read-only) */
vm_prot_t page_lock; /* Uses prohibited by data manager (O)
*/
vm_prot_t unlock_request; /* Outstanding unlock request (O) */
};
@@ -140,8 +156,6 @@ struct vm_page {
*/
extern
-vm_page_t vm_page_queue_free; /* memory free queue */
-extern
vm_page_t vm_page_queue_fictitious; /* fictitious free queue */
extern
queue_head_t vm_page_queue_active; /* active memory queue */
@@ -149,8 +163,6 @@ extern
queue_head_t vm_page_queue_inactive; /* inactive memory queue */
extern
-int vm_page_free_count; /* How many pages are free? */
-extern
int vm_page_fictitious_count;/* How many fictitious pages are free? */
extern
int vm_page_active_count; /* How many pages are active? */
@@ -196,25 +208,21 @@ extern void vm_page_bootstrap(
vm_offset_t *endp);
extern void vm_page_module_init(void);
-extern void vm_page_create(
- vm_offset_t start,
- vm_offset_t end);
extern vm_page_t vm_page_lookup(
vm_object_t object,
vm_offset_t offset);
extern vm_page_t vm_page_grab_fictitious(void);
-extern void vm_page_release_fictitious(vm_page_t);
-extern boolean_t vm_page_convert(vm_page_t, boolean_t);
+extern boolean_t vm_page_convert(vm_page_t *, boolean_t);
extern void vm_page_more_fictitious(void);
extern vm_page_t vm_page_grab(boolean_t);
-extern void vm_page_release(vm_page_t, boolean_t);
+extern vm_page_t vm_page_grab_contig(vm_size_t, unsigned int);
+extern void vm_page_free_contig(vm_page_t, vm_size_t);
extern void vm_page_wait(void (*)(void));
extern vm_page_t vm_page_alloc(
vm_object_t object,
vm_offset_t offset);
extern void vm_page_init(
- vm_page_t mem,
- vm_offset_t phys_addr);
+ vm_page_t mem);
extern void vm_page_free(vm_page_t);
extern void vm_page_activate(vm_page_t);
extern void vm_page_deactivate(vm_page_t);
@@ -312,4 +320,217 @@ extern unsigned int vm_page_info(
} \
MACRO_END
+/*
+ * Copyright (c) 2010-2014 Richard Braun.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ *
+ * Physical page management.
+ */
+
+/*
+ * Address/page conversion and rounding macros (not inline functions to
+ * be easily usable on both virtual and physical addresses, which may not
+ * have the same type size).
+ */
+#define vm_page_atop(addr) ((addr) >> PAGE_SHIFT)
+#define vm_page_ptoa(page) ((page) << PAGE_SHIFT)
+#define vm_page_trunc(addr) P2ALIGN(addr, PAGE_SIZE)
+#define vm_page_round(addr) P2ROUND(addr, PAGE_SIZE)
+#define vm_page_aligned(addr) P2ALIGNED(addr, PAGE_SIZE)
+
+/*
+ * Segment selectors.
+ *
+ * Selector-to-segment-list translation table :
+ * DMA DMA
+ * DMA32 DMA32 DMA
+ * DIRECTMAP DIRECTMAP DMA32 DMA
+ * HIGHMEM HIGHMEM DIRECTMAP DMA32 DMA
+ */
+#define VM_PAGE_SEL_DMA 0
+#define VM_PAGE_SEL_DMA32 1
+#define VM_PAGE_SEL_DIRECTMAP 2
+#define VM_PAGE_SEL_HIGHMEM 3
+
+/*
+ * Page usage types.
+ *
+ * Failing to allocate pmap pages will cause a kernel panic.
+ * TODO Obviously, this needs to be addressed, e.g. with a reserved pool of
+ * pages.
+ */
+#define VM_PT_FREE 0 /* Page unused */
+#define VM_PT_RESERVED 1 /* Page reserved at boot time */
+#define VM_PT_TABLE 2 /* Page is part of the page table */
+#define VM_PT_PMAP 3 /* Page stores pmap-specific data */
+#define VM_PT_KMEM 4 /* Page is part of a kmem slab */
+#define VM_PT_STACK 5 /* Type for generic kernel allocations */
+#define VM_PT_KERNEL 6 /* Type for generic kernel allocations */
+
+static inline unsigned short
+vm_page_type(const struct vm_page *page)
+{
+ return page->type;
+}
+
+void vm_page_set_type(struct vm_page *page, unsigned int order,
+ unsigned short type);
+
+static inline unsigned int
+vm_page_order(size_t size)
+{
+ return iorder2(vm_page_atop(vm_page_round(size)));
+}
+
+static inline phys_addr_t
+vm_page_to_pa(const struct vm_page *page)
+{
+ return page->phys_addr;
+}
+
+#if 0
+static inline unsigned long
+vm_page_direct_va(phys_addr_t pa)
+{
+ assert(pa < VM_PAGE_DIRECTMAP_LIMIT);
+ return ((unsigned long)pa + VM_MIN_DIRECTMAP_ADDRESS);
+}
+
+static inline phys_addr_t
+vm_page_direct_pa(unsigned long va)
+{
+ assert(va >= VM_MIN_DIRECTMAP_ADDRESS);
+ assert(va < VM_MAX_DIRECTMAP_ADDRESS);
+ return (va - VM_MIN_DIRECTMAP_ADDRESS);
+}
+
+static inline void *
+vm_page_direct_ptr(const struct vm_page *page)
+{
+ return (void *)vm_page_direct_va(vm_page_to_pa(page));
+}
+#endif
+
+/*
+ * Associate private data with a page.
+ */
+static inline void
+vm_page_set_priv(struct vm_page *page, void *priv)
+{
+ page->priv = priv;
+}
+
+static inline void *
+vm_page_get_priv(const struct vm_page *page)
+{
+ return page->priv;
+}
+
+/*
+ * Load physical memory into the vm_page module at boot time.
+ *
+ * The avail_start and avail_end parameters are used to maintain a simple
+ * heap for bootstrap allocations.
+ *
+ * All addresses must be page-aligned. Segments can be loaded in any order.
+ */
+void vm_page_load(unsigned int seg_index, phys_addr_t start, phys_addr_t end,
+ phys_addr_t avail_start, phys_addr_t avail_end);
+
+/*
+ * Return true if the vm_page module is completely initialized, false
+ * otherwise, in which case only vm_page_bootalloc() can be used for
+ * allocations.
+ */
+int vm_page_ready(void);
+
+/*
+ * Early allocation function.
+ *
+ * This function is used by the vm_resident module to implement
+ * pmap_steal_memory. It can be used after physical segments have been loaded
+ * and before the vm_page module is initialized.
+ */
+unsigned long vm_page_bootalloc(size_t size);
+
+/*
+ * Set up the vm_page module.
+ *
+ * Architecture-specific code must have loaded segments before calling this
+ * function. Segments must comply with the selector-to-segment-list table,
+ * e.g. HIGHMEM is loaded if and only if DIRECTMAP, DMA32 and DMA are loaded,
+ * notwithstanding segment aliasing.
+ *
+ * Once this function returns, the vm_page module is ready, and normal
+ * allocation functions can be used.
+ */
+void vm_page_setup(void);
+
+/*
+ * Make the given page managed by the vm_page module.
+ *
+ * If additional memory can be made usable after the VM system is initialized,
+ * it should be reported through this function.
+ */
+void vm_page_manage(struct vm_page *page);
+
+/*
+ * Return the page descriptor for the given physical address.
+ */
+struct vm_page * vm_page_lookup_pa(phys_addr_t pa);
+
+/*
+ * Allocate a block of 2^order physical pages.
+ *
+ * The selector is used to determine the segments from which allocation can
+ * be attempted.
+ *
+ * This function should only be used by the vm_resident module.
+ */
+struct vm_page * vm_page_alloc_pa(unsigned int order, unsigned int selector,
+ unsigned short type);
+
+/*
+ * Release a block of 2^order physical pages.
+ *
+ * This function should only be used by the vm_resident module.
+ */
+void vm_page_free_pa(struct vm_page *page, unsigned int order);
+
+/*
+ * Return the name of the given segment.
+ */
+const char * vm_page_seg_name(unsigned int seg_index);
+
+/*
+ * Display internal information about the module.
+ */
+void vm_page_info_all(void);
+
+/*
+ * Return the total amount of physical memory.
+ */
+phys_addr_t vm_page_mem_size(void);
+
+/*
+ * Return the amount of free (unused) pages.
+ *
+ * XXX This currently relies on the kernel being non preemptible and
+ * uniprocessor.
+ */
+unsigned long vm_page_mem_free(void);
+
#endif /* _VM_VM_PAGE_H_ */
diff --git a/vm/vm_pageout.c b/vm/vm_pageout.c
index 51a6a0d..72f96cb 100644
--- a/vm/vm_pageout.c
+++ b/vm/vm_pageout.c
@@ -82,7 +82,7 @@
* of active+inactive pages that should be inactive.
* The pageout daemon uses it to update vm_page_inactive_target.
*
- * If vm_page_free_count falls below vm_page_free_target and
+ * If the number of free pages falls below vm_page_free_target and
* vm_page_inactive_count is below vm_page_inactive_target,
* then the pageout daemon starts running.
*/
@@ -93,7 +93,7 @@
/*
* Once the pageout daemon starts running, it keeps going
- * until vm_page_free_count meets or exceeds vm_page_free_target.
+ * until the number of free pages meets or exceeds vm_page_free_target.
*/
#ifndef VM_PAGE_FREE_TARGET
@@ -101,7 +101,7 @@
#endif /* VM_PAGE_FREE_TARGET */
/*
- * The pageout daemon always starts running once vm_page_free_count
+ * The pageout daemon always starts running once the number of free pages
* falls below vm_page_free_min.
*/
@@ -125,7 +125,7 @@
#endif /* VM_PAGE_EXTERNAL_TARGET */
/*
- * When vm_page_free_count falls below vm_page_free_reserved,
+ * When the number of free pages falls below vm_page_free_reserved,
* only vm-privileged threads can allocate pages. vm-privilege
* allows the pageout daemon and default pager (and any other
* associated threads needed for default pageout) to continue
@@ -136,7 +136,7 @@
#endif /* VM_PAGE_FREE_RESERVED */
/*
- * When vm_page_free_count falls below vm_pageout_reserved_internal,
+ * When the number of free pages falls below vm_pageout_reserved_internal,
* the pageout daemon no longer trusts external pagers to clean pages.
* External pagers are probably all wedged waiting for a free page.
* It forcibly double-pages dirty pages belonging to external objects,
@@ -148,7 +148,7 @@
#endif /* VM_PAGEOUT_RESERVED_INTERNAL */
/*
- * When vm_page_free_count falls below vm_pageout_reserved_really,
+ * When the number of free pages falls below vm_pageout_reserved_really,
* the pageout daemon stops work entirely to let the default pager
* catch up (assuming the default pager has pages to clean).
* Beyond this point, it is too dangerous to consume memory
@@ -559,7 +559,7 @@ void vm_pageout_scan(void)
for (burst_count = 0;;) {
vm_page_t m;
vm_object_t object;
- unsigned int free_count;
+ unsigned long free_count;
/*
* Recalculate vm_page_inactivate_target.
@@ -630,7 +630,7 @@ void vm_pageout_scan(void)
*/
simple_lock(&vm_page_queue_free_lock);
- free_count = vm_page_free_count;
+ free_count = vm_page_mem_free();
if ((free_count >= vm_page_free_target) &&
(vm_page_external_count <= vm_page_external_target) &&
(vm_page_free_wanted == 0)) {
@@ -748,7 +748,12 @@ void vm_pageout_scan(void)
reclaim_page:
vm_page_free(m);
vm_page_unlock_queues();
- vm_object_unlock(object);
+
+ if (vm_object_collectable(object))
+ vm_object_collect(object);
+ else
+ vm_object_unlock(object);
+
continue;
}
@@ -910,7 +915,7 @@ void vm_pageout_continue(void)
void vm_pageout(void)
{
- int free_after_reserve;
+ unsigned long free_after_reserve;
current_thread()->vm_privilege = TRUE;
stack_privilege(current_thread());
@@ -946,7 +951,7 @@ void vm_pageout(void)
vm_pageout_reserved_really =
VM_PAGEOUT_RESERVED_REALLY(vm_page_free_reserved);
- free_after_reserve = vm_page_free_count - vm_page_free_reserved;
+ free_after_reserve = vm_page_mem_free() - vm_page_free_reserved;
if (vm_page_external_limit == 0)
vm_page_external_limit =
diff --git a/vm/vm_resident.c b/vm/vm_resident.c
index c70fa73..fa7a337 100644
--- a/vm/vm_resident.c
+++ b/vm/vm_resident.c
@@ -72,7 +72,7 @@
/*
* These variables record the values returned by vm_page_bootstrap,
* for debugging purposes. The implementation of pmap_steal_memory
- * and pmap_startup here also uses them internally.
+ * here also uses them internally.
*/
vm_offset_t virtual_space_start;
@@ -95,29 +95,18 @@ vm_page_bucket_t *vm_page_buckets; /* Array of
buckets */
unsigned int vm_page_bucket_count = 0; /* How big is array? */
unsigned int vm_page_hash_mask; /* Mask for hash function */
-/*
- * Resident page structures are initialized from
- * a template (see vm_page_alloc).
- *
- * When adding a new field to the virtual memory
- * object structure, be sure to add initialization
- * (see vm_page_bootstrap).
- */
-struct vm_page vm_page_template;
-
-/*
- * Resident pages that represent real memory
- * are allocated from a free list.
- */
-vm_page_t vm_page_queue_free;
vm_page_t vm_page_queue_fictitious;
decl_simple_lock_data(,vm_page_queue_free_lock)
unsigned int vm_page_free_wanted;
-int vm_page_free_count;
int vm_page_fictitious_count;
int vm_page_external_count;
-unsigned int vm_page_free_count_minimum; /* debugging */
+/*
+ * This variable isn't directly used. It's merely a placeholder for the
+ * address used to synchronize threads waiting for pages to become
+ * available. The real value is returned by vm_page_free_mem().
+ */
+unsigned int vm_page_free_avail;
/*
* Occasionally, the virtual memory system uses
@@ -192,48 +181,15 @@ void vm_page_bootstrap(
vm_offset_t *startp,
vm_offset_t *endp)
{
- vm_page_t m;
int i;
/*
- * Initialize the vm_page template.
- */
-
- m = &vm_page_template;
- m->object = VM_OBJECT_NULL; /* reset later */
- m->offset = 0; /* reset later */
- m->wire_count = 0;
-
- m->inactive = FALSE;
- m->active = FALSE;
- m->laundry = FALSE;
- m->free = FALSE;
- m->external = FALSE;
-
- m->busy = TRUE;
- m->wanted = FALSE;
- m->tabled = FALSE;
- m->fictitious = FALSE;
- m->private = FALSE;
- m->absent = FALSE;
- m->error = FALSE;
- m->dirty = FALSE;
- m->precious = FALSE;
- m->reference = FALSE;
-
- m->phys_addr = 0; /* reset later */
-
- m->page_lock = VM_PROT_NONE;
- m->unlock_request = VM_PROT_NONE;
-
- /*
* Initialize the page queues.
*/
simple_lock_init(&vm_page_queue_free_lock);
simple_lock_init(&vm_page_queue_lock);
- vm_page_queue_free = VM_PAGE_NULL;
vm_page_queue_fictitious = VM_PAGE_NULL;
queue_init(&vm_page_queue_active);
queue_init(&vm_page_queue_inactive);
@@ -241,12 +197,6 @@ void vm_page_bootstrap(
vm_page_free_wanted = 0;
/*
- * Steal memory for the kernel map entries.
- */
-
- kentry_data = pmap_steal_memory(kentry_data_size);
-
- /*
* Allocate (and initialize) the virtual-to-physical
* table hash buckets.
*
@@ -280,29 +230,19 @@ void vm_page_bootstrap(
simple_lock_init(&bucket->lock);
}
- /*
- * Machine-dependent code allocates the resident page table.
- * It uses vm_page_init to initialize the page frames.
- * The code also returns to us the virtual space available
- * to the kernel. We don't trust the pmap module
- * to get the alignment right.
- */
+ vm_page_setup();
- pmap_startup(&virtual_space_start, &virtual_space_end);
virtual_space_start = round_page(virtual_space_start);
virtual_space_end = trunc_page(virtual_space_end);
*startp = virtual_space_start;
*endp = virtual_space_end;
-
- /* printf("vm_page_bootstrap: %d free pages\n",
vm_page_free_count);*/
- vm_page_free_count_minimum = vm_page_free_count;
}
#ifndef MACHINE_PAGES
/*
- * We implement pmap_steal_memory and pmap_startup with the help
- * of two simpler functions, pmap_virtual_space and pmap_next_page.
+ * We implement pmap_steal_memory with the help
+ * of two simpler functions, pmap_virtual_space and vm_page_bootalloc.
*/
vm_offset_t pmap_steal_memory(
@@ -310,11 +250,7 @@ vm_offset_t pmap_steal_memory(
{
vm_offset_t addr, vaddr, paddr;
- /*
- * We round the size to an integer multiple.
- */
-
- size = (size + 3) &~ 3;
+ size = round_page(size);
/*
* If this is the first call to pmap_steal_memory,
@@ -347,8 +283,7 @@ vm_offset_t pmap_steal_memory(
for (vaddr = round_page(addr);
vaddr < addr + size;
vaddr += PAGE_SIZE) {
- if (!pmap_next_page(&paddr))
- panic("pmap_steal_memory");
+ paddr = vm_page_bootalloc(PAGE_SIZE);
/*
* XXX Logically, these mappings should be wired,
@@ -361,64 +296,6 @@ vm_offset_t pmap_steal_memory(
return addr;
}
-
-void pmap_startup(
- vm_offset_t *startp,
- vm_offset_t *endp)
-{
- unsigned int i, npages, pages_initialized;
- vm_page_t pages;
- vm_offset_t paddr;
-
- /*
- * We calculate how many page frames we will have
- * and then allocate the page structures in one chunk.
- */
-
- npages = ((PAGE_SIZE * pmap_free_pages() +
- (round_page(virtual_space_start) - virtual_space_start)) /
- (PAGE_SIZE + sizeof *pages));
-
- pages = (vm_page_t) pmap_steal_memory(npages * sizeof *pages);
-
- /*
- * Initialize the page frames.
- */
-
- for (i = 0, pages_initialized = 0; i < npages; i++) {
- if (!pmap_next_page(&paddr))
- break;
-
- vm_page_init(&pages[i], paddr);
- pages_initialized++;
- }
- i = 0;
- while (pmap_next_page(&paddr))
- i++;
- if (i)
- printf("%u memory page(s) left away\n", i);
-
- /*
- * Release pages in reverse order so that physical pages
- * initially get allocated in ascending addresses. This keeps
- * the devices (which must address physical memory) happy if
- * they require several consecutive pages.
- */
-
- for (i = pages_initialized; i > 0; i--) {
- vm_page_release(&pages[i - 1], FALSE);
- }
-
- /*
- * We have to re-align virtual_space_start,
- * because pmap_steal_memory has been using it.
- */
-
- virtual_space_start = round_page(virtual_space_start);
-
- *startp = virtual_space_start;
- *endp = virtual_space_end;
-}
#endif /* MACHINE_PAGES */
/*
@@ -430,35 +307,7 @@ void pmap_startup(
void vm_page_module_init(void)
{
kmem_cache_init(&vm_page_cache, "vm_page", sizeof(struct vm_page), 0,
- NULL, NULL, NULL, 0);
-}
-
-/*
- * Routine: vm_page_create
- * Purpose:
- * After the VM system is up, machine-dependent code
- * may stumble across more physical memory. For example,
- * memory that it was reserving for a frame buffer.
- * vm_page_create turns this memory into available pages.
- */
-
-void vm_page_create(
- vm_offset_t start,
- vm_offset_t end)
-{
- vm_offset_t paddr;
- vm_page_t m;
-
- for (paddr = round_page(start);
- paddr < trunc_page(end);
- paddr += PAGE_SIZE) {
- m = (vm_page_t) kmem_cache_alloc(&vm_page_cache);
- if (m == VM_PAGE_NULL)
- panic("vm_page_create");
-
- vm_page_init(m, paddr);
- vm_page_release(m, FALSE);
- }
+ NULL, 0);
}
/*
@@ -523,7 +372,7 @@ void vm_page_insert(
*/
object->resident_page_count++;
- assert(object->resident_page_count >= 0);
+ assert(object->resident_page_count != 0);
if (object->can_persist && (object->ref_count == 0))
vm_object_cached_pages_update(1);
@@ -630,7 +479,7 @@ void vm_page_replace(
*/
object->resident_page_count++;
- assert(object->resident_page_count >= 0);
+ assert(object->resident_page_count != 0);
if (object->can_persist && (object->ref_count == 0))
vm_object_cached_pages_update(1);
@@ -750,6 +599,33 @@ void vm_page_rename(
vm_page_unlock_queues();
}
+static void vm_page_init_template(vm_page_t m)
+{
+ m->object = VM_OBJECT_NULL; /* reset later */
+ m->offset = 0; /* reset later */
+ m->wire_count = 0;
+
+ m->inactive = FALSE;
+ m->active = FALSE;
+ m->laundry = FALSE;
+ m->free = FALSE;
+ m->external = FALSE;
+
+ m->busy = TRUE;
+ m->wanted = FALSE;
+ m->tabled = FALSE;
+ m->fictitious = FALSE;
+ m->private = FALSE;
+ m->absent = FALSE;
+ m->error = FALSE;
+ m->dirty = FALSE;
+ m->precious = FALSE;
+ m->reference = FALSE;
+
+ m->page_lock = VM_PROT_NONE;
+ m->unlock_request = VM_PROT_NONE;
+}
+
/*
* vm_page_init:
*
@@ -758,11 +634,9 @@ void vm_page_rename(
* so that it can be given to vm_page_release or vm_page_insert.
*/
void vm_page_init(
- vm_page_t mem,
- vm_offset_t phys_addr)
+ vm_page_t mem)
{
- *mem = vm_page_template;
- mem->phys_addr = phys_addr;
+ vm_page_init_template(mem);
}
/*
@@ -794,7 +668,7 @@ vm_page_t vm_page_grab_fictitious(void)
* Release a fictitious page to the free list.
*/
-void vm_page_release_fictitious(
+static void vm_page_release_fictitious(
vm_page_t m)
{
simple_lock(&vm_page_queue_free_lock);
@@ -826,7 +700,8 @@ void vm_page_more_fictitious(void)
if (m == VM_PAGE_NULL)
panic("vm_page_more_fictitious");
- vm_page_init(m, vm_page_fictitious_addr);
+ vm_page_init(m);
+ m->phys_addr = vm_page_fictitious_addr;
m->fictitious = TRUE;
vm_page_release_fictitious(m);
}
@@ -836,25 +711,46 @@ void vm_page_more_fictitious(void)
* vm_page_convert:
*
* Attempt to convert a fictitious page into a real page.
+ *
+ * The object referenced by *MP must be locked.
*/
boolean_t vm_page_convert(
- vm_page_t m,
+ struct vm_page **mp,
boolean_t external)
{
- vm_page_t real_m;
+ struct vm_page *real_m, *fict_m;
+ vm_object_t object;
+ vm_offset_t offset;
+
+ fict_m = *mp;
+
+ assert(fict_m->fictitious);
+ assert(fict_m->phys_addr == vm_page_fictitious_addr);
+ assert(!fict_m->active);
+ assert(!fict_m->inactive);
real_m = vm_page_grab(external);
if (real_m == VM_PAGE_NULL)
return FALSE;
- m->phys_addr = real_m->phys_addr;
- m->fictitious = FALSE;
+ object = fict_m->object;
+ offset = fict_m->offset;
+ vm_page_remove(fict_m);
- real_m->phys_addr = vm_page_fictitious_addr;
- real_m->fictitious = TRUE;
+ memcpy(&real_m->vm_page_header,
+ &fict_m->vm_page_header,
+ sizeof(*fict_m) - VM_PAGE_HEADER_SIZE);
+ real_m->fictitious = FALSE;
- vm_page_release_fictitious(real_m);
+ vm_page_insert(real_m, object, offset);
+
+ assert(real_m->phys_addr != vm_page_fictitious_addr);
+ assert(fict_m->fictitious);
+ assert(fict_m->phys_addr == vm_page_fictitious_addr);
+
+ vm_page_release_fictitious(fict_m);
+ *mp = real_m;
return TRUE;
}
@@ -878,7 +774,7 @@ vm_page_t vm_page_grab(
* for externally-managed pages.
*/
- if (((vm_page_free_count < vm_page_free_reserved)
+ if (((vm_page_mem_free() < vm_page_free_reserved)
|| (external
&& (vm_page_external_count > vm_page_external_limit)))
&& !current_thread()->vm_privilege) {
@@ -886,15 +782,16 @@ vm_page_t vm_page_grab(
return VM_PAGE_NULL;
}
- if (vm_page_queue_free == VM_PAGE_NULL)
- panic("vm_page_grab");
+ mem = vm_page_alloc_pa(0, VM_PAGE_SEL_DIRECTMAP, VM_PT_KERNEL);
+
+ if (mem == NULL) {
+ simple_unlock(&vm_page_queue_free_lock);
+ return NULL;
+ }
- if (--vm_page_free_count < vm_page_free_count_minimum)
- vm_page_free_count_minimum = vm_page_free_count;
if (external)
vm_page_external_count++;
- mem = vm_page_queue_free;
- vm_page_queue_free = (vm_page_t) mem->pageq.next;
+
mem->free = FALSE;
mem->extcounted = mem->external = external;
simple_unlock(&vm_page_queue_free_lock);
@@ -910,8 +807,8 @@ vm_page_t vm_page_grab(
* it doesn't really matter.
*/
- if ((vm_page_free_count < vm_page_free_min) ||
- ((vm_page_free_count < vm_page_free_target) &&
+ if ((vm_page_mem_free() < vm_page_free_min) ||
+ ((vm_page_mem_free() < vm_page_free_target) &&
(vm_page_inactive_count < vm_page_inactive_target)))
thread_wakeup((event_t) &vm_page_free_wanted);
@@ -928,208 +825,92 @@ vm_offset_t vm_page_grab_phys_addr(void)
}
/*
- * vm_page_grab_contiguous_pages:
- *
- * Take N pages off the free list, the pages should
- * cover a contiguous range of physical addresses.
- * [Used by device drivers to cope with DMA limitations]
+ * vm_page_release:
*
- * Returns the page descriptors in ascending order, or
- * Returns KERN_RESOURCE_SHORTAGE if it could not.
+ * Return a page to the free list.
*/
-/* Biggest phys page number for the pages we handle in VM */
-
-vm_size_t vm_page_big_pagenum = 0; /* Set this before call! */
-
-kern_return_t
-vm_page_grab_contiguous_pages(
- int npages,
- vm_page_t pages[],
- natural_t *bits,
- boolean_t external)
+static void vm_page_release(
+ vm_page_t mem,
+ boolean_t external)
{
- int first_set;
- int size, alloc_size;
- kern_return_t ret;
- vm_page_t mem, *prevmemp;
-
-#ifndef NBBY
-#define NBBY 8 /* size in bits of sizeof()`s unity */
-#endif
-
-#define NBPEL (sizeof(natural_t)*NBBY)
-
- size = (vm_page_big_pagenum + NBPEL - 1)
- & ~(NBPEL - 1); /* in bits */
-
- size = size / NBBY; /* in bytes */
-
- /*
- * If we are called before the VM system is fully functional
- * the invoker must provide us with the work space. [one bit
- * per page starting at phys 0 and up to vm_page_big_pagenum]
- */
- if (bits == 0) {
- alloc_size = round_page(size);
- if (kmem_alloc_wired(kernel_map,
- (vm_offset_t *)&bits,
- alloc_size)
- != KERN_SUCCESS)
- return KERN_RESOURCE_SHORTAGE;
- } else
- alloc_size = 0;
-
- memset(bits, 0, size);
-
- /*
- * A very large granularity call, its rare so that is ok
- */
simple_lock(&vm_page_queue_free_lock);
+ if (mem->free)
+ panic("vm_page_release");
+ mem->free = TRUE;
+ vm_page_free_pa(mem, 0);
+ if (external)
+ vm_page_external_count--;
/*
- * Do not dip into the reserved pool.
- */
-
- if ((vm_page_free_count < vm_page_free_reserved)
- || (vm_page_external_count >= vm_page_external_limit)) {
- printf_once("no more room for vm_page_grab_contiguous_pages");
- simple_unlock(&vm_page_queue_free_lock);
- return KERN_RESOURCE_SHORTAGE;
- }
-
- /*
- * First pass through, build a big bit-array of
- * the pages that are free. It is not going to
- * be too large anyways, in 4k we can fit info
- * for 32k pages.
+ * Check if we should wake up someone waiting for page.
+ * But don't bother waking them unless they can allocate.
+ *
+ * We wakeup only one thread, to prevent starvation.
+ * Because the scheduling system handles wait queues FIFO,
+ * if we wakeup all waiting threads, one greedy thread
+ * can starve multiple niceguy threads. When the threads
+ * all wakeup, the greedy threads runs first, grabs the page,
+ * and waits for another page. It will be the first to run
+ * when the next page is freed.
+ *
+ * However, there is a slight danger here.
+ * The thread we wake might not use the free page.
+ * Then the other threads could wait indefinitely
+ * while the page goes unused. To forestall this,
+ * the pageout daemon will keep making free pages
+ * as long as vm_page_free_wanted is non-zero.
*/
- mem = vm_page_queue_free;
- while (mem) {
- int word_index, bit_index;
- bit_index = (mem->phys_addr >> PAGE_SHIFT);
- word_index = bit_index / NBPEL;
- bit_index = bit_index - (word_index * NBPEL);
- bits[word_index] |= 1 << bit_index;
-
- mem = (vm_page_t) mem->pageq.next;
+ if ((vm_page_free_wanted > 0) &&
+ (vm_page_mem_free() >= vm_page_free_reserved)) {
+ vm_page_free_wanted--;
+ thread_wakeup_one((event_t) &vm_page_free_avail);
}
- /*
- * Second loop. Scan the bit array for NPAGES
- * contiguous bits. That gives us, if any,
- * the range of pages we will be grabbing off
- * the free list.
- */
- {
- int bits_so_far = 0, i;
+ simple_unlock(&vm_page_queue_free_lock);
+}
- first_set = 0;
+/*
+ * vm_page_grab_contig:
+ *
+ * Remove a block of contiguous pages from the free list.
+ * Returns VM_PAGE_NULL if the request fails.
+ */
- for (i = 0; i < size; i += sizeof(natural_t)) {
+vm_page_t vm_page_grab_contig(
+ vm_size_t size,
+ unsigned int selector)
+{
+ unsigned int i, order, nr_pages;
+ vm_page_t mem;
- natural_t v = bits[i / sizeof(natural_t)];
- int bitpos;
+ order = vm_page_order(size);
+ nr_pages = 1 << order;
- /*
- * Bitscan this one word
- */
- if (v) {
- /*
- * keep counting them beans ?
- */
- bitpos = 0;
+ simple_lock(&vm_page_queue_free_lock);
- if (bits_so_far) {
-count_ones:
- while (v & 1) {
- bitpos++;
- /*
- * got enough beans ?
- */
- if (++bits_so_far == npages)
- goto found_em;
- v >>= 1;
- }
- /* if we are being lucky, roll again */
- if (bitpos == NBPEL)
- continue;
- }
+ /*
+ * Only let privileged threads (involved in pageout)
+ * dip into the reserved pool or exceed the limit
+ * for externally-managed pages.
+ */
- /*
- * search for beans here
- */
- bits_so_far = 0;
- while ((bitpos < NBPEL) && ((v & 1) == 0)) {
- bitpos++;
- v >>= 1;
- }
- if (v & 1) {
- first_set = (i * NBBY) + bitpos;
- goto count_ones;
- }
- }
- /*
- * No luck
- */
- bits_so_far = 0;
- }
+ if (((vm_page_mem_free() - nr_pages) <= vm_page_free_reserved)
+ && !current_thread()->vm_privilege) {
+ simple_unlock(&vm_page_queue_free_lock);
+ return VM_PAGE_NULL;
}
- /*
- * We could not find enough contiguous pages.
- */
- simple_unlock(&vm_page_queue_free_lock);
+ /* TODO Allow caller to pass type */
+ mem = vm_page_alloc_pa(order, selector, VM_PT_KERNEL);
- printf_once("no contiguous room for vm_page_grab_contiguous_pages");
- ret = KERN_RESOURCE_SHORTAGE;
- goto out;
+ if (mem == NULL)
+ panic("vm_page_grab_contig");
- /*
- * Final pass. Now we know which pages we want.
- * Scan the list until we find them all, grab
- * pages as we go. FIRST_SET tells us where
- * in the bit-array our pages start.
- */
-found_em:
- vm_page_free_count -= npages;
- if (vm_page_free_count < vm_page_free_count_minimum)
- vm_page_free_count_minimum = vm_page_free_count;
- if (external)
- vm_page_external_count += npages;
- {
- vm_offset_t first_phys, last_phys;
-
- /* cache values for compare */
- first_phys = first_set << PAGE_SHIFT;
- last_phys = first_phys + (npages << PAGE_SHIFT);/* not included */
-
- /* running pointers */
- mem = vm_page_queue_free;
- prevmemp = &vm_page_queue_free;
-
- while (mem) {
-
- vm_offset_t addr;
-
- addr = mem->phys_addr;
-
- if ((addr >= first_phys) &&
- (addr < last_phys)) {
- *prevmemp = (vm_page_t) mem->pageq.next;
- pages[(addr - first_phys) >> PAGE_SHIFT] = mem;
- mem->free = FALSE;
- mem->extcounted = mem->external = external;
- /*
- * Got them all ?
- */
- if (--npages == 0) break;
- } else
- prevmemp = (vm_page_t *) &mem->pageq.next;
-
- mem = (vm_page_t) mem->pageq.next;
- }
+ for (i = 0; i < nr_pages; i++) {
+ mem[i].free = FALSE;
+ mem[i].extcounted = mem[i].external = 0;
}
simple_unlock(&vm_page_queue_free_lock);
@@ -1145,63 +926,42 @@ found_em:
* it doesn't really matter.
*/
- if ((vm_page_free_count < vm_page_free_min) ||
- ((vm_page_free_count < vm_page_free_target) &&
+ if ((vm_page_mem_free() < vm_page_free_min) ||
+ ((vm_page_mem_free() < vm_page_free_target) &&
(vm_page_inactive_count < vm_page_inactive_target)))
- thread_wakeup(&vm_page_free_wanted);
-
- ret = KERN_SUCCESS;
-out:
- if (alloc_size)
- kmem_free(kernel_map, (vm_offset_t) bits, alloc_size);
+ thread_wakeup((event_t) &vm_page_free_wanted);
- return ret;
+ return mem;
}
/*
- * vm_page_release:
+ * vm_page_free_contig:
*
- * Return a page to the free list.
+ * Return a block of contiguous pages to the free list.
*/
-void vm_page_release(
- vm_page_t mem,
- boolean_t external)
+void vm_page_free_contig(vm_page_t mem, vm_size_t size)
{
+ unsigned int i, order, nr_pages;
+
+ order = vm_page_order(size);
+ nr_pages = 1 << order;
+
simple_lock(&vm_page_queue_free_lock);
- if (mem->free)
- panic("vm_page_release");
- mem->free = TRUE;
- mem->pageq.next = (queue_entry_t) vm_page_queue_free;
- vm_page_queue_free = mem;
- vm_page_free_count++;
- if (external)
- vm_page_external_count--;
- /*
- * Check if we should wake up someone waiting for page.
- * But don't bother waking them unless they can allocate.
- *
- * We wakeup only one thread, to prevent starvation.
- * Because the scheduling system handles wait queues FIFO,
- * if we wakeup all waiting threads, one greedy thread
- * can starve multiple niceguy threads. When the threads
- * all wakeup, the greedy threads runs first, grabs the page,
- * and waits for another page. It will be the first to run
- * when the next page is freed.
- *
- * However, there is a slight danger here.
- * The thread we wake might not use the free page.
- * Then the other threads could wait indefinitely
- * while the page goes unused. To forestall this,
- * the pageout daemon will keep making free pages
- * as long as vm_page_free_wanted is non-zero.
- */
+ for (i = 0; i < nr_pages; i++) {
+ if (mem[i].free)
+ panic("vm_page_free_contig");
+
+ mem[i].free = TRUE;
+ }
+
+ vm_page_free_pa(mem, order);
if ((vm_page_free_wanted > 0) &&
- (vm_page_free_count >= vm_page_free_reserved)) {
+ (vm_page_mem_free() >= vm_page_free_reserved)) {
vm_page_free_wanted--;
- thread_wakeup_one((event_t) &vm_page_free_count);
+ thread_wakeup_one((event_t) &vm_page_free_avail);
}
simple_unlock(&vm_page_queue_free_lock);
@@ -1227,11 +987,11 @@ void vm_page_wait(
*/
simple_lock(&vm_page_queue_free_lock);
- if ((vm_page_free_count < vm_page_free_target)
+ if ((vm_page_mem_free() < vm_page_free_target)
|| (vm_page_external_count > vm_page_external_limit)) {
if (vm_page_free_wanted++ == 0)
thread_wakeup((event_t)&vm_page_free_wanted);
- assert_wait((event_t)&vm_page_free_count, FALSE);
+ assert_wait((event_t)&vm_page_free_avail, FALSE);
simple_unlock(&vm_page_queue_free_lock);
if (continuation != 0) {
counter(c_vm_page_wait_block_user++);
@@ -1310,12 +1070,13 @@ void vm_page_free(
*/
if (mem->private || mem->fictitious) {
- vm_page_init(mem, vm_page_fictitious_addr);
+ vm_page_init(mem);
+ mem->phys_addr = vm_page_fictitious_addr;
mem->fictitious = TRUE;
vm_page_release_fictitious(mem);
} else {
int external = mem->external && mem->extcounted;
- vm_page_init(mem, mem->phys_addr);
+ vm_page_init(mem);
vm_page_release(mem, external);
}
}
diff --git a/vm/vm_user.c b/vm/vm_user.c
index c71e9f5..e65f6d5 100644
--- a/vm/vm_user.c
+++ b/vm/vm_user.c
@@ -182,7 +182,7 @@ kern_return_t vm_statistics(
*stat = vm_stat;
stat->pagesize = PAGE_SIZE;
- stat->free_count = vm_page_free_count;
+ stat->free_count = vm_page_mem_free();
stat->active_count = vm_page_active_count;
stat->inactive_count = vm_page_inactive_count;
stat->wire_count = vm_page_wire_count;
--
Alioth's /usr/local/bin/git-commit-notice on
/srv/git.debian.org/git/pkg-hurd/gnumach.git