The document discusses the process for context switching between tasks in Rust. It explains that the current task is grabbed from thread-local storage and its ability to sleep is checked. The next task and cleanup function are prepared. Unsafe transmutes are used to get mutable references to tasks. The task contexts are swapped using a raw operation, placing the scheduler and next task in the proper locations. On the return swap, the cleanup function is immediately run.

2. Once Upon a Gash…
let mut prog = run::Process::new(program,
argv,
options);
Goal for this week: understand as deeply as possible
everything that happens to make this work.
5 November 2013
University of Virginia cs4414
1

3. Goal for Today and Thursday
run::Process::new(program, argv, options)
5 November 2013
University of Virginia cs4414
2

4. rust/src/libstd/run.rs
5 November 2013
University of Virginia cs4414
3

5. impl Process {
Note: code has been reformatted to
/**
remove some whitespace to fit on slide,
* Spawns a new Process.
not otherwise not changed.
* # Arguments
* * prog - The path to an executable.
* * args - Vector of arguments to pass to the child process.
* * options - Options to configure the environment of the process,
*
the working directory and the standard IO streams.
*/
pub fn new(prog: &str, args: &[~str], options: ProcessOptions) -> Process {
let ProcessOptions { env, dir, in_fd, out_fd, err_fd } = options;
let env = env.as_ref().map(|a| a.as_slice());
let cwd = dir.as_ref().map(|a| a.as_str().unwrap());
fn rtify(fd: Option<c_int>, input: bool) -> process::StdioContainer {
match fd { Some(fd) => process::InheritFd(fd),
None => process::CreatePipe(input, !input), } }
let rtio = [rtify(in_fd, true), rtify(out_fd, false), rtify(err_fd, false)];
let rtconfig = process::ProcessConfig { program: prog,
args: args, env: env, cwd: cwd, io: rtio, };
let inner = process::Process::new(rtconfig).unwrap();
Process { inner: inner }
}
5 November 2013
University of Virginia cs4414
4

6. rust/src/libstd/rt/io/process.rs
impl Process {
/// Creates a new pipe initialized, but not bound to any particular
/// source/destination
pub fn new(config: ProcessConfig) -> Option<Process> {
let config = Cell::new(config);
do with_local_io |io| {
match io.spawn(config.take()) {
Ok((p, io)) => Some(Process{
handle: p,
io: io.move_iter().map(|p|
p.map(|p| io::PipeStream::new(p))
).collect()
}),
Err(ioerr) => {
io_error::cond.raise(ioerr);
None
}
}
}
}
5 November 2013
University of Virginia cs4414
5

7. rust/src/libstd/rt/rtio.rs
5 November 2013
pub fn with_local_io<T>(f: &fn(&mut IoFactory) -> Option<T>) ->
Option<T> {
use rt::sched::Scheduler;
use rt::local::Local;
use rt::io::{io_error, standard_error, IoUnavailable};
unsafe {
let sched: *mut Scheduler = Local::unsafe_borrow();
let mut io = None;
(*sched).event_loop.io(|i| io = Some(i));
match io {
Some(io) => f(io),
None => {
do with_local_io |io| {
io_error::cond.raise(standard_error(IoUnavailable));
match io.spawn(config.take()) {
None
Ok((p, io)) => Some(Process{
}
handle: p,
io: io.move_iter().map(|p|
}
p.map(|p| io::PipeStream::new(p))
}
}
).collect()
}),
Err(ioerr) => …
University of Virginia cs4414
6

8. libstd/task/mod.rs
/**
* Creates and executes a new child task
*
* Sets up a new task with its own call stack and schedules it to run
* the provided unique closure. The task has the properties and behavior
* specified by the task_builder.
*
* # Failure
*
* When spawning into a new scheduler, the number of threads requested
* must be greater than zero.
*/
pub fn spawn(&mut self, f: ~fn()) {
…
5 November 2013
University of Virginia cs4414
7

9. libstd/task/mod.rs
pub fn spawn(&mut self, f: ~fn()) {
let gen_body = self.gen_body.take();
let notify_chan = self.opts.notify_chan.take();
let name = self.opts.name.take();
let x = self.consume();
let opts = TaskOpts { linked: x.opts.linked, supervised: x.opts.supervised,
watched: x.opts.watched, indestructible: x.opts.indestructible,
notify_chan: notify_chan, name: name, sched: x.opts.sched,
stack_size: x.opts.stack_size
};
let f = match gen_body {
Some(gen) => {
pub struct TaskBuilder {
gen(f)
opts: TaskOpts,
}
priv gen_body: Option<~fn(v: ~fn()) -> ~fn()>,
None => {
priv can_not_copy: Option<util::NonCopyable>,
f
priv consumed: bool,
}
}
};
spawn::spawn_raw(opts, f);
}
5 November 2013
University of Virginia cs4414
8

10. src/libstd/task/spawn.rs
pub fn spawn_raw(mut opts: TaskOpts, f: ~fn()) {
assert!(in_green_task_context());
… // ~130 lines
debug!("spawn calling run_task");
Scheduler::run_task(task);
}
5 November 2013
University of Virginia cs4414
9

11. Green Tasks?
pub fn in_green_task_context() -> bool {
unsafe {
let task: Option<*mut Task> = Local::try_unsafe_borrow();
match task {
Some(task) => {
match (*task).task_type {
GreenTask(_) => true,
_ => false
}
}
None => false
}
}
src/libstd/rt/mod.rs
}
5 November 2013
University of Virginia cs4414
???
Don’t really get
this…a good
explanation is
worth a
challenge!
10

12. zhttpto
16,000
Average Response Time (milliseconds)
35.632, 15106.5
14,000
PS3 Benchmarking
Sneak Preview
12,000
zhtta starting
37.113, 12641.3
37.434, 12406.5
34.018, 11759.8
Round 1: (request each file
once – no benefit to cache)
10,000
8,000
38.487, 7354
6,000
4,000
2,000
34.213, 1458.7
35.187, 1190.4
0
41.021, 929.2
43.314, 625.1
14.245, 15
0
5
10
15
20
25
30
35
40
45
50
Total Duration (seconds)
5 November 2013
University of Virginia cs4414
11

13. rt/sched.rs
pub fn run_task(task: ~Task) {
let sched: ~Scheduler = Local::take();
sched.process_task(task, Scheduler::switch_task);
}
fn process_task(mut ~self, mut task: ~Task, schedule_fn: SchedulingFn)
{
rtdebug!("processing a task");
let home = task.take_unwrap_home();
match home {
Sched(home_handle) => {
if home_handle.sched_id != self.sched_id() {
…
} else {
rtdebug!("running task here");
task.give_home(Sched(home_handle));
schedule_fn(self, task);
}
}
…
}
5 November 2013
University of Virginia cs4414
12

14. pub fn switch_running_tasks_and_then(~self, next_task: ~Task,
f: &fn(&mut Scheduler, BlockedTask)) {
// This is where we convert the BlockedTask-taking closure into one
// that takes just a Task, and is aware of the block-or-killed protocol.
do self.change_task_context(next_task) |sched, task| {
// Task might need to receive a kill signal instead of blocking.
// We can call the "and_then" only if it blocks successfully.
match BlockedTask::try_block(task) {
Left(killed_task) => sched.enqueue_task(killed_task),
Right(blocked_task) => f(sched, blocked_task),
}
}
}
fn switch_task(sched: ~Scheduler, task: ~Task) {
do sched.switch_running_tasks_and_then(task) |sched, last_task| {
sched.enqueue_blocked_task(last_task);
};
}
5 November 2013
University of Virginia cs4414
13

15. // However we still need an internal mutable pointer to the
// original task. The strategy here was "arrange memory, then
// get pointers", so we crawl back up the chain using
// transmute to eliminate borrowck errors.
unsafe {
// * Core Context Switching Functions
// The primary function for changing contexts. In the current
// design the scheduler is just a slightly modified GreenTask, so
// all context swaps are from Task to Task. The only difference
// between the various cases is where the inputs come from, and
// what is done with the resulting task. That is specified by the
// cleanup function f, which takes the scheduler and the
// old task as inputs.
let sched: &mut Scheduler =
transmute_mut_region(*next_task.sched.get_mut_ref());
let current_task: &mut Task = match sched.cleanup_job {
Some(CleanupJob { task: ref task, _ }) => {
let task_ptr: *~Task = task;
transmute_mut_region(*transmute_mut_unsafe(task_ptr))
}
None => {
rtabort!("no cleanup job");
}
};
pub fn change_task_context(mut ~self,
next_task: ~Task,
f: &fn(&mut Scheduler, ~Task)) {
// The current task is grabbed from TLS, not taken as an input.
// Doing an unsafe_take to avoid writing back a null pointer // We're going to call `put` later to do that.
let current_task: ~Task = unsafe { Local::unsafe_take() };
let (current_task_context, next_task_context) =
Scheduler::get_contexts(current_task, next_task);
// Check that the task is not in an atomically() section (e.g.,
// holding a pthread mutex, which could deadlock the scheduler).
current_task.death.assert_may_sleep();
// Done with everything - put the next task in TLS. This
// works because due to transmute the borrow checker
// believes that we have no internal pointers to
// next_task.
Local::put(next_task);
// These transmutes do something fishy with a closure.
let f_fake_region = unsafe {
transmute::<&fn(&mut Scheduler, ~Task),
&fn(&mut Scheduler, ~Task)>(f)
};
let f_opaque = ClosureConverter::from_fn(f_fake_region);
// The raw context swap operation. The next action taken
// will be running the cleanup job from the context of the
// next task.
Context::swap(current_task_context, next_task_context);
}
// The current task is placed inside an enum with the cleanup
// function. This enum is then placed inside the scheduler.
self.cleanup_job = Some(CleanupJob::new(current_task, f_opaque));
// When the context swaps back to this task we immediately
// run the cleanup job, as expected by the previously called
// swap_contexts function.
unsafe {
let task: *mut Task = Local::unsafe_borrow();
(*task).sched.get_mut_ref().run_cleanup_job();
// The scheduler is then placed inside the next task.
let mut next_task = next_task;
next_task.sched = Some(self);
// Must happen after running the cleanup job (of course).
(*task).death.check_killed((*task).unwinder.unwinding);
}
}
5 November 2013
University of Virginia cs4414
14

16. // The primary function for changing contexts. In the current
pub fn assert_may_sleep(&self) {
// design the scheduler is just a slightly modified GreenTask, so
if from Task to Task. != 0 {
// all context swaps are self.wont_sleep The only difference
rtabort!("illegal atomic-sleep: attempt
// between the various cases is where the inputs come from, and to reschedule while
using an Exclusive or LittleLock");
// what is done with the resulting task. That is specified by the
// cleanup function f, which takes the scheduler and the
}
// old task as inputs.
}
pub fn change_task_context(mut ~self,
next_task: ~Task,
f: &fn(&mut Scheduler, ~Task)) {
// The current task is grabbed from TLS, not taken as an input.
// Doing an unsafe_take to avoid writing back a null pointer // We're going to call `put` later to do that.
let current_task: ~Task = unsafe { Local::unsafe_take() };
// Check that the task is not in an atomically() section (e.g.,
// holding a pthread mutex, which could deadlock the scheduler).
current_task.death.assert_may_sleep();
// These transmutes do something fishy with a closure.
let f_fake_region = unsafe {
transmute::<&fn(&mut Scheduler, ~Task),
&fn(&mut Scheduler, ~Task)>(f)
};
5 November 2013
University of Virginia cs4414
15

17. src/libstd/rt/kill.rs
/// Enter a possibly-nested "atomic" section of code. Just for assertions.
/// All calls must be paired with a subsequent call to allow_deschedule.
#[inline]
pub fn inhibit_deschedule(&mut self) {
self.wont_sleep += 1;
}
/// Exit a possibly-nested "atomic" section of code. Just for assertions.
/// All calls must be paired with a preceding call to inhibit_deschedule.
#[inline]
pub fn allow_deschedule(&mut self) {
rtassert!(self.wont_sleep != 0);
self.wont_sleep -= 1;
}
5 November 2013
University of Virginia cs4414
16

18. struct KillHandleInner {
// Is the task running, blocked, or killed? Possible values:
// * KILL_RUNNING - Not unkillable, no kill pending.
// * KILL_KILLED - Kill pending.
// * <ptr>
- A transmuted blocked ~Task pointer.
// This flag is refcounted because it may also be referenced by a blocking
// concurrency primitive, used to wake the task normally, whose reference
// may outlive the handle's if the task is killed.
killed: KillFlagHandle,
// …
// Shared state between task and children for exit code propagation. These
// are here so we can re-use the kill handle to implement watched children
// tasks. Using a separate Arc-like would introduce extra atomic adds/subs
// into common spawn paths, so this is just for speed.
// Locklessly accessed; protected by the enclosing refcount's barriers.
any_child_failed: bool,
// A lazy list, consuming which may unwrap() many child tombstones.
child_tombstones: Option<~fn() -> bool>,
// Protects multiple children simultaneously creating tombstones.
graveyard_lock: LittleLock,
}
5 November 2013
University of Virginia cs4414
17

19. // * Core Context Switching Functions
// The primary function for changing contexts. In the current
// design the scheduler is just a slightly modified GreenTask, so
// all context swaps are from Task to Task. The only difference
// between the various cases is where the inputs come from, and
// what is done with the resulting task. That is specified by the
// cleanup function f, which takes the scheduler and the
// old task as inputs.
// However we still need an internal mutable pointer to the
// original task. The strategy here was "arrange memory, then
// get pointers", so we crawl back up the chain using
// transmute to eliminate borrowck errors.
unsafe {
let sched: &mut Scheduler =
transmute_mut_region(*next_task.sched.get_mut_ref());
let current_task: &mut Task = match sched.cleanup_job {
Some(CleanupJob { task: ref task, _ }) => {
let task_ptr: *~Task = task;
transmute_mut_region(*transmute_mut_unsafe(task_ptr))
}
None => {
rtabort!("no cleanup job");
}
};
pub fn change_task_context(mut ~self,
next_task: ~Task,
f: &fn(&mut Scheduler, ~Task)) {
// The current task is grabbed from TLS, not taken as an input.
// Doing an unsafe_take to avoid writing back a null pointer // We're going to call `put` later to do that.
let current_task: ~Task = unsafe { Local::unsafe_take() };
let (current_task_context, next_task_context) =
Scheduler::get_contexts(current_task, next_task);
// Check that the task is not in an atomically() section (e.g.,
// holding a pthread mutex, which could deadlock the scheduler).
current_task.death.assert_may_sleep();
// Done with everything - put the next task in TLS. This
// works because due to transmute the borrow checker
// believes that we have no internal pointers to
// next_task.
Local::put(next_task);
// These transmutes do something fishy with a closure.
let f_fake_region = unsafe {
transmute::<&fn(&mut Scheduler, ~Task),
&fn(&mut Scheduler, ~Task)>(f)
};
let f_opaque = ClosureConverter::from_fn(f_fake_region);
// The current task is placed inside an enum with the cleanup
// function. This enum is then placed inside the scheduler.
self.cleanup_job = Some(CleanupJob::new(current_task, f_opaque));
// The scheduler is then placed inside the next task.
let mut next_task = next_task;
next_task.sched = Some(self);
// The raw context swap operation. The next action taken
// will be running the cleanup job from the context of the
// next task.
Context::swap(current_task_context, next_task_context);
}
// When the context swaps back to this task we immediately
// run the cleanup job, as expected by the previously called
// swap_contexts function.
unsafe {
let task: *mut Task = Local::unsafe_borrow();
(*task).sched.get_mut_ref().run_cleanup_job();
// Must happen after running the cleanup job (of course).
(*task).death.check_killed((*task).unwinder.unwinding);
}
5 November 2013
}
University of Virginia cs4414
18

20. src/libstd/rt/context.rs
/* Switch contexts - Suspend the current execution context and resume another by saving the
registers values of the executing thread to a Context then loading the registers from a previously
saved Context. */
pub fn swap(out_context: &mut Context, in_context: &Context) {
rtdebug!("swapping contexts");
let out_regs: &mut Registers = match out_context {
&Context { regs: ~ref mut r, _ } => r };
let in_regs: &Registers = match in_context {
&Context { regs: ~ref r, _ } => r };
rtdebug!("noting the stack limit and doing raw swap");
unsafe {
// Right before we switch to the new context, set the new context’s stack limit in the
// OS-specified TLS slot. This also means that we cannot call any more rust functions after
// record_stack_bounds returns because they would all likely fail due to the limit being
// invalid for the current task. Lucky for us `swap_registers` is a C function so we don't
// have to worry about that!
match in_context.stack_bounds {
Some((lo, hi)) => record_stack_bounds(lo, hi),
// If we're going back to one of the original contexts or
// something that's possibly not a "normal task", then reset
// the stack limit to 0 to make morestack never fail
None => record_stack_bounds(0, uint::max_value),
}
swap_registers(out_regs, in_regs)
}
}
5 November 2013
University of Virginia cs4414
19

21. #[inline(always)]
pub unsafe fn record_stack_bounds(stack_lo: uint, stack_hi: uint) {
// When the old runtime had segmented stacks, it used a calculation that was
// "limit + RED_ZONE + FUDGE". The red zone was for things like dynamic
// symbol resolution, llvm function calls, etc. In theory this red zone
// value is 0, but it matters far less when we have gigantic stacks because
// we don't need to be so exact about our stack budget. The "fudge factor"
// was because LLVM doesn't emit a stack check for functions < 256 bytes in
// size. Again though, we have giant stacks, so we round all these
// calculations up to the nice round number of 20k.
record_sp_limit(stack_lo + RED_ZONE);
return target_record_stack_bounds(stack_lo, stack_hi);
#[cfg(not(windows))] #[cfg(not(target_arch = "x86_64"))] #[inline(always)]
unsafe fn target_record_stack_bounds(_stack_lo: uint, _stack_hi: uint) {}
#[cfg(windows, target_arch = "x86_64")] #[inline(always)]
unsafe fn target_record_stack_bounds(stack_lo: uint, stack_hi: uint) {
…
}
}
5 November 2013
University of Virginia cs4414
20

22. /// Records the current limit of the stack as specified by `end`.
///
/// This is stored in an OS-dependent location, likely inside of the thread
/// local storage. The location that the limit is stored is a pre-ordained
/// location because it's where LLVM has emitted code to check.
///
/// Note that this cannot be called under normal circumstances. This function is
/// changing the stack limit, so upon returning any further function calls will
/// possibly be triggering the morestack logic if you're not careful.
///
#[inline(always)]
pub unsafe fn record_sp_limit(limit: uint) {
return target_record_sp_limit(limit);
// x86-64
#[cfg(target_arch = "x86_64", target_os = "macos")] #[inline(always)]
unsafe fn target_record_sp_limit(limit: uint) { … }
#[cfg(target_arch = "x86_64", target_os = "linux")] #[inline(always)]
unsafe fn target_record_sp_limit(limit: uint) { … }
Why all the #[inline(always)] ?
5 November 2013
University of Virginia cs4414
21

23. /// Also note that this and all of the inside
functions are all flagged as "inline(always)"
because they're messing around with the stack
limits. This would be unfortunate for the
functions themselves to trigger a morestack
invocation (if they were an actual function call).
5 November 2013
University of Virginia cs4414
22

24. 25,000
Average Response Time (milliseconds)
zhttpto
PS3 Benchmarking
Sneak Preview
20,000
36.713, 21004.9
Round 2: re-request each file
15,000
zhtta starting
37.61, 12805.2
31.778, 11278.5
10,000
5,000
1.899, 324.5
1.673, 22
1.382, 0.3
0
0
5 November 2013
5
42.27, 1087.2
40.397, 826.545.66, 779
32.557, 265.9
10
15
20
25
30
35
Total Duration (seconds)
University of Virginia cs4414
40
45
50
23

25. Average Response Time (milliseconds)
1,000
900
700
PS3 Benchmarking
Sneak Preview
600
Round 2: re-request each file
800
40.397, 826.5
45.66, 779
500
400
1.899, 324.5
300
32.557, 265.9
200
100
1.673, 22
1.382, 0.3
0
0
5
10
15
20
25
30
35
40
45
50
Total Duration (seconds)
5 November 2013
University of Virginia cs4414
24

26. /// Records the current limit of the stack as specified by `end`.
/// This is stored in an OS-dependent location, likely inside of the thread
/// local storage. The location that the limit is stored is a pre-ordained
/// location because it's where LLVM has emitted code to check.
/// …
#[inline(always)]
pub unsafe fn record_sp_limit(limit: uint) {
return target_record_sp_limit(limit);
// x86-64
#[cfg(target_arch = "x86_64", target_os = "macos")] #[inline(always)]
unsafe fn target_record_sp_limit(limit: uint) {
asm!("movq $$0x60+90*8, %rsi
movq $0, %gs:(%rsi)" :: "r"(limit) : "rsi" : "volatile")
}
#[cfg(target_arch = "x86_64", target_os = "linux")] #[inline(always)]
unsafe fn target_record_sp_limit(limit: uint) {
asm!("movq $0, %fs:112" :: "r"(limit) :: "volatile")
}
#[cfg(target_arch = "x86_64", target_os = "win32")] #[inline(always)]
unsafe fn target_record_sp_limit(limit: uint) {
…
5 November 2013
University of Virginia cs4414
25

27. asm!("movq $$0x60+90*8, %rsi
movq $0, %gs:(%rsi)"
:: "r"(limit) : "rsi" : "volatile")
movq <source>, <destination>
q = “quad” (64-bit value)
$ cat asm.rs
fn main() {
let limit = 0xCAFEBABE; // choose your magic hex constants tastefully
unsafe {
asm!("movq $$0x60+90*8, %rsi
movq $0, %gs:(%rsi)" :: "r"(limit) : "rsi" : "volatile")
}
}
$ rustc -S asm.rs
5 November 2013
University of Virginia cs4414
26

28. asm.s (86 lines total)
.section
__TEXT,__text,regular,pure_instructions
.align 4, 0x90
__ZN4main19hcc4e1163d21f1c71af4v0.0E:
.cfi_startproc
Ltmp4:
cmpq %gs:816, %rsp
.cfi_def_cfa_register %rbp
ja LBB0_0
subq $16, %rsp
movabsq $24, %r10
movabsq $3405691582, %rax
movabsq $0, %r11
movq %rax, -8(%rbp)
callq ___morestack
movq -8(%rbp), %rax
ret
## InlineAsm Start
LBB0_0:
movq $0x60+90*8, %rsi
pushq %rbp
movq %rax, %gs:(%rsi)
Ltmp2:
## InlineAsm End
.cfi_def_cfa_offset 16
movq %rdi, -16(%rbp)
Ltmp3:
addq $16, %rsp
.cfi_offset %rbp, -16
popq %rbp
movq %rsp, %rbp
ret
.cfi_endproc
5 November 2013
University of Virginia cs4414
27

29. rustc –S asm.rs (86 lines total)
rustc –S –O asm.rs (76 lines total)
Ltmp4:
.cfi_def_cfa_register %rbp
subq $16, %rsp
movabsq $3405691582, %rax
Ltmp4:
movq %rax, -8(%rbp)
.cfi_def_cfa_register %rbp
movq -8(%rbp), %rax
movl $3405691582, %eax
## InlineAsm Start
## InlineAsm Start
movq $0x60+90*8, %rsi
movq $0x60+90*8, %rsi
movq %rax, %gs:(%rsi)
movq %rax, %gs:(%rsi)
## InlineAsm End
## InlineAsm End
movq %rdi, -16(%rbp)
popq %rbp
addq $16, %rsp
ret
popq %rbp
.cfi_endproc
ret
.cfi_endproc
%gs: segment register that holds address of processor data area
%gs + 0x60+90*8 holds stack limit
5 November 2013
University of Virginia cs4414
28

30. src/libstd/rt/context.rs
/* Switch contexts - Suspend the current execution context and resume another by saving the
registers values of the executing thread to a Context then loading the registers from a previously
saved Context. */
pub fn swap(out_context: &mut Context, in_context: &Context) {
rtdebug!("swapping contexts");
let out_regs: &mut Registers = match out_context {
&Context { regs: ~ref mut r, _ } => r };
let in_regs: &Registers = match in_context {
&Context { regs: ~ref r, _ } => r };
rtdebug!("noting the stack limit and doing raw swap");
unsafe {
// Right before we switch to the new context, set the new context’s stack limit in the
// OS-specified TLS slot. This also means that we cannot call any more rust functions after
// record_stack_bounds returns because they would all likely fail due to the limit being
// invalid for the current task. Lucky for us `swap_registers` is a C function so we don't
// have to worry about that!
match in_context.stack_bounds {
Some((lo, hi)) => record_stack_bounds(lo, hi),
// If we're going back to one of the original contexts or
// something that's possibly not a "normal task", then reset
// the stack limit to 0 to make morestack never fail
None => record_stack_bounds(0, uint::max_value),
}
swap_registers(out_regs, in_regs)
}
}
5 November 2013
University of Virginia cs4414
29

31. extern {
#[rust_stack]
fn swap_registers(out_regs: *mut Registers, in_regs: *Registers);
}
// Register contexts used in various architectures
//
// These structures all represent a context of one task throughout its
// execution. Each struct is a representation of the architecture's register
// set. When swapping between tasks, these register sets are used to save off
// the current registers into one struct, and load them all from another.
//
// Note that this is only used for context switching, which means that some of
// the registers may go unused. For example, for architectures with
// callee/caller saved registers, the context will only reflect the callee-saved
// registers. This is because the caller saved registers are already stored
// elsewhere on the stack (if it was necessary anyway).
//
// Additionally, there may be fields on various architectures which are unused
// entirely …
// These structures/functions are roughly in-sync with the source files inside
// of src/rt/arch/$arch. …
5 November 2013
University of Virginia cs4414
30

32. // Mark stack as non-executable
#if defined(__linux__) && defined(__ELF__)
.section
.note.GNU-stack, "", @progbits
#endif
src/rt/arch/x86_64/_context.S
#include "regs.h"
#define ARG0 RUSTRT_ARG0_S
#define ARG1 RUSTRT_ARG1_S
.text
/*
According to ABI documentation found at http://www.x86-64.org/documentation.html
and Microsoft discussion at http://msdn.microsoft.com/en-US/library/9z1stfyw%28v=VS.80%29.aspx.
BOTH CALLING CONVENTIONS
Callee save registers:
R12--R15, RDI, RSI, RBX, RBP, RSP
XMM0--XMM5
Caller save registers:
RAX, RCX, RDX, R8--R11
XMM6--XMM15
Floating point stack
MAC/AMD CALLING CONVENTIONS
…
5 November 2013
University of Virginia cs4414
31

33. // swap_registers(registers_t *oregs, registers_t *regs)
.globl SWAP_REGISTERS
SWAP_REGISTERS:
// n.b. when we enter, the return address is at the top of
// the stack (i.e., 0(%RSP)) and the argument is in
// RUSTRT_ARG0_S. We simply save all NV registers into oregs.
// We then restore all NV registers from regs. This restores
// the old stack pointer, which should include the proper
// return address. We can therefore just return normally to
// jump back into the old code.
#define RUSTRT_RBX 0
regs.h
#define RUSTRT_RSP 1
// Save instruction pointer:
#define RUSTRT_RBP 2
pop %rax
// RCX on Windows, RDI elsewhere
mov %rax, (RUSTRT_IP*8)(RUSTRT_ARG0_S)
#define RUSTRT_ARG0 3
#define RUSTRT_R12 4
// Save non-volatile integer registers:
#define RUSTRT_R13 5
// (including RSP)
#define RUSTRT_R14 6
mov %rbx, (RUSTRT_RBX*8)(ARG0)
#define RUSTRT_R15 7
mov %rsp, (RUSTRT_RSP*8)(ARG0)
#define RUSTRT_IP 8
mov %rbp, (RUSTRT_RBP*8)(ARG0)
…
mov %r12, (RUSTRT_R12*8)(ARG0)
# define RUSTRT_ARG0_S %rdi
mov %r13, (RUSTRT_R13*8)(ARG0)
…
…
5 November 2013
University of Virginia cs4414
32

34. #define ARG0 RUSTRT_ARG0_S
SWAP_REGISTERS:
// n.b. when we enter, the return address is at the top of
// the stack (i.e., 0(%RSP)) and the argument is in
// RUSTRT_ARG0_S. We simply save all NV registers into oregs.
// …
non-volatile: registers that must be
preserved by a called function
// Save instruction pointer:
pop %rax
mov %rax, (RUSTRT_IP*8)(RUSTRT_ARG0_S)
// Save non-volatile integer registers:
// (including RSP)
mov %rbx, (RUSTRT_RBX*8)(ARG0)
mov %rsp, (RUSTRT_RSP*8)(ARG0)
mov %rbp, (RUSTRT_RBP*8)(ARG0)
mov %r12, (RUSTRT_R12*8)(ARG0)
mov %r13, (RUSTRT_R13*8)(ARG0)
mov %r14, (RUSTRT_R14*8)(ARG0)
mov %r15, (RUSTRT_R15*8)(ARG0)
// Save 0th argument register:
mov ARG0, (RUSTRT_ARG0*8)(ARG0)
// Save non-volatile XMM registers:
movapd %xmm0, (RUSTRT_XMM0*8)(ARG0)
movapd %xmm1, (RUSTRT_XMM1*8)(ARG0)
movapd %xmm2, (RUSTRT_XMM2*8)(ARG0)
movapd %xmm3, (RUSTRT_XMM3*8)(ARG0)
movapd %xmm4, (RUSTRT_XMM4*8)(ARG0)
movapd %xmm5, (RUSTRT_XMM5*8)(ARG0)
All the non-volatile registers are now stored on the stack
5 November 2013
University of Virginia cs4414
33

35. SWAP_REGISTERS:
// …
pop %rax
mov %rax, (RUSTRT_IP*8)(RUSTRT_ARG0_S)
// Restore non-volatile integer registers:
mov (RUSTRT_RBX*8)(ARG1), %rbx
mov (RUSTRT_RSP*8)(ARG1), %rsp
mov (RUSTRT_RBP*8)(ARG1), %rbp
mov (RUSTRT_R12*8)(ARG1), %r12
// Save non-volatile integer registers: (including RSP)
mov (RUSTRT_R13*8)(ARG1), %r13
mov %rbx, (RUSTRT_RBX*8)(ARG0)
mov (RUSTRT_R14*8)(ARG1), %r14
mov %rsp, (RUSTRT_RSP*8)(ARG0)
mov (RUSTRT_R15*8)(ARG1), %r15
mov %rbp, (RUSTRT_RBP*8)(ARG0)
mov %r12, (RUSTRT_R12*8)(ARG0)
// Restore 0th argument register:
mov %r13, (RUSTRT_R13*8)(ARG0)
mov (RUSTRT_ARG0*8)(ARG1), ARG0
mov %r14, (RUSTRT_R14*8)(ARG0)
mov %r15, (RUSTRT_R15*8)(ARG0)
// Restore non-volatile XMM registers:
movapd (RUSTRT_XMM0*8)(ARG1), %xmm0
// Save 0th argument register:
movapd (RUSTRT_XMM1*8)(ARG1), %xmm1
mov ARG0, (RUSTRT_ARG0*8)(ARG0)
...
movapd (RUSTRT_XMM5*8)(ARG1), %xmm5
// Save non-volatile XMM registers:
movapd %xmm0, (RUSTRT_XMM0*8)(ARG0)
// Jump to the instruction pointer
movapd %xmm1, (RUSTRT_XMM1*8)(ARG0)
// found in regs:
…
jmp *(RUSTRT_IP*8)(ARG1)
movapd %xmm5, (RUSTRT_XMM5*8)(ARG0)
5 November 2013
University of Virginia cs4414
34

36. src/libstd/rt/context.rs
/* Switch contexts - Suspend the current execution context and resume another by saving the
registers values of the executing thread to a Context then loading the registers from a previously
saved Context. */
pub fn swap(out_context: &mut Context, in_context: &Context) {
rtdebug!("swapping contexts");
let out_regs: &mut Registers = match out_context {
&Context { regs: ~ref mut r, _ } => r };
let in_regs: &Registers = match in_context {
&Context { regs: ~ref r, _ } => r };
rtdebug!("noting the stack limit and doing raw swap");
unsafe {
// Right before we switch to the new context, set the new context’s stack limit in the
// OS-specified TLS slot. This also means that we cannot call any more rust functions after
// record_stack_bounds returns because they would all likely fail due to the limit being
// invalid for the current task. Lucky for us `swap_registers` is a C function so we don't
// have to worry about that!
match in_context.stack_bounds {
Some((lo, hi)) => record_stack_bounds(lo, hi),
// If we're going back to one of the original contexts or
// something that's possibly not a "normal task", then reset
// the stack limit to 0 to make morestack never fail
None => record_stack_bounds(0, uint::max_value),
}
swap_registers(out_regs, in_regs)
Have we created a new process yet?
}
}
5 November 2013
University of Virginia cs4414
35

37. 140,000
Average Response Time (milliseconds)
246.0, 130655.0
120,000
PS3 Benchmarking
Sneak Preview
100,000
Final Round: lots of concurrent
requests, many repeated files
80,000
199.9, 81272.5
167.3, 67649.4
60,000
40,000
20,000
13.2, 5701.3
9.7, 3908.1
44.0, 989.7
39.8, 960.8
5.5, 0.6
0
0
50
217.1, 3902.7
225.2, 531.3
100
150
200
250
300
Total Duration (seconds)
5 November 2013
University of Virginia cs4414
36

38. 140,000
Average Response Time (milliseconds)
246.0, 130655.0
120,000
PS3 Benchmarking
Sneak Preview
100,000
Final Round: lots of concurrent
requests, many repeated files
80,000
199.9, 81272.5
167.3, 67649.4
60,000
40,000
20,000
13.2, 5701.3
9.7, 3908.1
44.0, 989.7
39.8, 960.8
5.5, 0.6
0
0
50
217.1, 3902.7
225.2, 531.3
100
150
200
250
300
Total Duration (seconds)
5 November 2013
University of Virginia cs4414
37

39. 13.2, 5701.3
Average Response Time (milliseconds)
6,000
5,000
9.7, 3908.1
4,000
217.1, 3902.7
reference zhtta
3,000
Official Results will be Thursday!
2,000
44.0, 989.7
39.8, 960.8
1,000
225.2, 531.3
5.5, 0.6
0
0
50
100
150
200
Total Duration (seconds)
5 November 2013
University of Virginia cs4414
38

40. libstd/rt/io/native/process.rs
5 November 2013
/** A value representing a child process.
*
* The lifetime of this value is linked to the lifetime of the actual
* process - the Process destructor calls self.finish() which waits
* for the process to terminate.
*/
pub struct Process {
/// The unique id of the process (this should never be negative).
priv pid: pid_t,
/// A handle to the process - on unix this will always be NULL, …but on
/// windows it will be a HANDLE to the process, which will prevent the
/// pid being re-used until the handle is closed.
priv handle: *(),
/// Currently known stdin of the child, if any
priv input: Option<file::FileDesc>,
/// Currently known stdout of the child, if any
priv output: Option<file::FileDesc>,
/// Currently known stderr of the child, if any
priv error: Option<file::FileDesc>,
/// None until finish() is called.
priv exit_code: Option<int>,
}
University of Virginia cs4414
39

41. /// Creates a new process using native process-spawning abilities provided by the OS. Operations on this
/// process will be blocking instead of using the runtime for sleeping just this current task.
///
/// # Arguments
/// * prog - the program to run
/// * args - the arguments to pass to the program, not including the program itself
/// * env - an optional envrionment to specify for the child process. If
///
this value is `None`, then the child will inherit the parent’s environment
/// * cwd - an optionally specified current working directory of the child,
///
defaulting to the parent's current working directory
/// * stdin, stdout, stderr - These optionally specified file descriptors dictate where the stdin/out/err
/// of the child process will go. If these are `None`, then this module will bind the input/output to an
/// os pipe instead. This process takes ownership of these file descriptors, closing them upon
/// destruction of the process.
pub fn new(prog: &str, args: &[~str], env: Option<~[(~str, ~str)]>, cwd: Option<&Path>,
stdin: Option<file::fd_t>, stdout: Option<file::fd_t>, stderr: Option<file::fd_t>) -> Process {
… // 30 lines (next slide)
Process {
pid: res.pid, handle: res.handle,
input: in_pipe.map(|pipe| file::FileDesc::new(pipe.out)),
output: out_pipe.map(|pipe| file::FileDesc::new(pipe.input)),
error: err_pipe.map(|pipe| file::FileDesc::new(pipe.input)),
exit_code: None,
}
}
5 November 2013
University of Virginia cs4414
40

42. pub fn new(prog: &str, args: &[~str], env: Option<~[(~str, ~str)]>,
cwd: Option<&Path>, stdin: Option<file::fd_t>, stdout: Option<file::fd_t>,
stderr: Option<file::fd_t>) -> Process {
#[fixed_stack_segment]; #[inline(never)];
let (in_pipe, in_fd) = match stdin {
None => { let pipe = os::pipe(); (Some(pipe), pipe.input) },
Some(fd) => (None, fd) };
… // same for stdout, stderr
let res = spawn_process_os(prog, args, env, cwd, in_fd, out_fd, err_fd);
unsafe {
for pipe in in_pipe.iter() { libc::close(pipe.input); }
for pipe in out_pipe.iter() { libc::close(pipe.out); }
for pipe in err_pipe.iter() { libc::close(pipe.out); }
}
Process {
pid: res.pid, handle: res.handle,
input: in_pipe.map(|pipe| file::FileDesc::new(pipe.out)),
output: out_pipe.map(|pipe| file::FileDesc::new(pipe.input)),
error: err_pipe.map(|pipe| file::FileDesc::new(pipe.input)),
exit_code: None,
}
}
5 November 2013
University of Virginia cs4414
41

43. libstd/rt/io/native/process.rs
#[cfg(unix)]
fn spawn_process_os(prog: &str, args: &[~str],
env: Option<~[(~str, ~str)]>, dir: Option<&Path>,
in_fd: c_int, out_fd: c_int, err_fd: c_int) -> SpawnProcessResult {
#[fixed_stack_segment]; #[inline(never)];
use libc::funcs::posix88::unistd::{fork, dup2, close, chdir, execvp};
use libc::funcs::bsd44::getdtablesize;
mod rustrt {
#[abi = "cdecl"]
extern {
pub fn rust_unset_sigprocmask();
}
}
#[cfg(windows)]
unsafe fn set_environ(_envp: *c_void) {}
#[cfg(target_os = "macos")]
unsafe fn set_environ(envp: *c_void) {
externfn!(fn _NSGetEnviron() -> *mut *c_void);
*_NSGetEnviron() = envp;
}
…
5 November 2013
University of Virginia cs4414
42

44. libstd/rt/io/native/process.rs
#[cfg(unix)]
fn spawn_process_os(prog: &str, args: &[~str],
env: Option<~[(~str, ~str)]>, dir: Option<&Path>,
in_fd: c_int, out_fd: c_int, err_fd: c_int) -> SpawnProcessResult {
…
#[cfg(not(target_os = "macos"), not(windows))]
unsafe fn set_environ(envp: *c_void) {
extern {
static mut environ: *c_void;
}
environ = envp;
}
unsafe {
let pid = fork();
if pid < 0 {
fail!("failure in fork: {}", os::last_os_error());
} else if pid > 0 {
return SpawnProcessResult {pid: pid, handle: ptr::null()};
}
… // 25 lines of failure-handing code
}
5 November 2013
University of Virginia cs4414
43

45. Forking!
#[cfg(unix)]
fn spawn_process_os(prog: &str, args: &[~str],
env: Option<~[(~str, ~str)]>, dir: Option<&Path>,
in_fd: c_int, out_fd: c_int, err_fd: c_int) -> SpawnProcessResult {
…
use libc::funcs::posix88::unistd::{fork, dup2, close, chdir, execvp};
…
unsafe {
let pid = fork();
if pid < 0 {
fail!("failure in fork: {}", os::last_os_error());
} else if pid > 0 {
return SpawnProcessResult {pid: pid, handle: ptr::null()};
}
… // 25 lines of failure-handing code
}
5 November 2013
University of Virginia cs4414
44

46. src/libstd/libc.rs
5 November 2013
/*!
* Bindings for the C standard library and other platform libraries
*
* This module contains bindings to the C standard library,
* organized into modules by their defining standard.
* Additionally, it contains some assorted platform-specific definitions.
* For convenience, most functions and types are reexported from `std::libc`,
* so `pub use std::libc::*` will import the available
* C bindings as appropriate for the target platform. The exact
* set of functions available are platform specific.
*…
*/
…
#[nolink]
pub mod unistd {
use libc::types::common::c95::c_void;
use libc::types::os::arch::c95::{c_char, c_int, c_long, c_uint};
use libc::types::os::arch::c95::{size_t};
use libc::types::os::arch::posix88::{gid_t, off_t, pid_t};
use libc::types::os::arch::posix88::{ssize_t, uid_t};
…
University of Virginia cs4414
45

47. src/libstd/libc.rs
5 November 2013
pub mod unistd {
…
extern {
pub fn access(path: *c_char, amode: c_int) -> c_int;
pub fn alarm(seconds: c_uint) -> c_uint;
pub fn chdir(dir: *c_char) -> c_int;
pub fn chown(path: *c_char, uid: uid_t, gid: gid_t) -> c_int;
pub fn close(fd: c_int) -> c_int;
pub fn dup(fd: c_int) -> c_int;
pub fn dup2(src: c_int, dst: c_int) -> c_int;
pub fn execv(prog: *c_char, argv: **c_char) -> c_int;
pub fn execve(prog: *c_char, argv: **c_char, envp: **c_char)
-> c_int;
pub fn execvp(c: *c_char, argv: **c_char) -> c_int;
pub fn fork() -> pid_t;
pub fn fpathconf(filedes: c_int, name: c_int) -> c_long;
pub fn getcwd(buf: *mut c_char, size: size_t) -> *c_char;
pub fn getegid() -> gid_t;
pub fn geteuid() -> uid_t;
pub fn getgid() -> gid_t ;
…
University of Virginia cs4414
46

48. /*
* linux/kernel/fork.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
/*
* 'fork.c' contains the help-routines for the 'fork' system call
* (see also entry.S and others).
* Fork is rather simple, once you get the hang of it, but the memory
* management can be a bitch. See 'mm/memory.c': 'copy_page_range()'
*/
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/unistd.h>
…
1935 lines of C code
5 November 2013
University of Virginia cs4414
47

49. Rust Runtime
Recap
run::Process::new(program, argv, options)
spawn_process_os(prog, args, env, dir, in_fd, …)
fork()
libc: fork()
To be
continued
Thursday…
5 November 2013
linux kernel: fork syscall
University of Virginia cs4414
48