From 50b4d2f0ab05901a847d8e1318152308265e579b Mon Sep 17 00:00:00 2001 From: salmonik Date: Sun, 1 Nov 2026 16:33:05 +0100 Subject: usb: real-hardware rework — hubs, both EHCI controllers, keyboard, xHCI hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EHCI (OptiPlex 7010 / Q77 and similar Intel PCH): - enumerate *both* EHCI controllers (00:1a.0 + 00:1d.0), not just the first - hub support: read the hub class descriptor, power/reset each port via hub class requests, read the child speed, then drive the child with split transactions carrying the hub's address and port (TT). On 7-series PCH the root port presents the integrated Rate Matching Hub (8087:0024, class 9) and the mouse/keyboard sit behind it; without this the driver stopped at "no HID mouse/keyboard endpoint in config". - HID mouse *and* keyboard, boot protocol + idle, per-device periodic QH chained into the frame list, per-device report buffers and counters - periodic re-arm follows the standard shape (qTD re-activated, Next-qTD re-linked); the HC owns the status/toggle overlay - DMA from the page allocator inside [16 MiB, 4 GiB): EHCI programs 32-bit addresses and the page bitmap spans all RAM - 2 ms recovery after SET_ADDRESS xHCI: - scratchpad buffers (HCSPARAMS2) and MaxSlotsEn > 1, with per-slot EP0 and interrupt rings plus per-slot device contexts - DMA from the page allocator instead of the heap allocator - boot protocol + idle, mouse + keyboard, first-report diagnostics - transfer events are routed to every device before matching a waiter, so a second device's enumeration cannot swallow an armed device's report shared: - rsnix/src/hid.rs: HID usage -> scancode-set-1 tables, one scancode FIFO feeding ps2::feed_scancode, per-device boot-keyboard report differ - WM input pump polls PS/2 -> UHCI -> EHCI -> xHCI - memory::alloc_pages_range(count, min, max) for bounded DMA allocations QEMU coverage: make test-usb (EHCI + xHCI, mouse + keyboard). QEMU's EHCI split path delivers only the first report for FS devices, so repeated-report delivery and the RMH path remain hardware-test items. --- rsnix/src/ehci.rs | 1852 +++++++++++++++++++++++++++++------------------------ 1 file changed, 1007 insertions(+), 845 deletions(-) (limited to 'rsnix/src/ehci.rs') diff --git a/rsnix/src/ehci.rs b/rsnix/src/ehci.rs index 8baa0ad..5ee11cf 100644 --- a/rsnix/src/ehci.rs +++ b/rsnix/src/ehci.rs @@ -1,32 +1,37 @@ // --------------------------------------------------------------------------- -// rsdos EHCI (USB2.0 host controller) driver — mouse only. +// rsdos EHCI (USB 2.0 host controller) driver — HID mouse + keyboard. // -// Enumerates devices connected to an Intel-class EHCI (direct root-port -// connect), discovers a HID boot-mouse interrupt-IN endpoint, and drains its -// reports into ps2::MouseEvent. Uses the EHCI async schedule for control -// transfers and the periodic (frame-list) schedule for the interrupt IN -// endpoint. Polled: no IRQ/MSI — keep exactly one report outstanding and -// drain it from the cursor task. +// Real-hardware topology (Intel PCH, e.g. Q77/OptiPlex 7010): the USB2 +// controllers (00:1a.0, 00:1d.0) each present an *integrated Rate Matching +// Hub* on their root port (8087:0024, class 9). Full/low-speed devices +// (mice, keyboards) sit behind that hub, so the driver must // -// Why EHCI: on real hardware (e.g. a Q67/Q77-era OptiPlex) a USB mouse is a -// USB2 (full/low-speed) device attached to the EHCI USB2 controllers, not to -// the xHCI's USB3/SS ports. QEMU only exercises the xHCI path, so this class -// of real hardware requires its own driver. +// 1. enumerate every EHCI controller (not just the first), +// 2. enumerate a root-port hub, power/reset its ports and read the child +// speed via hub class requests, +// 3. drive the child through split transactions using the hub's USB +// address and the child port (TT), +// 4. arm a periodic interrupt-IN QH per HID device and poll reports. // -// Identity map (phys==virt) means any kernel-heap pointer is a valid DMA -// address; data structures must be 32-byte aligned for the EHCI. +// A device connected directly to a root port (QEMU's topology) uses the +// root-hub TT (hub address 0, port = root port + 1). +// +// Polled: no IRQ/MSI — one report outstanding per device, drained from the +// cursor task (mouse) and the WM input pump (keyboard). DMA structures are +// allocated from the page allocator (physical == CPU address, identity map) +// below 4 GiB because EHCI programs 32-bit DMA pointers. // --------------------------------------------------------------------------- -use crate::ps2::MouseEvent; +use crate::ps2::{KeyEvent, MouseEvent}; use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -// ---- Registers (host controller registers; opbase = base + CAPLENGTH) ---- +// ---- Registers (opbase = base + CAPLENGTH) ---- const OPCMD: usize = 0x00; const OPSTS: usize = 0x04; const OPINTR: usize = 0x08; const OPFRIDX: usize = 0x0C; const OPCFGFLAG: usize = 0x40; -const OP_PRTSC_BASE: usize = 0x44; // PORTSC #1 +const OP_PRTSC_BASE: usize = 0x44; const OP_PRTSC_STRIDE: usize = 0x04; const CMD_RUN: u32 = 1 << 0; @@ -42,79 +47,131 @@ const PRT_CONNECT: u32 = 1 << 0; const PRT_PED: u32 = 1 << 1; const PRT_RESET: u32 = 1 << 8; const PRT_POWER: u32 = 1 << 12; -const PRT_RWC: u32 = (1 << 17) | (1 << 18) | (1 << 16) | (1 << 20); // CSC|PEC|CRC|OCC +const PRT_RWC: u32 = (1 << 17) | (1 << 18) | (1 << 16) | (1 << 20); // ---- QH / QTD fields ---- const Q_TYPE_QH: u32 = 1 << 1; -const QT_TERMINATE: u32 = 1; // terminate = 1 in a link pointer +const QT_TERMINATE: u32 = 1; const QH_HEAD: u32 = 1 << 15; const QH_CONTROL_EP: u32 = 1 << 27; -const QH_DTC: u32 = 1 << 14; // data toggle comes from the qTDs (control endpoints) -const QH_RL: u32 = 4 << 28; // NAK reload count (Linux EHCI_TUNE_RL_{HS,TT}) +const QH_DTC: u32 = 1 << 14; +const QH_RL: u32 = 4 << 28; const QH_EPS_HIGH: u32 = 2 << 12; const QH_EPS_FULL: u32 = 0 << 12; const QH_EPS_LOW: u32 = 1 << 12; -const QH_MULT: u32 = 1 << 30; // MULT=1: one split transaction per uframe +const QH_MULT: u32 = 1 << 30; const TOK_IOC: u32 = 1 << 15; const TOK_ACTIVE: u32 = 1 << 7; const TOK_HALT: u32 = 1 << 6; const TOK_XACT: u32 = 1 << 3; -const TOK_MMF: u32 = 1 << 2; // missed microframe: split did not complete in its window +const TOK_MMF: u32 = 1 << 2; const PID_IN: u32 = 1; const PID_SETUP: u32 = 2; const CCERR_3: u32 = 3 << 10; -// ---- Static control state ---- -static EHCI_BASE: AtomicU32 = AtomicU32::new(0); -static N_PORTS: AtomicU32 = AtomicU32::new(0); +// ---- Limits / DMA layout ---- +const MAX_CTRL: usize = 2; // OptiPlex 7010: 00:1a.0 + 00:1d.0 +const MAX_DEV: usize = 4; // mouse + keyboard on each controller +const DMA_MIN: usize = 0x100_0000; // avoid <16 MiB (PCI-master hostile region) +const DMA_MAX: usize = 0x1_0000_0000; // 32-bit EHCI DMA pointers + +const BUF_DEV_DESC: usize = 0x000; // 64 B +const BUF_CFG: usize = 0x100; // 512 B (hubs can have long configs) +const BUF_HUB_DESC: usize = 0x300; // 64 B +const BUF_PORT_STATUS: usize = 0x340; // 16 B +const BUF_REPORT: usize = 0x400; // 64 B per device + +/// One EHCI controller instance (addresses are per controller). +#[derive(Clone, Copy)] +struct Ctrl { + base: usize, + opb: usize, + nports: u32, + async_head: *mut u32, + xfer_qh: *mut u32, + framelist: *mut u32, + qh_page: *mut u32, + buf_page: *mut u8, +} + +impl Ctrl { + const EMPTY: Ctrl = Ctrl { + base: 0, + opb: 0, + nports: 0, + async_head: core::ptr::null_mut(), + xfer_qh: core::ptr::null_mut(), + framelist: core::ptr::null_mut(), + qh_page: core::ptr::null_mut(), + buf_page: core::ptr::null_mut(), + }; +} + +#[derive(Clone, Copy, PartialEq)] +enum Kind { + Mouse, + Keyboard, +} + +/// Transaction translator for a device behind a hub (`None` = high-speed +/// device or direct root-port high-speed). +#[derive(Clone, Copy)] +struct Tt { + hub: u8, + port: u8, +} + +/// One armed HID device. +#[derive(Clone, Copy)] +struct Dev { + kind: Kind, + ctrl: u32, + qh: *mut u32, + iqtd: *mut u32, + report: *mut u8, + len: u32, + kbd: crate::hid::Keyboard, + armed: bool, + failures: u32, + reports: u32, +} + +impl Dev { + const EMPTY: Dev = Dev { + kind: Kind::Mouse, + ctrl: 0, + qh: core::ptr::null_mut(), + iqtd: core::ptr::null_mut(), + report: core::ptr::null_mut(), + len: 0, + kbd: crate::hid::Keyboard::new(), + armed: false, + failures: 0, + reports: 0, + }; +} + +static mut CTRLS: [Ctrl; MAX_CTRL] = [Ctrl::EMPTY; MAX_CTRL]; +static mut DEVS: [Dev; MAX_DEV] = [Dev::EMPTY; MAX_DEV]; +static N_CTRL: AtomicU32 = AtomicU32::new(0); +static N_DEV: AtomicU32 = AtomicU32::new(0); +/// Controller currently being initialized (registers/pointers resolve here). +static CUR: AtomicU32 = AtomicU32::new(0); static CTRL_UP: AtomicBool = AtomicBool::new(false); -static MOUSE_PRES: AtomicBool = AtomicBool::new(false); -static DEV_ADDR: AtomicU32 = AtomicU32::new(0); -static HID_EP: AtomicU32 = AtomicU32::new(0); // EP number (1..15), IN -static HID_MPS: AtomicU32 = AtomicU32::new(0); -static ARMED: AtomicBool = AtomicBool::new(false); -// When the last enumerate attempt ran (ms): a failed/glitched mouse must -// never trigger a re-enumeration hot loop. -static LAST_ENUM: AtomicU32 = AtomicU32::new(0); -// Root ports observed in full/low-speed mode with a UHCI companion present. -// Those ports are owned by the companion controller — the EHCI can never -// talk to them, so they are skipped instead of failing a 5 s descriptor -// timeout every second (the "stuck in a loop" the OptiPlex showed). -static FS_LS_PORTS: AtomicU32 = AtomicU32::new(0); -// One-shot: the first transfer failure renders a diagnostic QR on screen -// (photograph it); retries keep dumping serial but must not re-render. +static LAST_SCAN: AtomicU32 = AtomicU32::new(0); static QR_SHOWN: AtomicBool = AtomicBool::new(false); -// DEBUG: run the one-boot transfer variant matrix (real-hardware bisection -// of the EHCI System Error trigger). MATRIX_DONE gates it to one pass; -// XFER_TWEAK flips construction variables inside ctl_xfer. -// -// OFF in normal builds: with it on, `enumerate()` is replaced by the -// diagnostic matrix and returns false, i.e. the EHCI driver never brings a -// device up. Flip to true only when bisecting a hardware fault, and expect -// no USB input in that build. -static DEBUG_MATRIX: bool = false; -static MATRIX_DONE: AtomicBool = AtomicBool::new(false); -static XFER_TWEAK: AtomicU32 = AtomicU32::new(0); -// Low-region (<1 MB) control QH for the matrix's low-region variant. -static mut LOW_XFER: *mut u32 = core::ptr::null_mut(); - -// DMA buffers. Allocated once at init; base pointers recorded here. -static mut ASYNC_HEAD: *mut u32 = core::ptr::null_mut(); // QH (32 bytes min, align 32) -static mut XFER_QH: *mut u32 = core::ptr::null_mut(); // control QH -static mut FRAMELIST: *mut u32 = core::ptr::null_mut(); // 1024 entries, 4K aligned -static mut HID_QH: *mut u32 = core::ptr::null_mut(); - -// One dedicated 4 KiB DMA page for every transfer buffer (device descriptor, -// config descriptor, HID report). All DMA targets live in controller-visible -// pages — never in .bss adjacent to kernel statics — so an overrun (report -// longer than MPS, descriptor longer than requested, short-packet arithmetic -// slip) can only clobber spare DMA space, on QEMU or on real silicon. -static mut BUF_PAGE: *mut u8 = core::ptr::null_mut(); -const BUF_DEV_DESC: usize = 0x000; // 64 B incl. red zone -const BUF_CFG: usize = 0x100; // 320 B incl. red zone -const BUF_HID_REPORT: usize = 0x400; // 64 B incl. red zone + +unsafe fn ctrl_mut(i: usize) -> &'static mut Ctrl { + unsafe { &mut *core::ptr::addr_of_mut!(CTRLS[i]) } +} +unsafe fn dev_mut(i: usize) -> &'static mut Dev { + unsafe { &mut *core::ptr::addr_of_mut!(DEVS[i]) } +} +fn cur_ctrl() -> &'static mut Ctrl { + unsafe { ctrl_mut(CUR.load(Ordering::Relaxed) as usize) } +} unsafe fn rd(a: usize) -> u32 { unsafe { (a as *const u32).read_volatile() } @@ -123,20 +180,17 @@ unsafe fn wr(a: usize, v: u32) { unsafe { (a as *mut u32).write_volatile(v) } } -fn opbase() -> usize { - unsafe { - EHCI_BASE.load(Ordering::Relaxed) as usize - + (rd(EHCI_BASE.load(Ordering::Relaxed) as usize) & 0xff) as usize - } -} fn op(off: usize) -> usize { - opbase() + off + cur_ctrl().opb + off } fn prtsc(p: usize) -> usize { - opbase() + OP_PRTSC_BASE + p * OP_PRTSC_STRIDE + op(OP_PRTSC_BASE + p * OP_PRTSC_STRIDE) +} +fn hostpc(p: usize) -> u32 { + unsafe { rd(cur_ctrl().base + 0x84 + p * 4) } } -// ---- QH/QTD helpers: all writes go through raw dword pointers ---- +// ---- QH/QTD dword helpers ---- fn qh_next(qh: *mut u32) -> *mut u32 { qh.wrapping_add(0) } @@ -161,7 +215,6 @@ fn qh_token(qh: *mut u32) -> *mut u32 { fn qh_buf(qh: *mut u32) -> *mut u32 { qh.wrapping_add(7) } - fn link_of(ptr: usize) -> u32 { (ptr as u32) | Q_TYPE_QH } @@ -177,52 +230,64 @@ unsafe fn wait_recl(timeout: usize) { } } -/// Issue a single control transfer on the async schedule. `dev` is the USB -/// device address (0 for the first GET_DESCRIPTOR), `mps` the EP0 max packet, -/// `speed` the device speed (0=full, 1=low, 2=high), `port` the root port -/// index (0-based; used only for full/low-speed split QHs), `req` the 8-byte -/// setup (bmRequestType,bRequest,..), `dir_in` true for a device->host data -/// stage, `buf/len` the data buffer. Returns bytes transferred, or a -/// negative error code (-1 timeout, -2 HALT, -3 XactErr, -4 missed uframe). +fn sleep_ms(ms: u64) { + let t0 = crate::interrupts::uptime_ms(); + while crate::interrupts::uptime_ms().saturating_sub(t0) < ms { + core::hint::spin_loop(); + } +} + +/// 4 KiB-aligned DMA page below 4 GiB (EHCI uses 32-bit pointers). +unsafe fn dma_alloc(bytes: usize, max_addr: usize) -> *mut u8 { + unsafe { + let pages = bytes.div_ceil(4096).max(1); + let Some(phys) = crate::memory::alloc_pages_range(pages, DMA_MIN, max_addr) else { + return core::ptr::null_mut(); + }; + let p = phys as *mut u8; + core::ptr::write_bytes(p, 0, pages * 4096); + p + } +} + +fn ctrl_req(bmreq: u8, breq: u8, wval: u16, widx: u16, wlen: u16) -> [u8; 8] { + [ + bmreq, + breq, + (wval & 0xff) as u8, + (wval >> 8) as u8, + (widx & 0xff) as u8, + (widx >> 8) as u8, + (wlen & 0xff) as u8, + (wlen >> 8) as u8, + ] +} + +/// One control transfer on the async schedule. `tt` selects split +/// transactions (device behind a hub, or a full/low-speed root port). +/// Returns bytes transferred or a negative error (-1 timeout, -2 halt, +/// -3 setup halted, -4 data halted, -5 XactErr, -6 missed microframe). unsafe fn ctl_xfer( dev: u32, mps: u32, speed: u32, - port: usize, + tt: Option, req: &[u8; 8], dir_in: bool, buf: *mut u8, len: u32, + diag_port: usize, ) -> i32 { - // DEBUG matrix hook: XFER_TWEAK bits flip construction variables. - // bit0: no-ASE arm; bit1: no-split fields; bit2: no qtd chain; - // bit3: zero-length (no DATA stage); bit6: short timeout (1.2 s). - let len = if XFER_TWEAK.load(Ordering::Relaxed) & 8 != 0 { - 0 - } else { - len - }; unsafe { - let head = ASYNC_HEAD; - let tw0 = XFER_TWEAK.load(Ordering::Relaxed); - // DEBUG matrix: execute the same transfer from the low (<1 MB) page. - let xfer = if tw0 & 0x80 != 0 && !LOW_XFER.is_null() { - LOW_XFER - } else { - XFER_QH - }; + let head = cur_ctrl().async_head; + let xfer = cur_ctrl().xfer_qh; - // QH occupies dwords 0..11 (48 bytes) incl. the 5-dword buffer list. - // QEMU's NLPTR_GET masks qtd addresses to 32-byte alignment - // (& 0xffffffe0), so every qtd must sit on a 32-byte boundary. - // Place them at +0x40 / +0x60 / +0x80 (each 8 dwords = 32 bytes). - let st = xfer.add(0x40 / 4); // SETUP qtd at QH+0x40 (32-aligned) - let dt = st.add(0x20 / 4); // DATA qtd at QH+0x60 (32-aligned) - let sp = dt.add(0x20 / 4); // STATUS qtd at QH+0x80 (32-aligned) - - // SETUP token: toggle 0, len 8, PID_SETUP, active, IOC off. - core::ptr::write_volatile(st.add(0), 0u32); // next (chained below) - core::ptr::write_volatile(st.add(1), QT_TERMINATE); // alt: terminate + let st = xfer.add(0x40 / 4); // SETUP qtd (32-byte aligned) + let dt = st.add(0x20 / 4); // DATA qtd + let sp = dt.add(0x20 / 4); // STATUS qtd + + core::ptr::write_volatile(st.add(0), 0u32); + core::ptr::write_volatile(st.add(1), QT_TERMINATE); core::ptr::write_volatile( st.add(2), (8 << 16) | CCERR_3 | (PID_SETUP << 8) | TOK_ACTIVE, @@ -230,61 +295,45 @@ unsafe fn ctl_xfer( for i in 0..5 { core::ptr::write_volatile(st.add(3 + i), 0u32); } - // Setup packet buffer: 8 bytes at +0x110 (clear of the QH + qtds). let sbuf = xfer.add(0x110 / 4); let sb = core::slice::from_raw_parts_mut(sbuf.cast::(), 8); sb.copy_from_slice(req); core::ptr::write_volatile(st.add(3), sbuf as u32); - // DATA qtd. EHCI token PID field (bits 9:8): OUT=0, IN=1, SETUP=2. let data_pid_w = (if dir_in { 1u32 } else { 0u32 }) << 8; if len == 0 { - // no data stage core::ptr::write_volatile(dt.add(2), 0u32); } else { core::ptr::write_volatile(dt.add(0), sp as u32); - // altnext MUST be terminate (1): QEMU's ADVANCEQUEUE follows altnext - // whenever TBYTES != 0 after an IN (e.g. a short packet). altnext=0 - // is treated as *valid* (bit0==0) so the controller would jump to - // qtd address 0 and hang. Terminate keeps short transfers chained - // to the STATUS qtd, as required for control transfers. core::ptr::write_volatile(dt.add(1), QT_TERMINATE); core::ptr::write_volatile( dt.add(2), - ((len & 0x7fff) << 16) | (1 << 31) | CCERR_3 | (data_pid_w) | TOK_ACTIVE, + ((len & 0x7fff) << 16) | (1 << 31) | CCERR_3 | data_pid_w | TOK_ACTIVE, ); for i in 0..5 { core::ptr::write_volatile(dt.add(3 + i), 0u32); } - // data buffer core::ptr::write_volatile(dt.add(3), buf as u32); } - // STATUS qtd: toggle 1, len 0, PID opposite of the data stage; for a - // zero-length control the status is opposite of the setup direction. let setup_in = (req[0] & 0x80) != 0; let status_pid_w = if len > 0 { - if dir_in { - 0u32 - } else { - 1u32 - } // opposite of data + if dir_in { 0u32 } else { 1u32 } } else if setup_in { 0u32 } else { 1u32 } << 8; - core::ptr::write_volatile(sp.add(0), 1u32); // terminate + core::ptr::write_volatile(sp.add(0), 1u32); core::ptr::write_volatile(sp.add(1), QT_TERMINATE); core::ptr::write_volatile( sp.add(2), - (1 << 31) | CCERR_3 | (status_pid_w) | TOK_ACTIVE | TOK_IOC, + (1 << 31) | CCERR_3 | status_pid_w | TOK_ACTIVE | TOK_IOC, ); for i in 0..5 { core::ptr::write_volatile(sp.add(3 + i), 0u32); } - // Chain the qtd list. if len == 0 { core::ptr::write_volatile(st.add(0), sp as u32); } else { @@ -292,24 +341,18 @@ unsafe fn ctl_xfer( core::ptr::write_volatile(dt.add(0), sp as u32); } - // Configure the control QH (EP0). let eps = match speed { 2 => QH_EPS_HIGH, 1 => QH_EPS_LOW, _ => QH_EPS_FULL, }; let control_flag = if speed == 2 { 0 } else { QH_CONTROL_EP }; - // FS/LS split QH: DTC (toggles come from the qTDs), NAK reload, MULT=1, - // hub addr 0 (integrated root-hub TT) and the 1-based root port number - // in bits 29:23. The controller runs SSPLIT/CSPLIT pairs for every qTD - // on this QH; no S/C masks are needed on the async list (Linux and - // FreeBSD both leave them 0 for control/bulk). - let info1 = (mps & 0x7ff) << 16 | QH_RL | QH_DTC | control_flag | eps | dev; // devaddr in low 7 bits + let info1 = (mps & 0x7ff) << 16 | QH_RL | QH_DTC | control_flag | eps | dev; *qh_info1(xfer) = info1; - *qh_info2(xfer) = if speed == 2 || XFER_TWEAK.load(Ordering::Relaxed) & 2 != 0 { - 0 - } else { - QH_MULT | (((port as u32) + 1) << 23) + // Split fields: hub address bits 22:16, port bits 29:23, MULT bit 30. + *qh_info2(xfer) = match tt { + Some(t) if speed != 2 => QH_MULT | ((t.hub as u32) << 16) | ((t.port as u32) << 23), + _ => 0, }; *qh_current(xfer) = 0; *qh_next_qtd(xfer) = st as u32; @@ -319,44 +362,20 @@ unsafe fn ctl_xfer( *qh_buf(xfer).add(i) = 0; } - // Link: head -> xfer -> head (reclamation cycle). *qh_next(xfer) = link_of(head as usize); *qh_next(head) = link_of(xfer as usize); - if XFER_TWEAK.load(Ordering::Relaxed) & 4 != 0 { - *qh_next_qtd(xfer) = 1; // no qtd chain: HC treats the QH as idle - } - // Ensure the controller is running and both schedules enabled. - // Re-assert every transfer: QEMU may drop RUNSTOP, and re-setting the - // bits is what makes it (re)schedule the async walk. - // - // Clear the W1C status bits first so a failure dump reflects only - // this transfer's window (USBINT|USBERR|PCD|FR|SE). wr(op(OPSTS), 0x1F); - // Fence: make all DMA writes globally visible before the controller reads. core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); - let tw = XFER_TWEAK.load(Ordering::Relaxed); - if tw & 0x20 != 0 { - // wire-off: disable the port so no transaction can reach the wire. - wr(prtsc(port), rd(prtsc(port)) & !(1 << 1)); // PED=0 - } - wr( - op(OPCMD), - rd(op(OPCMD)) | CMD_RUN | (if tw & 1 == 0 { CMD_ASE } else { 0 }), - ); // no PSE — periodic not needed for control - let ctl_timeout = if tw & 0x40 != 0 { 1200u64 } else { 5000u64 }; - // Poll for completion. Watch the whole chain: a HALT on the SETUP - // or DATA qtd stops the queue and the STATUS qtd would never clear - // (which previously read as a plain timeout). Exit early naming the - // failing stage, and on any failure dump the controller + queue state - // for real-hardware iteration. + wr(op(OPCMD), rd(op(OPCMD)) | CMD_RUN | CMD_ASE); + let t0 = crate::interrupts::uptime_ms(); - let mut stage: i32 = 0; // 0=ok, 1=setup halted, 2=data halted, 3=timeout, 4=hc halted/SE + let mut stage: i32 = 0; loop { let stok = core::ptr::read_volatile(st.add(2).cast::()); let ptok = core::ptr::read_volatile(sp.add(2).cast::()); if ptok & TOK_ACTIVE == 0 { - break; // status completed + break; } if stok & TOK_HALT != 0 { stage = 1; @@ -369,15 +388,11 @@ unsafe fn ctl_xfer( break; } } - // Controller-level fault: the HC halted itself (HCH) or threw a - // system error (SE). Polling further is pointless — the qtds - // would sit ACTIVE forever and read back as a plain timeout. - // Catching it here names the window precisely. if rd(op(OPSTS)) & (STS_HCH | STS_SE) != 0 { stage = 4; break; } - if t0 + ctl_timeout < crate::interrupts::uptime_ms() { + if t0 + 5000 < crate::interrupts::uptime_ms() { stage = 3; break; } @@ -392,50 +407,34 @@ unsafe fn ctl_xfer( }; let ptok = core::ptr::read_volatile(sp.add(2).cast::()); crate::serial_println!( - "ehci: xfer fail stage={} (1=setup 2=data 3=timeout) speed={} dev={} dir_in={} len={}", + "ehci: xfer fail stage={} speed={} dev={} in={} len={} tt={:?}", stage, speed, dev, dir_in, - len + len, + tt.map(|t| (t.hub, t.port)) ); - let base = EHCI_BASE.load(Ordering::Relaxed) as usize; crate::serial_println!( "ehci: cmd={:#x} sts={:#x} prtsc{}={:#x} hostpc={:#x}", rd(op(OPCMD)), rd(op(OPSTS)), - port, - rd(prtsc(port)), - rd(base + 0x84 + port * 4) + diag_port, + rd(prtsc(diag_port)), + hostpc(diag_port) ); crate::serial_println!( - "ehci: qh i1={:#x} i2={:#x} cur={:#x} nqtd={:#x}", + "ehci: qh i1={:#x} i2={:#x} nqtd={:#x} setup={:#x} data={:#x} stat={:#x}", core::ptr::read_volatile(qh_info1(xfer)), core::ptr::read_volatile(qh_info2(xfer)), - core::ptr::read_volatile(qh_current(xfer)), - core::ptr::read_volatile(qh_next_qtd(xfer)) - ); - crate::serial_println!( - "ehci: tok setup={:#x} data={:#x} stat={:#x}", + core::ptr::read_volatile(qh_next_qtd(xfer)), stok, dtok, ptok ); - if stage == 4 { - crate::serial_println!( - "ehci: HC halted/SE after {} ms since arm (frindex={:#x} asyncbase={:#x})", - crate::interrupts::uptime_ms().wrapping_sub(t0), - rd(op(OPFRIDX)), - rd(op(0x18)) - ); - } - // One-shot on-screen QR of the whole diagnostic: band text plus a - // scannable matrix below. Used on hardware with no serial pipe — - // point a phone at the screen. Lock the klog screen echo so the - // retry loop cannot overwrite it (serial keeps logging). if !QR_SHOWN.swap(true, Ordering::Relaxed) { let l1 = alloc::format!( - "EHCI ctl fail stage={} t={}ms speed={} dev={} in={} len={}", + "EHCI fail stage={} t={}ms sp={} dev={} in={} len={}", stage, crate::interrupts::uptime_ms().wrapping_sub(t0), speed, @@ -446,27 +445,18 @@ unsafe fn ctl_xfer( let l2 = alloc::format!("cmd={:08x} sts={:08x}", rd(op(OPCMD)), rd(op(OPSTS))); let l3 = alloc::format!( "prtsc{}=0x{:08x} hp=0x{:08x}", - port, - rd(prtsc(port)), - rd(base + 0x84 + port * 4) + diag_port, + rd(prtsc(diag_port)), + hostpc(diag_port) ); let l4 = alloc::format!( - "qh1={:08x} qh2={:08x} cur={:08x} nqtd={:08x}", + "qh1={:08x} qh2={:08x} nqtd={:08x}", core::ptr::read_volatile(qh_info1(xfer)), core::ptr::read_volatile(qh_info2(xfer)), - core::ptr::read_volatile(qh_current(xfer)), core::ptr::read_volatile(qh_next_qtd(xfer)) ); let l5 = alloc::format!("setup={:08x} data={:08x} stat={:08x}", stok, dtok, ptok); - let l6 = alloc::format!( - "dma a={:08x} x={:08x} f={:08x} h={:08x} b={:08x}", - ASYNC_HEAD as usize, - XFER_QH as usize, - FRAMELIST as usize, - HID_QH as usize, - BUF_PAGE as usize - ); - let all = [l1, l2, l3, l4, l5, l6]; + let all = [l1, l2, l3, l4, l5]; let refs: alloc::vec::Vec<&str> = all.iter().map(|s| s.as_str()).collect(); crate::gop::dump_qr(&refs); crate::gop::qr_lock(); @@ -474,334 +464,720 @@ unsafe fn ctl_xfer( *qh_next(head) = link_of(head as usize); wait_recl(100); return match stage { - 1 => -3, // setup halted - 2 => -4, // data halted - _ => -1, // timeout + 1 => -3, + 2 => -4, + _ => -1, }; } - // Status completed — check its error bits. let tok = core::ptr::read_volatile(sp.add(2).cast::()); if tok & TOK_HALT != 0 { *qh_next(head) = link_of(head as usize); wait_recl(100); - return -2; // HALT — qtd halted on error (babble/DBE/...) + return -2; } if tok & TOK_XACT != 0 { *qh_next(head) = link_of(head as usize); wait_recl(100); - return -5; // XactErr — CRC/timeout on the transaction + return -5; } if tok & TOK_MMF != 0 { *qh_next(head) = link_of(head as usize); wait_recl(100); - return -6; // missed microframe — split window missed (retryable) + return -6; } - // Bytes transferred = requested - remaining on the DATA qtd. let mut got = len; if len > 0 { let dtok = core::ptr::read_volatile(dt.add(2).cast::()); let rem = ((dtok >> 16) & 0x7fff) as u32; got = len - rem.min(len); } - // Unlink: idle self-loop must keep the QH type bit, otherwise QEMU - // parses it as an iTD and resets the HC ("non queue head request"). *qh_next(head) = link_of(head as usize); wait_recl(100); - let _ = qh_next_qtd; got as i32 } } -// Control-transfer convenience using the standard control request layout. -fn ctrl_req(bmreq: u8, breq: u8, wval: u16, widx: u16, wlen: u16) -> [u8; 8] { - [ - bmreq, - breq, - (wval & 0xff) as u8, - (wval >> 8) as u8, - (widx & 0xff) as u8, - (widx >> 8) as u8, - (wlen & 0xff) as u8, - (wlen >> 8) as u8, - ] +// --------------------------------------------------------------------------- +// Descriptor parsing +// --------------------------------------------------------------------------- + +const HID_CLASS: u8 = 0x03; +const HID_SUBCLASS_BOOT: u8 = 0x01; +const HID_PROTO_KEYBOARD: u8 = 1; +const HID_PROTO_MOUSE: u8 = 2; + +/// Walk a config descriptor: find the first boot HID mouse/keyboard +/// interface and its interrupt-IN endpoint. +/// Returns (kind, interface number, ep address, mps, interval). +fn find_hid(cfg: &[u8]) -> Option<(Kind, u8, u8, u32, u32)> { + let mut i = 0usize; + let mut cand: Option<(Kind, u8)> = None; + while i + 2 <= cfg.len() { + let blen = cfg[i] as usize; + if blen < 2 { + break; + } + let bt = cfg[i + 1]; + if bt == 4 && blen >= 9 && i + 9 <= cfg.len() { + // interface: class/subclass/protocol at +5/+6/+7 + let iface = cfg[i + 2]; + let class = cfg[i + 5]; + let sub = cfg[i + 6]; + let proto = cfg[i + 7]; + cand = if class == HID_CLASS && sub == HID_SUBCLASS_BOOT { + match proto { + HID_PROTO_KEYBOARD => Some((Kind::Keyboard, iface)), + HID_PROTO_MOUSE => Some((Kind::Mouse, iface)), + _ => None, + } + } else { + None + }; + } else if bt == 5 && blen >= 7 && i + 7 <= cfg.len() { + if let Some((kind, iface)) = cand { + let addr = cfg[i + 2]; + let attrs = cfg[i + 3]; + let mps = (cfg[i + 4] as u32) | ((cfg[i + 5] as u32) << 8); + let interval = cfg[i + 6] as u32; + if (addr & 0x80) != 0 && (attrs & 0x03) == 0x03 { + return Some((kind, iface, addr, mps, interval)); + } + } + } + i += blen; + } + None +} + +/// Hub class descriptor: port count lives at offset 2. +fn hub_ports(desc: &[u8]) -> u32 { + if desc.len() >= 3 && desc[1] == 0x29 { + desc[2] as u32 + } else { + 0 + } } // --------------------------------------------------------------------------- -// Controller bring-up +// Hub port control (USB 2.0 hub class requests) // --------------------------------------------------------------------------- -/// Find an EHCI controller (class 0x0C class 0x03, prog_if 0x20). -fn find_ehci() -> Option<(u8, u8, u8)> { - for d in crate::pci::probe() { - if d.class == 0x0C && d.subclass == 0x03 && d.prog_if == 0x20 { - return Some((d.bus, d.device, d.func)); +const HUB_GET_STATUS: u8 = 0x00; +const HUB_CLEAR_FEATURE: u8 = 0x01; +const HUB_SET_FEATURE: u8 = 0x03; + +const PORT_RESET_FEAT: u16 = 4; +const PORT_POWER_FEAT: u16 = 8; +const C_PORT_RESET_FEAT: u16 = 20; + +const PS_CONNECTION: u32 = 1 << 0; +const PS_ENABLE: u32 = 1 << 1; +const PS_RESET: u32 = 1 << 4; +const PS_LOW_SPEED: u32 = 1 << 9; +const PS_HIGH_SPEED: u32 = 1 << 10; + +/// Read (status, change) for one hub port. +unsafe fn hub_port_status(hub_addr: u8, hub_mps: u32, port: u16) -> Option<(u32, u32)> { + unsafe { + let buf = cur_ctrl().buf_page.add(BUF_PORT_STATUS); + core::ptr::write_bytes(buf, 0, 8); + let req = ctrl_req(0xA3, HUB_GET_STATUS, 0, port, 4); + let r = ctl_xfer( + hub_addr as u32, + hub_mps, + 2, + None, + &req, + true, + buf, + 4, + port as usize, + ); + if r < 4 { + return None; } + let b = core::slice::from_raw_parts(buf, 4); + let st = u16::from_le_bytes([b[0], b[1]]) as u32; + let ch = u16::from_le_bytes([b[2], b[3]]) as u32; + Some((st, ch)) } - None } -/// Return the number of the first root port with a device attached, or usize::MAX. -fn find_connected_port() -> usize { - let np = N_PORTS.load(Ordering::Relaxed) as usize; - let skip = FS_LS_PORTS.load(Ordering::Relaxed); - for p in 0..np { - if skip & (1 << p) != 0 { - continue; // full/low-speed — owned by the companion UHCI +unsafe fn hub_feature(hub_addr: u8, hub_mps: u32, set: bool, feat: u16, port: u16) { + unsafe { + let req = ctrl_req( + if set { 0x23 } else { 0x23 }, + if set { HUB_SET_FEATURE } else { HUB_CLEAR_FEATURE }, + feat, + port, + 0, + ); + let _ = ctl_xfer( + hub_addr as u32, + hub_mps, + 2, + None, + &req, + false, + core::ptr::null_mut(), + 0, + port as usize, + ); + } +} + +// --------------------------------------------------------------------------- +// Enumeration +// --------------------------------------------------------------------------- + +static NEXT_DEV_ADDR: AtomicU32 = AtomicU32::new(1); + +fn take_addr() -> u8 { + NEXT_DEV_ADDR.fetch_add(1, Ordering::Relaxed) as u8 +} + +/// Register + arm one HID device. +unsafe fn register_hid( + ci: usize, + addr: u8, + kind: Kind, + ep: u8, + mps: u32, + interval: u32, + speed: u32, + tt: Option, +) -> bool { + unsafe { + let slot = N_DEV.load(Ordering::Relaxed) as usize; + if slot >= MAX_DEV { + return false; + } + let c = ctrl_mut(ci); + let qh = c.qh_page.add(slot * 0x100 / 4); + let iqtd = qh.add(0x40 / 4); + let report = c.buf_page.add(BUF_REPORT + slot * 64); + core::ptr::write_bytes(qh as *mut u8, 0, 0x100); + core::ptr::write_bytes(report, 0, 64); + + let len = mps.min(8).max(1); + let info1 = QH_RL + | (speed << 12) + | ((mps.max(4) & 0x7ff) << 16) + | (addr as u32 & 0x7f) + | (((ep as u32) & 0x0f) << 8); + let info2 = match tt { + Some(t) if speed != 2 => { + let (smask, cmask) = (0x01u32, 0x1Cu32 << 8); + smask | cmask | QH_MULT | ((t.hub as u32) << 16) | ((t.port as u32) << 23) + } + _ => 0xFF, // high-speed: schedule every microframe + }; + *qh_next(qh) = QT_TERMINATE; + *qh_info1(qh) = info1; + *qh_info2(qh) = info2; + *qh_current(qh) = 0; + *qh_alt_qtd(qh) = QT_TERMINATE; + *qh_token(qh) = 0; + for i in 0..5 { + *qh_buf(qh).add(i) = 0; } - let v = unsafe { rd(prtsc(p)) }; - if v & PRT_CONNECT != 0 { - return p; + core::ptr::write_volatile(iqtd.add(0), QT_TERMINATE); + core::ptr::write_volatile(iqtd.add(1), QT_TERMINATE); + core::ptr::write_volatile( + iqtd.add(2), + ((len & 0x7fff) << 16) | (PID_IN << 8) | TOK_ACTIVE | TOK_IOC, + ); + for i in 0..5 { + core::ptr::write_volatile(iqtd.add(3 + i), 0u32); } + core::ptr::write_volatile(iqtd.add(3), report as u32); + *qh_next_qtd(qh) = iqtd as u32; + + let d = dev_mut(slot); + d.kind = kind; + d.ctrl = ci as u32; + d.qh = qh; + d.iqtd = iqtd; + d.report = report; + d.len = len; + d.kbd = crate::hid::Keyboard::new(); + d.armed = true; + d.failures = 0; + d.reports = 0; + N_DEV.fetch_add(1, Ordering::Relaxed); + crate::serial_println!( + "ehci: HID {} armed (addr {} ep{:#x} mps {} interval {})", + if kind == Kind::Mouse { "mouse" } else { "keyboard" }, + addr, + ep, + mps, + interval + ); + true } - usize::MAX } -/// Clear the FS/LS port mask so discovery retries (shell `usb rescan`). -pub fn rescan() { - FS_LS_PORTS.store(0, Ordering::Relaxed); +/// Enumerate the device at `addr` (0 = unaddressed) on the current +/// controller. Handles HID devices and one level of USB hub. +/// Returns the number of HID devices registered. +unsafe fn enumerate(addr: u8, speed: u32, tt: Option, diag_port: usize) -> u32 { + unsafe { + let mut addr = addr; + let mut mps = if speed == 2 { 64u32 } else { 8u32 }; + let cbuf = cur_ctrl().buf_page; + + // 1. Device descriptor (learn bMaxPacketSize0 + class). + let dd = cbuf.add(BUF_DEV_DESC); + core::ptr::write_bytes(dd, 0, 64); + let req = ctrl_req(0x80, 6, 0x0100, 0, 18); + let mut got = ctl_xfer( + addr as u32, + mps, + speed, + tt, + &req, + true, + dd, + 18, + diag_port, + ); + if got < 8 { + crate::serial_println!( + "ehci: get-dev-desc failed (addr {} sp {} tt {:?}) code={}", + addr, + speed, + tt.map(|t| (t.hub, t.port)), + got + ); + return 0; + } + let dd = core::slice::from_raw_parts(dd, 18); + let class = dd[4]; + let mps0 = dd[7] as u32; + if mps0 != 0 { + mps = mps0; + } + + // 2. Assign an address (root/unaddressed devices only). + if addr == 0 { + let new = take_addr(); + let req = ctrl_req(0x00, 5, new as u16, 0, 0); + got = ctl_xfer( + 0, + mps, + speed, + tt, + &req, + false, + core::ptr::null_mut(), + 0, + diag_port, + ); + if got < 0 { + crate::serial_println!("ehci: set-address failed code={}", got); + return 0; + } + addr = new; + sleep_ms(2); // USB 2.0 §9.2.6.3: 2 ms recovery + } + + crate::serial_println!( + "ehci: dev {:04x}:{:04x} class={:#x} mps0={} addr={} sp={}{}", + u16::from_le_bytes([dd[8], dd[9]]), + u16::from_le_bytes([dd[10], dd[11]]), + class, + mps, + addr, + speed, + if tt.is_some() { " (split)" } else { "" } + ); + + // 3. Hub: configure it, then scan its ports for HID devices. + if class == 9 { + return scan_hub(addr, mps, diag_port); + } + + // 4. HID: read the config descriptor. + let cfg = cbuf.add(BUF_CFG); + core::ptr::write_bytes(cfg, 0, 512); + let req = ctrl_req(0x80, 6, 0x0200, 0, 512); + got = ctl_xfer(addr as u32, mps, speed, tt, &req, true, cfg, 512, diag_port); + if got < 9 { + crate::serial_println!("ehci: get-config failed code={}", got); + return 0; + } + let cfgs = core::slice::from_raw_parts(cfg, got as usize); + let Some((kind, iface, ep, epmps, interval)) = find_hid(cfgs) else { + crate::serial_println!("ehci: no HID mouse/keyboard endpoint in config"); + return 0; + }; + + // 5. SET_CONFIGURATION(config 1) then boot protocol for the interface. + let cfgval = cfgs[5]; + if cfgval == 0 { + return 0; + } + let req = ctrl_req(0x00, 9, cfgval as u16, 0, 0); + if ctl_xfer(addr as u32, mps, speed, tt, &req, false, core::ptr::null_mut(), 0, diag_port) + < 0 + { + crate::serial_println!("ehci: set-config failed"); + return 0; + } + // HID class SET_PROTOCOL(boot) — report layout becomes fixed. + let req = ctrl_req(0x21, 0x0B, 0, iface as u16, 0); + if ctl_xfer(addr as u32, mps, speed, tt, &req, false, core::ptr::null_mut(), 0, diag_port) + < 0 + { + crate::serial_println!("ehci: set-protocol(boot) stalled (continuing)"); + } + // HID SET_IDLE(0) — only report on change; some devices stall it. + let req = ctrl_req(0x21, 0x0A, 0, iface as u16, 0); + let _ = ctl_xfer(addr as u32, mps, speed, tt, &req, false, core::ptr::null_mut(), 0, diag_port); + + let ci = CUR.load(Ordering::Relaxed) as usize; + if register_hid(ci, addr, kind, ep, epmps, interval, speed, tt) { + 1 + } else { + 0 + } + } } -pub fn init() -> bool { - let Some((b, d, f)) = find_ehci() else { - crate::serial_println!("ehci: no EHCI controller"); - return false; - }; - crate::serial_println!("ehci: USB2 EHCI at {:02x}:{:02x}.{}", b, d, f); +/// Configure a USB hub, then enumerate every connected port's device. +unsafe fn scan_hub(hub_addr: u8, hub_mps: u32, diag_port: usize) -> u32 { + unsafe { + let cbuf = cur_ctrl().buf_page; + // Config + SET_CONFIGURATION. + let cfg = cbuf.add(BUF_CFG); + core::ptr::write_bytes(cfg, 0, 512); + let req = ctrl_req(0x80, 6, 0x0200, 0, 512); + let got = ctl_xfer(hub_addr as u32, hub_mps, 2, None, &req, true, cfg, 512, diag_port); + if got < 9 { + crate::serial_println!("ehci: hub get-config failed code={}", got); + return 0; + } + let cfgval = core::slice::from_raw_parts(cfg, got as usize)[5]; + let req = ctrl_req(0x00, 9, cfgval as u16, 0, 0); + if ctl_xfer(hub_addr as u32, hub_mps, 2, None, &req, false, core::ptr::null_mut(), 0, diag_port) + < 0 + { + crate::serial_println!("ehci: hub set-config failed"); + return 0; + } + + // Hub class descriptor -> port count. + let hd = cbuf.add(BUF_HUB_DESC); + core::ptr::write_bytes(hd, 0, 64); + let req = ctrl_req(0xA0, 6, 0x2900, 0, 9); + let got = ctl_xfer(hub_addr as u32, hub_mps, 2, None, &req, true, hd, 9, diag_port); + if got < 3 { + crate::serial_println!("ehci: hub descriptor failed code={}", got); + return 0; + } + let hd = core::slice::from_raw_parts(hd, got as usize); + let nports = hub_ports(hd); + crate::serial_println!("ehci: hub addr={} ports={}", hub_addr, nports); + // Power every port, then give the hub time to settle. + for p in 1..=nports as u16 { + hub_feature(hub_addr, hub_mps, true, PORT_POWER_FEAT, p); + } + sleep_ms(100); + + let mut found = 0u32; + for p in 1..=nports as u16 { + let Some((st, _ch)) = hub_port_status(hub_addr, hub_mps, p) else { + continue; + }; + if st & PS_CONNECTION == 0 { + continue; + } + // Reset the port and wait for the hub to finish. + hub_feature(hub_addr, hub_mps, true, PORT_RESET_FEAT, p); + let t0 = crate::interrupts::uptime_ms(); + let mut reset_done = false; + while crate::interrupts::uptime_ms().saturating_sub(t0) < 500 { + if let Some((s, _)) = hub_port_status(hub_addr, hub_mps, p) + && s & PS_RESET == 0 + { + reset_done = true; + break; + } + sleep_ms(10); + } + if !reset_done { + crate::serial_println!("ehci: hub port {} reset timeout", p); + continue; + } + hub_feature(hub_addr, hub_mps, false, C_PORT_RESET_FEAT, p); + + let Some((st, _)) = hub_port_status(hub_addr, hub_mps, p) else { + continue; + }; + if st & PS_ENABLE == 0 { + crate::serial_println!("ehci: hub port {} not enabled (st={:#x})", p, st); + continue; + } + let speed = if st & PS_HIGH_SPEED != 0 { + 2 + } else if st & PS_LOW_SPEED != 0 { + 1 + } else { + 0 + }; + let child_tt = if speed == 2 { + None + } else { + Some(Tt { + hub: hub_addr, + port: p as u8, + }) + }; + crate::serial_println!( + "ehci: hub port {}: {} device{}", + p, + match speed { + 2 => "high-speed", + 1 => "low-speed", + _ => "full-speed", + }, + if child_tt.is_some() { " (split)" } else { "" } + ); + if st & PS_CONNECTION == 0 { + continue; + } + found += enumerate(0, speed, child_tt, diag_port); + } + found + } +} + +/// Bring one controller up and scan it. +unsafe fn init_one(bus: u8, dev: u8, func: u8) -> bool { unsafe { - // Enable Memory Space + Bus Master, and disable INTx assertion - // (bit 10) — the driver is polled (OPINTR=0), and an unhandled - // EHCI interrupt on real hardware would fire a stray IRQ that the - // kernel's catch-all handler EOI's but cannot service. - let cmd = crate::pci::config_read(b, d, f, 0x04); + let ci = N_CTRL.load(Ordering::Relaxed) as usize; + if ci >= MAX_CTRL { + return false; + } + CUR.store(ci as u32, Ordering::Relaxed); + NEXT_DEV_ADDR.store(1, Ordering::Relaxed); + + // Memory space + bus master, INTx disabled (polled). + let cmd = crate::pci::config_read(bus, dev, func, 0x04); crate::pci::config_write( - b, - d, - f, + bus, + dev, + func, 0x04, (cmd & 0xFFFF_0000) | (cmd & 0xFFFF) | 0x6 | 0x400, ); - let bar = crate::pci::bar64(b, d, f, 0); + let bar = crate::pci::bar64(bus, dev, func, 0); let base = bar.unwrap_or(0) as usize; - EHCI_BASE.store(base as u32, Ordering::Relaxed); - let cap = rd(base); - let sp = rd(base + 4); - let hc = rd(base + 8); - let np = sp & 0xf; - N_PORTS.store(np, Ordering::Relaxed); - crate::serial_println!( - "ehci: mmio={:#x} caplen={:#x} ports={} 64-bit-caps={}", - base, - cap & 0xff, - np, - (hc & 1) != 0 - ); if base == 0 { + crate::serial_println!("ehci: bad BAR"); return false; } + let caplen = (rd(base) & 0xff) as usize; + let opb = base + caplen; + let nports = rd(base + 4) & 0xf; + crate::serial_println!( + "ehci: {:02x}:{:02x}.{} mmio={:#x} caplen={:#x} ports={} 64-bit-caps={}", + bus, + dev, + func, + base, + caplen, + nports, + (rd(base + 8) & 1) != 0 + ); - // Claim USB Legacy ownership from the BIOS before resetting the - // controller. The PCH BIOS drives USB keyboards through SMM - // legacy emulation while it owns the EHCI; resetting/reprogramming - // the controller underneath it silently kills the legacy keyboard - // (observed on the 7010) and the still-active SMI poll can collide - // with our async DMA — a Host System Error. The capability lives - // in PCI config space (EHCI spec §5.1); the claim walks it exactly - // like Linux/FreeBSD. Bounded, no-panic; reports what happened. - match crate::pci::claim_usb_legacy(b, d, f, base, 0x08, false) { - 0 => crate::serial_println!("ehci: USB legacy: already released (no claim needed)"), + // BIOS legacy ownership: resetting the controller while SMM owns it + // kills the firmware keyboard path and can collide with our DMA. + match crate::pci::claim_usb_legacy(bus, dev, func, base, 0x08, false) { + 0 => crate::serial_println!("ehci: USB legacy: already released"), 1 => crate::serial_println!("ehci: USB legacy handed over by BIOS"), - 2 => crate::serial_println!("ehci: USB legacy claim timed out (BIOS sem stuck)"), - _ => crate::serial_println!("ehci: USB legacy: no capability found"), + 2 => crate::serial_println!("ehci: USB legacy claim timed out"), + _ => crate::serial_println!("ehci: USB legacy: no capability"), + } + + { + let c = ctrl_mut(ci); + c.base = base; + c.opb = opb; + c.nports = nports; } - // Halt the controller. + // Halt, reset, wait for reset to clear. wr(op(OPCMD), 0); let mut t = 0; while t < 1_000_000 && (rd(op(OPSTS)) & STS_HCH) == 0 { t += 1; core::hint::spin_loop(); } - crate::serial_println!( - "ehci: halted {}", - if rd(op(OPSTS)) & STS_HCH != 0 { - "ok" - } else { - "TIMEOUT" - } - ); - // Ensure not in reset already; then issue HC reset. wr(op(OPCMD), CMD_HCRESET); t = 0; while t < 10_000_000 && (rd(op(OPCMD)) & CMD_HCRESET) != 0 { t += 1; core::hint::spin_loop(); } - crate::serial_println!( - "ehci: hcreset {}", - if rd(op(OPCMD)) & CMD_HCRESET == 0 { - "ok" - } else { - "TIMEOUT" - } - ); t = 0; while t < 10_000_000 && (rd(op(OPSTS)) & STS_HCH) == 0 { t += 1; core::hint::spin_loop(); } - crate::serial_println!( - "ehci: post-reset halt {}", - if rd(op(OPSTS)) & STS_HCH != 0 { - "ok" - } else { - "TIMEOUT" - } - ); - // Disable interrupts (polled). wr(op(OPINTR), 0); - // Clear HCHalted? After reset the HC is halted; running is set later. - - // Allocate the async head QH + control QH + frame list + HID structures. - ASYNC_HEAD = dma_alloc(0x400, 0x20) as *mut u32; // 1KB region - XFER_QH = ASYNC_HEAD.add(0x80 / 4); // control QH at +0x80 - // qtds live after XFER_QH's 11 dwords (44 bytes); space reserved. - FRAMELIST = dma_alloc(0x1000, 0x1000) as *mut u32; - HID_QH = dma_alloc(0x400, 0x20) as *mut u32; - // DEBUG matrix: low-region control QH (sub-1MB page) to test whether - // the SE is tied to the 16 MB DMA region (write protection) or not. - if let Some(lp) = crate::memory::alloc_pages_above(2, 0) { - LOW_XFER = lp as *mut u32; - crate::serial_println!("ehci: low-xfer debug page at {:#x}", lp); - } - BUF_PAGE = dma_alloc(0x1000, 0x20); - if BUF_PAGE.is_null() { - crate::serial_println!("ehci: buffer page allocation failed"); + + // DMA structures (<4 GiB, >=16 MiB). + let async_head = dma_alloc(0x400, DMA_MAX) as *mut u32; + let framelist = dma_alloc(0x1000, DMA_MAX) as *mut u32; + let qh_page = dma_alloc(0x1000, DMA_MAX) as *mut u32; + let buf_page = dma_alloc(0x1000, DMA_MAX); + if async_head.is_null() || framelist.is_null() || qh_page.is_null() || buf_page.is_null() { + crate::serial_println!("ehci: DMA allocation failed"); return false; } + { + let c = ctrl_mut(ci); + c.async_head = async_head; + c.xfer_qh = async_head.add(0x80 / 4); + c.framelist = framelist; + c.qh_page = qh_page; + c.buf_page = buf_page; + } crate::serial_println!( - "ehci: dma async={:#x} xfer={:#x} framelist={:#x} hid={:#x} buf={:#x}", - ASYNC_HEAD as usize, - XFER_QH as usize, - FRAMELIST as usize, - HID_QH as usize, - BUF_PAGE as usize + "ehci: dma async={:#x} framelist={:#x} qh={:#x} buf={:#x}", + async_head as usize, + framelist as usize, + qh_page as usize, + buf_page as usize ); - // Init async head QH: self-linked, marked as head of reclamation. - *qh_next(ASYNC_HEAD) = link_of(XFER_QH as usize); // circular: head → control QH → head - *qh_info1(ASYNC_HEAD) = QH_HEAD; - *qh_info2(ASYNC_HEAD) = 0; - *qh_current(ASYNC_HEAD) = 0; - *qh_next_qtd(ASYNC_HEAD) = 0; - - // Configure CONTROL QH as an idle EP0 (filled per transfer). - *qh_next(XFER_QH) = link_of(ASYNC_HEAD as usize); - *qh_info1(XFER_QH) = QH_EPS_FULL | (64 << 16); // HS mps 64 - *qh_info2(XFER_QH) = 0; - *qh_current(XFER_QH) = 0; - *qh_next_qtd(XFER_QH) = 0; // idle - *qh_alt_qtd(XFER_QH) = QT_TERMINATE; - *qh_token(XFER_QH) = 0; - - // Async schedule address. + let xfer_qh = cur_ctrl().xfer_qh; + *qh_next(async_head) = link_of(cur_ctrl().xfer_qh as usize); + *qh_info1(async_head) = QH_HEAD; + *qh_info2(async_head) = 0; + *qh_current(async_head) = 0; + *qh_next_qtd(async_head) = 0; + *qh_next(xfer_qh) = link_of(async_head as usize); + *qh_info1(xfer_qh) = QH_EPS_FULL | (64 << 16); + *qh_info2(xfer_qh) = 0; + *qh_current(xfer_qh) = 0; + *qh_next_qtd(xfer_qh) = 0; + *qh_alt_qtd(xfer_qh) = QT_TERMINATE; + *qh_token(xfer_qh) = 0; + for i in 0..5 { + *qh_buf(xfer_qh).add(i) = 0; + } + wr(op(OPFRIDX), 0); - // Set ASYNCLISTADDR (32-bit). Controller supports 64-bit (hcc&1) but we - // keep everything below 4 GB; CTRLDSSEGMENT=0 suffices. - // ASYNCLISTADDR low 32 bits: address of HEAD QH (lower 5 bits zero). - // With hcc 64-bit set, the controller reads the 64-bit address - // CTRLDSSEGMENT:ASYNCLISTADDR. Zero DSEG + low 32-bit pointer is fine - // for addresses < 4 GB. - let head_addr = ASYNC_HEAD as usize; - wr(op(0x10), 0); // CTRLDSSEGMENT - // ASYNCLISTADDR is at op+0x18 - wr(op(0x18), (head_addr as u32) & !0x1F); - // Periodic frame list address. - // Zero the frame list to avoid stale QH/ITD pointers causing - // QEMU to reset the controller when PSE is later enabled. - let fl = FRAMELIST as usize; + wr(op(0x10), 0); // CTRLDSSEGMENT: all our DMA is below 4 GiB + wr(op(0x18), (async_head as usize as u32) & !0x1F); for i in 0..1024 { - core::ptr::write_volatile(FRAMELIST.add(i), 0u32); + core::ptr::write_volatile(framelist.add(i), 0u32); } - wr(op(0x14), (fl as u32) & !0x3FF); - // Frame list size: FLS field (bits 2-3) stays 0 → 1024 entries. - // Intel/QEMU layout: PSE=bit4, ASE=bit5. - // NOTE: do NOT enable PSE here — the frame list is empty (all zeros). - // Enabling PSE on an empty frame list is fine (QEMU skips zero entries), - // but we want to avoid any race with stale OVMF entries. PSE will be - // enabled later in poll_mouse when the HID QH is linked into the list. - wr(op(OPCMD), 0x01u32); // CMD_RUN only, no PSE - wr(op(OPCMD), rd(op(OPCMD)) | CMD_RUN); - // Wait to leave halt. - let mut t = 0; + wr(op(0x14), (framelist as usize as u32) & !0x3FF); + wr(op(OPCMD), CMD_RUN); + t = 0; while t < 1_000_000 && (rd(op(OPSTS)) & STS_HCH) != 0 { t += 1; core::hint::spin_loop(); } - crate::serial_println!( - "ehci: running {}", - if rd(op(OPSTS)) & STS_HCH == 0 { - "ok" - } else { - "STUCK HALTED" - } - ); - // Configure flag: route all ports to the host (CF=1) and power them. - wr(op(OPCFGFLAG), 1); - for p in 0..np as usize { + wr(op(OPCFGFLAG), 1); // route all ports to the host + for p in 0..nports as usize { let psc = prtsc(p); let v = rd(psc); if v & PRT_POWER == 0 { wr(psc, v | PRT_POWER); } } + sleep_ms(20); + crate::serial_println!("ehci: controller up ({} ports)", nports); + + N_CTRL.store((ci + 1) as u32, Ordering::Relaxed); CTRL_UP.store(true, Ordering::Relaxed); - crate::serial_println!("ehci: controller up ({} ports)", np); - // DIAGNOSTIC: run the async schedule with an idle head (self-loop, - // no qtds) for 150 ms. If this faults (HCH/SE), the SE happens on - // schedule/DMA mechanics alone — nothing to do with the split - // transfer. If clean, the fault is specific to transfer execution. - *qh_next(ASYNC_HEAD) = link_of(ASYNC_HEAD as usize); - wr(op(OPSTS), 0x1F); - core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); - wr(op(OPCMD), rd(op(OPCMD)) | CMD_RUN | CMD_ASE); - let pt0 = crate::interrupts::uptime_ms(); - loop { - let s = rd(op(OPSTS)); - if s & (STS_HCH | STS_SE) != 0 { + // ---- scan ports ---- + for p in 0..nports as usize { + let v = rd(prtsc(p)); + if v & PRT_CONNECT == 0 { + continue; + } + let hs = reset_port(p); + if !hs && crate::uhci::controller_up() { crate::serial_println!( - "ehci: idle-probe FAULTED {}ms: cmd={:#x} sts={:#x} frindex={:#x} asyncbase={:#x} prtsc0={:#x}", - crate::interrupts::uptime_ms().wrapping_sub(pt0), - rd(op(OPCMD)), - s, - rd(op(OPFRIDX)), - rd(op(0x18)), - rd(prtsc(0)) + "ehci: port {} full/low-speed — owned by companion UHCI", + p ); - break; + continue; } - if crate::interrupts::uptime_ms().wrapping_sub(pt0) >= 150 { - crate::serial_println!("ehci: idle-probe ok (150ms, no fault)"); - break; + let speed = if hs { 2 } else { port_speed(p) }; + crate::serial_println!( + "ehci: port {} connected ({}{})", + p, + match speed { + 2 => "high-speed", + 1 => "low-speed", + _ => "full-speed", + }, + if hs { "" } else { ", root-port split" } + ); + let tt = if hs { + None + } else { + Some(Tt { + hub: 0, + port: (p + 1) as u8, + }) + }; + enumerate(0, speed, tt, p); + } + + arm_periodic(ci); + true + } +} + +/// Chain every armed device QH of this controller into its frame list and +/// enable the periodic schedule. +unsafe fn arm_periodic(ci: usize) { + unsafe { + CUR.store(ci as u32, Ordering::Relaxed); + let c = ctrl_mut(ci); + let mut first: *mut u32 = core::ptr::null_mut(); + let mut prev: *mut u32 = core::ptr::null_mut(); + let n = N_DEV.load(Ordering::Relaxed) as usize; + for i in 0..n { + let d = dev_mut(i); + if d.ctrl != ci as u32 || !d.armed { + continue; } - core::hint::spin_loop(); + if first.is_null() { + first = d.qh; + } else { + *qh_next(prev) = link_of(d.qh as usize); + } + prev = d.qh; + } + if prev.is_null() { + return; } + *qh_next(prev) = QT_TERMINATE; + for f in 0..1024 { + core::ptr::write_volatile(c.framelist.add(f), link_of(first as usize)); + } + core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); + wr(op(OPCMD), rd(op(OPCMD)) | CMD_RUN | CMD_PSE); + crate::serial_println!("ehci: periodic schedule enabled ({} device(s) on ctrl {})", n, ci); } - true } -/// Port reset on USB2 EHCI. Assert PR, hold it briefly (real EHCI clears -/// PR itself once reset signaling finishes; QEMU needs the driver to -/// deassert), then wait for the port to enable. Returns true if PED set -/// (high-speed device). Full/low-speed stay PED=0 and need a TT path. +/// Reset a root port. Returns true when the port enabled (high-speed). unsafe fn reset_port(p: usize) -> bool { unsafe { let psc = prtsc(p); let v = rd(psc); - // Put port in reset. wr(psc, (v | PRT_RESET) & !PRT_RWC); - // Hold ~50 ms, bailing early if the HC cleared PR on its own. let t0 = crate::interrupts::uptime_ms(); while crate::interrupts::uptime_ms().saturating_sub(t0) < 50 { core::hint::spin_loop(); @@ -809,470 +1185,256 @@ unsafe fn reset_port(p: usize) -> bool { break; } } - // Deassert explicitly (QEMU only completes the reset on deassert). wr(psc, (rd(psc) & !PRT_RESET) & !PRT_RWC); - // Wait briefly for the port to enable (PED). High-speed devices set - // PED on reset; full/low-speed stay PED=0 (they need split/TT). Do - // not spin forever — decide on the line state instead. let mut t = 0; while t < 2_000_000 && (rd(psc) & PRT_PED) == 0 { t += 1; core::hint::spin_loop(); } - // A high-speed device gains PED. Full/low-speed won't (needs companion/TT). rd(psc) & PRT_PED != 0 } } -/// Distinguish full- vs low-speed on a connected root port (for the split QH -/// EPS field). Intel PCH EHCIs expose the speed in HOSTPC (base+0x84+4p, -/// bits 27:26: 0=full, 1=low, 2=high); other controllers fall back to the -/// PORTSC bits 27:26. Only meaningful after the port has completed reset. -unsafe fn port_speed(p: usize) -> u32 { - unsafe { - let base = EHCI_BASE.load(Ordering::Relaxed) as usize; - let hp = rd(base + 0x84 + p * 4); - if hp != u32::MAX && hp != 0 { - let f = (hp >> 25) & 3; - if f != 3 { - return f; - } +fn port_speed(p: usize) -> u32 { + let hp = hostpc(p); + if hp != u32::MAX && hp != 0 { + let f = (hp >> 25) & 3; + if f != 3 { + return f; } - (rd(prtsc(p)) >> 26) & 3 } + unsafe { (rd(prtsc(p)) >> 26) & 3 } } // --------------------------------------------------------------------------- -// Enumeration + HID discovery +// Boot + polling // --------------------------------------------------------------------------- -const HID_CLASS: u8 = 0x03; - -fn find_hid_ep(cfg: &[u8]) -> Option<(u8, u32, u32, u8)> { - // returns (epaddr, mps, interval_bint, epnum) - let mut i = 0usize; - let mut in_iface = false; - while i + 2 <= cfg.len() { - let blen = cfg[i] as usize; - if blen == 0 { - break; - } - let bt = cfg[i + 1]; - if bt == 4 && i + 9 <= cfg.len() { - in_iface = cfg[i + 5] == HID_CLASS; - } else if bt == 5 && in_iface && i + 7 <= cfg.len() { - let addr = cfg[i + 2]; - let attrs = cfg[i + 3]; - let mps = (cfg[i + 4] as u32) | ((cfg[i + 5] as u32) << 8); - let bint = cfg[i + 6] as u32; - if (addr & 0x80) != 0 && (attrs & 0x03) == 0x03 { - return Some((addr, mps, bint, addr & 0x0f)); - } - } - i += blen; - } - None -} - -fn enumerate() -> bool { - let port = find_connected_port(); - if port == usize::MAX { - return false; - } - // DEBUG: one-boot transfer variant matrix replaces normal enumeration. - if DEBUG_MATRIX && !MATRIX_DONE.swap(true, Ordering::Relaxed) { - unsafe { - debug_xfer_matrix(port); - } - return false; // diagnostic build - } - unsafe { - // If a periodic schedule is live (a previous arm), stop it cleanly - // before touching the controller: point every frame at terminate, - // clear PSE and wait for the reclaim handshake. Re-enumerating while - // the old HID QH is still being walked interleaves the periodic walk - // with control transfers — fragile on any HC. - if rd(op(OPCMD)) & CMD_PSE != 0 { - let fl = FRAMELIST; - for f in 0..1024 { - core::ptr::write_volatile(fl.add(f), QT_TERMINATE); +pub fn init() -> bool { + let devs = crate::pci::probe(); + let mut up = false; + for d in devs { + if d.class == 0x0C && d.subclass == 0x03 && d.prog_if == 0x20 { + if N_CTRL.load(Ordering::Relaxed) as usize >= MAX_CTRL { + break; } - core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); - wr(op(OPCMD), rd(op(OPCMD)) & !CMD_PSE); - wait_recl(1000); - } - // Discovery state: nothing armed/present until the arm at the end. - ARMED.store(false, Ordering::Relaxed); - MOUSE_PRES.store(false, Ordering::Relaxed); - let hs = reset_port(port); - if !hs && crate::uhci::controller_up() { - // A real (non-QEMU) EHCI can never transfer to a full/low-speed - // device: the port is owned by the companion UHCI. Mark the - // port and stop — a descriptor timeout here costs 5 s and - // repeats every second, which is exactly the "stuck in a loop" - // the OptiPlex showed. (QEMU's usb-ehci accepts PED=0 with a - // high-speed device behind it, so this skip only triggers when - // a real UHCI companion is actually present.) - FS_LS_PORTS.store( - FS_LS_PORTS.load(Ordering::Relaxed) | (1 << port), - Ordering::Relaxed, - ); - crate::serial_println!( - "ehci: port {} full/low-speed — owned by companion UHCI (uhci driver handles it)", - port - ); - return false; - } - crate::serial_println!( - "ehci: port {} resolved{}", - port, - if hs { - " (high-speed)" - } else { - " (full/low — TT path)" + unsafe { + if init_one(d.bus, d.device, d.func) { + up = true; + } } - ); - // Speed for the QH EPS field: 2=HS, 1=LS, 0=FS. FS/LS on an EHCI - // root port must be serviced with split transactions (integrated TT). - let speed = if hs { 2 } else { port_speed(port) }; - let speed_name = match speed { - 2 => "high-speed", - 1 => "low-speed (split)", - _ => "full-speed (split)", - }; - let hostpc = unsafe { rd(EHCI_BASE.load(Ordering::Relaxed) as usize + 0x84 + port * 4) }; - crate::serial_println!( - "ehci: port {} speed: {} (hostpc={:#x})", - port, - speed_name, - hostpc - ); - let mps = if hs { 64u32 } else { 8u32 }; - - // 1. GET_DESCRIPTOR (device) — 18 bytes to learn idVendor etc. - let req = ctrl_req(0x80, 6, 0x0100, 0, 18); - let dd = BUF_PAGE.add(BUF_DEV_DESC); - let mut got = ctl_xfer(0, mps, speed, port, &req, true, dd, 18); - if got < 0 { - crate::serial_println!("ehci: get-dev-desc failed code={}", got); - return false; - } - if got < 8 { - crate::serial_println!("ehci: short dev-desc ({})", got); - return false; - } - let dd = dd as *const u8; - let mps0 = core::ptr::read_volatile(dd.add(7)) as u32; // bMaxPacketSize0 - let _ = mps0; - crate::serial_println!( - "ehci: dev {:04x}:{:04x} class={:#x} mps0={}", - (core::ptr::read_volatile(dd.add(9)) as u16) - | ((core::ptr::read_volatile(dd.add(8)) as u16) << 8), - (core::ptr::read_volatile(dd.add(11)) as u16) - | ((core::ptr::read_volatile(dd.add(10)) as u16) << 8), - core::ptr::read_volatile(dd.add(4)), - core::ptr::read_volatile(dd.add(7)) - ); - - // 2. SET_ADDRESS to 1. - let req = ctrl_req(0x00, 5, 1, 0, 0); - if ctl_xfer(0, mps, speed, port, &req, false, core::ptr::null_mut(), 0) < 0 { - crate::serial_println!("ehci: set-address failed"); - return false; - } - DEV_ADDR.store(1, Ordering::Relaxed); - - // 3. GET_CONFIG(0) — 256 bytes, enough for a HID descriptor. - let cfg = BUF_PAGE.add(BUF_CFG); - core::ptr::write_bytes(cfg, 0, 256); - let req = ctrl_req(0x80, 6, 0x0200, 0, 256); - got = ctl_xfer(1, mps, speed, port, &req, true, cfg, 256); - if got < 0 { - crate::serial_println!("ehci: get-config failed code={}", got); - return false; } - let Some((epaddr, epmps, bint, epnum)) = - find_hid_ep(core::slice::from_raw_parts(cfg as *const u8, 256)) - else { - crate::serial_println!("ehci: no HID mouse endpoint in config"); - return false; - }; + } + if !up { + crate::serial_println!("ehci: no EHCI controller"); + } else { crate::serial_println!( - "ehci: HID mouse IN ep{}({:#x}) mps={} interval={}", - epnum, - epaddr, - epmps, - bint - ); - - // 4. SET_CONFIGURATION(1). - let req = ctrl_req(0x00, 9, 1, 0, 0); - if ctl_xfer(1, mps, speed, port, &req, false, core::ptr::null_mut(), 0) < 0 { - crate::serial_println!("ehci: set-config failed"); - return false; - } - - // 5. Arm the HID interrupt-IN endpoint on the periodic schedule. - HID_EP.store(epnum as u32, Ordering::Relaxed); - HID_MPS.store(epmps, Ordering::Relaxed); - // Build HID QH for EP IN on device 1. - let hq = HID_QH; - // eps: speed (0/1/2) maps directly onto the QH EPS field (bits 13:12). - let info1 = QH_RL - | (speed << 12) - | ((epmps.max(4) & 0x7ff) << 16) - | (DEV_ADDR.load(Ordering::Relaxed) & 0x7f) - | (((epnum as u32) & 0x0f) << 8); - // Poll every frame. HS: SMASK covers every uframe. FS/LS: split — - // SSPLIT at uframe 0, CSPLIT at uframes 2..4 (Linux NEWSCHED), hub - // addr 0 (integrated root TT), 1-based root port in bits 29:23. - let (smask, cmask, split) = if speed == 2 { - (0xFFu32, 0u32, 0u32) - } else { - (0x01u32, 0x1Cu32 << 8, QH_MULT | (((port as u32) + 1) << 23)) - }; - *qh_next(hq) = QT_TERMINATE; // end of this frame's periodic list - *qh_info1(hq) = info1; - *qh_info2(hq) = smask | cmask | split; - *qh_current(hq) = 0; - let iqtd = hq.add(0x40 / 4); // qtd at +0x40 (must be 32-byte aligned) - // HID report qtd: len = epmps, PID IN. - core::ptr::write_volatile(iqtd.wrapping_add(0), QT_TERMINATE); - core::ptr::write_volatile(iqtd.wrapping_add(1), QT_TERMINATE); - core::ptr::write_volatile( - iqtd.wrapping_add(2), - ((epmps.max(4) & 0x7fff) << 16) | (PID_IN << 8) | TOK_ACTIVE | TOK_IOC, + "ehci: {} controller(s), {} HID device(s)", + N_CTRL.load(Ordering::Relaxed), + N_DEV.load(Ordering::Relaxed) ); - for i in 0..5 { - core::ptr::write_volatile(iqtd.add(3 + i), 0u32); - } - core::ptr::write_volatile(iqtd.wrapping_add(3), BUF_PAGE.add(BUF_HID_REPORT) as u32); - *qh_next_qtd(hq) = iqtd as u32; - *qh_alt_qtd(hq) = QT_TERMINATE; - *qh_token(hq) = 0; - // QEMU reads the report buffer from the qtd's bufptr (ehci_init_transfer), - // so the QH overlay's buffer list need not be pre-populated. - // Point every period frame entry at the HID QH. - let fl = FRAMELIST; - for f in 0..1024 { - core::ptr::write_volatile(fl.add(f), link_of(hq as usize)); - } - // Enable the periodic schedule so the frame list is actually walked. - // PSE was deliberately off until now (empty frame list during init). - core::sync::atomic::fence(core::sync::atomic::Ordering::SeqCst); - wr(op(OPCMD), rd(op(OPCMD)) | CMD_RUN | CMD_PSE); - ARMED.store(true, Ordering::Relaxed); - MOUSE_PRES.store(true, Ordering::Relaxed); - crate::serial_println!("ehci: HID boot mouse armed (device 1)"); - true - } -} - -// --------------------------------------------------------------------------- -// DEBUG: one-boot transfer variant matrix (real-hardware SE bisection). -// Each variant does a full HC reset + port reset, then one ctl_xfer with a -// single construction variable flipped (see XFER_TWEAK in ctl_xfer), and -// logs the outcome. A wedged port from an earlier SE cannot confound the -// comparisons because every variant restarts the controller. -// --------------------------------------------------------------------------- -unsafe fn hc_restart() { - unsafe { - wr(op(OPCMD), 0); - let mut t = 0; - while t < 1_000_000 && (rd(op(OPSTS)) & STS_HCH) == 0 { - t += 1; - core::hint::spin_loop(); - } - wr(op(OPCMD), CMD_HCRESET); - t = 0; - while t < 10_000_000 && (rd(op(OPCMD)) & CMD_HCRESET) != 0 { - t += 1; - core::hint::spin_loop(); - } - t = 0; - while t < 10_000_000 && (rd(op(OPSTS)) & STS_HCH) == 0 { - t += 1; - core::hint::spin_loop(); - } - wr(op(0x18), (ASYNC_HEAD as u32) & !0x1F); - wr(op(0x14), (FRAMELIST as u32) & !0x3FF); - wr(op(OPCFGFLAG), 1); - let np = N_PORTS.load(Ordering::Relaxed); - for p in 0..np as usize { - let psc = prtsc(p); - let v = rd(psc); - if v & PRT_POWER == 0 { - wr(psc, v | PRT_POWER); - } - } - wr(op(OPSTS), 0x1F); - wr(op(OPCMD), rd(op(OPCMD)) | CMD_RUN); - t = 0; - while t < 1_000_000 && (rd(op(OPSTS)) & STS_HCH) != 0 { - t += 1; - core::hint::spin_loop(); - } } + up } -unsafe fn debug_xfer_matrix(port: usize) { - unsafe { - QR_SHOWN.store(true, Ordering::Relaxed); // serial-only diagnostics - crate::serial_println!( - "ehci: ===== DEBUG xfer variant matrix (port {}) =====", - port - ); - let tweaks: [u32; 7] = [0, 1, 2, 4, 8, 0x20, 0x80]; - let names = [ - "base", - "no-ASE", - "no-split", - "no-qtd", - "len0", - "wire-off", - "low-region", - ]; - let ebase = EHCI_BASE.load(Ordering::Relaxed) as usize; - for (i, &tw) in tweaks.iter().enumerate() { - hc_restart(); - let _hs = reset_port(port); - let req: [u8; 8] = [0x80, 0x06, 0x00, 0x01, 0, 0, 0x12, 0]; // get_desc dev, 18B - let mut buf = [0u8; 18]; - XFER_TWEAK.store(tw | 0x40, Ordering::Relaxed); // + short timeout - let t0 = crate::interrupts::uptime_ms(); - let r = ctl_xfer(0, 8, 0, port, &req, true, buf.as_mut_ptr(), 18); - let dt = crate::interrupts::uptime_ms().wrapping_sub(t0); - crate::serial_println!( - "ehci: v{} {:<10}: ret={} t={}ms sts={:#x} cmd={:#x} prtsc0={:#x} hp={:#x}", - i, - names[i], - r, - dt, - rd(op(OPSTS)), - rd(op(OPCMD)), - rd(prtsc(0)), - rd(ebase + 0x84) - ); - XFER_TWEAK.store(0, Ordering::Relaxed); - let w0 = crate::interrupts::uptime_ms(); - while crate::interrupts::uptime_ms().wrapping_sub(w0) < 600 { - core::hint::spin_loop(); +pub fn rescan() { + // Hotplug is only handled while nothing is armed: re-run the boot scan. + if N_DEV.load(Ordering::Relaxed) == 0 { + let n = N_CTRL.load(Ordering::Relaxed) as usize; + for ci in 0..n { + unsafe { + CUR.store(ci as u32, Ordering::Relaxed); + let np = ctrl_mut(ci).nports as usize; + NEXT_DEV_ADDR.store((N_DEV.load(Ordering::Relaxed) + 1) as u32, Ordering::Relaxed); + for p in 0..np { + if rd(prtsc(p)) & PRT_CONNECT == 0 { + continue; + } + let hs = reset_port(p); + if !hs && crate::uhci::controller_up() { + continue; + } + let speed = if hs { 2 } else { port_speed(p) }; + let tt = if hs { + None + } else { + Some(Tt { + hub: 0, + port: (p + 1) as u8, + }) + }; + enumerate(0, speed, tt, p); + } + arm_periodic(ci); } } - crate::serial_println!("ehci: ===== matrix done ====="); } } -// --------------------------------------------------------------------------- -// Active interface (called by cursor task / shell) -// --------------------------------------------------------------------------- - pub fn controller_up() -> bool { CTRL_UP.load(Ordering::Relaxed) } + pub fn mouse_present() -> bool { - MOUSE_PRES.load(Ordering::Relaxed) + let n = N_DEV.load(Ordering::Relaxed) as usize; + for i in 0..n { + if unsafe { dev_mut(i) }.kind == Kind::Mouse { + return true; + } + } + false } pub fn poll_mouse() -> Option { unsafe { - // Hot path first. The qtd's ACTIVE bit (in controller-visible DMA - // memory) is the ground truth for "report outstanding"; a kernel - // static must never gate report delivery. - if ARMED.load(Ordering::Relaxed) { - let hq = HID_QH; - let iqtd = hq.add(0x40 / 4); // same 32-byte-aligned location as arming - let tok = core::ptr::read_volatile(iqtd.wrapping_add(2).cast::()); - if tok & TOK_ACTIVE == 0 { - if tok & (TOK_HALT | TOK_XACT) != 0 { - // Transfer error: drop the arm; a bounded re-enum follows. - ARMED.store(false, Ordering::Relaxed); - return None; - } - // Completed: decode the report from the dedicated DMA page. - let hb = BUF_PAGE.add(BUF_HID_REPORT) as *const u8; - let b0 = core::ptr::read_volatile(hb); - let b1 = core::ptr::read_volatile(hb.add(1)); - let b2 = core::ptr::read_volatile(hb.add(2)); - let ev = Some(MouseEvent { - dx: (b1 as i8) as i16, - dy: (b2 as i8) as i16, - left: b0 & 0x01 != 0, - right: b0 & 0x02 != 0, - middle: b0 & 0x04 != 0, - }); - // Re-arm the periodic interrupt-IN qtd: restore its token to - // ACTIVE and re-link it as the QH's next qtd so the HC fetches - // it again next frame (canonical one-qtd periodic re-arm). - core::ptr::write_volatile( - iqtd.wrapping_add(2), - ((HID_MPS.load(Ordering::Relaxed).max(4) & 0x7fff) << 16) - | (PID_IN << 8) - | TOK_ACTIVE - | TOK_IOC, + let n = N_DEV.load(Ordering::Relaxed) as usize; + for i in 0..n { + let d = dev_mut(i); + if d.kind != Kind::Mouse || !d.armed { + continue; + } + let tok = core::ptr::read_volatile(d.iqtd.add(2).cast::()); + if tok & TOK_ACTIVE != 0 { + continue; + } + if tok & (TOK_HALT | TOK_XACT) != 0 { + d.armed = false; + d.failures += 1; + continue; + } + let hb = d.report; + let b0 = core::ptr::read_volatile(hb); + let b1 = core::ptr::read_volatile(hb.add(1)); + let b2 = core::ptr::read_volatile(hb.add(2)); + let ev = MouseEvent { + dx: (b1 as i8) as i16, + dy: (b2 as i8) as i16, + left: b0 & 0x01 != 0, + right: b0 & 0x02 != 0, + middle: b0 & 0x04 != 0, + }; + d.reports += 1; + if d.reports == 1 { + crate::serial_println!( + "ehci: mouse report dx={} dy={} b={:#x}", + ev.dx, + ev.dy, + b0 ); - *qh_next_qtd(hq) = iqtd as u32; - return ev; } - return None; + rearm(d); + return Some(ev); } - // Not armed: bring the mouse up lazily, but never hot-loop — at most - // one enumerate attempt per second, and only while the controller is - // up. A transient glitch can delay a report; it must not wedge the - // system in a re-enumeration loop. - if CTRL_UP.load(Ordering::Relaxed) { + // Nothing armed: retry discovery at most once per second. + if N_DEV.load(Ordering::Relaxed) == 0 && CTRL_UP.load(Ordering::Relaxed) { let now = crate::interrupts::uptime_ms() as u32; - if now.wrapping_sub(LAST_ENUM.load(Ordering::Relaxed)) >= 1000 { - LAST_ENUM.store(now, Ordering::Relaxed); - enumerate(); + if now.wrapping_sub(LAST_SCAN.load(Ordering::Relaxed)) >= 1000 { + LAST_SCAN.store(now, Ordering::Relaxed); + rescan(); } } + None } - None } -/// Allocate DMA-safe memory above 16 MB. Uses the page allocator -/// (physical, identity-mapped addresses) instead of the heap allocator -/// (virtual addresses) because the EHCI controller's DMA engine needs -/// physical addresses that match what the CPU sees. The 16 MB floor -/// keeps the schedule structures out of the first 640 K, which some real -/// chipsets reject from PCI masters (controller System Error halt). -unsafe fn dma_alloc(bytes: usize, _align: usize) -> *mut u8 { +pub fn try_poll_key() -> Option { unsafe { - let pages = bytes.div_ceil(4096); - let Some(phys) = crate::memory::alloc_pages_above(pages, 0x100_0000) else { - return core::ptr::null_mut(); - }; - let p = phys as *mut u8; - core::ptr::write_bytes(p, 0, pages * 4096); - p + if let Some(b) = crate::hid::pop_byte() { + return crate::ps2::feed_scancode(b); + } + let n = N_DEV.load(Ordering::Relaxed) as usize; + for i in 0..n { + let d = dev_mut(i); + if d.kind != Kind::Keyboard || !d.armed { + continue; + } + let tok = core::ptr::read_volatile(d.iqtd.add(2).cast::()); + if tok & TOK_ACTIVE != 0 { + continue; + } + if tok & (TOK_HALT | TOK_XACT) != 0 { + d.armed = false; + d.failures += 1; + continue; + } + let mut rep = [0u8; 8]; + let len = (d.len as usize).min(8); + for (k, b) in rep.iter_mut().take(len).enumerate() { + *b = core::ptr::read_volatile(d.report.add(k)); + } + d.kbd.diff(&rep); + d.reports += 1; + if d.reports == 1 { + crate::serial_println!( + "ehci: kbd report mods={:#x} keys={:02x} {:02x} {:02x} {:02x} {:02x} {:02x}", + rep[0], rep[2], rep[3], rep[4], rep[5], rep[6], rep[7] + ); + } + rearm(d); + return crate::hid::pop_byte().and_then(crate::ps2::feed_scancode); + } + None } } -/// One-line status for the shell `usb` command. +/// Restore one device's periodic qtd to ACTIVE and re-link it. +/// +/// Standard EHCI re-arm (same shape as Linux's `qh_append_tds`): the HC owns +/// the QH status/current-qTD/toggle overlay, so firmware must not rewrite it +/// — only re-activate the qTD and point Next-qTD at it. +unsafe fn rearm(d: &mut Dev) { + unsafe { + core::ptr::write_volatile( + d.iqtd.add(2), + ((d.len & 0x7fff) << 16) | (PID_IN << 8) | TOK_ACTIVE | TOK_IOC, + ); + *qh_next_qtd(d.qh) = d.iqtd as u32; + } +} + +// --------------------------------------------------------------------------- +// Diagnostics (shell `usb`) +// --------------------------------------------------------------------------- + pub fn diagnose() -> alloc::string::String { let mut s = alloc::string::String::new(); + let nc = N_CTRL.load(Ordering::Relaxed); + let nd = N_DEV.load(Ordering::Relaxed); s.push_str(&alloc::format!( - "ehci mmio={:#x} ports={} mouse={} dev={} hid_ep={} armed={}\n", - EHCI_BASE.load(Ordering::Relaxed), - N_PORTS.load(Ordering::Relaxed), - if MOUSE_PRES.load(Ordering::Relaxed) { - "yes" - } else { - "no" - }, - DEV_ADDR.load(Ordering::Relaxed), - HID_EP.load(Ordering::Relaxed), - ARMED.load(Ordering::Relaxed) + "ehci: {} controller(s), {} device(s) up={}\n", + nc, + nd, + CTRL_UP.load(Ordering::Relaxed) )); - let np = N_PORTS.load(Ordering::Relaxed) as usize; - for p in 0..np.min(8) { - let v = unsafe { rd(prtsc(p)) }; + for c in 0..nc as usize { + let ct = unsafe { &*core::ptr::addr_of!(CTRLS[c]) }; + s.push_str(&alloc::format!( + " ctrl{} mmio={:#x} ports={}\n", + c, + ct.base, + ct.nports + )); + for p in 0..ct.nports.min(8) as usize { + let v = unsafe { rd(ct.opb + OP_PRTSC_BASE + p * OP_PRTSC_STRIDE) }; + s.push_str(&alloc::format!( + " port{}: {}{:#x}\n", + p, + if v & PRT_CONNECT != 0 { "*" } else { "" }, + v + )); + } + } + for i in 0..nd as usize { + let d = unsafe { &*core::ptr::addr_of!(DEVS[i]) }; s.push_str(&alloc::format!( - " eport{}: {}{:#x}\n", - p, - if v & PRT_CONNECT != 0 { "*" } else { "" }, - v & 0xffff + " {} dev{} ctrl={} armed={} fails={} reports={} qh={:#x}\n", + if d.kind == Kind::Mouse { "mouse" } else { "kbd" }, + i, + d.ctrl, + d.armed, + d.failures, + d.reports, + d.qh as usize )); } s -- cgit v1.2.3