x86_64 Hardware Breakpoint detours on macOS

hw_detour.c

#include <stdlib.h>
#include <stdio.h>
#include <stdbool.h>
#include <stdint.h>
#include <signal.h>

#include <mach/mach_init.h>
#include <mach/mach_port.h>
#include <mach/task.h>
#include <mach/thread_act.h>
#include <mach/machine/thread_status.h>

uint64_t orig;
uint64_t handler;
struct sigaction old_action;

// SIGTRAP handler
void breakpoint_handler(int signum, siginfo_t* info, ucontext_t* uap)
{
    puts("breakpoint hit.");

    if (uap && uap->uc_mcontext && uap->uc_mcontext->__ss.__rip == orig) {
        puts("caught sigtrap at original IP, redirecting thread.");
        uap->uc_mcontext->__ss.__rip = handler;
    } else if (old_action.sa_sigaction) {
        puts("caught unrelated sigtrap, calling original handler.");
        // Pass to original handler, if it exists
        (*old_action.sa_sigaction)(signum, info, uap);
    } else {
        puts("unhandled sigtrap, re-raising.");
        old_action.sa_handler = SIG_DFL;
        sigaction(SIGTRAP, &old_action, NULL);
        // This doesn't convey the details of the original trap but good enough
        raise(SIGTRAP);
    }
}

// Hooks can only be installed in the current process because a signal handler can only be installed in the current
// process
boolean_t thread_port_is_proc(thread_act_t thread) {
    thread_act_array_t a;
    natural_t c;
    // Get the threads of this task
    int rc = task_threads(mach_task_self(), &a, &c);
    if (rc != 0) {
        abort();
    }

    boolean_t r = false;
    for (natural_t i = 0; i < c; i++) {
        if (a[i] == thread) {
            r = true;
        }
        // "The calling task or thread also receives a send right to the kernel port for each listed thread."
        // So we must deallocate
        mach_port_deallocate(mach_task_self(), a[i]);
    }

    return r;
}

void create_hook(thread_act_t target_thread, uint64_t orig_ip, uint64_t handler_ip)
{
    if (!thread_port_is_proc(target_thread)) {
        abort();
    }

    x86_debug_state64_t dr;
    mach_msg_type_number_t dr_count = x86_DEBUG_STATE64_COUNT;

    kern_return_t rc = thread_get_state(target_thread, x86_DEBUG_STATE64, (thread_state_t)&dr, &dr_count);
    if (rc != 0) {
        printf("thread_get_state failed: %x\n", rc);
        abort();
    }

    // Set the address to break on and enable breakpoint
    dr.__dr0 = orig_ip;
    dr.__dr7 = (1 << 0);

    rc = thread_set_state(target_thread, x86_DEBUG_STATE64, (thread_state_t)&dr, dr_count);
    if (rc != 0) {
        printf("thread_get_state failed: %x\n", rc);
        abort();
    }

    // Install SIGTRAP breakpoint handler
    struct sigaction action;
    action.sa_sigaction = (void (*)(int, siginfo_t *, void *))breakpoint_handler;
    action.sa_flags = SA_SIGINFO | SA_64REGSET;
    sigemptyset(&action.sa_mask);

    old_action.sa_sigaction = 0;

    rc = sigaction(SIGTRAP, &action, &old_action);
    if (rc != 0) {
        abort();
    }

    orig = orig_ip;
    handler = handler_ip;
}

void clear_hook(thread_act_t target_thread)
{
    if (!thread_port_is_proc(target_thread))  {
        abort();
    }

    x86_debug_state64_t dr;
    mach_msg_type_number_t dr_count = x86_DEBUG_STATE64_COUNT;

    kern_return_t rc = thread_get_state(target_thread, x86_DEBUG_STATE64, (thread_state_t)&dr, &dr_count);
    if (rc != 0) {
        printf("thread_get_state failed: %x\n", rc);
        abort();
    }

    // Clear out the state and disable the breakpoint
    dr.__dr6 &= ~(1 << 0);
    dr.__dr7 &= ~(1 << 0);
    dr.__dr0 = 0;

    rc = thread_set_state(target_thread, x86_DEBUG_STATE64, (thread_state_t)&dr, dr_count);
    if (rc != 0) {
        printf("thread_get_state failed: %x\n", rc);
        abort();
    }

    rc = sigaction(SIGTRAP, &old_action, NULL);
    if (rc != 0) {
        printf("sigaction failed: %x\n", rc);
        abort();
    }
}

uint32_t dummy(char c)
{
    printf("dummy: c is %c\n", c);
    return 0xAAAA;
}

uint32_t hijack(char c)
{
    printf("hijack hit: c is %c\n", c);
    return 0xEEEE;
}

void dummy_sigtrap_handler(int _) {
    (void)(_);
    puts("hit old trap handler.");
}

int install_dummy_sigtrap_handler() {
    struct sigaction action;
    action.sa_handler = dummy_sigtrap_handler;
    action.sa_flags = 0;
    sigemptyset(&action.sa_mask);
    return sigaction(SIGTRAP, &action, NULL);
}

int main()
{
    if (install_dummy_sigtrap_handler() != 0) {
        puts("failed to install dummy sigtrap handler.");
        abort();
    }

    puts("exception redirecting function `dummy` to `hijack`.");
    create_hook(mach_thread_self(), (uint64_t)dummy, (uint64_t)hijack);

    printf("rv: %x\n", dummy('d'));

    puts("trying SIGTRAP, should hit previous sigtrap handler:");
    raise(SIGTRAP);

    puts("clearing exception redirect.");
    clear_hook(mach_thread_self());

    printf("rv: %x\n", dummy('e'));

    return(0);
}