Refactor dex2oat wrapper to solve long-standing issues (#515)

Manual library path injection via LD_LIBRARY_PATH has become unreliable due to symbol mismatches in core libraries (e.g., `libc++`) between the system and APEX partitions. Recent updates to `liblog` and `libbase` (in Android 16) have resulted in missing symbols like `__hash_memory` or `fmt` when the ART APEX binaries are forced to load system-partition shims.

This commit switches the wrapper to execute the runtime APEX linker directly (e.g., `/apex/com.android.runtime/bin/linker64`). By passing the dex2oat binary to the linker via `/proc/self/fd/`, the linker can properly initialize internal namespaces and resolve dependencies from the correct APEX and bootstrap locations.

Moreover, for the OatHeader hook, a bug introduced in 6703b45 is now fixed, where the target functions of PLT hooks are overwritten by the our helper functions.

Details of the refactored project are explained in README.

Author: JingMatrix

daemon/src/main/jni/logcat.cpp

@@ -229,7 +229,8 @@ void Logcat::ProcessBuffer(struct log_msg *buf) {
                      tag == "APatchD"sv || tag == "Dobby"sv || tag.starts_with("dex2oat"sv) ||
                      tag == "KernelSU"sv || tag == "LSPlant"sv || tag == "LSPlt"sv ||
                      tag.starts_with("LSPosed"sv) || tag == "Magisk"sv || tag == "SELinux"sv ||
-                     tag == "TEESimulator"sv || tag.starts_with("zygisk"sv))) [[unlikely]] {
+                     tag == "TEESimulator"sv || tag.starts_with("Vector"sv) ||
+                     tag.starts_with("zygisk"sv))) [[unlikely]] {
         verbose_print_count_ += PrintLogLine(entry, verbose_file_.get());
     }
     if (entry.pid == my_pid_ && tag == "LSPosedLogcat"sv) [[unlikely]] {

dex2oat/README.md

@@ -0,0 +1,37 @@
+# VectorDex2Oat
+
+VectorDex2Oat is a specialized wrapper and instrumentation suite for the Android `dex2oat` (Ahead-of-Time compiler) binary. It is designed to intercept the compilation process, force specific compiler behaviors (specifically disabling method inlining), and transparently spoof the resulting OAT metadata to hide the presence of the wrapper.
+
+## Overview
+
+In the Android Runtime (ART), `dex2oat` compiles DEX files into OAT files. Modern ART optimizations often inline methods, making it difficult for instrumentation tools to hook specific function calls. 
+
+This project consists of two primary components:
+1.  **dex2oat (Wrapper):** A replacement binary that intercepts the execution, communicates via Unix Domain Sockets to obtain the original compiler binary, and executes it with forced flags.
+2.  **liboat_hook.so (Hooker):** A shared library injected into the `dex2oat` process via `LD_PRELOAD` that utilizes PLT hooking to sanitize the OAT header's command-line metadata.
+
+## Key Features
+
+*   **Inlining Suppression:** Appends `--inline-max-code-units=0` to the compiler arguments, ensuring all methods remain discrete and hookable.
+*   **FD-Based Execution:** Executes the original `dex2oat` via the system linker using `/proc/self/fd/` paths, avoiding direct execution of files on the disk.
+*   **Metadata Spoofing:** Intercepts `art::OatHeader::ComputeChecksum` or `art::OatHeader::GetKeyValueStore` to remove traces of the wrapper and its injected flags from the final `.oat` file.
+*   **Abstract Socket Communication:** Uses the Linux Abstract Namespace for Unix sockets to coordinate file descriptor passing between the controller and the wrapper.
+
+## Architecture
+
+### The Wrapper [dex2oat.cpp](src/main/cpp/dex2oat.cpp)
+The wrapper acts as a "man-in-the-middle" for the compiler. When called by the system, it
+1.  connects to a predefined Unix socket (the stub name `5291374ceda0...` will be replaced during installation of `Vector`);
+2.  identifies the target architecture (32-bit vs 64-bit) and debug status;
+3.  receives File Descriptors (FDs) for both the original `dex2oat` binary and the `oat_hook` library;
+4.  reconstructs the command line, replacing the wrapper path with the original binary path and appending the "no-inline" flags;
+5.  clears `LD_LIBRARY_PATH` and sets `LD_PRELOAD` to the hooker library's FD;
+6.  invokes the dynamic linker (`linker64`) to execute the compiler.
+
+### The Hooker [oat_hook.cpp](src/main/cpp/oat_hook.cpp)
+The hooker library is preloaded into the compiler's address space. It uses the [LSPlt](https://github.com/JingMatrix/LSPlt) library to:
+1.  Scan the memory map to find the `dex2oat` binary.
+2.  Locate and hook internal ART functions:
+    *   [art::OatHeader::GetKeyValueStore](https://cs.android.com/android/platform/superproject/+/android-latest-release:art/runtime/oat/oat.cc;l=366)
+    *   [art::OatHeader::ComputeChecksum](https://cs.android.com/android/platform/superproject/+/android-latest-release:art/runtime/oat/oat.cc;l=366)
+3.  When the compiler attempts to write the "dex2oat-cmdline" key into the OAT header, the hooker intercepts the call, parses the key-value store, and removes the wrapper-specific flags and paths.

dex2oat/build.gradle.kts

@@ -1,47 +1,15 @@
-/*
- * This file is part of LSPosed.
- *
- * LSPosed is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LSPosed is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LSPosed.  If not, see <https://www.gnu.org/licenses/>.
- *
- * Copyright (C) 2022 LSPosed Contributors
- */
-
 plugins {
     alias(libs.plugins.agp.lib)
 }
 
 android {
-    namespace = "org.lsposed.dex2oat"
-
-    buildFeatures {
-        androidResources = false
-        buildConfig = false
-        prefab = true
-        prefabPublishing = true
-    }
+    namespace = "org.matrix.vector.dex2oat"
 
-    defaultConfig {
-        minSdk = 29
-    }
+    androidResources.enable = false
 
     externalNativeBuild {
         cmake {
             path("src/main/cpp/CMakeLists.txt")
         }
     }
-
-    prefab {
-        register("dex2oat")
-    }
 }

dex2oat/src/main/cpp/CMakeLists.txt

@@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.10)
 project(dex2oat)
 
 add_executable(dex2oat dex2oat.cpp)
-add_library(oat_hook SHARED oat_hook.cpp oat.cpp)
+add_library(oat_hook SHARED oat_hook.cpp)
 
 OPTION(LSPLT_BUILD_SHARED OFF)
 add_subdirectory(${EXTERNAL_ROOT}/lsplt/lsplt/src/main/jni external)

dex2oat/src/main/cpp/dex2oat.cpp

@@ -1,148 +1,195 @@
-/*
- * This file is part of LSPosed.
- *
- * LSPosed is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * LSPosed is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with LSPosed.  If not, see <https://www.gnu.org/licenses/>.
- *
- * Copyright (C) 2022 LSPosed Contributors
- */
-
-//
-// Created by Nullptr on 2022/4/1.
-//
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
 #include <sys/socket.h>
 #include <sys/un.h>
 #include <unistd.h>
 
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <string>
+#include <vector>
+
 #include "logging.h"
 
+// Access to the process environment variables
+extern "C" char **environ;
+
 #if defined(__LP64__)
 #define LP_SELECT(lp32, lp64) lp64
 #else
 #define LP_SELECT(lp32, lp64) lp32
 #endif
 
-#define ID_VEC(is64, is_debug) (((is64) << 1) | (is_debug))
+namespace {
 
-const char kSockName[] = "5291374ceda0aef7c5d86cd2a4f6a3ac\0";
+constexpr char kSockName[] = "5291374ceda0aef7c5d86cd2a4f6a3ac";
 
-static ssize_t xrecvmsg(int sockfd, struct msghdr *msg, int flags) {
-    int rec = recvmsg(sockfd, msg, flags);
+/**
+ * Calculates a vector ID based on architecture and debug status.
+ */
+inline int get_id_vec(bool is64, bool is_debug) {
+    return (static_cast<int>(is64) << 1) | static_cast<int>(is_debug);
+}
+
+/**
+ * Wraps recvmsg with error logging.
+ */
+ssize_t xrecvmsg(int sockfd, struct msghdr *msg, int flags) {
+    ssize_t rec = recvmsg(sockfd, msg, flags);
     if (rec < 0) {
         PLOGE("recvmsg");
     }
     return rec;
 }
 
-static void *recv_fds(int sockfd, char *cmsgbuf, size_t bufsz, int cnt) {
+/**
+ * Receives file descriptors passed over a Unix domain socket using SCM_RIGHTS.
+ *
+ * @return Pointer to the FD data on success, nullptr on failure.
+ */
+void *recv_fds(int sockfd, char *cmsgbuf, size_t bufsz, int cnt) {
     struct iovec iov = {
         .iov_base = &cnt,
         .iov_len = sizeof(cnt),
     };
-    struct msghdr msg = {
-        .msg_iov = &iov, .msg_iovlen = 1, .msg_control = cmsgbuf, .msg_controllen = bufsz};
+    struct msghdr msg = {.msg_name = nullptr,
+                         .msg_namelen = 0,
+                         .msg_iov = &iov,
+                         .msg_iovlen = 1,
+                         .msg_control = cmsgbuf,
+                         .msg_controllen = bufsz,
+                         .msg_flags = 0};
+
+    if (xrecvmsg(sockfd, &msg, MSG_WAITALL) < 0) return nullptr;
 
-    xrecvmsg(sockfd, &msg, MSG_WAITALL);
     struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
 
-    if (msg.msg_controllen != bufsz || cmsg == NULL ||
+    if (msg.msg_controllen != bufsz || cmsg == nullptr ||
         cmsg->cmsg_len != CMSG_LEN(sizeof(int) * cnt) || cmsg->cmsg_level != SOL_SOCKET ||
         cmsg->cmsg_type != SCM_RIGHTS) {
-        return NULL;
+        return nullptr;
     }
 
     return CMSG_DATA(cmsg);
 }
 
-static int recv_fd(int sockfd) {
+/**
+ * Helper to receive a single FD from the socket.
+ */
+int recv_fd(int sockfd) {
     char cmsgbuf[CMSG_SPACE(sizeof(int))];
-
     void *data = recv_fds(sockfd, cmsgbuf, sizeof(cmsgbuf), 1);
-    if (data == NULL) return -1;
+    if (data == nullptr) return -1;
 
     int result;
-    memcpy(&result, data, sizeof(int));
+    std::memcpy(&result, data, sizeof(int));
     return result;
 }
 
-static int read_int(int fd) {
+/**
+ * Reads an integer acknowledgment from the socket.
+ */
+int read_int(int fd) {
     int val;
     if (read(fd, &val, sizeof(val)) != sizeof(val)) return -1;
     return val;
 }
 
-static void write_int(int fd, int val) {
+/**
+ * Writes an integer command/ID to the socket.
+ */
+void write_int(int fd, int val) {
     if (fd < 0) return;
-    write(fd, &val, sizeof(val));
+    (void)write(fd, &val, sizeof(val));
 }
 
+}  // namespace
+
 int main(int argc, char **argv) {
     LOGD("dex2oat wrapper ppid=%d", getppid());
+
+    // Prepare Unix domain socket address (Abstract Namespace)
     struct sockaddr_un sock = {};
     sock.sun_family = AF_UNIX;
-    strlcpy(sock.sun_path + 1, kSockName, sizeof(sock.sun_path) - 1);
+    // sock.sun_path[0] is already \0, so we copy name into sun_path + 1
+    std::strncpy(sock.sun_path + 1, kSockName, sizeof(sock.sun_path) - 2);
+
+    // Abstract socket length: family + leading \0 + string length
+    socklen_t len = sizeof(sock.sun_family) + strlen(kSockName) + 1;
 
+    // 1. Get original dex2oat binary FD
     int sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-    size_t len = sizeof(sa_family_t) + strlen(sock.sun_path + 1) + 1;
-    if (connect(sock_fd, (struct sockaddr *)&sock, len)) {
+    if (connect(sock_fd, reinterpret_cast<struct sockaddr *>(&sock), len)) {
         PLOGE("failed to connect to %s", sock.sun_path + 1);
         return 1;
     }
-    write_int(sock_fd, ID_VEC(LP_SELECT(0, 1), strstr(argv[0], "dex2oatd") != NULL));
+
+    bool is_debug = (argv[0] != nullptr && std::strstr(argv[0], "dex2oatd") != nullptr);
+    write_int(sock_fd, get_id_vec(LP_SELECT(false, true), is_debug));
+
     int stock_fd = recv_fd(sock_fd);
-    read_int(sock_fd);
+    read_int(sock_fd);  // Sync
     close(sock_fd);
 
+    // 2. Get liboat_hook.so FD
     sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-    if (connect(sock_fd, (struct sockaddr *)&sock, len)) {
+    if (connect(sock_fd, reinterpret_cast<struct sockaddr *>(&sock), len)) {
         PLOGE("failed to connect to %s", sock.sun_path + 1);
         return 1;
     }
+
     write_int(sock_fd, LP_SELECT(4, 5));
     int hooker_fd = recv_fd(sock_fd);
-    read_int(sock_fd);
+    read_int(sock_fd);  // Sync
     close(sock_fd);
 
     if (hooker_fd == -1) {
-        PLOGE("failed to read liboat_hook.so");
+        LOGE("failed to read liboat_hook.so");
+    }
+    LOGD("sock: %s stock_fd: %d", sock.sun_path + 1, stock_fd);
+
+    // Prepare arguments for execve
+    // Logic: [linker] [/proc/self/fd/stock_fd] [original_args...] [--inline-max-code-units=0]
+    std::vector<const char *> exec_argv;
+
+    const char *linker_path =
+        LP_SELECT("/apex/com.android.runtime/bin/linker", "/apex/com.android.runtime/bin/linker64");
+
+    char stock_fd_path[64];
+    std::snprintf(stock_fd_path, sizeof(stock_fd_path), "/proc/self/fd/%d", stock_fd);
+
+    exec_argv.push_back(linker_path);
+    exec_argv.push_back(stock_fd_path);
+
+    // Append original arguments starting from argv[1]
+    for (int i = 1; i < argc; ++i) {
+        exec_argv.push_back(argv[i]);
     }
-    LOGD("sock: %s %d", sock.sun_path + 1, stock_fd);
-
-    const char *new_argv[argc + 2];
-    for (int i = 0; i < argc; i++) new_argv[i] = argv[i];
-    new_argv[argc] = "--inline-max-code-units=0";
-    new_argv[argc + 1] = NULL;
-
-    if (getenv("LD_LIBRARY_PATH") == NULL) {
-        char const *libenv = LP_SELECT(
-            "LD_LIBRARY_PATH=/apex/com.android.art/lib:/apex/com.android.os.statsd/lib",
-            "LD_LIBRARY_PATH=/apex/com.android.art/lib64:/apex/com.android.os.statsd/lib64");
-        putenv((char *)libenv);
+
+    // Append hooking flags to disable inline, which is our purpose of this wrapper, since we cannot
+    // hook inlined target methods.
+    exec_argv.push_back("--inline-max-code-units=0");
+    exec_argv.push_back(nullptr);
+
+    // Setup Environment variables
+    // Clear LD_LIBRARY_PATH to let the linker use internal config
+    unsetenv("LD_LIBRARY_PATH");
+
+    // Set LD_PRELOAD to point to the hooker library FD
+    std::string preload_val = "LD_PRELOAD=/proc/self/fd/" + std::to_string(hooker_fd);
+    setenv("LD_PRELOAD", ("/proc/self/fd/" + std::to_string(hooker_fd)).c_str(), 1);
+
+    // Pass original argv[0] as DEX2OAT_CMD
+    if (argv[0]) {
+        setenv("DEX2OAT_CMD", argv[0], 1);
+        LOGD("DEX2OAT_CMD set to %s", argv[0]);
     }
 
-    // Set LD_PRELOAD to load liboat_hook.so
-    const int STRING_BUFFER = 50;
-    char env_str[STRING_BUFFER];
-    snprintf(env_str, STRING_BUFFER, "LD_PRELOAD=/proc/%d/fd/%d", getpid(), hooker_fd);
-    putenv(env_str);
-    LOGD("Set env %s", env_str);
+    LOGI("Executing via linker: %s executing %s", linker_path, stock_fd_path);
 
-    fexecve(stock_fd, (char **)new_argv, environ);
+    // Perform the execution
+    execve(linker_path, const_cast<char *const *>(exec_argv.data()), environ);
 
-    PLOGE("fexecve failed");
+    // If we reach here, execve failed
+    PLOGE("execve failed");
     return 2;
 }

dex2oat/src/main/cpp/include/logging.h

@@ -1,10 +1,10 @@
 #pragma once
 
-#include <errno.h>
 #include <android/log.h>
+#include <errno.h>
 
 #ifndef LOG_TAG
-#define LOG_TAG "LSPosedDex2Oat"
+#define LOG_TAG "VectorDex2Oat"
 #endif
 
 #ifdef LOG_DISABLED
@@ -15,11 +15,7 @@
 #define LOGE(...) 0
 #else
 #ifndef NDEBUG
-#define LOGD(fmt, ...)                                                                             \
-    __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG,                                                \
-                        "%s:%d#%s"                                                                 \
-                        ": " fmt,                                                                  \
-                        __FILE_NAME__, __LINE__, __PRETTY_FUNCTION__ __VA_OPT__(, ) __VA_ARGS__)
+#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__)
 #define LOGV(fmt, ...)                                                                             \
     __android_log_print(ANDROID_LOG_VERBOSE, LOG_TAG,                                              \
                         "%s:%d#%s"                                                                 \