[SP 2025-09-01] RPC Binder: shutdown on ENOMEM

We were expecting crashes in these case, but explicitly
shut down. More work needs to be done to ignore transactions
in out of memory conditions.

Bug: 404210068
Bug: 414720799
Bug: 416734088
Test: binderRpcTest
Change-Id: Iaf9a34b4031fb7b9807c962bcc67de8cd9102088
Merged-In: Iaf9a34b4031fb7b9807c962bcc67de8cd9102088
(cherry picked from commit cba4e3642a8a58d54481ed4c14f179bcc7f9ae70)

1 files changed, 11 insertions, 1 deletions

diff --git a/libs/binder/RpcState.cpp b/libs/binder/RpcState.cpp
index 03d974d186..1ecf3c3628 100644
--- a/libs/binder/RpcState.cpp
+++ b/libs/binder/RpcState.cpp
@@ -687,7 +687,13 @@ status_t RpcState::waitForReply(const sp<RpcSession::RpcConnection>& connection,
     memset(&rpcReply, 0, sizeof(RpcWireReply)); // zero because of potential short read
 
     CommandData data(command.bodySize - rpcReplyWireSize);
-    if (!data.valid()) return NO_MEMORY;
+    if (!data.valid()) {
+        // b/404210068 - if we run out of memory, the wire protocol gets messed up.
+        // so shutdown. We would need to read all the transaction data anyway and
+        // send a reply still to gracefully recover.
+        (void)session->shutdownAndWait(false);
+        return NO_MEMORY;
+    }
 
     iovec iovs[]{
             {&rpcReply, rpcReplyWireSize},
@@ -844,6 +850,10 @@ status_t RpcState::processTransact(
 
     CommandData transactionData(command.bodySize);
     if (!transactionData.valid()) {
+        // b/404210068 - if we run out of memory, the wire protocol gets messed up.
+        // so shutdown. We would need to read all the transaction data anyway and
+        // send a reply still to gracefully recover.
+        (void)session->shutdownAndWait(false);
         return NO_MEMORY;
     }
     iovec iov{transactionData.data(), transactionData.size()};