libbinder: RPC avoid abort on too big allocations

We may want some more heuristic checks here, since allocations that are
just barely too big may interrupt the operations of other threads that
can't handle failing allocations, but this is a start so that too-big
allocations won't cause an abort.

Bug: 182938024
Test: binderRpcTest + running WIP fuzzer
Change-Id: Id9f1b63962cd22b7c799b14e595c47ea8628354f

1 files changed, 26 insertions, 9 deletions

diff --git a/libs/binder/RpcState.cpp b/libs/binder/RpcState.cpp
index 96190dc03c..976351c6c4 100644
--- a/libs/binder/RpcState.cpp
+++ b/libs/binder/RpcState.cpp
@@ -326,7 +326,11 @@ status_t RpcState::transact(const base::unique_fd& fd, const RpcAddress& address
             .asyncNumber = asyncNumber,
     };
 
-    std::vector<uint8_t> transactionData(sizeof(RpcWireTransaction) + data.dataSize());
+    ByteVec transactionData(sizeof(RpcWireTransaction) + data.dataSize());
+    if (!transactionData.valid()) {
+        return NO_MEMORY;
+    }
+
     memcpy(transactionData.data() + 0, &transaction, sizeof(RpcWireTransaction));
     memcpy(transactionData.data() + sizeof(RpcWireTransaction), data.data(), data.dataSize());
 
@@ -379,9 +383,12 @@ status_t RpcState::waitForReply(const base::unique_fd& fd, const sp<RpcSession>&
         if (status != OK) return status;
     }
 
-    uint8_t* data = new uint8_t[command.bodySize];
+    ByteVec data(command.bodySize);
+    if (!data.valid()) {
+        return NO_MEMORY;
+    }
 
-    if (!rpcRec(fd, "reply body", data, command.bodySize)) {
+    if (!rpcRec(fd, "reply body", data.data(), command.bodySize)) {
         return DEAD_OBJECT;
     }
 
@@ -391,9 +398,10 @@ status_t RpcState::waitForReply(const base::unique_fd& fd, const sp<RpcSession>&
         terminate();
         return BAD_VALUE;
     }
-    RpcWireReply* rpcReply = reinterpret_cast<RpcWireReply*>(data);
+    RpcWireReply* rpcReply = reinterpret_cast<RpcWireReply*>(data.data());
     if (rpcReply->status != OK) return rpcReply->status;
 
+    data.release();
     reply->ipcSetDataReference(rpcReply->data, command.bodySize - offsetof(RpcWireReply, data),
                                nullptr, 0, cleanup_reply_data);
 
@@ -461,7 +469,10 @@ status_t RpcState::processTransact(const base::unique_fd& fd, const sp<RpcSessio
                                    const RpcWireHeader& command) {
     LOG_ALWAYS_FATAL_IF(command.command != RPC_COMMAND_TRANSACT, "command: %d", command.command);
 
-    std::vector<uint8_t> transactionData(command.bodySize);
+    ByteVec transactionData(command.bodySize);
+    if (!transactionData.valid()) {
+        return NO_MEMORY;
+    }
     if (!rpcRec(fd, "transaction body", transactionData.data(), transactionData.size())) {
         return DEAD_OBJECT;
     }
@@ -479,7 +490,7 @@ static void do_nothing_to_transact_data(Parcel* p, const uint8_t* data, size_t d
 }
 
 status_t RpcState::processTransactInternal(const base::unique_fd& fd, const sp<RpcSession>& session,
-                                           std::vector<uint8_t>&& transactionData) {
+                                           ByteVec transactionData) {
     if (transactionData.size() < sizeof(RpcWireTransaction)) {
         ALOGE("Expecting %zu but got %zu bytes for RpcWireTransaction. Terminating!",
               sizeof(RpcWireTransaction), transactionData.size());
@@ -630,7 +641,7 @@ status_t RpcState::processTransactInternal(const base::unique_fd& fd, const sp<R
                 // justification for const_cast (consider avoiding priority_queue):
                 // - AsyncTodo operator< doesn't depend on 'data' object
                 // - gotta go fast
-                std::vector<uint8_t> data = std::move(
+                ByteVec data = std::move(
                         const_cast<BinderNode::AsyncTodo&>(it->second.asyncTodo.top()).data);
                 it->second.asyncTodo.pop();
                 _l.unlock();
@@ -644,7 +655,10 @@ status_t RpcState::processTransactInternal(const base::unique_fd& fd, const sp<R
             .status = replyStatus,
     };
 
-    std::vector<uint8_t> replyData(sizeof(RpcWireReply) + reply.dataSize());
+    ByteVec replyData(sizeof(RpcWireReply) + reply.dataSize());
+    if (!replyData.valid()) {
+        return NO_MEMORY;
+    }
     memcpy(replyData.data() + 0, &rpcReply, sizeof(RpcWireReply));
     memcpy(replyData.data() + sizeof(RpcWireReply), reply.data(), reply.dataSize());
 
@@ -671,7 +685,10 @@ status_t RpcState::processTransactInternal(const base::unique_fd& fd, const sp<R
 status_t RpcState::processDecStrong(const base::unique_fd& fd, const RpcWireHeader& command) {
     LOG_ALWAYS_FATAL_IF(command.command != RPC_COMMAND_DEC_STRONG, "command: %d", command.command);
 
-    std::vector<uint8_t> commandData(command.bodySize);
+    ByteVec commandData(command.bodySize);
+    if (!commandData.valid()) {
+        return NO_MEMORY;
+    }
     if (!rpcRec(fd, "dec ref body", commandData.data(), commandData.size())) {
         return DEAD_OBJECT;
     }