test/ti-agent/ti_utf.h

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ART_TEST_TI_AGENT_TI_UTF_H_
#define ART_TEST_TI_AGENT_TI_UTF_H_

#include <inttypes.h>
#include <string.h>

#include "android-base/logging.h"
#include "ti_macros.h"

namespace art {
namespace ti {

inline size_t CountModifiedUtf8Chars(const char* utf8, size_t byte_count) {
  DCHECK_LE(byte_count, strlen(utf8));
  size_t len = 0;
  const char* end = utf8 + byte_count;
  for (; utf8 < end; ++utf8) {
    int ic = *utf8;
    len++;
    if (LIKELY((ic & 0x80) == 0)) {
      // One-byte encoding.
      continue;
    }
    // Two- or three-byte encoding.
    utf8++;
    if ((ic & 0x20) == 0) {
      // Two-byte encoding.
      continue;
    }
    utf8++;
    if ((ic & 0x10) == 0) {
      // Three-byte encoding.
      continue;
    }

    // Four-byte encoding: needs to be converted into a surrogate
    // pair.
    utf8++;
    len++;
  }
  return len;
}

inline uint16_t GetTrailingUtf16Char(uint32_t maybe_pair) {
  return static_cast<uint16_t>(maybe_pair >> 16);
}

inline uint16_t GetLeadingUtf16Char(uint32_t maybe_pair) {
  return static_cast<uint16_t>(maybe_pair & 0x0000FFFF);
}

inline uint32_t GetUtf16FromUtf8(const char** utf8_data_in) {
  const uint8_t one = *(*utf8_data_in)++;
  if ((one & 0x80) == 0) {
    // one-byte encoding
    return one;
  }

  const uint8_t two = *(*utf8_data_in)++;
  if ((one & 0x20) == 0) {
    // two-byte encoding
    return ((one & 0x1f) << 6) | (two & 0x3f);
  }

  const uint8_t three = *(*utf8_data_in)++;
  if ((one & 0x10) == 0) {
    return ((one & 0x0f) << 12) | ((two & 0x3f) << 6) | (three & 0x3f);
  }

  // Four byte encodings need special handling. We'll have
  // to convert them into a surrogate pair.
  const uint8_t four = *(*utf8_data_in)++;

  // Since this is a 4 byte UTF-8 sequence, it will lie between
  // U+10000 and U+1FFFFF.
  //
  // TODO: What do we do about values in (U+10FFFF, U+1FFFFF) ? The
  // spec says they're invalid but nobody appears to check for them.
  const uint32_t code_point = ((one & 0x0f) << 18) | ((two & 0x3f) << 12)
      | ((three & 0x3f) << 6) | (four & 0x3f);

  uint32_t surrogate_pair = 0;
  // Step two: Write out the high (leading) surrogate to the bottom 16 bits
  // of the of the 32 bit type.
  surrogate_pair |= ((code_point >> 10) + 0xd7c0) & 0xffff;
  // Step three : Write out the low (trailing) surrogate to the top 16 bits.
  surrogate_pair |= ((code_point & 0x03ff) + 0xdc00) << 16;

  return surrogate_pair;
}

// Note: This is a copy of the code in `libdexfile`.
template <bool kUseShortZero, bool kUse4ByteSequence, bool kReplaceBadSurrogates, typename Append>
inline void ConvertUtf16ToUtf8(const uint16_t* utf16, size_t char_count, Append&& append) {
  static_assert(kUse4ByteSequence || !kReplaceBadSurrogates);

  // Use local helpers instead of macros from `libicu` to avoid the dependency on `libicu`.
  auto is_lead = [](uint16_t ch) ALWAYS_INLINE { return (ch & 0xfc00u) == 0xd800u; };
  auto is_trail = [](uint16_t ch) ALWAYS_INLINE { return (ch & 0xfc00u) == 0xdc00u; };
  auto is_surrogate = [](uint16_t ch) ALWAYS_INLINE { return (ch & 0xf800u) == 0xd800u; };
  auto is_surrogate_lead = [](uint16_t ch) ALWAYS_INLINE { return (ch & 0x0400u) == 0u; };
  auto get_supplementary = [](uint16_t lead, uint16_t trail) ALWAYS_INLINE {
    constexpr uint32_t offset = (0xd800u << 10) + 0xdc00u - 0x10000u;
    return (static_cast<uint32_t>(lead) << 10) + static_cast<uint32_t>(trail) - offset;
  };

  for (size_t i = 0u; i < char_count; ++i) {
    auto has_trail = [&]() { return i + 1u != char_count && is_trail(utf16[i + 1u]); };

    uint16_t ch = utf16[i];
    if (ch < 0x80u && (kUseShortZero || ch != 0u)) {
      // One byte.
      append(ch);
    } else if (ch < 0x800u) {
      // Two bytes.
      append((ch >> 6) | 0xc0);
      append((ch & 0x3f) | 0x80);
    } else if (kReplaceBadSurrogates
                   ? is_surrogate(ch)
                   : kUse4ByteSequence && is_lead(ch) && has_trail()) {
      if (kReplaceBadSurrogates && (!is_surrogate_lead(ch) || !has_trail())) {
        append('?');
      } else {
        // We have a *valid* surrogate pair.
        uint32_t code_point = get_supplementary(ch, utf16[i + 1u]);
        ++i;  //  Consume the leading surrogate.
        // Four bytes.
        append((code_point >> 18) | 0xf0);
        append(((code_point >> 12) & 0x3f) | 0x80);
        append(((code_point >> 6) & 0x3f) | 0x80);
        append((code_point & 0x3f) | 0x80);
      }
    } else {
      // Three bytes.
      append((ch >> 12) | 0xe0);
      append(((ch >> 6) & 0x3f) | 0x80);
      append((ch & 0x3f) | 0x80);
    }
  }
}

inline void ConvertUtf16ToModifiedUtf8(char* utf8_out,
                                       size_t byte_count,
                                       const uint16_t* utf16_in,
                                       size_t char_count) {
  if (LIKELY(byte_count == char_count)) {
    // Common case where all characters are ASCII.
    const uint16_t *utf16_end = utf16_in + char_count;
    for (const uint16_t *p = utf16_in; p < utf16_end;) {
      *utf8_out++ = static_cast<char>(*p++);
    }
    return;
  }

  // String contains non-ASCII characters.
  // FIXME: We should not emit 4-byte sequences. Bug: 192935764
  auto append = [&](char c) { *utf8_out++ = c; };
  ConvertUtf16ToUtf8</*kUseShortZero=*/ false,
                     /*kUse4ByteSequence=*/ true,
                     /*kReplaceBadSurrogates=*/ false>(utf16_in, char_count, append);
}

inline size_t CountModifiedUtf8BytesInUtf16(const uint16_t* chars, size_t char_count) {
  // FIXME: We should not emit 4-byte sequences. Bug: 192935764
  size_t result = 0;
  auto append = [&](char c ATTRIBUTE_UNUSED) { ++result; };
  ConvertUtf16ToUtf8</*kUseShortZero=*/ false,
                     /*kUse4ByteSequence=*/ true,
                     /*kReplaceBadSurrogates=*/ false>(chars, char_count, append);
  return result;
}

}  // namespace ti
}  // namespace art

#endif  // ART_TEST_TI_AGENT_TI_UTF_H_