Track whether noncharacters are in a string + wp_has_noncharacters()

Author: dmsnell

src/wp-includes/compat-utf8.php

@@ -35,19 +35,21 @@
  * @since 6.9.0
  * @access private
  *
- * @param string   $bytes           UTF-8 encoded string which might include invalid spans of bytes.
- * @param int      $at              Where to start scanning.
- * @param int      $invalid_length  Will be set to how many bytes are to be ignored after `$at`.
- * @param int|null $max_bytes       Stop scanning after this many bytes have been seen.
- * @param int|null $max_code_points Stop scanning after this many code points have been seen.
+ * @param string   $bytes             UTF-8 encoded string which might include invalid spans of bytes.
+ * @param int      $at                Where to start scanning.
+ * @param int      $invalid_length    Will be set to how many bytes are to be ignored after `$at`.
+ * @param int|null $max_bytes         Stop scanning after this many bytes have been seen.
+ * @param int|null $max_code_points   Stop scanning after this many code points have been seen.
+ * @param bool     $has_noncharacters Set to indicate if scanned string contained noncharacters.
  * @return int How many code points were successfully scanned.
  */
-function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max_bytes = null, ?int $max_code_points = null ): int {
-	$byte_length    = strlen( $bytes );
-	$end            = min( $byte_length, $at + ( $max_bytes ?? PHP_INT_MAX ) );
-	$invalid_length = 0;
-	$count          = 0;
-	$max_count      = $max_code_points ?? PHP_INT_MAX;
+function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max_bytes = null, ?int $max_code_points = null, ?bool &$has_noncharacters = null ): int {
+	$byte_length       = strlen( $bytes );
+	$end               = min( $byte_length, $at + ( $max_bytes ?? PHP_INT_MAX ) );
+	$invalid_length    = 0;
+	$count             = 0;
+	$max_count         = $max_code_points ?? PHP_INT_MAX;
+	$has_noncharacters = false;
 
 	for ( $i = $at; $i < $end && $count <= $max_count; $i++ ) {
 		/*
@@ -145,6 +147,18 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
 		) {
 			++$count;
 			$i += 2;
+
+			// Covers the range U+FDD0–U+FDEF, U+FD.
+			$is_noncharacter = (
+				0xEF === $b1 &&
+				(
+					( 0xB7 === $b2 && $b3 >= 0x90 && $b3 <= 0xAF ) ||
+					( 0xBF === $b2 && ( 0xBE === $b3 || 0xBF === $b3 ) )
+				)
+			);
+
+			$has_noncharacters = $has_noncharacters || $is_noncharacter;
+
 			continue;
 		}
 
@@ -162,6 +176,14 @@ function _wp_scan_utf8( string $bytes, int &$at, int &$invalid_length, ?int $max
 		) {
 			++$count;
 			$i += 3;
+
+			$is_noncharacter = (
+				( 0x0F === ( $b2 & 0x0F ) ) &&
+				0xBF === $b3 &&
+				( 0xBE === $b4 || 0xBF === $b4 )
+			);
+			$has_noncharacters = $has_noncharacters || $is_noncharacter;
+
 			continue;
 		}
 

src/wp-includes/utf8.php

@@ -133,3 +133,30 @@ function wp_scrub_utf8( $text ) {
 		return _wp_scrub_utf8_fallback( $text );
 	}
 endif;
+
+/**
+ * Returns whether the given string contains Unicode noncharacters.
+ *
+ * Noncharacters:
+ *  - U+FDD0–U+FDEF
+ *  - U+FFFE–U+FFFF
+ *  - U+1FFFx, U+2FFFx, …, U+FFFFx, U+10FFFx (where x is either E or F)
+ *
+ * @todo link to Unicode noncharacter spec.
+ *
+ * @param string $text Are there noncharacters in this string?
+ * @return bool Whether noncharacters were found in the string.
+ */
+function wp_has_noncharacters( string $text ): bool {
+	$at                = 0;
+	$invalid_length    = 0;
+	$has_noncharacters = false;
+	$end               = strlen( $text );
+
+	while ( $at < $end && !$has_noncharacters ) {
+		_wp_scan_utf8( $text, $at, $invalid_length, null, $has_noncharacters );
+		$at += $invalid_length;
+	}
+
+	return $has_noncharacters;
+}