key_length = $key_length; // Start by grouping words. $groups = array(); $shorts = array(); foreach ( $mappings as $word => $mapping ) { if ( self::MAX_LENGTH <= strlen( $word ) || self::MAX_LENGTH <= strlen( $mapping ) ) { _doing_it_wrong( __METHOD__, sprintf( /* translators: 1: maximum byte length (a count) */ __( 'Token Map tokens and substitutions must all be shorter than %1$d bytes.' ), self::MAX_LENGTH ), '6.6.0' ); return null; } $length = strlen( $word ); if ( $key_length >= $length ) { $shorts[] = $word; } else { $group = substr( $word, 0, $key_length ); if ( ! isset( $groups[ $group ] ) ) { $groups[ $group ] = array(); } $groups[ $group ][] = array( substr( $word, $key_length ), $mapping ); } } /* * Sort the words to ensure that no smaller substring of a match masks the full match. * For example, `Cap` should not match before `CapitalDifferentialD`. */ usort( $shorts, 'WP_Token_Map::longest_first_then_alphabetical' ); foreach ( $groups as $group_key => $group ) { usort( $groups[ $group_key ], static function ( array $a, array $b ): int { return self::longest_first_then_alphabetical( $a[0], $b[0] ); } ); } // Finally construct the optimized lookups. foreach ( $shorts as $word ) { $map->small_words .= str_pad( $word, $key_length + 1, "\x00", STR_PAD_RIGHT ); $map->small_mappings[] = $mappings[ $word ]; } $group_keys = array_keys( $groups ); sort( $group_keys ); foreach ( $group_keys as $group ) { $map->groups .= "{$group}\x00"; $group_string = ''; foreach ( $groups[ $group ] as $group_word ) { list( $word, $mapping ) = $group_word; $word_length = pack( 'C', strlen( $word ) ); $mapping_length = pack( 'C', strlen( $mapping ) ); $group_string .= "{$word_length}{$word}{$mapping_length}{$mapping}"; } $map->large_words[] = $group_string; } return $map; } public static function from_precomputed_table( $state ): ?WP_Token_Map { $has_necessary_state = isset( $state['storage_version'], $state['key_length'], $state['groups'], $state['large_words'], $state['small_words'], $state['small_mappings'] ); if ( ! $has_necessary_state ) { _doing_it_wrong( __METHOD__, __( 'Missing required inputs to pre-computed WP_Token_Map.' ), '6.6.0' ); return null; } if ( self::STORAGE_VERSION !== $state['storage_version'] ) { _doing_it_wrong( __METHOD__, /* translators: 1: version string, 2: version string. */ sprintf( __( 'Loaded version \'%1$s\' incompatible with expected version \'%2$s\'.' ), $state['storage_version'], self::STORAGE_VERSION ), '6.6.0' ); return null; } $map = new WP_Token_Map(); $map->key_length = $state['key_length']; $map->groups = $state['groups']; $map->large_words = $state['large_words']; $map->small_words = $state['small_words']; $map->small_mappings = $state['small_mappings']; return $map; } public function contains( string $word, string $case_sensitivity = 'case-sensitive' ): bool { $ignore_case = 'ascii-case-insensitive' === $case_sensitivity; if ( $this->key_length >= strlen( $word ) ) { if ( 0 === strlen( $this->small_words ) ) { return false; } $term = str_pad( $word, $this->key_length + 1, "\x00", STR_PAD_RIGHT ); $word_at = $ignore_case ? stripos( $this->small_words, $term ) : strpos( $this->small_words, $term ); if ( false === $word_at ) { return false; } return true; } $group_key = substr( $word, 0, $this->key_length ); $group_at = $ignore_case ? stripos( $this->groups, $group_key ) : strpos( $this->groups, $group_key ); if ( false === $group_at ) { return false; } $group = $this->large_words[ $group_at / ( $this->key_length + 1 ) ]; $group_length = strlen( $group ); $slug = substr( $word, $this->key_length ); $length = strlen( $slug ); $at = 0; while ( $at < $group_length ) { $token_length = unpack( 'C', $group[ $at++ ] )[1]; $token_at = $at; $at += $token_length; $mapping_length = unpack( 'C', $group[ $at++ ] )[1]; $mapping_at = $at; if ( $token_length === $length && 0 === substr_compare( $group, $slug, $token_at, $token_length, $ignore_case ) ) { return true; } $at = $mapping_at + $mapping_length; } return false; } public function read_token( string $text, int $offset = 0, &$matched_token_byte_length = null, $case_sensitivity = 'case-sensitive' ): ?string { $ignore_case = 'ascii-case-insensitive' === $case_sensitivity; $text_length = strlen( $text ); // Search for a long word first, if the text is long enough, and if that fails, a short one. if ( $text_length > $this->key_length ) { $group_key = substr( $text, $offset, $this->key_length ); $group_at = $ignore_case ? stripos( $this->groups, $group_key ) : strpos( $this->groups, $group_key ); if ( false === $group_at ) { // Perhaps a short word then. return strlen( $this->small_words ) > 0 ? $this->read_small_token( $text, $offset, $matched_token_byte_length, $case_sensitivity ) : null; } $group = $this->large_words[ $group_at / ( $this->key_length + 1 ) ]; $group_length = strlen( $group ); $at = 0; while ( $at < $group_length ) { $token_length = unpack( 'C', $group[ $at++ ] )[1]; $token = substr( $group, $at, $token_length ); $at += $token_length; $mapping_length = unpack( 'C', $group[ $at++ ] )[1]; $mapping_at = $at; if ( 0 === substr_compare( $text, $token, $offset + $this->key_length, $token_length, $ignore_case ) ) { $matched_token_byte_length = $this->key_length + $token_length; return substr( $group, $mapping_at, $mapping_length ); } $at = $mapping_at + $mapping_length; } } // Perhaps a short word then. return strlen( $this->small_words ) > 0 ? $this->read_small_token( $text, $offset, $matched_token_byte_length, $case_sensitivity ) : null; } private function read_small_token( string $text, int $offset = 0, &$matched_token_byte_length = null, $case_sensitivity = 'case-sensitive' ): ?string { $ignore_case = 'ascii-case-insensitive' === $case_sensitivity; $small_length = strlen( $this->small_words ); $search_text = substr( $text, $offset, $this->key_length ); if ( $ignore_case ) { $search_text = strtoupper( $search_text ); } $starting_char = $search_text[0]; $at = 0; while ( $at < $small_length ) { if ( $starting_char !== $this->small_words[ $at ] && ( ! $ignore_case || strtoupper( $this->small_words[ $at ] ) !== $starting_char ) ) { $at += $this->key_length + 1; continue; } for ( $adjust = 1; $adjust < $this->key_length; $adjust++ ) { if ( "\x00" === $this->small_words[ $at + $adjust ] ) { $matched_token_byte_length = $adjust; return $this->small_mappings[ $at / ( $this->key_length + 1 ) ]; } if ( $search_text[ $adjust ] !== $this->small_words[ $at + $adjust ] && ( ! $ignore_case || strtoupper( $this->small_words[ $at + $adjust ] !== $search_text[ $adjust ] ) ) ) { $at += $this->key_length + 1; continue 2; } } $matched_token_byte_length = $adjust; return $this->small_mappings[ $at / ( $this->key_length + 1 ) ]; } return null; } public function to_array(): array { $tokens = array(); $at = 0; $small_mapping = 0; $small_length = strlen( $this->small_words ); while ( $at < $small_length ) { $key = rtrim( substr( $this->small_words, $at, $this->key_length + 1 ), "\x00" ); $value = $this->small_mappings[ $small_mapping++ ]; $tokens[ $key ] = $value; $at += $this->key_length + 1; } foreach ( $this->large_words as $index => $group ) { $prefix = substr( $this->groups, $index * ( $this->key_length + 1 ), 2 ); $group_length = strlen( $group ); $at = 0; while ( $at < $group_length ) { $length = unpack( 'C', $group[ $at++ ] )[1]; $key = $prefix . substr( $group, $at, $length ); $at += $length; $length = unpack( 'C', $group[ $at++ ] )[1]; $value = substr( $group, $at, $length ); $tokens[ $key ] = $value; $at += $length; } } return $tokens; } public function precomputed_php_source_table( string $indent = "\t" ): string { $i1 = $indent; $i2 = $i1 . $indent; $i3 = $i2 . $indent; $class_version = self::STORAGE_VERSION; $output = self::class . "::from_precomputed_table(\n"; $output .= "{$i1}array(\n"; $output .= "{$i2}\"storage_version\" => \"{$class_version}\",\n"; $output .= "{$i2}\"key_length\" => {$this->key_length},\n"; $group_line = str_replace( "\x00", "\\x00", $this->groups ); $output .= "{$i2}\"groups\" => \"{$group_line}\",\n"; $output .= "{$i2}\"large_words\" => array(\n"; $prefixes = explode( "\x00", $this->groups ); foreach ( $prefixes as $index => $prefix ) { if ( '' === $prefix ) { break; } $group = $this->large_words[ $index ]; $group_length = strlen( $group ); $comment_line = "{$i3}//"; $data_line = "{$i3}\""; $at = 0; while ( $at < $group_length ) { $token_length = unpack( 'C', $group[ $at++ ] )[1]; $token = substr( $group, $at, $token_length ); $at += $token_length; $mapping_length = unpack( 'C', $group[ $at++ ] )[1]; $mapping = substr( $group, $at, $mapping_length ); $at += $mapping_length; $token_digits = str_pad( dechex( $token_length ), 2, '0', STR_PAD_LEFT ); $mapping_digits = str_pad( dechex( $mapping_length ), 2, '0', STR_PAD_LEFT ); $mapping = preg_replace_callback( "~[\\x00-\\x1f\\x22\\x5c]~", static function ( $match_result ) { switch ( $match_result[0] ) { case '"': return '\\"'; case '\\': return '\\\\'; default: $hex = dechex( ord( $match_result[0] ) ); return "\\x{$hex}"; } }, $mapping ); $comment_line .= " {$prefix}{$token}[{$mapping}]"; $data_line .= "\\x{$token_digits}{$token}\\x{$mapping_digits}{$mapping}"; } $comment_line .= ".\n"; $data_line .= "\",\n"; $output .= $comment_line; $output .= $data_line; } $output .= "{$i2}),\n"; $small_words = array(); $small_length = strlen( $this->small_words ); $at = 0; while ( $at < $small_length ) { $small_words[] = substr( $this->small_words, $at, $this->key_length + 1 ); $at += $this->key_length + 1; } $small_text = str_replace( "\x00", '\x00', implode( '', $small_words ) ); $output .= "{$i2}\"small_words\" => \"{$small_text}\",\n"; $output .= "{$i2}\"small_mappings\" => array(\n"; foreach ( $this->small_mappings as $mapping ) { $output .= "{$i3}\"{$mapping}\",\n"; } $output .= "{$i2})\n"; $output .= "{$i1})\n"; $output .= ')'; return $output; } private static function longest_first_then_alphabetical( string $a, string $b ): int { if ( $a === $b ) { return 0; } $length_a = strlen( $a ); $length_b = strlen( $b ); // Longer strings are less-than for comparison's sake. if ( $length_a !== $length_b ) { return $length_b - $length_a; } return strcmp( $a, $b ); } }