uimaj-core/src/main/java/org/apache/uima/internal/util/TextStringTokenizer.java [404:427]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public int getCharType(char c) {
    // First, check user-defined lists in the order end-of-sentence
    // delimiter, separator character, whitespace and finally regular
    // character that can be part of a word.
    if (Arrays.binarySearch(eosDels, c) >= 0) {
      return EOS;
    }
    if (Arrays.binarySearch(separators, c) >= 0) {
      return SEP;
    }
    if (Arrays.binarySearch(whitespace, c) >= 0) {
      return WSP;
    }
    if (Arrays.binarySearch(wordChars, c) >= 0) {
      return WCH;
    }

    // If we get here, we check if it's Unicode whitespace.
    // Otherwise, we consider it a word character.
    if (Character.isWhitespace(c)) {
      return WSP;
    }
    return WCH;
  }
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



uimaj-core/src/main/java/org/apache/uima/internal/util/TextTokenizer.java [382:405]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public int getCharType(char c) {
    // First, check user-defined lists in the order end-of-sentence
    // delimiter, separator character, whitespace and finally regular
    // character that can be part of a word.
    if (Arrays.binarySearch(eosDels, c) >= 0) {
      return EOS;
    }
    if (Arrays.binarySearch(separators, c) >= 0) {
      return SEP;
    }
    if (Arrays.binarySearch(whitespace, c) >= 0) {
      return WSP;
    }
    if (Arrays.binarySearch(wordChars, c) >= 0) {
      return WCH;
    }

    // If we get here, we check if it's Unicode whitespace.
    // Otherwise, we consider it a word character.
    if (Character.isWhitespace(c)) {
      return WSP;
    }
    return WCH;
  }
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -