<?php
/**
 * Multi Byte Character uni-gram Analyzer
 * like Java CJKAnalyzer
 *
 * @version 20080616
 * @author twk (http://nonn-et-twk.net/twk/)
 */

// Requires analyzer.php
// @see http://framework.zend.com/issues/browse/ZF-991
require_once 'Zend/Search/Lucene/Analysis/Analyzer.php';

/** Zend_Search_Lucene_Analysis_Analyzer_Common */
//require_once 'Zend/Search/Lucene/Analysis/Analyzer/Common.php';


/**
 * @category   Twk
 * @package    Twk_Search_Lucene
 * @subpackage Analysis
 * @license    http://framework.zend.com/license/new-bsd     New BSD License
 */

class Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram extends Zend_Search_Lucene_Analysis_Analyzer_Common
{
	/**
	 * Current char position in an UTF-8 stream
	 *
	 * @var integer
	 */
	private $_position;

	/**
	 * Current binary position in an UTF-8 stream
	 *
	 * @var integer
	 */
	private $_bytePosition;

	private $_lastPosition;
	private $_lastBytePosition;

	/**
	 * Stream length
	 *
	 * @var integer
	 */
	private $_streamLength;

	const CATEGORY_ALNUM = 1;
	const CATEGORY_PUNCT = 2;
	const CATEGORY_CNTRL = 3;
	const CATEGORY_SPACE = 4;
	const CATEGORY_SBCS_OTHER = 5;
	const CATEGORY_MBCS = 0x10;
	const CATEGORY_HIRAGANA = 0x11;
	const CATEGORY_KATAKANA = 0x12;
	const CATEGORY_JAPANESE_KANJI = 0x13;
	const CATEGORY_FULLWIDTH_ALNUM = 0x14;
	const CATEGORY_JAPANESE_PUNCT = 0x14;
	const CATEGORY_MBCS_OTHER = 0x15;

	// 2009/2/27 Kenji Suzuki added for concere5
	public function __construct()
	{
		require_once 'Zend/Search/Lucene/Analysis/TokenFilter/LowerCase.php';
		$this->addFilter(new Zend_Search_Lucene_Analysis_TokenFilter_LowerCaseUtf8());
	}

	/**
	 * Reset token stream
	 */
	public function reset()
	{
		$this->_position     = 0;
		$this->_bytePosition = 0;

		$this->_lastPosition = 0;
		$this->_lastBytePosition = 0;

		// convert input into UTF-8
		if (strcasecmp($this->_encoding, 'utf8' ) != 0  &&
		strcasecmp($this->_encoding, 'utf-8') != 0 ) {
			$this->_input = iconv($this->_encoding, 'UTF-8', $this->_input);
			$this->_encoding = 'UTF-8';
		}

		// Get UTF-8 string length.
		// It also checks if it's a correct utf-8 string
		$this->_streamLength = iconv_strlen($this->_input, 'UTF-8');
	}

	/**
	 * @return CATEGORY_xxx
	 */
	private static function _getCharCategory($char)
	{
		if (!self::_isMbcs($char))
		{
			if (ctype_alnum($char))
			$category = self::CATEGORY_ALNUM;
			else if (ctype_punct($char))
			$category = self::CATEGORY_PUNCT;
			else if (ctype_cntrl($char))
			$category = self::CATEGORY_CNTRL;
			else if (ctype_space($char))
			$category = self::CATEGORY_SPACE;
			else
			$category = self::CATEGORY_SBCS_OTHER;
		}
		else
		{
			if (extension_loaded('mbstring'))
			{
				// @see http://ablog.seesaa.net/article/20969848.html
				// 記号は代表的な物のみ指定している。pregと/u使っても書けそう
				if (mb_ereg_match('^[ぁ-ん]+$', $char))
				$category = self::CATEGORY_HIRAGANA;
				else if (mb_ereg_match('^[ァ-ヴー]+$', $char))
				$category = self::CATEGORY_KATAKANA;
				else if (mb_ereg_match('^[一-龠々〆ヵヶ]+$', $char))
				$category = self::CATEGORY_JAPANESE_KANJI;
				else if (mb_ereg_match('^[、。！？（）「」『』【】]+$', $char))
				$category = self::CATEGORY_JAPANESE_PUNCT;
				else if (mb_ereg_match('^[ａ-ｚＡ-Ｚ０-９]+$', $char))
				$category = self::CATEGORY_FULLWIDTH_ALNUM;
				else
				$category = self::CATEGORY_MBCS_OTHER;
			}
			else
			{
				$category = self::CATEGORY_MBCS_OTHER;
			}
		}

		return $category;
	}

	/**
	 * Check, how that character is
	 *
	 * @param string $char
	 * @return boolean
	 */
	private static function _isWhiteSpaceCategory($char)
	{
		static $whiteSpaceCategories = array(self::CATEGORY_PUNCT, self::CATEGORY_CNTRL, self::CATEGORY_SPACE, self::CATEGORY_JAPANESE_PUNCT);

		$category = self::_getCharCategory($char);
		//echo $category, "\n";

		return in_array($category, $whiteSpaceCategories);
	}

	/**
	 * Get next UTF-8 char
	 * _position and _bytePosition are modified
	 * last positions are stored to _lastPosition and _lastBytePosition
	 *
	 * @param string $char
	 * @return boolean
	 */
	private function _nextChar()
	{
		$this->_lastBytePosition = $this->_bytePosition;
		$this->_lastPosition = $this->_position;
			
		$char = $this->_input[$this->_bytePosition++];

		$ordChar = ord($char);
		if (( $ordChar & 0xC0 ) == 0xC0) {
			$addBytes = 1;
			if ($ordChar & 0x20 ) {
				$addBytes++;
				if ($ordChar & 0x10 ) {
					$addBytes++;
				}
			}
			$char .= substr($this->_input, $this->_bytePosition, $addBytes);
			$this->_bytePosition += $addBytes;
		}

		$this->_position++;

		return $char;
	}

	private function _undoLastChar()
	{
		$this->_bytePosition = $this->_lastBytePosition;
		$this->_position = $this->_lastPosition;
	}

	/**
	 * @return boolean
	 */
	private static function _isMbcs($char)
	{
		return strlen($char) > 1;
	}

	/**
	 * Tokenization stream API
	 * Get next token
	 * Returns null at the end of stream
	 *
	 * @return Zend_Search_Lucene_Analysis_Token|null
	 */
	public function nextToken()
	{
		if ($this->_input === null) {
			return null;
		}

		while ($this->_position < $this->_streamLength) {
			// skip white spaces
			while ($this->_position < $this->_streamLength &&
			self::_isWhiteSpaceCategory($char = $this->_nextChar())) {
				$char = '';
			}
			if ($char === '') {
				return null;
			}

			$wasMbcs = $this->_isMbcs($char);
			$this->_undoLastChar();
			// echo $this->_position, '=', $char;

			// read token
			$termStartPosition = $this->_position;
			$termText = '';
			$isTokenDivided = false;
			while ($this->_position < $this->_streamLength) {
				$char = $this->_nextChar();
				if (self::_isWhiteSpaceCategory($char))
				{
					$isTokenDivided = true;
					break;
				}
				else if (!$wasMbcs)
				{
					if ($this->_isMbcs($char))
					{
						$isTokenDivided = true;
						break;
					}
				}
				else
				{
					if (!$this->_isMbcs($char) || iconv_strlen($termText, 'UTF-8') >= 1)
					{
						$isTokenDivided = true;
						break;
					}
				}
				$termText .= $char;
			}

			// Empty token, end of stream.
			if ($termText == '') {
				return null;
			}

			// if not End, undo the last char
			// echo $this->_position ,"<=", $this->_streamLength;
			if ($isTokenDivided) {
				$this->_undoLastChar();
			}

			$termEndPosition = $this->_position;
			$token = new Zend_Search_Lucene_Analysis_Token(
			$termText,
			$termStartPosition,
			$termEndPosition);
			$token = $this->normalize($token);
			if ($token !== null) {
				// echo "$termText $termStartPosition $termEndPosition"
				return $token;
			}

			// Continue if token is skipped
		}

		return null;
	}
}


/*
// ここから先は直接実行の時のみ動きます。チェックは甘め
if (!count(debug_backtrace()))
{
	$fp = fopen(__FILE__, 'r');
	fseek($fp, __COMPILER_HALT_OFFSET__ + 2); // 2 for ? and >
	echo stream_get_contents($fp);

	$text = '日本でWindows 95が発売されたのはいつでしょう?';
	$analyzer = new Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram();
	$a = $analyzer->tokenize($text, 'UTF-8');
	print_r($a);
}

__halt_compiler();?>
<html>
<head>
<title>Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram</title>
</head>
<body>
<h1>Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram</h1>
<pre>
	Zend_Search_Lucene用日本語アナライザー
	作者 twk

	<a
	href="http://coderepos.org/share/browser/lang/php/ZendFramework_ext/library/Twk/Search/Lucene/Analysis/Analyzer/Common/">配布場所</a>
	
	コード例:
	<code>
		$text = '日本でWindows 95が発売されたのはいつでしょう?';
		$analyzer = new Twk_Search_Lucene_Analysis_Analyzer_Common_Utf8MbcsUnigram();
		$a = $analyzer-&gt;tokenize($text, 'UTF-8');
		print_r($a);	
	</code>
</pre>
</body>
</html>
*/
