{*********************************************************

 SlavaNap source code.

 Copyright 2001,2002 by SlavaNap development team
 Released under GNU General Public License

 Latest version is available at
 http://www.slavanap.org

**********************************************************

 Unit: keywords

 Functions to manage/search keywords list

*********************************************************}
unit keywords;

interface

uses
 Windows, Classes2, SysUtils, stypes, constants, slavastrings, class_cmdlist,
 class_cmd2list, keywords2, share;

{$I defines.pas}

procedure InitKeywords; // should be called before first share/search command
procedure FreeKeywords; // should be called after all files are unshared
function  SplitToKeywords(str: String; list: TNapCmdList; limit: Integer): Integer;
procedure SplitToKeywordsEx(inc_str, exc_str: String; inc_list, exc_list: TNapCmdList; limit: Integer);
function  GetKeywordIndex(keyword: String): Integer;
procedure LoadDirs;
procedure SaveDirs;
procedure FreeDirs;

var
 // global list of keywords.
 // sorted this way (3-dimentional array to search it faster):
 // 1 - type of shared file: SHARED_xxx constants
 // 2 - first character of keyword
 // 3 - length of keyword
 db_keywords: Array[0..SHARED_ARRAY-1,0..KEYWORDS_FIRST_COUNT-1,KEYWORD_LEN_MIN..KEYWORD_LEN_MAX] of PKeywordList;

implementation

uses
 vars, handler;

procedure Split2ByteString(str: PString);//2バイトキーワードを変換して区切る
var
 i: Integer;
 t,tprev: TMbcsByteType;
begin
  i:=1;
  while i<=Length(str^) do
  begin
    t:=ByteType(str^,i);
    tprev:=ByteType(str^,i-1);
    if (t=mbSingleByte) and (tprev=mbSingleByte) then
    begin
      if ((str^[i] in [#$30..#$7A]) and (str^[i-1] in [#$A6..#$DF])) or
         ((str^[i] in [#$A6..#$DF]) and (str^[i-1] in [#$30..#$7A])) then
      begin //半角英数字と半角カタカナの間に区切り
        Insert('.',str^,i);
        Inc(i);
      end                    
      else if ((str^[i] in [#$30..#$39]) and (str^[i-1] in [#$40..#$7A])) or
              ((str^[i] in [#$40..#$7A]) and (str^[i-1] in [#$30..#$39])) then
      begin //数字とアルファベット+'@'の間に区切り
        Insert('.',str^,i);
        Inc(i);
      end;
    end
    else if (t=mbSingleByte) and (tprev=mbTrailByte) then
    begin
      if (str^[i]='ﾞ') and (str^[i-2]=#$82) and (StrScan(HIRAGANA_D,str^[i-1])<>nil) then
      begin //濁音になれる全角ひらがな+'ﾞ'を濁音に変換
        Inc(str^[i-1]);
        Delete(str^,i,1);
        Dec(i);
      end
      else if (str^[i]='ﾟ') and (str^[i-2]=#$82) and (StrScan(HIRAGANA_H,str^[i-1])<>nil) then
      begin //半濁音になれる全角ひらがな+'ﾟ'を半濁音に変換
        Inc(str^[i-1],2);
        Delete(str^,i,1);
        Dec(i);
      end
      else
      begin //全角文字と半角文字の間に区切り
        Insert('.',str^,i);
        Inc(i);
      end;
    end
    else if t=mbLeadByte then
    begin
      if (str^[i]=#$81) and (str^[i+1] in [#$40..#$49,#$4C..#$51,#$5C..#$FC]) then
      begin //全角記号→区切り(例外:゛゜ヽヾゝゞ〃仝々〆〇ー)
        Delete(str^,i,2);
        Insert('.',str^,i);
      end
      else if (str^[i]=#$81) and (str^[i+1] in [#$4A,#$4B,#$5B]) then
      begin //゛゜ーを半角にする
        if str^[i+1]=#$4A then str^[i+1]:=#$DE
        else if str^[i+1]=#$4B then str^[i+1]:=#$DF
        else if str^[i+1]=#$5B then str^[i+1]:=#$B0;
        Delete(str^,i,1);
        Dec(i);
      end
      else if (str^[i]=#$82) and (str^[i+1] in [#$4F..#$58,#$60..#$79,#$81..#$9A]) then
      begin //全角英数字を半角にする
        str^[i+1]:=EISUUJI[Ord(str^[i+1])-$4F+1];
        Delete(str^,i,1);
        Dec(i);
      end
      else if (str^[i]=#$83) and (str^[i+1] in [#$40..#$96]) then
      begin //全角カタカナを半角にする
        str^[i]:=KATAKANA[(Ord(str^[i+1])-$40)*2+1];
        str^[i+1]:=KATAKANA[(Ord(str^[i+1])-$40)*2+2];
        if str^[i]=' ' then Delete(str^,i,1);
        Dec(i);
      end
      else if (tprev=mbSingleByte) and (i>1) and (str^[i-1]<>' ') and (str^[i-1]<>'-') then
      begin //半角文字(空白・'-'以外)と全角文字の間に区切り
        Insert('.',str^,i);
        Inc(i);
      end
      else if tprev=mbTrailByte then
      begin
        if ((str^[i]>=#$88) and (str^[i-2]<#$88)) or
           ((str^[i]<#$88) and (str^[i-2]>=#$88)) then
        begin //漢字とそれ以外の全角文字の間に区切り
          Insert('.',str^,i);
          Inc(i);
        end
        else if ((str^[i]=#$8F) and (str^[i+1]=#$97) and (str^[i+2]=#$8E) and (str^[i+3]=#$71)) or //'女子'
           ((str^[i]=#$8F) and (str^[i+1]=#$AC) and (str^[i+2]=#$8A) and (str^[i+3]=#$77)) or //'小学'
           ((str^[i]=#$92) and (str^[i+1]=#$86) and (str^[i+2]=#$8A) and (str^[i+3]=#$77)) or //'中学'
           ((str^[i]=#$93) and (str^[i+1]=#$90) and (str^[i+2]=#$8E) and (str^[i+3]=#$42)) then //'盗撮'
        begin //特定の漢字の直前に区切り
          Insert('.',str^,i);
          Inc(i,4);
        end;
      end;
    end
    else if t=mbTrailByte then
    begin
      if ((str^[i-1]=#$8D) and (str^[i]=#$CB)) or //'才'
         ((str^[i-1]=#$8D) and (str^[i]=#$CE)) or //'歳'
         ((str^[i-3]=#$8D) and (str^[i-2]=#$82) and (str^[i-1]=#$90) and (str^[i]=#$B6)) or //'高生'
         ((str^[i-3]=#$8A) and (str^[i-2]=#$77) and (str^[i-1]=#$90) and (str^[i]=#$B6)) then //'学生'
      begin //特定の漢字の直後に区切り
        Insert('.',str^,i+1);
        Inc(i);
      end
    end;
    Inc(i);
  end;
end;

function SplitToKeywords(str: String; list: TNapCmdList; limit: Integer): Integer;
var
 i, start, count, added, crc: Integer;
 spacing: Boolean;
 c: Char;
 s: String;
 item: String;
begin // extract up to $limit keywords from string and add it to list with CRC
 // before calling this function you should add ' ' at the end of string or you might loose your last keyword
 tmp_pos:=12262;
 Split2ByteString(@str);
 tmp_pos:=1280;
 result:=0;
 if block_weird_xnapfile and (AnsiPos('?', str)<>0) then exit;
 spacing:=true;
 for i:=1 to Length(str) do
 begin
   c:=str[i];
   if spacing then
   begin // searching for beginning of keyword
//     if pos(c,KEYWORDS_FIRST)>0 then // found our first keyword character
     if pos(c,KEYWORDS_SEPARATORS)=0 then // found our first keyword character
     begin
       start:=i;
       spacing:=false;
     end;
   end
   else
   begin // searching for end of keyword
     if pos(c,KEYWORDS_SEPARATORS)>0 then // found spacing
     if ByteType(str,i)=mbSingleByte then
     begin
       spacing:=true;
       count:=i-start;
       if count>=KEYWORD_LEN_MIN then
       begin
         if count>KEYWORD_LEN_MAX then count:=KEYWORD_LEN_MAX;
         item:=Copy(str,start,count);
         crc:=StringCRC(item,false);
         if list.FindItem(crc,item)=-1 then
         begin
           list.AddCmd(crc,item);
           inc(result);
           if result>=limit then exit;
         end;
       end;
     end;
   end;
 end;
 tmp_pos:=1281;
end;

procedure SplitToKeywordsEx(inc_str, exc_str: String; inc_list, exc_list: TNapCmdList; limit: Integer);
var 
 i, j, start, count, added, crc: Integer;
 spacing, hardspacing, excluding, last_exc: Boolean;
 c: Char;
 s: String;
 item: String; 
begin // extract up to $limit keywords from string and add it to list with CRC
 // before calling this function you should add ' ' at the end of string or you might loose your last keyword
 tmp_pos:=12262;
 Split2byteString(@inc_str);
 Split2byteString(@exc_str);
 tmp_pos:=1280;
 spacing:=true;
 hardspacing:=true;
 excluding:=false;
 last_exc:=false;
 for i:=1 to Length(inc_str) do
 begin
   c:=inc_str[i];
   if spacing then
   begin // searching for beginning of keyword
//     if pos(c,KEYWORDS_FIRST)>0 then // found our first keyword character
     if pos(c,KEYWORDS_SEPARATORS)=0 then // found our first keyword character
     begin
       start:=i;
       spacing:=false;
       hardspacing:=false;
       excluding:=last_exc;
       last_exc:=false;
     end
     else
     begin
      if (c='-') and hardspacing and (ByteType(inc_str,i)=mbSingleByte) then
        last_exc:=true
      else
        last_exc:=false;
      if (c=#$20) and (ByteType(inc_str,i)=mbSingleByte) then hardspacing:=true
      else hardspacing:=false;
     end;
   end
   else
   begin // searching for end of keyword
     last_exc:=false;
     hardspacing:=false;
     if pos(c,KEYWORDS_SEPARATORS)>0 then // found spacing
     if ByteType(inc_str,i)=mbSingleByte then
     begin
       spacing:=true;
       if (c=#$20) and (ByteType(inc_str,i)=mbSingleByte) then hardspacing:=true;
       count:=i-start;
       if count>=KEYWORD_LEN_MIN then
       begin
         if count>KEYWORD_LEN_MAX then count:=KEYWORD_LEN_MAX; 
         item:=Copy(inc_str,start,count); 
         crc:=StringCRC(item,false); 
         if excluding then
         begin
           if exc_list.FindItem(crc,item)=-1 then
            exc_list.AddCmd(crc,item);
           j:=inc_list.FindItem(crc,item);
           if j<>-1 then
            inc_list.Delete(j);
           if not hardspacing then last_exc:=True;
         end
         else
         begin
           if (inc_list.FindItem(crc,item)=-1) and (inc_list.count<limit) then
             inc_list.AddCmd(crc,item);
           j:=exc_list.FindItem(crc,item);
           if j<>-1 then
            exc_list.Delete(j);
         end;
       end; 
     end; 
   end; 
 end;
 spacing:=true;
 hardspacing:=false;
 for i:=1 to Length(exc_str) do 
 begin
   c:=exc_str[i];
   if spacing then
   begin // searching for beginning of keyword 
//     if pos(c,KEYWORDS_FIRST)>0 then // found our first keyword character
     if pos(c,KEYWORDS_SEPARATORS)=0 then // found our first keyword character
     begin
       start:=i;
       spacing:=false;
     end;
   end 
   else 
   begin // searching for end of keyword
     if pos(c,KEYWORDS_SEPARATORS)>0 then // found spacing
     if ByteType(exc_str,i)=mbSingleByte then
     begin
       spacing:=true;
       count:=i-start;
       if count>=KEYWORD_LEN_MIN then
       begin 
         if count>KEYWORD_LEN_MAX then count:=KEYWORD_LEN_MAX;
         item:=Copy(exc_str,start,count);
         crc:=StringCRC(item,false);
         if exc_list.FindItem(crc,item)=-1 then
         begin
           exc_list.AddCmd(crc,item);
         end;
         j:=inc_list.FindItem(crc,item);
         if j<>-1 then
          inc_list.Delete(j);
       end;
     end;
   end;
 end;
 tmp_pos:=1281;
end;

function GetKeywordIndex(keyword: String): Integer;
// returns index of keyword or -1 if keyword is invalid
begin
 if Length(keyword)<1 then Result:=KEYWORDS_NOINDEX
 else if pos(keyword[1],KEYWORDS_FIRST)=0 then result:=KEYWORDS_FIRST_COUNT-1//2byte char
 else result:=pos(keyword[1],KEYWORDS_FIRST)-1;
end;

procedure InitKeywords;
var
 i,j,k: Integer;
begin
 for i:=0 to SHARED_ARRAY-1 do
  for j:=0 to KEYWORDS_FIRST_COUNT-1 do
   for k:=KEYWORD_LEN_MIN to KEYWORD_LEN_MAX do
    db_keywords[i,j,k]:=KWList_CreateList;
end;

procedure FreeKeywords;
var
 i,j,k: Integer;
 ATime: Integer;
begin
 ATime := GetTickCount;
 for i:=0 to SHARED_ARRAY-1 do
  for j:=0 to KEYWORDS_FIRST_COUNT-1 do
   for k:=KEYWORD_LEN_MIN to KEYWORD_LEN_MAX do
   begin
    KWList_FreeList(db_keywords[i,j,k]);
    db_keywords[i,j,k]:=nil;
   end;
 DebugLog('Raptime of FreeKeywords=' + IntToStr(GetTickCount - ATime), True);
end;

procedure LoadDirs;
var
 list: TMyStringList;
 i: Integer;
 str: String;
begin
 if db_dirs<>nil then db_dirs.Clear
 else db_dirs:=TNapCmdList.Create;
 list:=TMyStringList.Create;
 try
  list.LoadFromFile(ApplicationDir+'dirs');
  except
 end;
 for i:=0 to list.count-1 do
  if Length(list.Strings[i])>1 then
   if list.Strings[i][1]<>'#' then
   begin
    str:=AnsiLowerCase(list.Strings[i]);
    db_dirs.AddCmd(StringCRC(str,false),str);
   end;
 list.Free;
end;

procedure SaveDirs;
var
 list: TMyStringList;
 i: Integer;
begin
 if db_dirs=nil then exit;
 list:=TMyStringList.Create;
 for i:=0 to db_dirs.Count-1 do
  list.Add(PNapCmd(db_dirs.Items[i])^.cmd);
 list.Sort;
 list.Insert(0,'# インデックスしない、よくあるディレクトリの一覧');
 list.Insert(1,'# ');
 list.Insert(2,'# よく使われるディレクトリを共有から除外すれば、');
 list.Insert(3,'# 無駄な検索結果が減って、サーバーが使うメモリが少なくてすみます。');
 list.Insert(4,'# ');
 try
  list.SaveToFile(ApplicationDir+'dirs');
  except
 end;
 list.Free;
end;

procedure FreeDirs;
begin
 if db_dirs<>nil then db_dirs.Free;
 db_dirs:=nil;
end;

initialization
 db_dirs:=nil;
end.
