{

FILE * fp = fopen ( g_sTmpfile, "w+" );

if ( !fp )

return false;

fprintf ( fp,

"AT&T => AT&T\n"

" AT & T => AT & T \n"

"standarten fuehrer => Standartenfuehrer\n"

"standarten fuhrer => Standartenfuehrer\n"

"OS/2 => OS/2\n"

"Ms-Dos => MS-DOS\n"

"MS DOS => MS-DOS\n"

"feat. => featuring\n"

"U.S. => US\n"

"U.S.A. => USA\n"

"U.S.B. => USB\n"

"U.S.D. => USD\n"

"U.S.P. => USP\n"

"U.S.A.F. => USAF\n"

"life:) => life:)\n"

"; => ;\n"

);

if ( sMagic )

fprintf ( fp, "%s => test\n", sMagic );

fclose ( fp );

return true;

{

const char * sPrefix = "testing tokenizer";

// test exceptions more

{

#ifndef NDEBUG

const char * sMagic = "\xD1\x82\xD0\xB5\xD1\x81\xD1\x82\xD1\x82\xD1\x82";

#endif

assert ( CreateSynonymsFile ( sMagic ) );

ISphTokenizer * pTokenizer = CreateTestTokenizer ( TOK_EXCEPTIONS | TOK_NO_SHORT );

CSphString sError;

assert ( pTokenizer->SetBlendChars ( "+, U+23", sError ) );

const char * dTests[] =

{

// for completeness...

"AT&T!!!", "AT&T", "!", "!", "!", NULL, // exceptions vs specials

"U.S.AB U.S.A. U.S.B.U.S.D.U.S.U.S.A.F.", "US", "ab", "USA", "USB", "USD", "US", "USAF", NULL,

"Y.M.C.A.", "y", "m", "c", "a", NULL,

"B&E's", "b", "e", "s", NULL,

// exceptions vs spaces

"AT & T", "AT & T", NULL,

"AT & T", "AT & T", NULL,

"AT & T", "AT & T", NULL,

"AT$&$T", "at", "t", NULL,

// prefix fun

"U.S.A.X.", "USA", "x", NULL,

"U.X.U.S.A.", "u", "x", "USA", NULL,

// exceptions vs blended

"#test this", "#test", "test", "this", NULL,

"#test this", "#test", "test", "this", NULL,

"test#that", "test#that", "test", "that", NULL,

"1+2", "1+2", "1", "2", NULL,

"te.st#this", "te", "st#this", "st", "this", NULL,

"U.boat", "u", "boat", NULL,

// regressions

";foo bar", ";", "foo", "bar", NULL,

NULL

};

for ( int iCur=0; dTests[iCur]; )

{

printf ( "%s, exceptions, line=%s\n", sPrefix, dTests[iCur] );

pTokenizer->SetBuffer ( (BYTE*)dTests[iCur], strlen ( dTests[iCur] ) );

iCur++;

for ( BYTE * pToken=pTokenizer->GetToken(); pToken; pToken=pTokenizer->GetToken() )

{

assert ( dTests[iCur] && strcmp ( (const char*)pToken, dTests[iCur] )==0 );

iCur++;

}

assert ( dTests[iCur]==NULL );

iCur++;

}

// query mode tokenizer tests

ISphTokenizer * pQtok = pTokenizer->Clone ( SPH_CLONE_QUERY_LIGHTWEIGHT );

pQtok->SetBuffer ( (BYTE*)"life:)", 7 );

assert ( strcmp ( (char*)pQtok->GetToken(), "life:)" )==0 );

assert ( pQtok->GetToken()==NULL );

pQtok->SetBuffer ( (BYTE*)"life:\\)", 8 );

assert ( strcmp ( (char*)pQtok->GetToken(), "life:)" )==0 );

assert ( pQtok->GetToken()==NULL );

}

for ( int iRun=1; iRun<=3; iRun++ )

{

// simple "one-line" tests

const char * sMagic = "\xD1\x82\xD0\xB5\xD1\x81\xD1\x82\xD1\x82\xD1\x82";

assert ( CreateSynonymsFile ( sMagic ) );

ISphTokenizer * pTokenizer = CreateTestTokenizer ( ( iRun>=2 ) ? TOK_EXCEPTIONS : 0 );

const char * dTests[] =

{

"1", "", NULL, // test that empty strings work

"1", "this is my rifle", "this", "is", "my", "rifle", NULL, // test that tokenizing works

"1", "This is MY rifle", "this", "is", "my", "rifle", NULL, // test that folding works

"1", "i-phone", "i-phone", NULL, // test that duals (specials in the middle of the word) work ok

"1", "i phone", "phone", NULL, // test that short words are skipped

"1", "this is m", "this", "is", NULL, // test that short words at the end are skipped

"1", "the -phone", "the", "-", "phone", NULL, // test that specials work

"1", "the!phone", "the", "!", "phone", NULL, // test that specials work

"1", "i!phone", "!", "phone", NULL, // test that short words preceding specials are skipped

"1", "/-hi", "-", "hi", NULL, // test that synonym-dual but folded-special chars work ok

"2", "AT&T", "AT&T", NULL, // test that synonyms work

"2", "AT & T", "AT & T", NULL, // test that synonyms with spaces work

"2", "AT & T", "AT & T", NULL, // test that synonyms with continuous spaces work

"2", "-AT&T", "-", "AT&T", NULL, // test that synonyms with specials work

"2", "AT&", "at", NULL, // test that synonyms prefixes are not lost on eof

"2", "AT&tee.yo", "at", "tee", "yo", NULL, // test that non-synonyms with partially matching prefixes work

"2", "standarten fuehrer", "Standartenfuehrer", NULL,

"2", "standarten fuhrer", "Standartenfuehrer", NULL,

"2", "standarten fuehrerr", "standarten", "fuehrerr", NULL,

"2", "standarten fuehrer Stirlitz", "Standartenfuehrer", "stirlitz", NULL,

"2", "standarten fuehrer Zog", "Standartenfuehrer", "zog", NULL,

"2", "stand\\arten fue\\hrer Zog", "Standartenfuehrer", "zog", NULL,

"2", "OS/2 vs OS/360 vs Ms-Dos", "OS/2", "vs", "os", "360", "vs", "MS-DOS", NULL,

"2", "AT ", "at", NULL, // test that prefix-whitespace-eof combo does not hang

"2", "AT&T&TT", "AT&T", "tt", NULL,

"2", "http://OS/2", "http", "OS/2", NULL,

"2", "AT*&*T", "at", NULL,

"2", "# OS/2's system install", "OS/2", "system", "install", NULL,

"2", "IBM-s/OS/2/Merlin", "ibm-s", "OS/2", "merlin", NULL,

"2", "U.S.A", "US", NULL,

"2", "AT&T!", "AT&T", "!", NULL, // exceptions vs specials

"2", "AT&T!!!", "AT&T", "!", "!", "!", NULL, // exceptions vs specials

"2", "U.S.A.!", "USA", "!", NULL, // exceptions vs specials

"2", "MS DOSS feat.Deskview.MS DOS", "ms", "doss", "featuring", "deskview", "MS-DOS", NULL,

"2", sMagic, "test", NULL,

"2", "U.S. U.S.A. U.S.A.F.", "US", "USA", "USAF", NULL,

"2", "U.S.AB U.S.A. U.S.B.U.S.D.U.S.U.S.A.F.", "US", "ab", "USA", "USB", "USD", "US", "USAF", NULL,

"3", "phon\\e", "phone", NULL,

"3", "\\thephone", "thephone", NULL,

"3", "the\\!phone", "the", "phone", NULL,

"3", "\\!phone", "phone", NULL,

"3", "\\\\phone", "phone", NULL, // the correct behavior if '\' is not in charset

"3", "pho\\\\ne", "pho", "ne", NULL,

"3", "phon\\\\e", "phon", NULL,

"3", "trailing\\", "trailing", NULL,

NULL

};

for ( int iCur=0; dTests[iCur] && atoi ( dTests[iCur++] )<=iRun; )

{

printf ( "%s, run=%d, line=%s\n", sPrefix, iRun, dTests[iCur] );

pTokenizer->SetBuffer ( (BYTE*)dTests[iCur], strlen ( dTests[iCur] ) );

iCur++;

for ( BYTE * pToken=pTokenizer->GetToken(); pToken; pToken=pTokenizer->GetToken() )

{

assert ( dTests[iCur] && strcmp ( (const char*)pToken, dTests[iCur] )==0 );

iCur++;

}

assert ( dTests[iCur]==NULL );

iCur++;

}

// test misc one-liners

const char * dTests2[] =

{

"\xC2\x80\xC2\x81\xC2\x82", "\xC2\x80\xC2\x81\xC2\x82", NULL,

NULL

};

for ( int iCur=0; dTests2[iCur]; )

{

printf ( "%s, run=%d, line=%s\n", sPrefix, iRun, dTests2[iCur] );

pTokenizer->SetBuffer ( (BYTE*)dTests2[iCur], strlen ( dTests2[iCur] ) );

iCur++;

for ( BYTE * pToken=pTokenizer->GetToken(); pToken; pToken=pTokenizer->GetToken() )

{

assert ( dTests2[iCur] && strcmp ( (const char*)pToken, dTests2[iCur] )==0 );

iCur++;

}

assert ( dTests2[iCur]==NULL );

iCur++;

}

// test that decoder does not go over the buffer boundary on errors

printf ( "%s for proper UTF-8 error handling\n", sPrefix );

const char * sLine3 = "hi\xd0\xffh";

pTokenizer->SetBuffer ( (BYTE*)sLine3, 4 );

assert ( !strcmp ( (char*)pTokenizer->GetToken(), "hi" ) );

// test uberlong tokens

printf ( "%s for uberlong token handling\n", sPrefix );

const int UBERLONG = 4096;

char * sLine4 = new char [ UBERLONG+1 ];

memset ( sLine4, 'a', UBERLONG );

sLine4[UBERLONG] = '\0';

char sTok4[SPH_MAX_WORD_LEN+1];

memset ( sTok4, 'a', SPH_MAX_WORD_LEN );

sTok4[SPH_MAX_WORD_LEN] = '\0';

pTokenizer->SetBuffer ( (BYTE*)sLine4, strlen(sLine4) );

assert ( !strcmp ( (char*)pTokenizer->GetToken(), sTok4 ) );

assert ( pTokenizer->GetToken()==NULL );

// test short word callbacks

printf ( "%s for short token handling\n", sPrefix );

ISphTokenizer * pShortTokenizer = pTokenizer->Clone ( SPH_CLONE_QUERY );

pShortTokenizer->AddPlainChar ( '*' );

CSphTokenizerSettings tSettings = pShortTokenizer->GetSettings();

tSettings.m_iMinWordLen = 5;

pShortTokenizer->Setup ( tSettings );

const char * dTestsShort[] =

{

"ab*", "ab*", NULL,

"*ab", "*ab", NULL,

"abcdef", "abcdef", NULL,

"ab *ab* abc", "*ab*", NULL,

NULL

};

for ( int iCur=0; dTestsShort[iCur]; )

{

pShortTokenizer->SetBuffer ( (BYTE*)(dTestsShort [iCur]), strlen ( (const char*)dTestsShort [iCur] ) );

iCur++;

for ( BYTE * pToken=pShortTokenizer->GetToken(); pToken; pToken=pShortTokenizer->GetToken() )

{

assert ( dTestsShort[iCur] && strcmp ( (const char*)pToken, dTestsShort[iCur] )==0 );

iCur++;

}

assert ( !dTestsShort [iCur] );

iCur++;

}

SafeDelete ( pShortTokenizer );

// test uberlong synonym-only tokens

if ( iRun==2 )

{

printf ( "%s for uberlong synonym-only char token handling\n", sPrefix );

memset ( sLine4, '/', UBERLONG );

sLine4[UBERLONG] = '\0';

pTokenizer->SetBuffer ( (BYTE*)sLine4, strlen(sLine4) );

assert ( pTokenizer->GetToken()==NULL );

printf ( "%s for uberlong synonym token handling\n", sPrefix );

for ( int i=0; i<UBERLONG-3; i+=3 )

{

sLine4[i+0] = 'a';

sLine4[i+1] = 'a';

sLine4[i+2] = '/';

sLine4[i+3] = '\0';

}

pTokenizer->SetBuffer ( (BYTE*)sLine4, strlen(sLine4) );

for ( int i=0; i<UBERLONG-3; i+=3 )

assert ( !strcmp ( (char*)pTokenizer->GetToken(), "aa" ) );

assert ( pTokenizer->GetToken()==NULL );

}

SafeDeleteArray ( sLine4 );

// test boundaries

printf ( "%s for boundaries handling, run=%d\n", sPrefix, iRun );

CSphString sError;

assert ( pTokenizer->SetBoundary ( "?", sError ) );

char sLine5[] = "hello world? testing boundaries?";

pTokenizer->SetBuffer ( (BYTE*)sLine5, strlen(sLine5) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "hello" ) ); assert ( !pTokenizer->GetBoundary() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "world" ) ); assert ( !pTokenizer->GetBoundary() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "testing" ) ); assert ( pTokenizer->GetBoundary() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "boundaries" ) ); assert ( !pTokenizer->GetBoundary() );

// test specials vs token start/end ptrs

printf ( "%s vs specials vs token start/end ptrs\n", sPrefix );

char sLine6[] = "abc!def";

pTokenizer->SetBuffer ( (BYTE*)sLine6, strlen(sLine6) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "abc" ) );

assert ( *pTokenizer->GetTokenStart()=='a' );

assert ( *pTokenizer->GetTokenEnd()=='!' );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "!" ) );

assert ( *pTokenizer->GetTokenStart()=='!' );

assert ( *pTokenizer->GetTokenEnd()=='d' );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "def" ) );

assert ( *pTokenizer->GetTokenStart()=='d' );

assert ( *pTokenizer->GetTokenEnd()=='\0' );

// test embedded zeroes

printf ( "%s vs embedded zeroes\n", sPrefix );

char sLine7[] = "abc\0\0\0defgh";

pTokenizer->SetBuffer ( (BYTE*)sLine7, 9 );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "abc" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "def" ) );

assert ( !pTokenizer->GetToken() );

assert ( !pTokenizer->GetToken() );

assert ( !pTokenizer->GetToken() );

// done

SafeDelete ( pTokenizer );

}

// test blended

printf ( "%s vs escaping vs blend_chars edge cases\n", sPrefix );

CSphString sError;

ISphTokenizer * pTokenizer = CreateTestTokenizer ( 0 );

assert ( pTokenizer->SetBlendChars ( "., @", sError ) );

pTokenizer->AddSpecials ( "()!-\"@" );

char sTest1[] = "(texas.\\\")";

pTokenizer->SetBuffer ( (BYTE*)sTest1, strlen(sTest1) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "(" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "texas." ) );

assert ( pTokenizer->TokenIsBlended() );

pTokenizer->SkipBlended ();

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), ")" ) );

assert ( pTokenizer->GetToken()==NULL );

char sTest2[] = "\"series 2003\\-\\\"\"";

printf ( "test %s\n", sTest2 );

pTokenizer->SetBuffer ( (BYTE*)sTest2, strlen(sTest2) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "\"" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "series" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "2003-" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "\"" ) );

assert ( pTokenizer->GetToken()==NULL );

char sTest3[] = "aa lock.up bb";

printf ( "test %s\n", sTest3 );

pTokenizer->SetBuffer ( (BYTE*)sTest3, strlen(sTest3) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "aa" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( !pTokenizer->TokenIsBlendedPart() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "lock.up" ) );

assert ( pTokenizer->TokenIsBlended() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "lock" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( pTokenizer->TokenIsBlendedPart() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "up" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( pTokenizer->TokenIsBlendedPart() );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "bb" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( !pTokenizer->TokenIsBlendedPart() );

char sTest4[] = "3.rd text";

printf ( "test %s\n", sTest4 );

pTokenizer->SetBuffer ( (BYTE*)sTest4, strlen(sTest4) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "3.rd" ) );

assert ( pTokenizer->TokenIsBlended() );

assert ( pTokenizer->SkipBlended()==1 );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "text" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( !pTokenizer->GetToken() );

char sTest5[] = "123\\@rd text";

printf ( "test %s\n", sTest5 );

pTokenizer->SetBuffer ( (BYTE*)sTest5, strlen(sTest5) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "123@rd" ) );

assert ( pTokenizer->TokenIsBlended() );

assert ( pTokenizer->SkipBlended()==2 );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "text" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( !pTokenizer->GetToken() );

char sTest6[] = "at.ta\\.c.da\\.bl.ok yo pest";

printf ( "test %s\n", sTest6 );

pTokenizer->SetBuffer ( (BYTE*)sTest6, strlen(sTest6) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "at.ta.c.da.bl.ok" ) );

assert ( pTokenizer->TokenIsBlended() );

assert ( pTokenizer->SkipBlended()==5 );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "yo" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "pest" ) );

assert ( !pTokenizer->GetToken() );

char sTest7[] = "3\\@rd text";

printf ( "test %s\n", sTest7 );

pTokenizer->SetBuffer ( (BYTE*)sTest7, strlen(sTest7) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "3@rd" ) );

assert ( pTokenizer->TokenIsBlended() );

assert ( pTokenizer->SkipBlended()==1 ); // because 3 is overshort!

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "text" ) );

assert ( !pTokenizer->TokenIsBlended() );

assert ( !pTokenizer->GetToken() );

// blended/special vs query mode vs modifier.. hell, this is complicated

SafeDelete ( pTokenizer );

pTokenizer = CreateTestTokenizer ( TOK_NO_DASH );

assert ( pTokenizer->SetBlendChars ( "., -", sError ) );

pTokenizer->AddSpecials ( "-" );

pTokenizer->AddPlainChar ( '=' );

assert ( pTokenizer->SetBlendMode ( "trim_none, skip_pure", sError ) );

char sTest10[] = "hello =- =world";

printf ( "test %s\n", sTest10 );

pTokenizer->SetBuffer ( (BYTE*)sTest10, strlen(sTest10) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "hello" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "=world" ) );

SafeDelete ( pTokenizer );

printf ( "test utf8 len 1\n" );

assert ( sphUTF8Len ( "ab\0cd", 256 )==2 );

printf ( "test utf8 len 2\n" );

assert ( sphUTF8Len ( "", 256 )==0 && sphUTF8Len ( NULL, 256 )==0 );

printf ( "test noascii case\n" );

pTokenizer = sphCreateUTF8Tokenizer();

assert ( pTokenizer->SetCaseFolding ( "U+410..U+42F->U+430..U+44F, U+430..U+44F, U+401->U+451, U+451", sError ) );

char sTest20[] = "abc \xD0\xBE\xD0\xBF\xD0\xB0\x58\xD1\x87\xD0\xB0 def";

pTokenizer->SetBuffer ( (BYTE*)sTest20, strlen(sTest20) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "\xD0\xBE\xD0\xBF\xD0\xB0" ) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "\xD1\x87\xD0\xB0" ) );

assert ( !pTokenizer->GetToken() );

SafeDelete ( pTokenizer );

printf ( "test utf8 4-bytes codepoint\n" );

BYTE sTest21[] = "\xF4\x80\x80\x80\x32\x34\x20";

BYTE sTest22[] = "\xEC\x97\xB0";

BYTE sRes21[SPH_MAX_WORD_LEN];

memset ( sRes21, 0, sizeof(sRes21) );

const BYTE * pTest21 = sTest21;

int iCode21 = sphUTF8Decode ( pTest21 );

assert ( sphUTF8Encode ( sRes21, iCode21 )==4 );

assert ( sTest21[0]==sRes21[0] && sTest21[1]==sRes21[1] && sTest21[2]==sRes21[2] && sTest21[3]==sRes21[3] );

memset ( sRes21, 0, sizeof(sRes21) );

BYTE * pRes21 = sRes21;

SPH_UTF8_ENCODE ( pRes21, iCode21 );

assert ( sTest21[0]==sRes21[0] && sTest21[1]==sRes21[1] && sTest21[2]==sRes21[2] && sTest21[3]==sRes21[3] );

memset ( sRes21, 0, sizeof(sRes21) );

pTest21 = sTest22;

int iCode22 = sphUTF8Decode ( pTest21 );

assert ( iCode22==0xC5F0 );

assert ( sphUTF8Encode ( sRes21, iCode22 )==3 );

assert ( memcmp ( sTest22, sRes21, sizeof(sTest22) )==0 );

memset ( sRes21, 0, sizeof(sRes21) );

pRes21 = sRes21;

SPH_UTF8_ENCODE ( pRes21, iCode22 );

assert ( memcmp ( sTest22, sRes21, sizeof(sTest22) )==0 );

pTokenizer = sphCreateUTF8Tokenizer();

pTokenizer->SetBuffer ( (BYTE*)sTest21, sizeof(sTest21) );

assert ( !strcmp ( (const char*)pTokenizer->GetToken(), "\xF4\x80\x80\x80\x32\x34" ) );

delete pTokenizer;

pTokenizer = sphCreateUTF8NgramTokenizer ();

assert ( !pTokenizer->SetNgramChars ( "2..4", sError ) );

assert ( pTokenizer->SetCaseFolding ( "0..9, A..Z->a..z, _, a..z", sError ) );

assert ( pTokenizer->SetNgramChars ( "U+410..U+42F->U+430..U+44F, U+430..U+44F", sError ) );

delete pTokenizer;

{

const int iPasses = 1000;

int iTokens = 0;

CSphVector<int> dTimes;

// do several benchmark passes

for ( int iPass=0; iPass<iPasses; iPass++ )

{

int64_t tmTime = -sphMicroTimer();

pTokenizer->SetBuffer ( sData, iBytes );

while ( pTokenizer->GetToken() )

iTokens++;

tmTime += sphMicroTimer();

dTimes.Add ( (int)tmTime ); // 2 bil usec == 2000 sec, should be enough for one pass

}

iTokens /= iPasses;

// analyse results

int64_t iMin = INT_MAX, iAvg = 0;

ARRAY_FOREACH ( i, dTimes )

{

if ( dTimes[i]<iMin )

iMin = dTimes[i];

iAvg += dTimes[i];

}

// report

printf ( "%d bytes, %d tokens, max %.1f MB/sec, avg %.1f MB/sec\n",

iBytes, iTokens, double(iBytes)/iMin, double(iBytes*iPasses)/iAvg );

{

printf ( "benchmarking tokenizer\n" );

if ( !CreateSynonymsFile ( NULL ) )

{

printf ( "benchmark failed: error writing temp synonyms file\n" );

return;

}

int iBytes = 0;

CSphString sError;

for ( int iRun=1; iRun<=2; iRun++ )

{

char * sData = LoadFile ( "./configure", &iBytes, true );

ISphTokenizer * pTokenizer = sphCreateUTF8Tokenizer ();

// pTokenizer->SetCaseFolding ( "-, 0..9, A..Z->a..z, _, a..z", sError );

if ( iRun==2 )

pTokenizer->LoadSynonyms ( g_sTmpfile, NULL, sError );

pTokenizer->AddSpecials ( "!-" );

printf ( "run %d: ", iRun );

BenchTokenizer ( pTokenizer, (BYTE*)sData, iBytes );

SafeDeleteArray ( sData );

SafeDelete ( pTokenizer );

}

char * sData = LoadFile ( "./utf8.txt", &iBytes, false );

if ( sData )

{

ISphTokenizer * pTokenizer = sphCreateUTF8Tokenizer ();

printf ( "run 3: " );

BenchTokenizer ( pTokenizer, (BYTE*)sData, iBytes );

SafeDelete ( pTokenizer );

}

SafeDeleteArray ( sData );

{

const char * sTests[][4] =

{

// source-data, index-attrs, remove-elements, expected-results

{ "<?xml broken piece of shit/>should be indexed", "", "", "should be indexed" },

{ "<?xml crazily=\"broken>shit\">still should be indexed", "", "", "shit\">still should be indexed" },

{ "MOVING? HATE TO PACK ?HIRE A TRUCK WE\'LL DO THE REST! CLICK HERE<?<?<?", "", "", "MOVING? HATE TO PACK ?HIRE A TRUCK WE'LL DO THE REST! CLICK HERE" },

{ "<html>trivial test</html>", "", "", " trivial test " },

{ "<html>lets <img src=\"g/smth.jpg\" alt=\"nice picture\">index attrs</html>", "img=alt", "", " lets nice picture index attrs " },

{ "<html> lets also<script> whatever here; a<b</script>remove scripts", "", "script, style", " lets also remove scripts" },

{ "testing in<b><font color='red'>line</font> ele<em>men</em>ts", "", "", "testing inline elements" },

{ "testing non<p>inline</h1>elements", "", "", "testing non inline elements" },

{ "testing&nbsp;entities&amp;stuff", "", "", "testing entities&stuff" },

{ "testing &#1040;&#1041;&#1042; utf encoding", "", "", "testing \xD0\x90\xD0\x91\xD0\x92 utf encoding" },

{ "testing <1 <\" <\x80 <\xe0 <\xff </3 malformed tags", "", "", "testing <1 <\" <\x80 <\xe0 <\xff </3 malformed tags" },

{ "testing comm<!--comm-->ents", "", "", "testing comments" },

{ "&lt; &gt; &thetasym; &somethingverylong; &the", "", "", "< > \xCF\x91 &somethingverylong; &the" },

{ "testing <img src=\"g/smth.jpg\" alt=\"nice picture\" rel=anotherattr junk=throwaway>inline tags vs attr indexing", "img=alt,rel", "", "testing nice picture anotherattr inline tags vs attr indexing" },

{ "this <?php $code = \"must be stripped\"; ?> away", "", "", "this away" },

{ "<a href=\"http://www.com\">content1</a>", "a=title", "", "content1" },

{ "<a href=\"http://www.com\" title=\"my test title\">content2</a>", "a=title", "", "my test title content2" },

{ "testing <img src=\"g/smth.jpg\" alt=\"nice picture\" rel=anotherattr junk=\"throwaway\">inline tags vs attr indexing", "img=alt,rel", "", "testing nice picture anotherattr inline tags vs attr indexing" },

{ "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\"><html>test</html>", "", "", " test " },

{ "<!smth \"that>can<break\"><html>test</html>", "", "", " test " },

{ "<TABLE CLASS=\"MSONORMALTABLE\" STYLE=\"BORDER-COLLAPSE: COLLAPSE; MARGIN-LEFT: ID=\"TABLE76\"><TR><TD>ohai</TD></TR></TABLE>", "", "", " ohai " },

{ "ohai2<table class", "", "", "ohai2 " },

{ "ohai<table class>3", "", "", "ohai 3" },

{ "ohai<table class >4", "", "", "ohai 4" },

{ "ohai<table class =>5", "", "", "ohai 5" },

{ "ohai<table class =\"smth><tr><td>6</td></tr></table> some more content", "", "", "ohai 6 some more content" },

{ "ohai<table nowrap class=\"a>b\">7", "", "", "ohai 7" },

{ "ohai<table nowrap class =\"a>b\">8", "", "", "ohai 8" },

{ "ohai<table nowrap class= \"a>b\">9", "", "", "ohai 9" },

{ "ohai<table now rap class=\"a>b\">10", "", "", "ohai 10" },

{ "ohai<table class = \"smth><tr><td>6</td><td class=\"test\">11</td></tr></table> gimme more", "", "", "ohai 11 gimme more" },

{ "<P ALIGN=\"LEFT STYLE=\"MARGIN:0IN 0IN .0001PT;TEXT-ALIGN:LEFT;\"><B><FONT SIZE=\"2\" FACE=\"TIMES NEW ROMAN\" STYLE=\"FONT-SIZE:10.0PT;FONT-WEIGHT:BOLD;\">Commission File Number: 333-155507", "", "", " Commission File Number: 333-155507" },

{ "<TD NOWRAP ALIGN=RIGHT STYLE=\"BORDER-BOTTOM: #000000 1PX SOLID; BORDER-TOP: #000000 1PX SOLID;\"\"><B>SGX", "", "", " SGX" },

{ "tango & cash", "", "", "tango & cash" },

{ "<font CLASS=\"MSONORMALTABLE\" STYLE=\"BORDER-COLLAPSE: COLLAPSE; MARGIN-LEFT: ID=\"TABLE76\">ahoy\"mate", "font=zzz", "", "ahoy\"mate" },

{ "ahoy<font class =>2", "font=zzz", "", "ahoy2" },

{ "ahoy<font class =\"smth><b>3</b></font>there", "font=zzz", "", "ahoy3there" },

{ "ahoy<font nowrap class=\"a>b\">4", "font=zzz", "", "ahoy4" },

{ "ahoy<font now rap class=\"a>b\">5", "font=zzz", "", "ahoy5" },

{ "ahoy<font class = \"smth><b><i>6</i><b class=\"test\">seven</b></i></font>eight", "font=zzz", "", "ahoyseveneight" },

{ "testing &#xC0; &#x2116; &#x0116;1 numbers utf encoding", "", "", "testing \xC3\x80 \xE2\x84\x96 \xC4\x96\x31 numbers utf encoding" }

};

int nTests = (int)(sizeof(sTests)/sizeof(sTests[0]));

for ( int iTest=0; iTest<nTests; iTest++ )

{

printf ( "testing HTML stripper, test %d/%d... ", 1+iTest, nTests );

CSphString sError;

CSphHTMLStripper tStripper ( true );

assert ( tStripper.SetIndexedAttrs ( sTests[iTest][1], sError ) );

assert ( tStripper.SetRemovedElements ( sTests[iTest][2], sError ) );

CSphString sBuf ( sTests[iTest][0] );

tStripper.Strip ( (BYTE*)sBuf.cstr() );

assert ( strcmp ( sBuf.cstr(), sTests[iTest][3] )==0 );

printf ( "ok\n" );

}

{

printf ( "benchmarking HTML stripper\n" );

FILE * fp = fopen ( "doc/sphinx.html", "rb" );

if ( !fp )

{

printf ( "benchmark failed: unable to read doc/sphinx.html\n" );

return;

}

const int MAX_SIZE = 1048576;

char * sBuf = new char [ MAX_SIZE ];

int iLen = fread ( sBuf, 1, MAX_SIZE-1, fp );

fclose ( fp );

char * sRef = new char [ MAX_SIZE ];

sBuf[iLen] = '\0';

memcpy ( sRef, sBuf, iLen+1 );

for ( int iRun=0; iRun<2; iRun++ )

{

CSphString sError;

CSphHTMLStripper tStripper ( true );

if ( iRun==1 )

tStripper.SetRemovedElements ( "style, script", sError );

const int iPasses = 50;

int64_t tmTime = -sphMicroTimer();

for ( int iPass=0; iPass<iPasses; iPass++ )

{

tStripper.Strip ( (BYTE*)sBuf );

memcpy ( sBuf, sRef, iLen+1 );

}

tmTime += sphMicroTimer();

tmTime /= iPasses;

printf ( "run %d: %d bytes, %d.%03d ms, %.3f MB/sec\n", iRun, iLen, (int)(tmTime/1000), (int)(tmTime%1000), float(iLen)/tmTime );

}

SafeDeleteArray ( sBuf );

SafeDeleteArray ( sRef );

{

CSphColumnInfo tCol;

tCol.m_eAttrType = SPH_ATTR_INTEGER;

CSphSchema tSchema;

tCol.m_sName = "aaa"; tSchema.AddAttr ( tCol, false );

tCol.m_sName = "bbb"; tSchema.AddAttr ( tCol, false );

tCol.m_sName = "ccc"; tSchema.AddAttr ( tCol, false );

CSphRowitem * pRow = new CSphRowitem [ tSchema.GetRowSize() ];

for ( int i=0; i<tSchema.GetRowSize(); i++ )

pRow[i] = 1+i;

CSphMatch tMatch;

tMatch.m_uDocID = 123;

tMatch.m_iWeight = 456;

tMatch.m_pStatic = pRow;

struct ExprTest_t

{

const char * m_sExpr;

float m_fValue;

};

ExprTest_t dTests[] =

{

{ "ccc/2", 1.5f },

{ "1*2*3*4*5*6*7*8*9*10", 3628800.0f },

{ "aaa+bbb*sin(0)*ccc", 1.0f },

{ "if(pow(sqrt(2),2)=2,123,456)", 123.0f },

{ "if(2<2,3,4)", 4.0f },

{ "if(2>=2,3,4)", 3.0f },

{ "pow(7,5)", 16807.f },

{ "sqrt(3)", 1.7320508f },

{ "log2((2+2)*(2+2))", 4.0f },

{ "min(3,15)", 3.0f },

{ "max(3,15)", 15.0f },

{ "if(3<15,bbb,ccc)", 2.0f },

{ "@id+@weight", 579.0f },

{ "abs(-3-ccc)", 6.0f },

{ "(aaa+bbb)*(ccc-aaa)", 6.0f },

{ "(((aaa)))", 1.0f },

{ "aaa-bbb*ccc", -5.0f },

{ " aaa -\tbbb *\t\t\tccc ", -5.0f },

{ "bbb+123*aaa", 125.0f },

{ "2.000*2e+1+2", 42.0f },

{ "3<5", 1.0f },

{ "1 + 2*3 > 4*4", 0.0f },

{ "aaa/-bbb", -0.5f, },

{ "-10*-10", 100.0f },

{ "aaa+-bbb*-5", 11.0f },

{ "-aaa>-bbb", 1.0f },

{ "1-aaa+2-3+4", 3.0f },

{ "bbb/1*2/6*3", 2.0f },

{ "(aaa+bbb)/sqrt(3)/sqrt(3)", 1.0f },

{ "aaa-bbb-2", -3.0f },

{ "ccc/2*4/bbb", 3.0f },

{ "(2+(aaa*bbb))+3", 7.0f }

};

const int nTests = sizeof(dTests)/sizeof(dTests[0]);

for ( int iTest=0; iTest<nTests; iTest++ )

{

printf ( "testing expression evaluation, test %d/%d... ", 1+iTest, nTests );

CSphString sError;

CSphScopedPtr<ISphExpr> pExpr ( sphExprParse ( dTests[iTest].m_sExpr, tSchema, NULL, NULL, sError, NULL ) );

if ( !pExpr.Ptr() )

{

printf ( "FAILED; %s\n", sError.cstr() );

assert ( 0 );

}

float fValue = pExpr->Eval(tMatch);

if ( fabs ( fValue - dTests[iTest].m_fValue )>=0.0001f )

{

printf ( "FAILED; expected %.3f, got %.3f\n", dTests[iTest].m_fValue, fValue );

assert ( 0 );

}

printf ( "ok\n" );

}

SafeDeleteArray ( pRow );

{

printf ( "benchmarking expressions\n" );

CSphColumnInfo tCol;

tCol.m_eAttrType = SPH_ATTR_INTEGER;

CSphSchema tSchema;

tCol.m_sName = "aaa"; tSchema.AddAttr ( tCol, false );

tCol.m_sName = "bbb"; tSchema.AddAttr ( tCol, false );

tCol.m_sName = "ccc"; tSchema.AddAttr ( tCol, false );

CSphRowitem * pRow = new CSphRowitem [ tSchema.GetRowSize() ];

for ( int i=0; i<tSchema.GetRowSize(); i++ )

pRow[i] = 1+i;

CSphMatch tMatch;

tMatch.m_uDocID = 123;

tMatch.m_iWeight = 456;

tMatch.m_pStatic = pRow;

struct ExprBench_t

{

const char * m_sExpr;

float ( *m_pFunc )( const CSphMatch & );

};

ExprBench_t dBench[] =

{

{ "aaa+bbb*(ccc)-1", ExprNative1 },

{ "aaa+bbb*ccc*2-3/4*5/6*bbb", ExprNative2 },

{ "sqrt(2)", ExprNative3 }

};

for ( int iRun=0; iRun<int(sizeof(dBench)/sizeof(dBench[0])); iRun++ )

{

printf ( "run %d: ", iRun+1 );

ESphAttr uType;

CSphString sError;

CSphScopedPtr<ISphExpr> pExpr ( sphExprParse ( dBench[iRun].m_sExpr, tSchema, &uType, NULL, sError, NULL ) );

if ( !pExpr.Ptr() )

{

printf ( "FAILED; %s\n", sError.cstr() );

return;

}

const int NRUNS = 1000000;

volatile float fValue = 0.0f;

int64_t tmTime = sphMicroTimer();

for ( int i=0; i<NRUNS; i++ ) fValue += pExpr->Eval(tMatch);

tmTime = sphMicroTimer() - tmTime;

int64_t tmTimeInt = sphMicroTimer();

if ( uType==SPH_ATTR_INTEGER )

{

int uValue = 0;

for ( int i=0; i<NRUNS; i++ ) uValue += pExpr->IntEval(tMatch);

}

tmTimeInt = sphMicroTimer() - tmTimeInt;

int64_t tmTimeNative = sphMicroTimer();

for ( int i=0; i<NRUNS; i++ ) fValue += dBench[iRun].m_pFunc ( tMatch );

tmTimeNative = sphMicroTimer() - tmTimeNative;

if ( uType==SPH_ATTR_INTEGER )

printf ( "int-eval %.1fM/sec, ", float(NRUNS)/tmTimeInt );

printf ( "flt-eval %.1fM/sec, native %.1fM/sec\n",

float(NRUNS)/tmTime,

float(NRUNS)/tmTimeNative );

}

SafeDeleteArray ( pRow );

{

bool IsLess ( const XQKeyword_t * pA, const XQKeyword_t * pB )

{

return pA->m_iAtomPos<pB->m_iAtomPos;

}

{

assert ( pNode );