fix not support scws compile error

hetao29 · Apr 19, 2017 · Apr 19, 2017 · Apr 19, 2017 · Apr 19, 2017 · Apr 20, 2017
commit bb75ac91e55f8c83c52daf710b20d1baa849e78d
diff --git a/configure b/configure
@@ -608,6 +608,8 @@ LTLIBOBJS
 CONFDIR
 USE_RLP_FALSE
 USE_RLP_TRUE
+USE_SCWS_FALSE
+USE_SCWS_TRUE
 USE_RE2_FALSE
 USE_RE2_TRUE
 LIBRE2_PATH
@@ -653,6 +655,7 @@ DEPDIR
 OBJEXT
 EXEEXT
 ac_ct_CC
+ac_cv_use_scws
 CPPFLAGS
 LDFLAGS
 CFLAGS
@@ -745,6 +748,7 @@ with_re2
 with_re2_includes
 with_re2_libs
 with_rlp
+with_scws
 with_iconv
 with_unixodbc
 enable_mem_override
@@ -1417,6 +1421,8 @@ Optional Packages:
   --with-re2-libs         path to RE2 libraries
   --with-rlp              compile with RLP library support (default is
                           disabled)
+  --with-scws             compile with scws library support (default is
+                          disabled)
   --with-iconv            compile with iconv support (default is autodetect)
   --with-unixodbc         compile with UnixODBC support (default is
                           autodetect)
@@ -8222,6 +8228,44 @@ fi
 
 
 
+# Check whether --with-scws was given.
+if test "${with_scws+set}" = set; then :
+  withval=$with_scws; ac_cv_use_scws=$withval
+else
+  ac_cv_use_scws=no
+
+fi
+
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to compile with scws library support" >&5
+$as_echo_n "checking whether to compile with scws library support... " >&6; }
+if test x$ac_cv_use_scws != xno; then
+	if test -d $withval && test -f $withval/include/scws/scws.h; then
+		{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
+$as_echo "yes" >&6; }
+
+$as_echo "#define USE_SCWS 1" >>confdefs.h
+
+	else
+		as_fn_error $? "missing SCWS sources from libscws" "$LINENO" 5
+	fi
+else
+	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
+$as_echo "no" >&6; }
+
+$as_echo "#define USE_SCWS 0" >>confdefs.h
+
+fi
+ if test x$ac_cv_use_scws != xno; then
+  USE_SCWS_TRUE=
+  USE_SCWS_FALSE='#'
+else
+  USE_SCWS_TRUE='#'
+  USE_SCWS_FALSE=
+fi
+
+
+
 
 got_expat=0
 dl_expat=0
@@ -9102,6 +9146,10 @@ if test -z "${USE_RLP_TRUE}" && test -z "${USE_RLP_FALSE}"; then
   as_fn_error $? "conditional \"USE_RLP\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
 fi
+if test -z "${USE_SCWS_TRUE}" && test -z "${USE_SCWS_FALSE}"; then
+  as_fn_error $? "conditional \"USE_SCWS\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 
 : "${CONFIG_STATUS=./config.status}"
 ac_write_fail=0

diff --git a/configure.ac b/configure.ac
@@ -551,6 +551,28 @@ fi
 AM_CONDITIONAL(USE_RLP, test x$ac_cv_use_rlp != xno)
 
 
+dnl ---
+
+AC_ARG_WITH([scws],
+	AC_HELP_STRING([--with-scws], [compile with scws library support (default is disabled)]),
+	[ac_cv_use_scws=$withval], [ac_cv_use_scws=no]
+)
+
+AC_MSG_CHECKING([whether to compile with scws library support])
+if test x$ac_cv_use_scws != xno; then
+	if test -d $ac_cv_use_scws && test -f $ac_cv_use_scws/include/scws/scws.h; then
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(USE_SCWS, 1, [scws library support])
+	else
+		AC_MSG_ERROR([missing scws sources from libscws])
+	fi
+else
+	AC_MSG_RESULT([no])
+	AC_DEFINE(USE_SCWS, 0, [scws library support])
+fi
+AM_CONDITIONAL(USE_SCWS, test x$ac_cv_use_scws != xno)
+
+
 dnl ---
 
 got_expat=0

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -3,7 +3,7 @@ SRC_SPHINX = sphinx.cpp sphinxexcerpt.cpp sphinxquery.cpp \
 	sphinxutils.cpp sphinxstd.cpp sphinxsort.cpp sphinxexpr.cpp sphinxfilter.cpp \
 	sphinxsearch.cpp sphinxrt.cpp sphinxjson.cpp sphinxudf.c sphinxaot.cpp sphinxplugin.cpp
 
-ARFLAGS = crU
+ARFLAGS = cr
 noinst_LIBRARIES = libsphinx.a
 libsphinx_a_SOURCES = $(SRC_SPHINX)
 
@@ -30,6 +30,14 @@ RLP_LIBS =
 RLP_INC =
 endif
 
-AM_CPPFLAGS = $(LIBRE2_CFLAGS) $(RLP_INC) -DSYSCONFDIR="\"$(sysconfdir)\"" -DDATADIR="\"$(localstatedir)/data\""
-COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(PGSQL_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS)
+if USE_SCWS
+SCWS_LIBS = -L@ac_cv_use_scws@/lib/ -lscws
+SCWS_INC = -I@ac_cv_use_scws@/include
+else
+SCWS_LIBS =
+SCWS_INC =
+endif
+
+AM_CPPFLAGS = $(LIBRE2_CFLAGS) $(RLP_INC) $(SCWS_INC) -DSYSCONFDIR="\"$(sysconfdir)\"" -DDATADIR="\"$(localstatedir)/data\""
+COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $S(CWS_LIBS) $(PGSQL_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS) $(SCWS_LIBS)
 LDADD = $(COMMON_LIBS)
diff --git a/src/Makefile.in b/src/Makefile.in
@@ -180,6 +180,7 @@ abs_top_builddir = @abs_top_builddir@
 abs_top_srcdir = @abs_top_srcdir@
 ac_ct_CC = @ac_ct_CC@
 ac_ct_CXX = @ac_ct_CXX@
+#ac_cv_use_scws = @ac_cv_use_scws@
 am__include = @am__include@
 am__leading_dot = @am__leading_dot@
 am__quote = @am__quote@
@@ -223,7 +224,7 @@ SRC_SPHINX = sphinx.cpp sphinxexcerpt.cpp sphinxquery.cpp \
 	sphinxutils.cpp sphinxstd.cpp sphinxsort.cpp sphinxexpr.cpp sphinxfilter.cpp \
 	sphinxsearch.cpp sphinxrt.cpp sphinxjson.cpp sphinxudf.c sphinxaot.cpp sphinxplugin.cpp
 
-ARFLAGS = crU
+ARFLAGS = cr
 noinst_LIBRARIES = libsphinx.a
 libsphinx_a_SOURCES = $(SRC_SPHINX)
 indexer_SOURCES = indexer.cpp
@@ -237,8 +238,12 @@ BUILT_SOURCES = extract-version
 @USE_RLP_TRUE@RLP_LIBS = -L$(top_srcdir)/rlp/lib/amd64-glibc25-gcc42 -lbtrlpc -lbtrlpcore -lbtutils
 @USE_RLP_FALSE@RLP_INC = 
 @USE_RLP_TRUE@RLP_INC = -I$(top_srcdir)/rlp/rlp/include -I$(top_srcdir)/rlp/utilities/include -D_REENTRANT
-AM_CPPFLAGS = $(LIBRE2_CFLAGS) $(RLP_INC) -DSYSCONFDIR="\"$(sysconfdir)\"" -DDATADIR="\"$(localstatedir)/data\""
-COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(PGSQL_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS)
+@USE_SCWS_FALSE@SCWS_LIBS = 
+@USE_SCWS_TRUE@SCWS_LIBS = -L@ac_cv_use_scws@/lib/ -lscws
+@USE_SCWS_FALSE@SCWS_INC = 
+@USE_SCWS_TRUE@SCWS_INC = -I@ac_cv_use_scws@/include
+AM_CPPFLAGS = $(LIBRE2_CFLAGS) $(RLP_INC) $(SCWS_INC) -DSYSCONFDIR="\"$(sysconfdir)\"" -DDATADIR="\"$(localstatedir)/data\""
+COMMON_LIBS = libsphinx.a $(LIBSTEMMER_LIBS) $(MYSQL_LIBS) $(SCWS_LIBS) $(PGSQL_LIBS) $(LIBRE2_LIBS) $(RLP_LIBS) $(SCWS_LIBS)
 LDADD = $(COMMON_LIBS)
 all: $(BUILT_SOURCES)
 	$(MAKE) $(AM_MAKEFLAGS) all-am

diff --git a/src/sphinx.cpp b/src/sphinx.cpp
@@ -2576,6 +2576,7 @@ class CSphTokenizer_UTF8Ngram : public CSphTokenizer_UTF8<IS_QUERY>
 
 
 /// SCWS tokenizer
+#if USE_SCWS
 template < bool IS_QUERY >
 class CSphTokenizer_SCWS : public CSphTokenizerBase2
 {
@@ -2588,11 +2589,12 @@ class CSphTokenizer_SCWS : public CSphTokenizerBase2
         virtual int                 GetCodepointLength ( int iCode ) const;
         virtual int                 GetMaxCodepointLength () const { return m_tLC.GetMaxCodepointLength(); }
 
-      
+
     scws_t s; 
     scws_res_t cur;
-};
 
+};
+#endif
 
 struct CSphNormalForm
 {
@@ -3811,11 +3813,13 @@ ISphTokenizer * sphCreateUTF8NgramTokenizer ()
 {
 	return new CSphTokenizer_UTF8Ngram<false> ();
 }
+
+#if USE_SCWS
 ISphTokenizer * sphCreateUTF8SCWSTokenizer ()
 {
         return new CSphTokenizer_SCWS<false> ();
 }
-
+#endif
 
 /////////////////////////////////////////////////////////////////////////////
 
@@ -4414,8 +4418,13 @@ bool LoadTokenizerSettings ( CSphReader & tReader, CSphTokenizerSettings & tSett
 
 	tSettings.m_iType = tReader.GetByte ();
 
-        if ( tSettings.m_iType!=TOKENIZER_UTF8 && tSettings.m_iType!=TOKENIZER_NGRAM && tSettings.m_iType!=TOKENIZER_SCWS)
-	{
+        if ( 
+                tSettings.m_iType!=TOKENIZER_UTF8 
+                && tSettings.m_iType!=TOKENIZER_NGRAM  
+#if USE_SCWS
+                && tSettings.m_iType!=TOKENIZER_SCWS
+#endif
+        ){
 		sWarning = "can't load an old index with SBCS tokenizer";
 		return false;
 	}
@@ -4746,9 +4755,9 @@ ISphTokenizer * ISphTokenizer::Create ( const CSphTokenizerSettings & tSettings,
 	switch ( tSettings.m_iType )
 	{
 		case TOKENIZER_UTF8:	pTokenizer = sphCreateUTF8Tokenizer (); break;
-
+#if USE_SCWS
 		case TOKENIZER_SCWS:	pTokenizer = sphCreateUTF8SCWSTokenizer (); break;
-                
+#endif               
 		case TOKENIZER_NGRAM:	pTokenizer = sphCreateUTF8NgramTokenizer (); break;
 		default:
 			sError.SetSprintf ( "failed to create tokenizer (unknown charset type '%d')", tSettings.m_iType );
@@ -6445,16 +6454,24 @@ BYTE * CSphTokenizer_UTF8Ngram<IS_QUERY>::GetToken ()
 
 /////////////////////////////////////////////////////////////////////////////
 
+#if USE_SCWS
+
+
+
 template < bool IS_QUERY >
 CSphTokenizer_SCWS<IS_QUERY>::CSphTokenizer_SCWS ()
 {
+
         s = scws_new();
+
 }
 template < bool IS_QUERY >
 CSphTokenizer_SCWS<IS_QUERY>::~CSphTokenizer_SCWS ()
 {
+
         scws_free_result(cur);
         scws_free(s);
+
 }
 
 
@@ -6536,7 +6553,7 @@ int CSphTokenizer_SCWS<IS_QUERY>::GetCodepointLength ( int iCode ) const
         assert ( iBytes>=2 && iBytes<=4 );
         return iBytes;
 }
-
+#endif
 //////////////////////////////////////////////////////////////////////////
 
 CSphMultiformTokenizer::CSphMultiformTokenizer ( ISphTokenizer * pTokenizer, const CSphMultiformContainer * pContainer )

diff --git a/src/sphinx.h b/src/sphinx.h
@@ -504,11 +504,11 @@ struct CSphTokenizerSettings
 	CSphString			m_sBlendMode;
 
 
-
+ #if USE_SCWS       
         CSphString                      m_scwsDict;
         CSphString                      m_scwsRule;
         int                      m_scwsMulti;
-        
+#endif      
 
 	CSphString			m_sIndexingPlugin;	///< this tokenizer wants an external plugin to process its raw output
 

diff --git a/src/sphinxutils.cpp b/src/sphinxutils.cpp
@@ -592,12 +592,12 @@ static KeyDesc_t g_dKeysIndex[] =
 	{ "rlp_context",			0, NULL },
 	{ "ondisk_attrs",			0, NULL },
 	{ "index_token_filter",		0, NULL },
-
+#if USE_SCWS
 	{ "scws",		0, NULL },
 	{ "scws_dict",		0, NULL },
 	{ "scws_rule",		0, NULL },
 	{ "scws_multi",		0, NULL },
-
+#endif
 	{ NULL,						0, NULL }
 };
 
@@ -1269,15 +1269,15 @@ void sphConfTokenizer ( const CSphConfigSection & hIndex, CSphTokenizerSettings
 		else
 			sphWarning ( "ngram_chars specified, but ngram_len=0; IGNORED" );
 	}
-
+#if USE_SCWS
         if ( hIndex ( "scws" ) )
         {
                 tSettings.m_iType = TOKENIZER_SCWS;
                 tSettings.m_scwsDict = hIndex.GetStr ( "scws_dict" );
                 tSettings.m_scwsRule = hIndex.GetStr ( "scws_rule" );
                 tSettings.m_scwsMulti = hIndex.GetInt ( "scws_multi",0 );
         }
-
+#endif
 	tSettings.m_sCaseFolding = hIndex.GetStr ( "charset_table" );
 	tSettings.m_iMinWordLen = Max ( hIndex.GetInt ( "min_word_len", 1 ), 1 );
 	tSettings.m_sNgramChars = hIndex.GetStr ( "ngram_chars" );

diff --git a/src/sphinxutils.h b/src/sphinxutils.h
@@ -147,7 +147,9 @@ enum
 	// where was TOKENIZER_SBCS=1 once
 	TOKENIZER_UTF8		= 2,
 	TOKENIZER_NGRAM	= 3
+#if USE_SCWS
 	,TOKENIZER_SCWS	= 4
+#endif
 };
 
 /// load config file