Skip to content

Commit 276fc6f

Browse files
FrayxRulezFrayxRulez
committed
Detect messages language
1 parent b50674b commit 276fc6f

100 files changed

Lines changed: 13201 additions & 12 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.gitmodules‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[submodule "Unigram/Libraries/libutf"]
2+
path = Unigram/Libraries/libutf
3+
url = https://android.googlesource.com/platform/external/libutf
4+
[submodule "Unigram/Libraries/flatbuffers"]
5+
path = Unigram/Libraries/flatbuffers
6+
url = https://github.com/google/flatbuffers.git

‎Unigram/Libraries/flatbuffers‎

Submodule flatbuffers added at a9a295f

‎Unigram/Libraries/libtextclassifier/lang_id/common/embedding-feature-extractor.cc‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Copyright (C) 2018 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "lang_id/common/embedding-feature-extractor.h"
18+
19+
#include <stddef.h>
20+
21+
#include <string>
22+
#include <vector>
23+
24+
#include "lang_id/common/fel/feature-extractor.h"
25+
#include "lang_id/common/fel/feature-types.h"
26+
#include "lang_id/common/fel/task-context.h"
27+
#include "lang_id/common/lite_base/integral-types.h"
28+
#include "lang_id/common/lite_base/logging.h"
29+
#include "lang_id/common/lite_strings/numbers.h"
30+
#include "lang_id/common/lite_strings/str-split.h"
31+
#include "lang_id/common/lite_strings/stringpiece.h"
32+
33+
namespace libtextclassifier3 {
34+
namespace mobile {
35+
36+
bool GenericEmbeddingFeatureExtractor::Setup(TaskContext *context) {
37+
// Don't use version to determine how to get feature FML.
38+
const std::string features = context->Get(GetParamName("features"), "");
39+
const std::string embedding_names =
40+
context->Get(GetParamName("embedding_names"), "");
41+
const std::string embedding_dims =
42+
context->Get(GetParamName("embedding_dims"), "");
43+
44+
// NOTE: unfortunately, LiteStrSplit returns a vector of StringPieces pointing
45+
// to the original string, in this case |features|, which is local to this
46+
// method. We need to explicitly create new strings.
47+
for (StringPiece sp : LiteStrSplit(features, ';')) {
48+
embedding_fml_.emplace_back(sp);
49+
}
50+
51+
// Same here.
52+
for (StringPiece sp : LiteStrSplit(embedding_names, ';')) {
53+
embedding_names_.emplace_back(sp);
54+
}
55+
56+
std::vector<StringPiece> dim_strs = LiteStrSplit(embedding_dims, ';');
57+
for (const auto &dim_str : dim_strs) {
58+
int dim = 0;
59+
if (!LiteAtoi(dim_str, &dim)) {
60+
SAFTM_LOG(ERROR) << "Unable to parse " << dim_str;
61+
return false;
62+
}
63+
embedding_dims_.push_back(dim);
64+
}
65+
return true;
66+
}
67+
68+
bool GenericEmbeddingFeatureExtractor::Init(TaskContext *context) {
69+
return true;
70+
}
71+
72+
} // namespace mobile
73+
} // namespace nlp_saft

‎Unigram/Libraries/libtextclassifier/lang_id/common/embedding-feature-extractor.h‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
* Copyright (C) 2018 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef NLP_SAFT_COMPONENTS_COMMON_MOBILE_EMBEDDING_FEATURE_EXTRACTOR_H_
18+
#define NLP_SAFT_COMPONENTS_COMMON_MOBILE_EMBEDDING_FEATURE_EXTRACTOR_H_
19+
20+
#include <memory>
21+
#include <string>
22+
#include <vector>
23+
24+
#include "lang_id/common/fel/feature-extractor.h"
25+
#include "lang_id/common/fel/task-context.h"
26+
#include "lang_id/common/fel/workspace.h"
27+
#include "lang_id/common/lite_base/attributes.h"
28+
29+
namespace libtextclassifier3 {
30+
namespace mobile {
31+
32+
// An EmbeddingFeatureExtractor manages the extraction of features for
33+
// embedding-based models. It wraps a sequence of underlying classes of feature
34+
// extractors, along with associated predicate maps. Each class of feature
35+
// extractors is associated with a name, e.g., "words", "labels", "tags".
36+
//
37+
// The class is split between a generic abstract version,
38+
// GenericEmbeddingFeatureExtractor (that can be initialized without knowing the
39+
// signature of the ExtractFeatures method) and a typed version.
40+
//
41+
// The predicate maps must be initialized before use: they can be loaded using
42+
// Read() or updated via UpdateMapsForExample.
43+
class GenericEmbeddingFeatureExtractor {
44+
public:
45+
// Constructs this GenericEmbeddingFeatureExtractor.
46+
//
47+
// |arg_prefix| is a string prefix for the relevant TaskContext parameters, to
48+
// avoid name clashes. See GetParamName().
49+
explicit GenericEmbeddingFeatureExtractor(const std::string &arg_prefix)
50+
: arg_prefix_(arg_prefix) {}
51+
52+
virtual ~GenericEmbeddingFeatureExtractor() {}
53+
54+
// Sets/inits up predicate maps and embedding space names that are common for
55+
// all embedding based feature extractors.
56+
//
57+
// Returns true on success, false otherwise.
58+
SAFTM_MUST_USE_RESULT virtual bool Setup(TaskContext *context);
59+
SAFTM_MUST_USE_RESULT virtual bool Init(TaskContext *context);
60+
61+
// Requests workspace for the underlying feature extractors. This is
62+
// implemented in the typed class.
63+
virtual void RequestWorkspaces(WorkspaceRegistry *registry) = 0;
64+
65+
// Returns number of embedding spaces.
66+
int NumEmbeddings() const { return embedding_dims_.size(); }
67+
68+
const std::vector<std::string> &embedding_fml() const {
69+
return embedding_fml_;
70+
}
71+
72+
// Get parameter name by concatenating the prefix and the original name.
73+
std::string GetParamName(const std::string &param_name) const {
74+
std::string full_name = arg_prefix_;
75+
full_name.push_back('_');
76+
full_name.append(param_name);
77+
return full_name;
78+
}
79+
80+
private:
81+
// Prefix for TaskContext parameters.
82+
const std::string arg_prefix_;
83+
84+
// Embedding space names for parameter sharing.
85+
std::vector<std::string> embedding_names_;
86+
87+
// FML strings for each feature extractor.
88+
std::vector<std::string> embedding_fml_;
89+
90+
// Size of each of the embedding spaces (maximum predicate id).
91+
std::vector<int> embedding_sizes_;
92+
93+
// Embedding dimensions of the embedding spaces (i.e. 32, 64 etc.)
94+
std::vector<int> embedding_dims_;
95+
};
96+
97+
// Templated, object-specific implementation of the
98+
// EmbeddingFeatureExtractor. EXTRACTOR should be a FeatureExtractor<OBJ,
99+
// ARGS...> class that has the appropriate FeatureTraits() to ensure that
100+
// locator type features work.
101+
//
102+
// Note: for backwards compatibility purposes, this always reads the FML spec
103+
// from "<prefix>_features".
104+
template <class EXTRACTOR, class OBJ, class... ARGS>
105+
class EmbeddingFeatureExtractor : public GenericEmbeddingFeatureExtractor {
106+
public:
107+
// Constructs this EmbeddingFeatureExtractor.
108+
//
109+
// |arg_prefix| is a string prefix for the relevant TaskContext parameters, to
110+
// avoid name clashes. See GetParamName().
111+
explicit EmbeddingFeatureExtractor(const std::string &arg_prefix)
112+
: GenericEmbeddingFeatureExtractor(arg_prefix) {}
113+
114+
// Sets up all predicate maps, feature extractors, and flags.
115+
SAFTM_MUST_USE_RESULT bool Setup(TaskContext *context) override {
116+
if (!GenericEmbeddingFeatureExtractor::Setup(context)) {
117+
return false;
118+
}
119+
feature_extractors_.resize(embedding_fml().size());
120+
for (int i = 0; i < embedding_fml().size(); ++i) {
121+
feature_extractors_[i].reset(new EXTRACTOR());
122+
if (!feature_extractors_[i]->Parse(embedding_fml()[i])) return false;
123+
if (!feature_extractors_[i]->Setup(context)) return false;
124+
}
125+
return true;
126+
}
127+
128+
// Initializes resources needed by the feature extractors.
129+
SAFTM_MUST_USE_RESULT bool Init(TaskContext *context) override {
130+
if (!GenericEmbeddingFeatureExtractor::Init(context)) return false;
131+
for (auto &feature_extractor : feature_extractors_) {
132+
if (!feature_extractor->Init(context)) return false;
133+
}
134+
return true;
135+
}
136+
137+
// Requests workspaces from the registry. Must be called after Init(), and
138+
// before Preprocess().
139+
void RequestWorkspaces(WorkspaceRegistry *registry) override {
140+
for (auto &feature_extractor : feature_extractors_) {
141+
feature_extractor->RequestWorkspaces(registry);
142+
}
143+
}
144+
145+
// Must be called on the object one state for each sentence, before any
146+
// feature extraction (e.g., UpdateMapsForExample, ExtractFeatures).
147+
void Preprocess(WorkspaceSet *workspaces, OBJ *obj) const {
148+
for (auto &feature_extractor : feature_extractors_) {
149+
feature_extractor->Preprocess(workspaces, obj);
150+
}
151+
}
152+
153+
// Extracts features using the extractors. Note that features must already
154+
// be initialized to the correct number of feature extractors. No predicate
155+
// mapping is applied.
156+
void ExtractFeatures(const WorkspaceSet &workspaces, const OBJ &obj,
157+
ARGS... args,
158+
std::vector<FeatureVector> *features) const {
159+
// DCHECK(features != nullptr);
160+
// DCHECK_EQ(features->size(), feature_extractors_.size());
161+
for (int i = 0; i < feature_extractors_.size(); ++i) {
162+
(*features)[i].clear();
163+
feature_extractors_[i]->ExtractFeatures(workspaces, obj, args...,
164+
&(*features)[i]);
165+
}
166+
}
167+
168+
private:
169+
// Templated feature extractor class.
170+
std::vector<std::unique_ptr<EXTRACTOR>> feature_extractors_;
171+
};
172+
173+
} // namespace mobile
174+
} // namespace nlp_saft
175+
176+
#endif // NLP_SAFT_COMPONENTS_COMMON_MOBILE_EMBEDDING_FEATURE_EXTRACTOR_H_

‎Unigram/Libraries/libtextclassifier/lang_id/common/embedding-feature-interface.h‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Copyright (C) 2018 The Android Open Source Project
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef NLP_SAFT_COMPONENTS_COMMON_MOBILE_EMBEDDING_FEATURE_INTERFACE_H_
18+
#define NLP_SAFT_COMPONENTS_COMMON_MOBILE_EMBEDDING_FEATURE_INTERFACE_H_
19+
20+
#include <string>
21+
#include <vector>
22+
23+
#include "lang_id/common/embedding-feature-extractor.h"
24+
#include "lang_id/common/fel/feature-extractor.h"
25+
#include "lang_id/common/fel/task-context.h"
26+
#include "lang_id/common/fel/workspace.h"
27+
#include "lang_id/common/lite_base/attributes.h"
28+
29+
namespace libtextclassifier3 {
30+
namespace mobile {
31+
32+
template <class EXTRACTOR, class OBJ, class... ARGS>
33+
class EmbeddingFeatureInterface {
34+
public:
35+
// Constructs this EmbeddingFeatureInterface.
36+
//
37+
// |arg_prefix| is a string prefix for the TaskContext parameters, passed to
38+
// |the underlying EmbeddingFeatureExtractor.
39+
explicit EmbeddingFeatureInterface(const std::string &arg_prefix)
40+
: feature_extractor_(arg_prefix) {}
41+
42+
// Sets up feature extractors and flags for processing (inference).
43+
SAFTM_MUST_USE_RESULT bool SetupForProcessing(TaskContext *context) {
44+
return feature_extractor_.Setup(context);
45+
}
46+
47+
// Initializes feature extractor resources for processing (inference)
48+
// including requesting a workspace for caching extracted features.
49+
SAFTM_MUST_USE_RESULT bool InitForProcessing(TaskContext *context) {
50+
if (!feature_extractor_.Init(context)) return false;
51+
feature_extractor_.RequestWorkspaces(&workspace_registry_);
52+
return true;
53+
}
54+
55+
// Preprocesses *obj using the internal workspace registry.
56+
void Preprocess(WorkspaceSet *workspace, OBJ *obj) const {
57+
workspace->Reset(workspace_registry_);
58+
feature_extractor_.Preprocess(workspace, obj);
59+
}
60+
61+
// Extract features from |obj|. On return, FeatureVector features[i]
62+
// contains the features for the embedding space #i.
63+
//
64+
// This function uses the precomputed info from |workspace|. Usage pattern:
65+
//
66+
// EmbeddingFeatureInterface<...> feature_interface;
67+
// ...
68+
// OBJ obj;
69+
// WorkspaceSet workspace;
70+
// feature_interface.Preprocess(&workspace, &obj);
71+
//
72+
// // For the same obj, but with different args:
73+
// std::vector<FeatureVector> features;
74+
// feature_interface.GetFeatures(obj, args, workspace, &features);
75+
//
76+
// This pattern is useful (more efficient) if you can pre-compute some info
77+
// for the entire |obj|, which is reused by the feature extraction performed
78+
// for different args. If that is not the case, you can use the simpler
79+
// version GetFeaturesNoCaching below.
80+
void GetFeatures(const OBJ &obj, ARGS... args, const WorkspaceSet &workspace,
81+
std::vector<FeatureVector> *features) const {
82+
feature_extractor_.ExtractFeatures(workspace, obj, args..., features);
83+
}
84+
85+
// Simpler version of GetFeatures(), for cases when there is no opportunity to
86+
// reuse computation between feature extractions for the same |obj|, but with
87+
// different |args|. Returns the extracted features. For more info, see the
88+
// doc for GetFeatures().
89+
std::vector<FeatureVector> GetFeaturesNoCaching(OBJ *obj,
90+
ARGS... args) const {
91+
// Technically, we still use a workspace, because
92+
// feature_extractor_.ExtractFeatures requires one. But there is no real
93+
// caching here, as we start from scratch for each call to ExtractFeatures.
94+
WorkspaceSet workspace;
95+
Preprocess(&workspace, obj);
96+
std::vector<FeatureVector> features(NumEmbeddings());
97+
GetFeatures(*obj, args..., workspace, &features);
98+
return features;
99+
}
100+
101+
// Returns number of embedding spaces.
102+
int NumEmbeddings() const { return feature_extractor_.NumEmbeddings(); }
103+
104+
private:
105+
// Typed feature extractor for embeddings.
106+
EmbeddingFeatureExtractor<EXTRACTOR, OBJ, ARGS...> feature_extractor_;
107+
108+
// The registry of shared workspaces in the feature extractor.
109+
WorkspaceRegistry workspace_registry_;
110+
};
111+
112+
} // namespace mobile
113+
} // namespace nlp_saft
114+
115+
#endif // NLP_SAFT_COMPONENTS_COMMON_MOBILE_EMBEDDING_FEATURE_INTERFACE_H_

0 commit comments

Comments
 (0)