Skip to content

Commit 68747e9

Browse files
gxlzlihaogxlzlihao
committed
change the code to support the latest design
1 parent dc5cd95 commit 68747e9

11 files changed

Lines changed: 411 additions & 410 deletions

File tree

‎include/parser.hrl‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
%% input_type is the datatype to which this datatype is designed.
88
%% for instance, application/json
99

10-
-record(parser, {resource_id,
11-
stream_id,
10+
-record(parser, {stream_id,
1211
input_parser,
1312
input_type}).

‎include/poller.hrl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
%%here comes the definition of the structure of poller`s information
22

3-
-record(pollerInfo, {resourceid,
3+
-record(pollerInfo, {stream_id,
44
name,
5-
url,
5+
uri,
66
frequency,
77
pid}).
88

‎include/state.hrl‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
%%here comes the state`s information of the gen_server
22

3-
-record(state, {resourceid,
4-
url,
5-
parserslist}).
3+
-record(state, {stream_id,
4+
uri,
5+
parser}).

scripts/python/cgi-bin/resource.py renamed to scripts/python/cgi-bin/humidity.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
{"resource": "polling-resource",
1111
"streams":
1212
{
13-
"temperature": {"value": %f, "timestamp": "%s"},
1413
"humidity": {"value": %f, "timestamp": "%s"}
1514
}
1615
}

‎scripts/python/cgi-bin/temperature.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env python
2+
import datetime
3+
import random
4+
timestamp = datetime.datetime.now()
5+
humidity = random.random()
6+
temperature = random.random() * 100
7+
print "Content-Type: application/json"
8+
print
9+
print """\
10+
{"resource": "polling-resource",
11+
"streams":
12+
{
13+
"temperature": {"value": %f, "timestamp": "%s"}
14+
}
15+
}
16+
""" % (temperature, timestamp.strftime("%Y-%m-%d %H:%M:%S"), humidity, timestamp.strftime("%Y-%m-%d %H:%M:%S"))

‎src/parser.erl‎

Lines changed: 24 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -17,54 +17,7 @@
1717
%% ====================================================================
1818
%% API functions
1919
%% ====================================================================
20-
-export([applyParser/3]).
21-
22-
%% @doc
23-
%% Function: applyParser/3
24-
%% Purpose: used relative parsers to extract the values we want from the data. In the current design, the datapoint
25-
%% is a list of values, we care more on the values than the keys. ParsersList contains the regular expressions used to
26-
%% parse the different data.
27-
%% Returns: ok | {error, ErrMsg}
28-
%% @end
29-
-spec applyParser(list(),any() ,string()) -> ok | {error, string()}.
30-
applyParser(ParsersList, Data, ContentType) ->
31-
Parsers = get_parsers_with_content_type(ContentType, ParsersList, []),
32-
33-
%% only for testing
34-
%% erlang:display("the length of appropriate parsers: "++integer_to_list(length(Parsers))),
35-
36-
case Parsers == [] of
37-
false ->
38-
doParsing(ParsersList, Data, ContentType);
39-
_ -> %%the parsers for this type do not exist.
40-
{error, "the parsers for this type do not exist"}
41-
end.
42-
43-
44-
45-
%% ====================================================================
46-
%% Internal functions
47-
%% ====================================================================
48-
49-
%% @doc
50-
%% Function: processParser/2
51-
%% Purpose: transformat the format of the parser
52-
%% Example: ["streams","temperature","value"] => "streams.temperature.value"
53-
%% Returns: string()
54-
%% @end
55-
-spec processParser(list(string()), list(string())) -> list(string()).
56-
processParser([Item|Tail], Res)->
57-
case Item=="/" of
58-
true->
59-
processParser(Tail, Res);
60-
_ ->
61-
case Tail of
62-
[]->
63-
Res++Item;
64-
_ ->
65-
processParser(Tail, Res++Item++".")
66-
end
67-
end.
20+
-export([parseJson/2, parseText/2]).
6821

6922
%% @doc
7023
%% Function: parseJson/2
@@ -76,7 +29,6 @@ processParser([Item|Tail], Res)->
7629
parseJson(Parser, Data) ->
7730

7831
%%extract the data from the coming data
79-
ResourceId = Parser#parser.resource_id,
8032
StreamId = Parser#parser.stream_id,
8133
InputType = Parser#parser.input_type,
8234
InputParser = Parser#parser.input_parser,
@@ -123,7 +75,7 @@ parseJson(Parser, Data) ->
12375
{error, Reason};
12476
{ok,List} ->
12577
%% only for testing
126-
%% erlang:display("the final data which is inserted into the database: "++FinalJson),
78+
erlang:display("the final data which is inserted into the database: "++FinalJson),
12779

12880
ok
12981
end.
@@ -140,48 +92,29 @@ parseText(Parser, Data) ->
14092
%% return the status of the transaction, ok or {error, ErrMsg}
14193
ok.
14294

143-
%% @doc
144-
%% Function: doParsing/3
145-
%% Purpose: call parse function using the patterns contained in ParsersList to parse the data.
146-
%% Returns: ok | {error, ErrMsg}
147-
%% Side effects: Stores the newly parsed data points in the DB
148-
%% @end
149-
-spec doParsing(list(), any(), string()) -> ok | {error, string()}.
150-
doParsing([], Data, ContentType) -> ok;
151-
doParsing([Parser|ParsersList], Data, ContentType) ->
152-
case ContentType of
153-
"application/json"->
154-
TempRes = parseJson(Parser, Data),
155-
case TempRes of
156-
ok ->
157-
doParsing(ParsersList, Data, ContentType);
158-
{error, ErrMsg} ->
159-
%%parsing fails
160-
TempRes
161-
end;
162-
"plain/text"->
163-
TempRes = parseText(Parser, Data),
164-
case TempRes of
165-
ok ->
166-
doParsing(ParsersList, Data, ContentType);
167-
{error, ErrMsg} ->
168-
%%parsing fails
169-
TempRes
170-
end;
171-
_ ->
172-
%%error the content`s type doesn`t exist
173-
{error, "the content`s type doesn`t exist"}
174-
end.
95+
96+
97+
98+
%% ====================================================================
99+
%% Internal functions
100+
%% ====================================================================
175101

176102
%% @doc
177-
%% Function: get_parsers_with_content_type/3
178-
%% Purpose: find relative patterns for the specific content type.
179-
%% Returns: The patterns suitable for the current content type
103+
%% Function: processParser/2
104+
%% Purpose: transformat the format of the parser
105+
%% Example: ["streams","temperature","value"] => "streams.temperature.value"
106+
%% Returns: string()
180107
%% @end
181-
-spec get_parsers_with_content_type(string(), list(), list()) -> list().
182-
get_parsers_with_content_type(_Content_type, [], L) -> L;
183-
get_parsers_with_content_type(Content_type, [Parser|Parsers], L) ->
184-
case Parser#parser.input_type == Content_type of
185-
true -> get_parsers_with_content_type(Content_type, Parsers, [Parser|L]);
186-
_ -> get_parsers_with_content_type(Content_type, Parsers, L)
108+
-spec processParser(list(string()), list(string())) -> list(string()).
109+
processParser([Item|Tail], Res)->
110+
case Item=="/" of
111+
true->
112+
processParser(Tail, Res);
113+
_ ->
114+
case Tail of
115+
[]->
116+
Res++Item;
117+
_ ->
118+
processParser(Tail, Res++Item++".")
119+
end
187120
end.

‎src/poll_help.erl‎

Lines changed: 54 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@
1717
-include("poller.hrl").
1818
-include("parser.hrl").
1919

20-
-export([get_resources_using_polling/0,
21-
json_to_record_resources/1,
22-
json_to_record_resource/1,
23-
get_parsers_by_id/1,
20+
-export([get_streams_using_polling/0,
21+
json_to_record_streams/1,
22+
json_to_record_stream/1,
23+
get_parser_by_id/1,
2424
post_datapoint/2,
2525
get_datapoint/1]).
2626

@@ -44,7 +44,7 @@ get_datapoint(StreamId)->
4444
_->
4545
Id = StreamId
4646
end,
47-
case erlastic_search:search_limit(?ES_INDEX, "datapoint", "streamid:" ++ Id, 100) of
47+
case erlastic_search:search_limit(?ES_INDEX, "datapoint", "stream_id:" ++ Id, 100) of
4848
{ok, Result} ->
4949
FinalJson = lib_json:get_list_and_add_id(Result, data),
5050
lib_json:get_field(FinalJson, "data");
@@ -63,9 +63,9 @@ get_datapoint(StreamId)->
6363
post_datapoint(StreamId, Value)->
6464
case is_integer(StreamId) of
6565
true->
66-
FieldValue1 = {"streamid",integer_to_binary(StreamId)};
66+
FieldValue1 = {"stream_id",integer_to_binary(StreamId)};
6767
_ ->
68-
FieldValue1 = {"streamid",list_to_binary(StreamId)}
68+
FieldValue1 = {"stream_id",list_to_binary(StreamId)}
6969
end,
7070
case is_integer(Value) of
7171
true->
@@ -99,82 +99,91 @@ post_datapoint(StreamId, Value)->
9999
end.
100100

101101
%% @doc
102-
%% Function: get_parsers_by_id/1
103-
%% Purpose: get parsers list according to specific resource id
104-
%% Returns: {error, ErrMsg} | [#parser .....]
102+
%% Function: get_parser_by_id/1
103+
%% Purpose: get parser according to specific stream id
104+
%% Returns: {error, ErrMsg} | #parser
105105
%% @end
106-
-spec get_parsers_by_id(string()) -> tuple() | list().
107-
get_parsers_by_id(ResourceId)->
108-
case erlastic_search:search_limit(?ES_INDEX, "parser", "resource_id:" ++ ResourceId, 100) of
106+
-spec get_parser_by_id(string()) -> tuple() | tuple().
107+
get_parser_by_id(StreamId)->
108+
case erlastic_search:search_limit(?ES_INDEX, "parser", "stream_id:" ++ StreamId, 100) of
109109
{ok, Result} ->
110110
EncodedResult = lib_json:encode(Result),
111111
case re:run(EncodedResult, "\"max_score\":null", [{capture, first, list}]) of
112-
{match, _} -> {error, "parsers not found"};
112+
{match, _} -> {error, "the parser not found"};
113113
nomatch -> FinalJsonList = lib_json:get_field(lib_json:get_list_and_add_id(Result), "hits"),
114-
115-
%%transform this json list to record list
116-
jsonToRecord(FinalJsonList, [])
114+
case length(FinalJsonList) of
115+
1 ->
116+
Item = lists:nth(1, FinalJsonList),
117+
#parser{stream_id = lib_json:get_field(Item, "stream_id"),
118+
input_parser = binary_to_list(lib_json:get_field(Item, "input_parser")),
119+
input_type = binary_to_list(lib_json:get_field(Item, "input_type"))
120+
};
121+
_ ->
122+
erlang:display("multiple parsers exist for this stream id"),
123+
{error, "multiple parsers exit for this stream id"}
124+
end
125+
%% transform this json list to record list
126+
%% jsonToRecord(FinalJsonList, [])
117127
end;
118128
_ ->
119-
erlang:display("an error happens: parsers not found"),
129+
erlang:display("an error happens: the parser not found"),
120130
{error, "parsers not found!!"}
121131
end.
122132

123133
%% @doc
124-
%% Function: get_resources_using_polling/0
125-
%% Purpose: Retrieves all resources from Elastic Search that are using polling.
126-
%% Returns: [] | [Resource, ... ] | {error, Reason}.
134+
%% Function: get_streams_using_polling/0
135+
%% Purpose: Retrieves all streams from Elastic Search that are using polling.
136+
%% Returns: [] | [Stream, ... ] | {error, Reason}.
127137
%% @end
128-
-spec get_resources_using_polling() -> [] | [string()] | {error, term()}.
129-
get_resources_using_polling() ->
130-
%% TODO : Post a query to ES to retrieve resources that have a polling-URL.
131-
138+
-spec get_streams_using_polling() -> [] | [string()] | {error, term()}.
139+
get_streams_using_polling() ->
132140
JsonQuery = "{\"query\" : {\"filtered\" : " ++
133-
"{ \"filter\" : {\"exists\" : {\"field\" : \"url\"}}}}}",
141+
"{ \"filter\" : {\"exists\" : {\"field\" : \"uri\"}}}}}",
134142

135143
case erlastic_search:search_json(#erls_params{},
136144
?ES_INDEX,
137-
"resource",
145+
"stream",
138146
JsonQuery) of
139147
{error, Reason} -> {error, Reason};
140148
{ok, Result} ->
141149
lib_json:get_field(Result, "hits.hits")
142150
end.
151+
143152

144153

145154

146155
%% @doc
147-
%% Function: json_to_record_resources/1
148-
%% Purpose: Converts a list of resource Jsons to a list of pollerInfo records
149-
%% Returns: [] | [Resource, ...]
156+
%% Function: json_to_record_streams/1
157+
%% Purpose: Converts a list of stream Jsons to a list of pollerInfo records
158+
%% Returns: [] | [Stream, ...]
150159
%% @end
151-
-spec json_to_record_resources([json()]) -> [] | [record()].
152-
json_to_record_resources([]) -> [];
153-
json_to_record_resources([H|T]) ->
154-
[json_to_record_resource(H) | json_to_record_resources(T)].
160+
-spec json_to_record_streams([json()]) -> [] | [record()].
161+
json_to_record_streams([]) -> [];
162+
json_to_record_streams([H|T]) ->
163+
[json_to_record_stream(H) | json_to_record_streams(T)].
155164

156165

157166

158167

159168
%% @doc
160-
%% Function: json_to_record_resource/1
161-
%% Purpose: Converts a resource Json to a pollerInfo record
162-
%% Returns: Resource
169+
%% Function: json_to_record_stream/1
170+
%% Purpose: Converts a stream Json to a pollerInfo record
171+
%% Returns: #pollerInfo{}
163172
%% @end
164-
-spec json_to_record_resource(Resource::json()) -> record().
165-
json_to_record_resource(Resource) ->
166-
Name = case lib_json:get_field(Resource, "_source.name") of
173+
-spec json_to_record_stream(Stream::json()) -> record().
174+
json_to_record_stream(Stream) ->
175+
Name = case lib_json:get_field(Stream, "_source.name") of
167176
undefined -> undefined;
168177
N -> binary_to_list(N)
169178
end,
170-
Url = case lib_json:get_field(Resource, "_source.url") of
179+
Uri = case lib_json:get_field(Stream, "_source.uri") of
171180
undefined -> undefined;
172181
U -> binary_to_list(U)
173182
end,
174-
#pollerInfo{resourceid = binary_to_list(lib_json:get_field(Resource, "_id")),
183+
#pollerInfo{stream_id = binary_to_list(lib_json:get_field(Stream, "_id")),
175184
name = Name,
176-
url = Url,
177-
frequency = lib_json:get_field(Resource, "_source.polling_freq")}.
185+
uri = Uri,
186+
frequency = lib_json:get_field(Stream, "_source.polling_freq")}.
178187

179188

180189
%% ====================================================================
@@ -191,8 +200,7 @@ json_to_record_resource(Resource) ->
191200
jsonToRecord([], Res)->
192201
Res;
193202
jsonToRecord([Item|Tail], Res)->
194-
Tmp = #parser{resource_id = lib_json:get_field(Item, "resource_id"),
195-
stream_id = lib_json:get_field(Item, "stream_id"),
203+
Tmp = #parser{stream_id = lib_json:get_field(Item, "stream_id"),
196204
input_parser = binary_to_list(lib_json:get_field(Item, "input_parser")),
197205
input_type = binary_to_list(lib_json:get_field(Item, "input_type"))
198206
},

0 commit comments

Comments
 (0)