Repository navigation
Expand file tree
/
Copy pathlogparser.py
More file actions
121 lines (89 loc) · 4.74 KB
/
Copy pathlogparser.py
File metadata and controls
121 lines (89 loc) · 4.74 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
import re
import time
from collections import namedtuple
from datetime import datetime
class LogDetails(namedtuple('LogDetails', 'client_ip, user, timestamp, http_method, url, http_protocol, status_code, domain')):
def to_tags(self, **kwargs):
tags = self._asdict()
del tags['timestamp']
del tags['url']
tags.update(kwargs)
return tags
WILDCARD = '*'
class LogRequests(object):
def __init__(self, logger, line):
self.details = _get_log_details(logger, line)
if not self.details:
return None
self.url_group = _get_url_group(self.details.url)
def parse_request_total(logger, line):
log_req = LogRequests(logger, line)
return 'artifactory.requests.total', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
def parse_status_code(logger, line):
log_req = LogRequests(logger, line)
if log_req.details[6].startswith('20'):
return 'artifactory.requests.2xx', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[6].startswith('40'):
return 'artifactory.requests.4xx', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[6].startswith('50'):
return 'artifactory.requests.5xx', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
def parse_http_method(logger, line):
log_req = LogRequests(logger, line)
if log_req.details[3] == 'GET':
return 'artifactory.requests.get', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[3] == 'PUT':
return 'artifactory.requests.put', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[3] == 'POST':
return 'artifactory.requests.post', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[3] == 'DELETE':
return 'artifactory.requests.delete', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
def parse_api_call(logger, line):
log_req = LogRequests(logger, line)
if log_req.details[4].startswith(r'/api') and log_req.details[3] == 'GET':
return 'artifactory.requests.api.get', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[4].startswith(r'/api') and log_req.details[3] == 'PUT':
return 'artifactory.requests.api.put', log_req.details.timestamp, 1, log_req.details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[4].startswith(r'/api') and log_req.details[3] == 'POST':
return 'artifactory.requests.api.post', log_req.details.timestamp, 1,log_req. details.to_tags(metric_type='counter', url_group=log_req.url_group)
elif log_req.details[4].startswith(r'/api') and log_req.details[3] == 'DELETE':
return 'artifactory.requests.api.delete', log_req.details.timestamp, 1,log_req. details.to_tags(metric_type='counter', url_group=log_req.url_group)
def _get_log_details(logger, line):
if not line:
return None
try:
details = _parse_line(line)
except Exception:
logger.exception('Failed to parse log line')
return None
if _should_skip_log(details.url):
return None
return details
def _get_url_group(url):
default = 'other'
if url.startswith('/a/' + WILDCARD):
parts = url.split('/')
return parts[3] if len(parts) >= 4 else default
return default
def _should_skip_log(url):
return re.search(r'^/static/', url)
def _parse_line(line):
match = re.match(r'(?P<date>[0-9]+)', line)
string_date = match.group('date')
date = datetime.strptime(string_date, "%Y%m%d%H%M%S")
# First two dummy args are from the date being split
parts = line.split('|')
client_ip, user, http_method, url, http_protocol, status_code = parts[3], parts[4], parts[5], parts[6], parts[7], parts[8]
timestamp = time.mktime(date.timetuple())
domain = _extract_domain(url)
url = _sanitize_url(url)
return LogDetails(client_ip, user, timestamp, http_method, url, http_protocol, status_code, domain)
def _sanitize_url(url):
url = re.sub(r'/a/[0-9a-z-]+', '/a/{}'.format(WILDCARD), url)
# Remove URL params
url = re.sub(r'\?[^ ]*', '', url)
return url
def _extract_domain(url):
match = re.search(r'/a/(?P<domain>[0-9a-z-]+)', url)
if not match:
return ''
return match.group('domain')