Skip to content

Commit e3e8051

Browse files
clintyclinty
committed
Generate plugins list through https requests
The devstack plugins list can be generated through web requests in environments (such as the proposal slave) that lack copies of all the relevant git repositories. One downside to this is that there is no way of getting the last modification time of the plugin. Change-Id: I2c5c9282a8ad80014cad171a4dfbdc8f26044cd1
1 parent 4bcf652 commit e3e8051

3 files changed

Lines changed: 102 additions & 22 deletions

File tree

‎data/devstack-plugins-registry.header‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@ The following are plugins that a script has found in the openstack/
1414
namespace, which includes but is not limited to official OpenStack
1515
projects.
1616

17-
+------------------+------------------------------------------------------------+------------+
18-
|Plugin Name |URL |Date |
19-
+------------------+------------------------------------------------------------+------------+
17+
+------------------+-------------------------------------------------------------------------+
18+
|Plugin Name |URL |
19+
+------------------+-------------------------------------------------------------------------+

‎tools/generate-devstack-plugins-list.py‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#! /usr/bin/env python
2+
3+
# Copyright 2016 Hewlett Packard Enterprise Development Company, L.P.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License"); you may
6+
# not use this file except in compliance with the License. You may obtain
7+
# a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
# License for the specific language governing permissions and limitations
15+
# under the License.
16+
17+
# This script is intended to be run as part of a periodic proposal bot
18+
# job in OpenStack infrastructure.
19+
#
20+
# In order to function correctly, the environment in which the
21+
# script runs must have
22+
# * network access to the review.openstack.org Gerrit API
23+
# working directory
24+
# * network access to https://git.openstack.org/cgit
25+
26+
import json
27+
import requests
28+
29+
url = 'https://review.openstack.org/projects/'
30+
31+
# This is what a project looks like
32+
'''
33+
"openstack-attic/akanda": {
34+
"id": "openstack-attic%2Fakanda",
35+
"state": "READ_ONLY"
36+
},
37+
'''
38+
39+
def is_in_openstack_namespace(proj):
40+
return proj.startswith('openstack/')
41+
42+
# Rather than returning a 404 for a nonexistent file, cgit delivers a
43+
# 0-byte response to a GET request. It also does not provide a
44+
# Content-Length in a HEAD response, so the way we tell if a file exists
45+
# is to check the length of the entire GET response body.
46+
def has_devstack_plugin(proj):
47+
r = requests.get("https://git.openstack.org/cgit/%s/plain/devstack/plugin.sh" % proj)
48+
if len(r.text) > 0:
49+
return True
50+
else:
51+
False
52+
53+
r = requests.get(url)
54+
projects = sorted(filter(is_in_openstack_namespace, json.loads(r.text[4:])))
55+
56+
found_plugins = filter(has_devstack_plugin, projects)
57+
58+
for project in found_plugins:
59+
print project[10:]

‎tools/generate-devstack-plugins-list.sh‎

Lines changed: 40 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,19 @@
1919
#
2020
# In order to function correctly, the environment in which the
2121
# script runs must have
22+
# * a writable doc/source directory relative to the current
23+
# working directory
24+
# AND ( (
2225
# * git
2326
# * all git repos meant to be searched for plugins cloned and
2427
# at the desired level of up-to-datedness
25-
# * a writable doc/source directory relative to the current
28+
# * the environment variable git_dir pointing to the location
29+
# * of said git repositories
30+
# ) OR (
31+
# * network access to the review.openstack.org Gerrit API
2632
# working directory
33+
# * network access to https://git.openstack.org/cgit
34+
# ))
2735
#
2836
# If a file named data/devstack-plugins-registry.header or
2937
# data/devstack-plugins-registry.footer is found relative to the
@@ -35,25 +43,38 @@ declare -A plugins
3543

3644
test -r data/devstack-plugins-registry.header && cat data/devstack-plugins-registry.header
3745

38-
pushd ${git_dir:-/opt/openstack} >/dev/null
39-
for i in *; do
40-
pushd ${i} >/dev/null
41-
if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
42-
test -n "$output" && plugins[$i]=${output}
43-
fi
46+
if test -n "$git_dir"; then
47+
pushd ${git_dir} >/dev/null
48+
for i in *; do
49+
pushd ${i} >/dev/null
50+
if output="$(git log --diff-filter=A --format='%cd' --date=short -1 -- devstack/plugin.sh)"; then
51+
test -n "$output" && plugins[$i]=${output}
52+
fi
53+
popd >/dev/null
54+
done
4455
popd >/dev/null
45-
done
46-
popd >/dev/null
47-
48-
sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
49-
50-
for k in "${sorted_plugins[@]}"; do
51-
project=${k:0:18}
52-
giturl="git://git.openstack.org/openstack/${k:0:26}"
53-
pdate="${plugins[$k]}"
54-
printf "|%-18s|%-60s|%-12s|\n" "${project}" "${giturl}" "${pdate}"
55-
printf "+------------------+------------------------------------------------------------+------------+\n"
56-
done
56+
57+
sorted_plugins=( $(for k in "${!plugins[@]}"; do echo "$k"; done | sort))
58+
59+
for k in "${sorted_plugins[@]}"; do
60+
project=${k:0:18}
61+
giturl="git://git.openstack.org/openstack/${k:0:26}"
62+
pdate="${plugins[$k]}"
63+
printf "|%-18s|%-60s (%-10s)|\n" "${project}" "${giturl}" "${pdate}"
64+
printf "+------------------+-------------------------------------------------------------------------+\n"
65+
done
66+
67+
else
68+
sorted_plugins=$(python tools/generate-devstack-plugins-list.py)
69+
70+
for k in ${sorted_plugins}; do
71+
project=${k:0:18}
72+
giturl="git://git.openstack.org/openstack/${k:0:26}"
73+
printf "|%-18s|%-73s|\n" "${project}" "${giturl}"
74+
printf "+------------------+-------------------------------------------------------------------------+\n"
75+
done
76+
77+
fi
5778

5879
test -r data/devstack-plugins-registry.footer && cat data/devstack-plugins-registry.footer
5980
) > doc/source/plugin-registry.rst

0 commit comments

Comments
 (0)