Skip to content

Rewriting bigquery samples to use proper argparse and main execution #89

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
theacodes merged 6 commits into from
Sep 15, 2015
Merged

Rewriting bigquery samples to use proper argparse and main execution #89

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
40339f2
Updating all bigquery samples to new main pattern
Sep 14, 2015
39dab6b
Fixing bigquery tests
Sep 14, 2015
ad20eca
Fixing pep8, removing mock_input_answers
Sep 14, 2015
e4f827c
Moving to module-based execution
Sep 15, 2015
7b5f96c
Fixing inconsistences in style
Sep 15, 2015
2e5d57c
Removing utils, putting depending code directly in samples
Sep 15, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Updating all bigquery samples to new main pattern
  • Loading branch information
Jon Wayne Parrott
Jon Wayne Parrott committed Sep 15, 2015
commit 40339f2c1827aa6aaccc188b3d44ad21432d030c
4 changes: 0 additions & 4 deletions appengine/ndb/transactions/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,3 @@ def add_note():
return ('Already there<br><a href="%s">Return</a>'
% flask.url_for('main_page', page_name=page_name))
return flask.redirect(flask.url_for('main_page', page_name=page_name))


if __name__ == '__main__':
app.run()
46 changes: 29 additions & 17 deletions bigquery/samples/async_query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import json
import uuid

from bigquery.samples.utils import get_service
from bigquery.samples.utils import paging
from bigquery.samples.utils import poll_job
from six.moves import input
from utils import get_service, paging, poll_job


# [START async_query]
Expand All @@ -43,7 +41,7 @@ def async_query(service, project_id, query, batch=False, num_retries=5):


# [START run]
def run(project_id, query_string, batch, num_retries, interval):
def main(project_id, query_string, batch, num_retries, interval):
service = get_service()

query_job = async_query(service,
Expand All @@ -63,21 +61,35 @@ def run(project_id, query_string, batch, num_retries, interval):
num_retries=num_retries,
**query_job['jobReference']):

yield json.dumps(page['rows'])
print(json.dumps(page['rows']))
# [END run]


# [START main]
def main():
project_id = input("Enter the project ID: ")
query_string = input("Enter the Bigquery SQL Query: ")
batch = input("Run query as batch (y/n)?: ") in (
'True', 'true', 'y', 'Y', 'yes', 'Yes')
num_retries = int(input(
"Enter number of times to retry in case of 500 error: "))
interval = input(
"Enter how often to poll the query for completion (seconds): ")
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Loads data into BigQuery.')
parser.add_argument('project_id', help='Your Google Cloud project ID.')
parser.add_argument('query', help='BigQuery SQL Query.')
parser.add_argument(
'-b', '--batch', help='Run query in batch mode.', action='store_true')
parser.add_argument(
'-r', '--num_retries',
help='Number of times to retry in case of 500 error.',
type=int,
default=5)
parser.add_argument(
'-p', '--poll_interval',
help='How often to poll the query for completion (seconds).',
type=int,
default=1)

for result in run(project_id, query_string, batch, num_retries, interval):
print(result)
args = parser.parse_args()

main(
args.project_id,
args.query,
args.batch,
args.num_retries,
args.poll_interval)
# [END main]
66 changes: 39 additions & 27 deletions bigquery/samples/export_data_to_cloud_storage.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import uuid

from bigquery.samples.utils import get_service
from bigquery.samples.utils import poll_job
from six.moves import input
from utils import get_service, poll_job


# [START export_table]
def export_table(service, cloud_storage_path,
projectId, datasetId, tableId,
project_id, dataset_id, table_id,

@elibixby elibixby Sep 14, 2015

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to give my thinking behind this. Camel case matches the field names in the json objects. So you could make these kwargs and then do something like:

tableRef = fetch_some_bigquery_object()['tableReference']
export_table(service, cloud_storage, **tableRef)

Probably not worth inconsistent/camel case naming, but something to consider.

@theacodes theacodes Sep 15, 2015

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True, but I'd rather follow python convention where possible:

Names that are visible to the user as public parts of the API should follow conventions that reflect usage rather than implementation.
...
mixedCase is allowed only in contexts where that's already the prevailing style (e.g. threading.py), to retain backwards compatibility.

export_format="CSV",
num_retries=5):
"""
Expand All @@ -42,35 +41,34 @@ def export_table(service, cloud_storage_path,
# don't accidentally duplicate export
job_data = {
'jobReference': {
'projectId': projectId,
'projectId': project_id,
'jobId': str(uuid.uuid4())
},
'configuration': {
'extract': {
'sourceTable': {
'projectId': projectId,
'datasetId': datasetId,
'tableId': tableId,
'projectId': project_id,
'datasetId': dataset_id,
'tableId': table_id,
},
'destinationUris': [cloud_storage_path],
'destinationFormat': export_format
}
}
}
return service.jobs().insert(
projectId=projectId,
projectId=project_id,
body=job_data).execute(num_retries=num_retries)
# [END export_table]


# [START run]
def run(cloud_storage_path,
projectId, datasetId, tableId,
num_retries, interval, export_format="CSV"):
def main(cloud_storage_path, project_id, dataset_id, table_id,
num_retries, interval, export_format="CSV"):

bigquery = get_service()
resource = export_table(bigquery, cloud_storage_path,
projectId, datasetId, tableId,
project_id, dataset_id, table_id,
num_retries=num_retries,
export_format=export_format)
poll_job(bigquery,
Expand All @@ -82,20 +80,34 @@ def run(cloud_storage_path,


# [START main]
def main():
projectId = input("Enter the project ID: ")
datasetId = input("Enter a dataset ID: ")
tableId = input("Enter a table name to copy: ")
cloud_storage_path = input(
"Enter a Google Cloud Storage URI: ")
interval = input(
"Enter how often to poll the job (in seconds): ")
num_retries = input(
"Enter the number of retries in case of 500 error: ")
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Exports data from BigQuery to Google Cloud Storage.')
parser.add_argument('project_id', help='Your Google Cloud project ID.')
parser.add_argument('dataset_id', help='BigQuery dataset to export.')
parser.add_argument('table_id', help='BigQuery table to export.')
parser.add_argument(
'gcs_path',
help=('Google Cloud Storage path to store the exported data. For '
'example, gs://mybucket/mydata.csv'))
parser.add_argument(
'-p', '--poll_interval',
help='How often to poll the query for completion (seconds).',
type=int,
default=1)
parser.add_argument(
'-r', '--num_retries',
help='Number of times to retry in case of 500 error.',
type=int,
default=5)

run(cloud_storage_path,
projectId, datasetId, tableId,
num_retries, interval)
args = parser.parse_args()

print('Done exporting!')
main(
args.gcs_path,
args.project_id,
args.dataset_id,
args.table_id,
args.num_retries,
args.poll_interval)
# [END main]
22 changes: 14 additions & 8 deletions bigquery/samples/getting_started.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@
words.
"""
# [START all]
import argparse

from apiclient.discovery import build
from apiclient.errors import HttpError

from oauth2client.client import GoogleCredentials


Expand All @@ -38,10 +39,11 @@ def main(project_id):
# [START run_query]
query_request = bigquery_service.jobs()
query_data = {
'query': ('SELECT TOP(corpus, 10) as title, '
'COUNT(*) as unique_words '
'FROM [publicdata:samples.shakespeare];')
}
'query': (
'SELECT TOP(corpus, 10) as title, '
'COUNT(*) as unique_words '
'FROM [publicdata:samples.shakespeare];')
}

query_response = query_request.query(
projectId=project_id,
Expand All @@ -60,7 +62,11 @@ def main(project_id):


if __name__ == '__main__':
# The id of the project to run queries under.
project_id = input("Enter the project ID: ")
main(project_id)
parser = argparse.ArgumentParser(
description='Queries the public BigQuery Shakespeare dataset.')
parser.add_argument('project_id', help='Your Google Cloud Project ID.')

args = parser.parse_args()

main(args.project_id)
# [END all]
29 changes: 15 additions & 14 deletions bigquery/samples/list_datasets_projects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,21 @@
application-default-credentials#howtheywork
[2] https://cloud.google.com/sdk/
[3] https://console.developers.google.com
""" # NOQA

For more information on the BigQuery API you can visit:

https://developers.google.com/bigquery/docs/overview

For more information on the BigQuery API Python library surface you
can visit:

https://developers.google.com/resources/api-libraries/documentation/
bigquery/v2/python/latest/

For information on the Python Client Library visit:

https://developers.google.com/api-client-library/python/start/get_started
"""

import argparse
from pprint import pprint
Expand Down Expand Up @@ -81,19 +95,6 @@ def main(project_id):
list_datasets(service, project_id)
list_projects(service)


# For more information on the BigQuery API you can visit:
#
# https://developers.google.com/bigquery/docs/overview
#
# For more information on the BigQuery API Python library surface you
# can visit:
#
# https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/python/latest/
#
# For information on the Python Client Library visit:
#
# https://developers.google.com/api-client-library/python/start/get_started
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Lists BigQuery datasets and projects.')
Expand Down
50 changes: 32 additions & 18 deletions bigquery/samples/load_data_by_post.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import json

from bigquery.samples.utils import get_service, poll_job
from utils import get_service, poll_job
import httplib2
from oauth2client.client import GoogleCredentials
from six.moves import input


# [START make_post]
Expand Down Expand Up @@ -71,38 +71,52 @@ def make_post(http, schema, data, projectId, datasetId, tableId):


# [START main]
def main():
def main(project_id, dataset_id, table_name, schema_path, data_path):
credentials = GoogleCredentials.get_application_default()
http = credentials.authorize(httplib2.Http())
projectId = input('Enter the project ID: ')
datasetId = input('Enter a dataset ID: ')
tableId = input('Enter a table name to load the data to: ')
schema_path = input(
'Enter the path to the schema file for the table: ')

with open(schema_path, 'r') as schema_file:
schema = schema_file.read()

data_path = input('Enter the path to the data file: ')

with open(data_path, 'r') as data_file:
data = data_file.read()

resp, content = make_post(http,
schema,
data,
projectId,
datasetId,
tableId)
resp, content = make_post(
http,
schema,
data,
project_id,
dataset_id,
table_name)

if resp.status == 200:
job_resource = json.loads(content)
service = get_service(credentials)
service = get_service()
poll_job(service, **job_resource['jobReference'])
print("Success!")
else:
print("Http error code: {}".format(resp.status))
# [END main]

if __name__ == '__main__':
main()
parser = argparse.ArgumentParser(
description='Loads data into BigQuery.')
parser.add_argument('project_id', help='Your Google Cloud project ID.')
parser.add_argument('dataset_id', help='A BigQuery dataset ID.')
parser.add_argument(
'table_name', help='Name of the table to load data into.')
parser.add_argument(
'schema_file',
help='Path to a schema file describing the table schema.')
parser.add_argument(
'data_file',
help='Path to the data file.')

args = parser.parse_args()

main(
args.project_id,
args.dataset_id,
args.table_name,
args.schema_file,
args.data_file)
Loading