Commit fc9b60b9 authored by michitaro's avatar michitaro
Browse files

la2016対応

parent 0c49f85a
# Command line interface to SQL Database
This tool allows you to send a query and retrieve data from your terminal. This will be useful for sequential data retrieval.
## Basic Usage
```sh
echo "SELECT now();" > 1.sql
python hscReleaseQuery.py --user $YOUR_ID 1.sql --format csv > 1.csv
```
## Advanced Usage
```
usage: hscSspQuery.py [-h] --user USER [--release-version {hscla}]
[--delete-job]
[--format {csv,csv.gz,sqlite3,fits,numpygres-fits,fast-fits}]
[--nomail] [--password-env PASSWORD_ENV] [--preview]
[--skip-syntax-check] [--api-url API_URL]
[--login-url LOGIN_URL]
sql-file
positional arguments:
sql-file SQL file
optional arguments:
-h, --help show this help message and exit
--user USER, -u USER specify your account (default: None)
--release-version {hscla}, -r {hscla}
specify release version (default: hscla)
--delete-job, -D delete the job you submitted after your downloading
(default: False)
--format {csv,csv.gz,sqlite3,fits,numpygres-fits,fast-fits}, -f {csv,csv.gz,sqlite3,fits,numpygres-fits,fast-fits}
specify output format (default: csv)
--nomail, -M suppress email notice (default: False)
--password-env PASSWORD_ENV
specify the environment variable which contains the
password (default: HSC_LA_PASSWORD)
--preview, -p quick mode (short timeout) (default: False)
--skip-syntax-check, -S
skip syntax check (default: False)
--api-url API_URL for developers (default: https://hscla.mtk.nao.ac.jp/d
atasearch/api/catalog_jobs/)
--login-url LOGIN_URL
for developers (default:
https://hscla.mtk.nao.ac.jp/account/api/session)
```
\ No newline at end of file
import argparse
import csv
import getpass
import http.cookies
import json
import os
import os.path
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Dict, Optional
version = 20190924.1
args = None
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--user', '-u', required=True,
help='specify your account')
parser.add_argument('--release-version', '-r', choices='hscla2014 hscla2016'.split(), default='hscla2016',
help='specify release version')
parser.add_argument('--delete-job', '-D', action='store_true',
help='delete the job you submitted after your downloading')
parser.add_argument('--format', '-f', dest='out_format', default='csv', choices=['csv', 'csv.gz', 'sqlite3', 'fits', 'numpygres-fits', 'fast-fits'],
help='specify output format')
parser.add_argument('--nomail', '-M', action='store_true',
help='suppress email notice')
parser.add_argument('--password-env', default='HSC_LA_PASSWORD',
help='specify the environment variable which contains the password')
parser.add_argument('--preview', '-p', action='store_true',
help='quick mode (short timeout)')
parser.add_argument('--skip-syntax-check', '-S', action='store_true',
help='skip syntax check')
parser.add_argument('--api-url', default='https://hscla.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
help='for developers')
parser.add_argument('--login-url', default='https://hscla.mtk.nao.ac.jp/account/api/session',
help='for developers')
parser.add_argument('sql-file', type=argparse.FileType('r'),
help='SQL file')
global args
args = parser.parse_args()
credential = {'account_name': args.user, 'password': getPassword()}
httpJsonPost.credential = credential
httpJsonPost.login_url = args.login_url
sql = args.__dict__['sql-file'].read()
job = None
try:
if args.preview:
preview(credential, sql, sys.stdout)
else:
job = submitJob(credential, sql, args.out_format)
blockUntilJobFinishes(credential, job['id'])
download(credential, job['id'], sys.stdout.buffer)
if args.delete_job:
deleteJob(credential, job['id'])
except urllib.error.HTTPError as e:
if e.code == 401:
print('invalid id or password.', file=sys.stderr)
if e.code == 406:
print(e.read(), file=sys.stderr)
else:
print(e, file=sys.stderr)
except QueryError as e:
print(e, file=sys.stderr)
except KeyboardInterrupt:
if job is not None:
jobCancel(credential, job['id'])
raise
else:
sys.exit(0)
sys.exit(1)
class QueryError(Exception):
pass
class HttpJsonPost:
def __init__(self):
self.credential: Optional[Dict] = None
self._session: Optional[str] = None
self.login_url: Optional[str] = None
def _httpPost(self, url, postData, headers):
req = urllib.request.Request(url, postData.encode('utf-8'), headers)
res = urllib.request.urlopen(req)
return res
def __call__(self, url, data):
assert (self.login_url is None) == (self.credential is None)
data['clientVersion'] = version
headers = {'Content-type': 'application/json'}
if self.credential:
if self._session is None and self.login_url:
try:
res = self._httpPost(
self.login_url,
json.dumps({
'email': self.credential['account_name'],
'password': self.credential['password']
}), headers)
except urllib.error.HTTPError as e:
if e.code == 422:
print('invalid id or password.', file=sys.stderr)
exit(1)
raise
self._session = http.cookies.SimpleCookie(res.headers['Set-Cookie'])['LAAUTH_SESSION'].value
headers['Cookie'] = f'LAAUTH_SESSION={self._session}'
postData = json.dumps(data)
return self._httpPost(url, postData, headers)
httpJsonPost = HttpJsonPost()
def submitJob(credential, sql, out_format):
url = args.api_url + 'submit'
catalog_job = {
'sql': sql,
'out_format': out_format,
'include_metainfo_to_body': True,
'release_version': args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job,
'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobStatus(credential, job_id):
url = args.api_url + 'status'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobCancel(credential, job_id):
url = args.api_url + 'cancel'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def preview(credential, sql, out):
url = args.api_url + 'preview'
catalog_job = {
'sql': sql,
'release_version': args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job}
res = httpJsonPost(url, postData)
result = json.load(res)
writer = csv.writer(out)
# writer.writerow(result['result']['fields'])
for row in result['result']['rows']:
writer.writerow(row)
if result['result']['count'] > len(result['result']['rows']):
raise QueryError('only top %d records are displayed !' % len(result['result']['rows']))
def blockUntilJobFinishes(credential, job_id):
max_interval = 5 * 60 # sec.
interval = 1
while True:
time.sleep(interval)
job = jobStatus(credential, job_id)
if job['status'] == 'error':
raise QueryError('query error: ' + job['error'])
if job['status'] == 'done':
break
interval *= 2
if interval > max_interval:
interval = max_interval
def download(credential, job_id, out):
url = args.api_url + 'download'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
bufSize = 64 * 1 << 10 # 64k
while True:
buf = res.read(bufSize)
out.write(buf)
if len(buf) < bufSize:
break
def deleteJob(credential, job_id):
url = args.api_url + 'delete'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def getPassword():
password_from_envvar = os.environ.get(args.password_env, '')
if password_from_envvar != '':
return password_from_envvar
else:
return getpass.getpass('password? ')
if __name__ == '__main__':
main()
import json
import argparse
import urllib2
import time
import sys
import csv
import getpass
import os
import os.path
version = 20190924.1
args = None
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--user', '-u', required=True,
help='specify your STARS account')
parser.add_argument('--release-version', '-r', choices='hscla2014 hscla2016'.split(), default='hscla2016',
help='specify release version')
parser.add_argument('--delete-job', '-D', action='store_true',
help='delete the job you submitted after your downloading')
parser.add_argument('--format', '-f', dest='out_format', default='csv', choices=['csv', 'csv.gz', 'sqlite3', 'fits', 'numpygres-fits'],
help='specify output format')
parser.add_argument('--nomail', '-M', action='store_true',
help='suppress email notice')
parser.add_argument('--password-env', default='HSC_LA_PASSWORD',
help='specify the environment variable which contains the password')
parser.add_argument('--preview', '-p', action='store_true',
help='quick mode (short timeout)')
parser.add_argument('--skip-syntax-check', '-S', action='store_true',
help='skip syntax check')
parser.add_argument('--api-url', default='https://hscla.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
help='for developers')
parser.add_argument('--login-url', default='https://hscla.mtk.nao.ac.jp/account/api/session',
help='for developers')
parser.add_argument('sql-file', type=argparse.FileType('r'),
help='SQL file')
global args
args = parser.parse_args()
credential = {'account_name': args.user, 'password': getPassword()}
httpJsonPost.credential = credential
httpJsonPost.login_url = args.login_url
sql = args.__dict__['sql-file'].read()
job = None
try:
if args.preview:
preview(credential, sql, sys.stdout)
else:
job = submitJob(credential, sql, args.out_format)
blockUntilJobFinishes(credential, job['id'])
download(credential, job['id'], sys.stdout)
if args.delete_job:
deleteJob(credential, job['id'])
except urllib2.HTTPError, e:
if e.code == 401:
print >> sys.stderr, 'invalid id or password.'
if e.code == 406:
print >> sys.stderr, e.read()
else:
print >> sys.stderr, e
except QueryError, e:
print >> sys.stderr, e
except KeyboardInterrupt:
if job is not None:
jobCancel(credential, job['id'])
raise
else:
sys.exit(0)
sys.exit(1)
class QueryError(Exception):
pass
class HttpJsonPost:
def __init__(self):
self.credential = None
self._session = None
self.login_url = None
def _httpPost(self, url, postData, headers):
req = urllib2.Request(url, postData, headers)
res = urllib2.urlopen(req)
return res
def __call__(self, url, data):
assert (self.login_url is None) == (self.credential is None)
data['clientVersion'] = version
headers = {'Content-type': 'application/json'}
if self.credential:
import Cookie
if self._session is None and self.login_url:
try:
res = self._httpPost(
self.login_url,
json.dumps({
'email': self.credential['account_name'],
'password': self.credential['password']
}), headers)
except urllib2.HTTPError as e:
if e.code == 422:
print >> sys.stderr, 'invalid id or password.'
exit(1)
raise
self._session = Cookie.SimpleCookie(res.headers['Set-Cookie'])['LAAUTH_SESSION'].value
headers['Cookie'] = 'LAAUTH_SESSION=' + self._session
postData = json.dumps(data)
return self._httpPost(url, postData, headers)
httpJsonPost = HttpJsonPost()
def submitJob(credential, sql, out_format):
url = args.api_url + 'submit'
catalog_job = {
'sql' : sql,
'out_format' : out_format,
'include_metainfo_to_body': True,
'release_version' : args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job, 'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobStatus(credential, job_id):
url = args.api_url + 'status'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobCancel(credential, job_id):
url = args.api_url + 'cancel'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def preview(credential, sql, out):
url = args.api_url + 'preview'
catalog_job = {
'sql' : sql,
'release_version' : args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job}
res = httpJsonPost(url, postData)
result = json.load(res)
writer = csv.writer(out)
# writer.writerow(result['result']['fields'])
for row in result['result']['rows']:
writer.writerow(row)
if result['result']['count'] > len(result['result']['rows']):
raise QueryError, 'only top %d records are displayed !' % len(result['result']['rows'])
def blockUntilJobFinishes(credential, job_id):
max_interval = 5 * 60 # sec.
interval = 1
while True:
time.sleep(interval)
job = jobStatus(credential, job_id)
if job['status'] == 'error':
raise QueryError, 'query error: ' + job['error']
if job['status'] == 'done':
break
interval *= 2
if interval > max_interval:
interval = max_interval
def download(credential, job_id, out):
url = args.api_url + 'download'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
bufSize = 64 * 1<<10 # 64k
while True:
buf = res.read(bufSize)
out.write(buf)
if len(buf) < bufSize:
break
def deleteJob(credential, job_id):
url = args.api_url + 'delete'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def getPassword():
password_from_envvar = os.environ.get(args.password_env, '')
if password_from_envvar != '':
return password_from_envvar
else:
return getpass.getpass('password? ')
if __name__ == '__main__':
main()
/coords.txt
/png2
/png3
\ No newline at end of file
# Postage Stamps in Color
If you want postage-stamps of your objects in color, you can upload an object list with this tool.
## Basic Usage
```sh
cat > coords.txt <<EOT
# ra dec outfile(optional)
4.62630 16.44671 a.png
4.63742 16.44747 b.png
4.62537 16.44345 c.png
4.65146 16.44694 d.png
4.64854 16.44695 e.png
4.65057 16.44706 f.png
4.62701 16.44535 g.png
4.65151 16.44745 h.png
4.63186 16.44911 i.png
4.63830 16.44271 j.png
EOT
python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
```
## Advanced Usage
```
usage: colorPostage.py [-h] --outDir OUTDIR --user USER [--filters FILTERS FILTERS FILTERS] [--fov FOV] [--rerun {la2014}] [--color {hsc,sdss}] input
positional arguments:
input
optional arguments:
-h, --help show this help message and exit
--outDir OUTDIR, -o OUTDIR
--user USER, -u USER
--filters FILTERS FILTERS FILTERS, -f FILTERS FILTERS FILTERS
--fov FOV
--rerun {la2014}
--color {hsc,sdss}
```
\ No newline at end of file
#!/bin/env python3
'''
> head coords.txt
# ra dec outfile(optional)
4.62630 16.44671 a.png
4.63742 16.44747 b.png
4.62537 16.44345 c.png
4.65146 16.44694 d.png
4.64854 16.44695 e.png
4.65057 16.44706 f.png
4.62701 16.44535 g.png
4.65151 16.44745 h.png
4.63186 16.44911 i.png
4.63830 16.44271 j.png
> python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
'''
import PIL.Image
import numpy
import argparse
import tarfile
import subprocess
import tempfile
import getpass
import os
import os.path
import contextlib
import logging
logging.basicConfig(level=logging.INFO)
try:
import pyfits
except:
import astropy.io.fits as pyfits
FLUXMAG0 = 10 ** (27 / 2.5)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--outDir', '-o', required=True)
parser.add_argument('--user', '-u', required=True)
parser.add_argument('--filters', '-f',
nargs=3, default=['HSC-I', 'HSC-R', 'HSC-G'])
parser.add_argument('--fov', default='30asec')
parser.add_argument('--rerun', default='any',
choices='la2016 la2014'.split())
parser.add_argument('--color', choices='hsc sdss'.split(), default='hsc')
parser.add_argument('input', type=argparse.FileType('r'))
args = parser.parse_args()
password = getpass.getpass('Password: ')
checkPassword(args.user, password)
coords, outs = loadCoords(args.input)
mkdir_p(args.outDir)
batchSize = 300
for batchI, batchCoords in enumerate(batch(coords, batchSize)):
with requestFileFor(batchCoords, args.filters, args.fov, args.rerun) as requestFile:
tarMembers = queryTar(args.user, password, requestFile)
for i, rgb in rgbBundle(tarMembers):
j = batchI * batchSize + i
outFile = os.path.join(args.outDir, outs[j])
logging.info('-> {}'.format(outFile))
makeColorPng(rgb, outFile, args.color)
TOP_PAGE = 'https://hscla.mtk.nao.ac.jp/das_cutout/la2014/'
API = 'https://hscla.mtk.nao.ac.jp/das_cutout/la2014/cgi-bin/cutout'
def loadCoords(input):
import re
comment = re.compile('\s*(?:$|#)')
num = 1