Commit 7944fb8a authored by michitaro's avatar michitaro
Browse files

add dr4 support

parent db853c64
import json
import argparse
import urllib2
import time
import sys
import csv
import getpass
import os
import os.path
import re
import ssl
version = 20190924.1
args = None
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--user', '-u', required=True,
help='specify your STARS account')
parser.add_argument('--release-version', '-r', choices='dr4 dr3 dr2 dr1 dr_early'.split(), required=True,
help='specify release version')
parser.add_argument('--delete-job', '-D', action='store_true',
help='delete the job you submitted after your downloading')
parser.add_argument('--format', '-f', dest='out_format', default='csv', choices=['csv', 'csv.gz', 'sqlite3', 'fits', 'numpygres-fits'],
help='specify output format')
parser.add_argument('--nomail', '-M', action='store_true',
help='suppress email notice')
parser.add_argument('--password-env', default='HSC_SSP_CAS_PASSWORD',
help='specify the environment variable that has STARS password as its content')
parser.add_argument('--preview', '-p', action='store_true',
help='quick mode (short timeout)')
parser.add_argument('--skip-syntax-check', '-S', action='store_true',
help='skip syntax check')
parser.add_argument('--api-url', default='https://hscdata.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
help='for developers')
parser.add_argument('sql-file', type=argparse.FileType('r'),
help='SQL file')
global args
args = parser.parse_args()
credential = {'account_name': args.user, 'password': getPassword()}
sql = args.__dict__['sql-file'].read()
job = None
try:
if args.preview:
preview(credential, sql, sys.stdout)
else:
job = submitJob(credential, sql, args.out_format)
blockUntilJobFinishes(credential, job['id'])
download(credential, job['id'], sys.stdout)
if args.delete_job:
deleteJob(credential, job['id'])
except urllib2.HTTPError, e:
if e.code == 401:
print >> sys.stderr, 'invalid id or password.'
if e.code == 406:
print >> sys.stderr, e.read()
else:
print >> sys.stderr, e
except QueryError, e:
print >> sys.stderr, e
except KeyboardInterrupt:
if job is not None:
jobCancel(credential, job['id'])
raise
else:
sys.exit(0)
sys.exit(1)
class QueryError(Exception):
pass
def httpJsonPost(url, data):
data['clientVersion'] = version
postData = json.dumps(data)
return httpPost(url, postData, {'Content-type': 'application/json'})
def httpPost(url, postData, headers):
req = urllib2.Request(url, postData, headers)
res = urllib2.urlopen(req)
return res
def submitJob(credential, sql, out_format):
url = args.api_url + 'submit'
catalog_job = {
'sql' : sql,
'out_format' : out_format,
'include_metainfo_to_body': True,
'release_version' : args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job, 'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobStatus(credential, job_id):
url = args.api_url + 'status'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobCancel(credential, job_id):
url = args.api_url + 'cancel'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def preview(credential, sql, out):
url = args.api_url + 'preview'
catalog_job = {
'sql' : sql,
'release_version' : args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job}
res = httpJsonPost(url, postData)
result = json.load(res)
writer = csv.writer(out)
# writer.writerow(result['result']['fields'])
for row in result['result']['rows']:
writer.writerow(row)
if result['result']['count'] > len(result['result']['rows']):
raise QueryError, 'only top %d records are displayed !' % len(result['result']['rows'])
def blockUntilJobFinishes(credential, job_id):
max_interval = 5 * 60 # sec.
interval = 1
while True:
time.sleep(interval)
job = jobStatus(credential, job_id)
if job['status'] == 'error':
raise QueryError, 'query error: ' + job['error']
if job['status'] == 'done':
break
interval *= 2
if interval > max_interval:
interval = max_interval
def download(credential, job_id, out):
url = args.api_url + 'download'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
bufSize = 64 * 1<<10 # 64k
while True:
buf = res.read(bufSize)
out.write(buf)
if len(buf) < bufSize:
break
def deleteJob(credential, job_id):
url = args.api_url + 'delete'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def getPassword():
password_from_envvar = os.environ.get(args.password_env, '')
if password_from_envvar != '':
return password_from_envvar
else:
return getpass.getpass('password? ')
if __name__ == '__main__':
main()
import json
import argparse
import urllib.request, urllib.error, urllib.parse
import time
import sys
import csv
import getpass
import os
import os.path
import re
import ssl
version = 20190924.1
args = None
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--user', '-u', required=True,
help='specify your STARS account')
parser.add_argument('--release-version', '-r', choices='dr4 dr3 dr3-citus dr2 dr1 dr_early'.split(), required=True,
help='specify release version')
parser.add_argument('--delete-job', '-D', action='store_true',
help='delete the job you submitted after your downloading')
parser.add_argument('--format', '-f', dest='out_format', default='csv', choices=['csv', 'csv.gz', 'sqlite3', 'fits', 'numpygres-fits', 'fast-fits'],
help='specify output format')
parser.add_argument('--nomail', '-M', action='store_true',
help='suppress email notice')
parser.add_argument('--password-env', default='HSC_SSP_CAS_PASSWORD',
help='specify the environment variable that has STARS password as its content')
parser.add_argument('--preview', '-p', action='store_true',
help='quick mode (short timeout)')
parser.add_argument('--skip-syntax-check', '-S', action='store_true',
help='skip syntax check')
parser.add_argument('--api-url', default='https://hscdata.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
help='for developers')
parser.add_argument('sql-file', type=argparse.FileType('r'),
help='SQL file')
global args
args = parser.parse_args()
credential = {'account_name': args.user, 'password': getPassword()}
sql = args.__dict__['sql-file'].read()
job = None
try:
if args.preview:
preview(credential, sql, sys.stdout.buffer)
else:
job = submitJob(credential, sql, args.out_format)
blockUntilJobFinishes(credential, job['id'])
download(credential, job['id'], sys.stdout.buffer)
if args.delete_job:
deleteJob(credential, job['id'])
except urllib.error.HTTPError as e:
if e.code == 401:
print('invalid id or password.', file=sys.stderr)
if e.code == 406:
print(e.read(), file=sys.stderr)
else:
print(e, file=sys.stderr)
except QueryError as e:
print(e, file=sys.stderr)
except KeyboardInterrupt:
if job is not None:
jobCancel(credential, job['id'])
raise
else:
sys.exit(0)
sys.exit(1)
class QueryError(Exception):
pass
def httpJsonPost(url, data):
data['clientVersion'] = version
postData = json.dumps(data)
return httpPost(url, postData, {'Content-type': 'application/json'})
def httpPost(url, postData, headers):
req = urllib.request.Request(url, postData.encode('utf-8'), headers)
res = urllib.request.urlopen(req)
return res
def submitJob(credential, sql, out_format):
url = args.api_url + 'submit'
catalog_job = {
'sql' : sql,
'out_format' : out_format,
'include_metainfo_to_body': True,
'release_version' : args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job, 'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobStatus(credential, job_id):
url = args.api_url + 'status'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
def jobCancel(credential, job_id):
url = args.api_url + 'cancel'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def preview(credential, sql, out):
url = args.api_url + 'preview'
catalog_job = {
'sql' : sql,
'release_version' : args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job}
res = httpJsonPost(url, postData)
result = json.load(res)
writer = csv.writer(out)
# writer.writerow(result['result']['fields'])
for row in result['result']['rows']:
writer.writerow(row)
if result['result']['count'] > len(result['result']['rows']):
raise QueryError('only top %d records are displayed !' % len(result['result']['rows']))
def blockUntilJobFinishes(credential, job_id):
max_interval = 5 * 60 # sec.
interval = 1
while True:
time.sleep(interval)
job = jobStatus(credential, job_id)
if job['status'] == 'error':
raise QueryError('query error: ' + job['error'])
if job['status'] == 'done':
break
interval *= 2
if interval > max_interval:
interval = max_interval
def download(credential, job_id, out):
url = args.api_url + 'download'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
bufSize = 64 * 1<<10 # 64k
while True:
buf = res.read(bufSize)
out.write(buf)
if len(buf) < bufSize:
break
def deleteJob(credential, job_id):
url = args.api_url + 'delete'
postData = {'credential': credential, 'id': job_id}
httpJsonPost(url, postData)
def getPassword():
password_from_envvar = os.environ.get(args.password_env, '')
if password_from_envvar != '':
return password_from_envvar
else:
return getpass.getpass('password? ')
if __name__ == '__main__':
main()
/coords.txt
/png2
/png3
\ No newline at end of file
# Postage Stamps in Color
If you want postage-stamps of your objects in color, you can upload an object list with this tool.
## Basic Usage
```sh
cat > coords.txt <<EOT
# ra dec outfile(optional)
33.995270 -5.011639 a.png
33.994442 -4.996707 b.png
33.994669 -5.001553 c.png
33.996395 -5.008107 d.png
33.995679 -4.993945 e.png
33.997352 -5.010902 f.png
33.997315 -5.012523 g.png
33.997438 -5.011647 h.png
33.997379 -5.010878 i.png
33.996636 -5.008742 j.png
EOT
python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
```
## Advanced Usage
```
usage: colorPostage.py [-h] --outDir OUTDIR --user USER
[--filters FILTERS FILTERS FILTERS] [--fov FOV]
[--rerun {any,pdr2_dud,pdr2_wide}] [--color {hsc,sdss}]
input
positional arguments:
input
optional arguments:
-h, --help show this help message and exit
--outDir OUTDIR, -o OUTDIR
--user USER, -u USER
--filters FILTERS FILTERS FILTERS, -f FILTERS FILTERS FILTERS
--fov FOV
--rerun {any,pdr2_dud,pdr2_wide}
--color {hsc,sdss}
```
\ No newline at end of file
#!/bin/env python3
'''
> head coords.txt
# ra dec outfile(optional)
33.995270 -5.011639 a.png
33.994442 -4.996707 b.png
33.994669 -5.001553 c.png
33.996395 -5.008107 d.png
33.995679 -4.993945 e.png
33.997352 -5.010902 f.png
33.997315 -5.012523 g.png
33.997438 -5.011647 h.png
33.997379 -5.010878 i.png
33.996636 -5.008742 j.png
> python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
'''
import PIL.Image
import numpy
import argparse
import tarfile
import subprocess
import tempfile
import getpass
import os
import os.path
import contextlib
import logging
logging.basicConfig(level=logging.INFO)
try:
import pyfits
except:
import astropy.io.fits as pyfits
FLUXMAG0 = 10 ** (27 / 2.5)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--outDir', '-o', required=True)
parser.add_argument('--user', '-u', required=True)
parser.add_argument('--filters', '-f',
nargs=3, default=['HSC-I', 'HSC-R', 'HSC-G'])
parser.add_argument('--fov', default='30asec')
parser.add_argument('--rerun', default='any',
choices='any s21a_dud s21a_wide s20a_dud s20a_wide s19a_wide s19a_dud'.split())
parser.add_argument('--color', choices='hsc sdss'.split(), default='hsc')
parser.add_argument('input', type=argparse.FileType('r'))
args = parser.parse_args()
password = getpass.getpass('Password for internal-release: ')
checkPassword(args.user, password)
coords, outs = loadCoords(args.input)
mkdir_p(args.outDir)
batchSize = 300
for batchI, batchCoords in enumerate(batch(coords, batchSize)):
with requestFileFor(batchCoords, args.filters, args.fov, args.rerun) as requestFile:
tarMembers = queryTar(args.user, password, requestFile)
for i, rgb in rgbBundle(tarMembers):
j = batchI * batchSize + i
outFile = os.path.join(args.outDir, outs[j])
logging.info('-> {}'.format(outFile))
makeColorPng(rgb, outFile, args.color)
TOP_PAGE = 'https://hscdata.mtk.nao.ac.jp/das_quarry/dr4/'
API = 'https://hscdata.mtk.nao.ac.jp/das_quarry/dr4/cgi-bin/cutout'
def loadCoords(input):
import re
comment = re.compile('\s*(?:$|#)')
num = 1
coords = []
outs = []
for line in input:
if comment.match(line):
continue
cols = line.split()
if len(cols) == 2:
ra, dec = cols
out = '{}.png'.format(num)
else:
ra, dec, out = cols
ra = float(ra)
dec = float(dec)
num += 1
coords.append([ra, dec])
outs.append(out)
return coords, outs
@contextlib.contextmanager
def requestFileFor(coords, filters, fov, rerun):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write('#? filter ra dec sw sh rerun\n'.encode('utf-8'))
for coord in coords:
for filterName in filters:
tmp.write('{} {} {} {} {} {}\n'.format(
filterName, coord[0], coord[1], fov, fov, rerun).encode())
tmp.flush()
yield tmp.name
def batch(arr, n):
i = 0
while i < len(arr):
yield arr[i: i + n]
i += n
def rgbBundle(files):
mktmp = tempfile.NamedTemporaryFile
with mktmp() as r, mktmp() as g, mktmp() as b:
lastObjNum = 0
rgb = {}
for info, fileObj in files:
resNum = int(os.path.basename(info.name).split('-')[0])
objNum = (resNum - 2) // 3
if lastObjNum != objNum:
yield lastObjNum, rgb
rgb.clear()
lastObjNum = objNum
ch = 'gbr'[resNum % 3]
dst = locals()[ch]
copyFileObj(fileObj, dst)
rgb[ch] = dst.name
yield objNum, rgb
def copyFileObj(src, dst):
dst.seek(0)
dst.write(src.read())
dst.truncate()
def checkPassword(user, password):
with tempfile.NamedTemporaryFile() as netrc:
netrc.write('machine hscdata.mtk.nao.ac.jp login {} password {}'.format(
user, password).encode('ascii'))
netrc.flush()
httpCode = subprocess.check_output(
['curl', '--netrc-file', netrc.name, '-o', os.devnull, '-w', '%{http_code}', '-s', TOP_PAGE]).strip()
if httpCode == b'401':
raise RuntimeError('Account or Password is not correct')
def queryTar(user, password, requestFile):
with tempfile.NamedTemporaryFile() as netrc:
netrc.write('machine hscdata.mtk.nao.ac.jp login {} password {}'.format(
user, password).encode('ascii'))
netrc.flush()
pipe = subprocess.Popen([
'curl', '--netrc-file', netrc.name,
'--form', 'list=@{}'.format(requestFile),
'--silent',
API,
], stdout=subprocess.PIPE)
with tarfile.open(fileobj=pipe.stdout, mode='r|*') as tar:
while True:
info = tar.next()
if info is None:
break
logging.info('extracting {}...'.format(info.name))
f = tar.extractfile(info)
yield info, f
f.close()
pipe.wait()
def makeColorPng(rgb, out, color):
if len(rgb) == 0:
return
with pyfits.open(list(rgb.values())[0]) as hdul:
template = hdul[1].data
layers = [numpy.zeros_like(template) for i in range(3)]