Commit 2694ebbb authored by michitaro's avatar michitaro
Browse files

legacy archive対応

parent b15cde97
......@@ -21,7 +21,7 @@ def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--user', '-u', required=True,
help='specify your STARS account')
parser.add_argument('--release-version', '-r', choices='dr3 dr2 dr1 dr_early'.split(), required=True,
parser.add_argument('--release-version', '-r', choices='hscla'.split(), default='hscla',
help='specify release version')
parser.add_argument('--delete-job', '-D', action='store_true',
help='delete the job you submitted after your downloading')
......@@ -29,13 +29,15 @@ def main():
help='specify output format')
parser.add_argument('--nomail', '-M', action='store_true',
help='suppress email notice')
parser.add_argument('--password-env', default='HSC_SSP_CAS_PASSWORD',
help='specify the environment variable that has STARS password as its content')
parser.add_argument('--password-env', default='HSC_LA_PASSWORD',
help='specify the environment variable which contains the password')
parser.add_argument('--preview', '-p', action='store_true',
help='quick mode (short timeout)')
parser.add_argument('--skip-syntax-check', '-S', action='store_true',
help='skip syntax check')
parser.add_argument('--api-url', default='https://hscdata.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
parser.add_argument('--api-url', default='https://hscla.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
help='for developers')
parser.add_argument('--login-url', default='https://hscla.mtk.nao.ac.jp/account/api/session',
help='for developers')
parser.add_argument('sql-file', type=argparse.FileType('r'),
help='SQL file')
......@@ -44,6 +46,8 @@ def main():
args = parser.parse_args()
credential = {'account_name': args.user, 'password': getPassword()}
httpJsonPost.credential = credential
httpJsonPost.login_url = args.login_url
sql = args.__dict__['sql-file'].read()
job = None
......@@ -80,16 +84,43 @@ class QueryError(Exception):
pass
def httpJsonPost(url, data):
data['clientVersion'] = version
postData = json.dumps(data)
return httpPost(url, postData, {'Content-type': 'application/json'})
def httpPost(url, postData, headers):
req = urllib2.Request(url, postData, headers)
res = urllib2.urlopen(req)
return res
class HttpJsonPost:
def __init__(self):
self.credential = None
self._session = None
self.login_url = None
def _httpPost(self, url, postData, headers):
req = urllib2.Request(url, postData, headers)
res = urllib2.urlopen(req)
return res
def __call__(self, url, data):
assert (self.login_url is None) == (self.credential is None)
data['clientVersion'] = version
headers = {'Content-type': 'application/json'}
if self.credential:
import Cookie
if self._session is None and self.login_url:
try:
res = self._httpPost(
self.login_url,
json.dumps({
'email': self.credential['account_name'],
'password': self.credential['password']
}), headers)
except urllib2.HTTPError as e:
if e.code == 422:
print >> sys.stderr, 'invalid id or password.'
exit(1)
raise
self._session = Cookie.SimpleCookie(res.headers['Set-Cookie'])['LAAUTH_SESSION'].value
headers['Cookie'] = 'LAAUTH_SESSION=' + self._session
postData = json.dumps(data)
return self._httpPost(url, postData, headers)
httpJsonPost = HttpJsonPost()
def submitJob(credential, sql, out_format):
......
import json
import argparse
import urllib.request, urllib.error, urllib.parse
import time
import sys
import csv
import getpass
import http.cookies
import json
import os
import os.path
import re
import ssl
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
from typing import Dict, Optional
version = 20190924.1
......@@ -20,8 +21,8 @@ args = None
def main():
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--user', '-u', required=True,
help='specify your STARS account')
parser.add_argument('--release-version', '-r', choices='dr3 dr3-citus dr2 dr1 dr_early'.split(), required=True,
help='specify your account')
parser.add_argument('--release-version', '-r', choices='hscla'.split(), default='hscla',
help='specify release version')
parser.add_argument('--delete-job', '-D', action='store_true',
help='delete the job you submitted after your downloading')
......@@ -29,13 +30,15 @@ def main():
help='specify output format')
parser.add_argument('--nomail', '-M', action='store_true',
help='suppress email notice')
parser.add_argument('--password-env', default='HSC_SSP_CAS_PASSWORD',
help='specify the environment variable that has STARS password as its content')
parser.add_argument('--password-env', default='HSC_LA_PASSWORD',
help='specify the environment variable which contains the password')
parser.add_argument('--preview', '-p', action='store_true',
help='quick mode (short timeout)')
parser.add_argument('--skip-syntax-check', '-S', action='store_true',
help='skip syntax check')
parser.add_argument('--api-url', default='https://hscdata.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
parser.add_argument('--api-url', default='https://hscla.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
help='for developers')
parser.add_argument('--login-url', default='https://hscla.mtk.nao.ac.jp/account/api/session',
help='for developers')
parser.add_argument('sql-file', type=argparse.FileType('r'),
help='SQL file')
......@@ -44,13 +47,16 @@ def main():
args = parser.parse_args()
credential = {'account_name': args.user, 'password': getPassword()}
httpJsonPost.credential = credential
httpJsonPost.login_url = args.login_url
sql = args.__dict__['sql-file'].read()
job = None
try:
if args.preview:
preview(credential, sql, sys.stdout.buffer)
preview(credential, sql, sys.stdout)
else:
job = submitJob(credential, sql, args.out_format)
blockUntilJobFinishes(credential, job['id'])
......@@ -80,27 +86,54 @@ class QueryError(Exception):
pass
def httpJsonPost(url, data):
data['clientVersion'] = version
postData = json.dumps(data)
return httpPost(url, postData, {'Content-type': 'application/json'})
def httpPost(url, postData, headers):
req = urllib.request.Request(url, postData.encode('utf-8'), headers)
res = urllib.request.urlopen(req)
return res
class HttpJsonPost:
def __init__(self):
self.credential: Optional[Dict] = None
self._session: Optional[str] = None
self.login_url: Optional[str] = None
def _httpPost(self, url, postData, headers):
req = urllib.request.Request(url, postData.encode('utf-8'), headers)
res = urllib.request.urlopen(req)
return res
def __call__(self, url, data):
assert (self.login_url is None) == (self.credential is None)
data['clientVersion'] = version
headers = {'Content-type': 'application/json'}
if self.credential:
if self._session is None and self.login_url:
try:
res = self._httpPost(
self.login_url,
json.dumps({
'email': self.credential['account_name'],
'password': self.credential['password']
}), headers)
except urllib.error.HTTPError as e:
if e.code == 422:
print('invalid id or password.', file=sys.stderr)
exit(1)
raise
self._session = http.cookies.SimpleCookie(res.headers['Set-Cookie'])['LAAUTH_SESSION'].value
headers['Cookie'] = f'LAAUTH_SESSION={self._session}'
postData = json.dumps(data)
return self._httpPost(url, postData, headers)
httpJsonPost = HttpJsonPost()
def submitJob(credential, sql, out_format):
url = args.api_url + 'submit'
catalog_job = {
'sql' : sql,
'out_format' : out_format,
'sql': sql,
'out_format': out_format,
'include_metainfo_to_body': True,
'release_version' : args.release_version,
'release_version': args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job, 'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
postData = {'credential': credential, 'catalog_job': catalog_job,
'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
res = httpJsonPost(url, postData)
job = json.load(res)
return job
......@@ -123,8 +156,8 @@ def jobCancel(credential, job_id):
def preview(credential, sql, out):
url = args.api_url + 'preview'
catalog_job = {
'sql' : sql,
'release_version' : args.release_version,
'sql': sql,
'release_version': args.release_version,
}
postData = {'credential': credential, 'catalog_job': catalog_job}
res = httpJsonPost(url, postData)
......@@ -140,7 +173,7 @@ def preview(credential, sql, out):
def blockUntilJobFinishes(credential, job_id):
max_interval = 5 * 60 # sec.
max_interval = 5 * 60 # sec.
interval = 1
while True:
time.sleep(interval)
......@@ -158,7 +191,7 @@ def download(credential, job_id, out):
url = args.api_url + 'download'
postData = {'credential': credential, 'id': job_id}
res = httpJsonPost(url, postData)
bufSize = 64 * 1<<10 # 64k
bufSize = 64 * 1 << 10 # 64k
while True:
buf = res.read(bufSize)
out.write(buf)
......
/coords.txt
/png2
/png3
\ No newline at end of file
# Postage Stamps in Color
If you want postage-stamps of your objects in color, you can upload an object list with this tool.
## Basic Usage
```sh
cat > coords.txt <<EOT
# ra dec outfile(optional)
4.62630 16.44671 a.png
4.63742 16.44747 b.png
4.62537 16.44345 c.png
4.65146 16.44694 d.png
4.64854 16.44695 e.png
4.65057 16.44706 f.png
4.62701 16.44535 g.png
4.65151 16.44745 h.png
4.63186 16.44911 i.png
4.63830 16.44271 j.png
EOT
python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
```
## Advanced Usage
```
usage: colorPostage.py [-h] --outDir OUTDIR --user USER [--filters FILTERS FILTERS FILTERS] [--fov FOV] [--rerun {la2014}] [--color {hsc,sdss}] input
positional arguments:
input
optional arguments:
-h, --help show this help message and exit
--outDir OUTDIR, -o OUTDIR
--user USER, -u USER
--filters FILTERS FILTERS FILTERS, -f FILTERS FILTERS FILTERS
--fov FOV
--rerun {la2014}
--color {hsc,sdss}
```
\ No newline at end of file
#!/bin/env python3
'''
> head coords.txt
# ra dec outfile(optional)
4.62630 16.44671 a.png
4.63742 16.44747 b.png
4.62537 16.44345 c.png
4.65146 16.44694 d.png
4.64854 16.44695 e.png
4.65057 16.44706 f.png
4.62701 16.44535 g.png
4.65151 16.44745 h.png
4.63186 16.44911 i.png
4.63830 16.44271 j.png
> python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
'''
import PIL.Image
import numpy
import argparse
import tarfile
import subprocess
import tempfile
import getpass
import os
import os.path
import contextlib
import logging
logging.basicConfig(level=logging.INFO)
try:
import pyfits
except:
import astropy.io.fits as pyfits
FLUXMAG0 = 10 ** (27 / 2.5)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--outDir', '-o', required=True)
parser.add_argument('--user', '-u', required=True)
parser.add_argument('--filters', '-f',
nargs=3, default=['HSC-I', 'HSC-R', 'HSC-G'])
parser.add_argument('--fov', default='30asec')
parser.add_argument('--rerun', default='any',
choices='la2014'.split())
parser.add_argument('--color', choices='hsc sdss'.split(), default='hsc')
parser.add_argument('input', type=argparse.FileType('r'))
args = parser.parse_args()
password = getpass.getpass('Password: ')
checkPassword(args.user, password)
coords, outs = loadCoords(args.input)
mkdir_p(args.outDir)
batchSize = 300
for batchI, batchCoords in enumerate(batch(coords, batchSize)):
with requestFileFor(batchCoords, args.filters, args.fov, args.rerun) as requestFile:
tarMembers = queryTar(args.user, password, requestFile)
for i, rgb in rgbBundle(tarMembers):
j = batchI * batchSize + i
outFile = os.path.join(args.outDir, outs[j])
logging.info('-> {}'.format(outFile))
makeColorPng(rgb, outFile, args.color)
TOP_PAGE = 'https://hscla.mtk.nao.ac.jp/das_cutout/la2014/'
API = 'https://hscla.mtk.nao.ac.jp/das_cutout/la2014/cgi-bin/cutout'
def loadCoords(input):
import re
comment = re.compile('\s*(?:$|#)')
num = 1
coords = []
outs = []
for line in input:
if comment.match(line):
continue
cols = line.split()
if len(cols) == 2:
ra, dec = cols
out = '{}.png'.format(num)
else:
ra, dec, out = cols
ra = float(ra)
dec = float(dec)
num += 1
coords.append([ra, dec])
outs.append(out)
return coords, outs
@contextlib.contextmanager
def requestFileFor(coords, filters, fov, rerun):
with tempfile.NamedTemporaryFile() as tmp:
tmp.write('#? filter ra dec sw sh rerun\n'.encode('utf-8'))
for coord in coords:
for filterName in filters:
tmp.write('{} {} {} {} {} {}\n'.format(
filterName, coord[0], coord[1], fov, fov, rerun).encode())
tmp.flush()
yield tmp.name
def batch(arr, n):
i = 0
while i < len(arr):
yield arr[i: i + n]
i += n
def rgbBundle(files):
mktmp = tempfile.NamedTemporaryFile
with mktmp() as r, mktmp() as g, mktmp() as b:
lastObjNum = 0
rgb = {}
for info, fileObj in files:
resNum = int(os.path.basename(info.name).split('-')[0])
objNum = (resNum - 2) // 3
if lastObjNum != objNum:
yield lastObjNum, rgb
rgb.clear()
lastObjNum = objNum
ch = 'gbr'[resNum % 3]
dst = locals()[ch]
copyFileObj(fileObj, dst)
rgb[ch] = dst.name
yield objNum, rgb
def copyFileObj(src, dst):
dst.seek(0)
dst.write(src.read())
dst.truncate()
def checkPassword(user, password):
with tempfile.NamedTemporaryFile() as netrc:
netrc.write('machine hscla.mtk.nao.ac.jp login {} password {}'.format(
user, password).encode('ascii'))
netrc.flush()
httpCode = subprocess.check_output(
['curl', '--netrc-file', netrc.name, '-o', os.devnull, '-w', '%{http_code}', '-s', TOP_PAGE]).strip()
if httpCode != b'200':
raise RuntimeError('Account or Password is not correct')
def queryTar(user, password, requestFile):
with tempfile.NamedTemporaryFile() as netrc:
netrc.write(
'machine hscla.mtk.nao.ac.jp login {} password {}'.format(user, password).encode('ascii'))
netrc.flush()
pipe = subprocess.Popen([
'curl', '--netrc-file', netrc.name,
'-L',
# '--dump-header', '-',
'--form', 'list=@{}'.format(requestFile),
'--silent',
API,
], stdout=subprocess.PIPE)
with tarfile.open(fileobj=pipe.stdout, mode='r|*') as tar:
while True:
info = tar.next()
if info is None:
break
logging.info('extracting {}...'.format(info.name))
f = tar.extractfile(info)
yield info, f
f.close()
pipe.wait()
def makeColorPng(rgb, out, color):
if len(rgb) == 0:
return
with pyfits.open(list(rgb.values())[0]) as hdul:
template = hdul[1].data
layers = [numpy.zeros_like(template) for i in range(3)]
for i, ch in enumerate('rgb'):
if ch in rgb:
with pyfits.open(rgb[ch]) as hdul:
x = scale(hdul[1].data, FLUXMAG0)
layers[i] = x
if color == 'hsc':
layers = hscColor(layers)
elif color == 'sdss':
layers = sdssColor(layers)
else:
assert False
layers = numpy.array(layers)
layers[layers < 0] = 0
layers[layers > 1] = 1
layers = layers.transpose((1, 2, 0))[::-1, :, :]
layers = numpy.array(255 * layers, dtype=numpy.uint8)
img = PIL.Image.fromarray(layers)
img.save(out)
def scale(x, fluxMag0):
mag0 = 19
scale = 10 ** (0.4 * mag0) / fluxMag0
x *= scale
return x
def hscColor(rgb):
u_min = -0.05
u_max = 2. / 3.
u_a = numpy.exp(10.)
for i, x in enumerate(rgb):
x = numpy.arcsinh(u_a*x) / numpy.arcsinh(u_a)
x = (x - u_min) / (u_max - u_min)
rgb[i] = x
return rgb
def sdssColor(rgb):
u_a = numpy.exp(10.)
u_b = 0.05
r, g, b = rgb
I = (r + g + b) / 3.
for i, x in enumerate(rgb):
x = numpy.arcsinh(u_a * I) / numpy.arcsinh(u_a) / I * x
x += u_b
rgb[i] = x
return rgb
def mkdir_p(d):
try:
os.makedirs(d)
except:
pass
if __name__ == '__main__':
main()
cat > coords.txt <<EOT
# ra dec outfile(optional)
4.62630 16.44671 a.png
4.63742 16.44747 b.png
4.62537 16.44345 c.png
4.65146 16.44694 d.png
4.64854 16.44695 e.png
4.65057 16.44706 f.png
4.62701 16.44535 g.png
4.65151 16.44745 h.png
4.63186 16.44911 i.png
4.63830 16.44271 j.png
EOT
~/anaconda3/bin/python colorPostage.py --user michitaro.nike@gmail.com --outDir png3 ./coords.txt
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment