la2016対応

la2016/catalogQuery/README.md

+# Command line interface to SQL Database
+This tool allows you to send a query and retrieve data from your terminal.  This will be useful for sequential data retrieval.
+
+## Basic Usage
+
+```sh
+echo "SELECT now();" > 1.sql
+python hscReleaseQuery.py --user $YOUR_ID 1.sql --format csv > 1.csv
+```
+
+## Advanced Usage
+```
+usage: hscSspQuery.py [-h] --user USER [--release-version {hscla}]
+                       [--delete-job]
+                       [--format {csv,csv.gz,sqlite3,fits,numpygres-fits,fast-fits}]
+                       [--nomail] [--password-env PASSWORD_ENV] [--preview]
+                       [--skip-syntax-check] [--api-url API_URL]
+                       [--login-url LOGIN_URL]
+                       sql-file
+
+positional arguments:
+  sql-file              SQL file
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --user USER, -u USER  specify your account (default: None)
+  --release-version {hscla}, -r {hscla}
+                        specify release version (default: hscla)
+  --delete-job, -D      delete the job you submitted after your downloading
+                        (default: False)
+  --format {csv,csv.gz,sqlite3,fits,numpygres-fits,fast-fits}, -f {csv,csv.gz,sqlite3,fits,numpygres-fits,fast-fits}
+                        specify output format (default: csv)
+  --nomail, -M          suppress email notice (default: False)
+  --password-env PASSWORD_ENV
+                        specify the environment variable which contains the
+                        password (default: HSC_LA_PASSWORD)
+  --preview, -p         quick mode (short timeout) (default: False)
+  --skip-syntax-check, -S
+                        skip syntax check (default: False)
+  --api-url API_URL     for developers (default: https://hscla.mtk.nao.ac.jp/d
+                        atasearch/api/catalog_jobs/)
+  --login-url LOGIN_URL
+                        for developers (default:
+                        https://hscla.mtk.nao.ac.jp/account/api/session)
+
+```
\ No newline at end of file

la2016/catalogQuery/hscSspQuery.py

+import argparse
+import csv
+import getpass
+import http.cookies
+import json
+import os
+import os.path
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Dict, Optional
+
+version = 20190924.1
+
+
+args = None
+
+
+def main():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--user', '-u', required=True,
+                        help='specify your account')
+    parser.add_argument('--release-version', '-r', choices='hscla2014 hscla2016'.split(), default='hscla2016',
+                        help='specify release version')
+    parser.add_argument('--delete-job', '-D', action='store_true',
+                        help='delete the job you submitted after your downloading')
+    parser.add_argument('--format', '-f', dest='out_format', default='csv', choices=['csv', 'csv.gz', 'sqlite3', 'fits', 'numpygres-fits', 'fast-fits'],
+                        help='specify output format')
+    parser.add_argument('--nomail', '-M', action='store_true',
+                        help='suppress email notice')
+    parser.add_argument('--password-env', default='HSC_LA_PASSWORD',
+                        help='specify the environment variable which contains the password')
+    parser.add_argument('--preview', '-p', action='store_true',
+                        help='quick mode (short timeout)')
+    parser.add_argument('--skip-syntax-check', '-S', action='store_true',
+                        help='skip syntax check')
+    parser.add_argument('--api-url', default='https://hscla.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
+                        help='for developers')
+    parser.add_argument('--login-url', default='https://hscla.mtk.nao.ac.jp/account/api/session',
+                        help='for developers')
+    parser.add_argument('sql-file', type=argparse.FileType('r'),
+                        help='SQL file')
+
+    global args
+    args = parser.parse_args()
+
+    credential = {'account_name': args.user, 'password': getPassword()}
+    httpJsonPost.credential = credential
+    httpJsonPost.login_url = args.login_url
+
+    sql = args.__dict__['sql-file'].read()
+
+    job = None
+
+    try:
+        if args.preview:
+            preview(credential, sql, sys.stdout)
+        else:
+            job = submitJob(credential, sql, args.out_format)
+            blockUntilJobFinishes(credential, job['id'])
+            download(credential, job['id'], sys.stdout.buffer)
+            if args.delete_job:
+                deleteJob(credential, job['id'])
+    except urllib.error.HTTPError as e:
+        if e.code == 401:
+            print('invalid id or password.', file=sys.stderr)
+        if e.code == 406:
+            print(e.read(), file=sys.stderr)
+        else:
+            print(e, file=sys.stderr)
+    except QueryError as e:
+        print(e, file=sys.stderr)
+    except KeyboardInterrupt:
+        if job is not None:
+            jobCancel(credential, job['id'])
+        raise
+    else:
+        sys.exit(0)
+
+    sys.exit(1)
+
+
+class QueryError(Exception):
+    pass
+
+
+class HttpJsonPost:
+    def __init__(self):
+        self.credential: Optional[Dict] = None
+        self._session: Optional[str] = None
+        self.login_url: Optional[str] = None
+
+    def _httpPost(self, url, postData, headers):
+        req = urllib.request.Request(url, postData.encode('utf-8'), headers)
+        res = urllib.request.urlopen(req)
+        return res
+
+    def __call__(self, url, data):
+        assert (self.login_url is None) == (self.credential is None)
+        data['clientVersion'] = version
+        headers = {'Content-type': 'application/json'}
+        if self.credential:
+            if self._session is None and self.login_url:
+                try:
+                    res = self._httpPost(
+                        self.login_url,
+                        json.dumps({
+                            'email': self.credential['account_name'],
+                            'password': self.credential['password']
+                        }), headers)
+                except urllib.error.HTTPError as e:
+                    if e.code == 422:
+                        print('invalid id or password.', file=sys.stderr)
+                        exit(1)
+                    raise
+                self._session = http.cookies.SimpleCookie(res.headers['Set-Cookie'])['LAAUTH_SESSION'].value
+            headers['Cookie'] = f'LAAUTH_SESSION={self._session}'
+        postData = json.dumps(data)
+        return self._httpPost(url, postData, headers)
+
+
+httpJsonPost = HttpJsonPost()
+
+
+def submitJob(credential, sql, out_format):
+    url = args.api_url + 'submit'
+    catalog_job = {
+        'sql': sql,
+        'out_format': out_format,
+        'include_metainfo_to_body': True,
+        'release_version': args.release_version,
+    }
+    postData = {'credential': credential, 'catalog_job': catalog_job,
+                'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
+    res = httpJsonPost(url, postData)
+    job = json.load(res)
+    return job
+
+
+def jobStatus(credential, job_id):
+    url = args.api_url + 'status'
+    postData = {'credential': credential, 'id': job_id}
+    res = httpJsonPost(url, postData)
+    job = json.load(res)
+    return job
+
+
+def jobCancel(credential, job_id):
+    url = args.api_url + 'cancel'
+    postData = {'credential': credential, 'id': job_id}
+    httpJsonPost(url, postData)
+
+
+def preview(credential, sql, out):
+    url = args.api_url + 'preview'
+    catalog_job = {
+        'sql': sql,
+        'release_version': args.release_version,
+    }
+    postData = {'credential': credential, 'catalog_job': catalog_job}
+    res = httpJsonPost(url, postData)
+    result = json.load(res)
+
+    writer = csv.writer(out)
+    # writer.writerow(result['result']['fields'])
+    for row in result['result']['rows']:
+        writer.writerow(row)
+
+    if result['result']['count'] > len(result['result']['rows']):
+        raise QueryError('only top %d records are displayed !' % len(result['result']['rows']))
+
+
+def blockUntilJobFinishes(credential, job_id):
+    max_interval = 5 * 60  # sec.
+    interval = 1
+    while True:
+        time.sleep(interval)
+        job = jobStatus(credential, job_id)
+        if job['status'] == 'error':
+            raise QueryError('query error: ' + job['error'])
+        if job['status'] == 'done':
+            break
+        interval *= 2
+        if interval > max_interval:
+            interval = max_interval
+
+
+def download(credential, job_id, out):
+    url = args.api_url + 'download'
+    postData = {'credential': credential, 'id': job_id}
+    res = httpJsonPost(url, postData)
+    bufSize = 64 * 1 << 10  # 64k
+    while True:
+        buf = res.read(bufSize)
+        out.write(buf)
+        if len(buf) < bufSize:
+            break
+
+
+def deleteJob(credential, job_id):
+    url = args.api_url + 'delete'
+    postData = {'credential': credential, 'id': job_id}
+    httpJsonPost(url, postData)
+
+
+def getPassword():
+    password_from_envvar = os.environ.get(args.password_env, '')
+    if password_from_envvar != '':
+        return password_from_envvar
+    else:
+        return getpass.getpass('password? ')
+
+
+if __name__ == '__main__':
+    main()

la2016/catalogQuery/hscSspQuery.python2.py

+import json
+import argparse
+import urllib2
+import time
+import sys
+import csv
+import getpass
+import os
+import os.path
+
+
+version = 20190924.1
+
+
+args = None
+
+
+def main():
+    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--user', '-u', required=True,
+                        help='specify your STARS account')
+    parser.add_argument('--release-version', '-r', choices='hscla2014 hscla2016'.split(), default='hscla2016',
+                        help='specify release version')
+    parser.add_argument('--delete-job', '-D', action='store_true',
+                        help='delete the job you submitted after your downloading')
+    parser.add_argument('--format', '-f', dest='out_format', default='csv', choices=['csv', 'csv.gz', 'sqlite3', 'fits', 'numpygres-fits'],
+                        help='specify output format')
+    parser.add_argument('--nomail', '-M', action='store_true',
+                        help='suppress email notice')
+    parser.add_argument('--password-env', default='HSC_LA_PASSWORD',
+                        help='specify the environment variable which contains the password')
+    parser.add_argument('--preview', '-p', action='store_true',
+                        help='quick mode (short timeout)')
+    parser.add_argument('--skip-syntax-check', '-S', action='store_true',
+                        help='skip syntax check')
+    parser.add_argument('--api-url', default='https://hscla.mtk.nao.ac.jp/datasearch/api/catalog_jobs/',
+                        help='for developers')
+    parser.add_argument('--login-url', default='https://hscla.mtk.nao.ac.jp/account/api/session',
+                        help='for developers')
+    parser.add_argument('sql-file', type=argparse.FileType('r'),
+                        help='SQL file')
+
+    global args
+    args = parser.parse_args()
+
+    credential = {'account_name': args.user, 'password': getPassword()}
+    httpJsonPost.credential = credential
+    httpJsonPost.login_url = args.login_url
+    sql = args.__dict__['sql-file'].read()
+
+    job = None
+
+    try:
+        if args.preview:
+            preview(credential, sql, sys.stdout)
+        else:
+            job = submitJob(credential, sql, args.out_format)
+            blockUntilJobFinishes(credential, job['id'])
+            download(credential, job['id'], sys.stdout)
+            if args.delete_job:
+                deleteJob(credential, job['id'])
+    except urllib2.HTTPError, e:
+        if e.code == 401:
+            print >> sys.stderr, 'invalid id or password.'
+        if e.code == 406:
+            print >> sys.stderr, e.read()
+        else:
+            print >> sys.stderr, e
+    except QueryError, e:
+        print >> sys.stderr, e
+    except KeyboardInterrupt:
+        if job is not None:
+            jobCancel(credential, job['id'])
+        raise
+    else:
+        sys.exit(0)
+
+    sys.exit(1)
+
+
+class QueryError(Exception):
+    pass
+
+
+class HttpJsonPost:
+    def __init__(self):
+        self.credential = None
+        self._session = None
+        self.login_url = None
+
+    def _httpPost(self, url, postData, headers):
+        req = urllib2.Request(url, postData, headers)
+        res = urllib2.urlopen(req)
+        return res
+
+    def __call__(self, url, data):
+        assert (self.login_url is None) == (self.credential is None)
+        data['clientVersion'] = version
+        headers = {'Content-type': 'application/json'}
+        if self.credential:
+            import Cookie
+            if self._session is None and self.login_url:
+                try:
+                    res = self._httpPost(
+                        self.login_url,
+                        json.dumps({
+                            'email': self.credential['account_name'],
+                            'password': self.credential['password']
+                        }), headers)
+                except urllib2.HTTPError as e:
+                    if e.code == 422:
+                        print >> sys.stderr, 'invalid id or password.'
+                        exit(1)
+                    raise
+                self._session = Cookie.SimpleCookie(res.headers['Set-Cookie'])['LAAUTH_SESSION'].value
+            headers['Cookie'] = 'LAAUTH_SESSION=' + self._session
+        postData = json.dumps(data)
+        return self._httpPost(url, postData, headers)
+
+
+httpJsonPost = HttpJsonPost()
+
+
+def submitJob(credential, sql, out_format):
+    url = args.api_url + 'submit'
+    catalog_job = {
+        'sql'                     : sql,
+        'out_format'              : out_format,
+        'include_metainfo_to_body': True,
+        'release_version'         : args.release_version,
+    }
+    postData = {'credential': credential, 'catalog_job': catalog_job, 'nomail': args.nomail, 'skip_syntax_check': args.skip_syntax_check}
+    res = httpJsonPost(url, postData)
+    job = json.load(res)
+    return job
+
+
+def jobStatus(credential, job_id):
+    url = args.api_url + 'status'
+    postData = {'credential': credential, 'id': job_id}
+    res = httpJsonPost(url, postData)
+    job = json.load(res)
+    return job
+
+
+def jobCancel(credential, job_id):
+    url = args.api_url + 'cancel'
+    postData = {'credential': credential, 'id': job_id}
+    httpJsonPost(url, postData)
+
+
+def preview(credential, sql, out):
+    url = args.api_url + 'preview'
+    catalog_job = {
+        'sql'             : sql,
+        'release_version' : args.release_version,
+    }
+    postData = {'credential': credential, 'catalog_job': catalog_job}
+    res = httpJsonPost(url, postData)
+    result = json.load(res)
+
+    writer = csv.writer(out)
+    # writer.writerow(result['result']['fields'])
+    for row in result['result']['rows']:
+        writer.writerow(row)
+
+    if result['result']['count'] > len(result['result']['rows']):
+        raise QueryError, 'only top %d records are displayed !' % len(result['result']['rows'])
+
+
+def blockUntilJobFinishes(credential, job_id):
+    max_interval = 5 * 60 # sec.
+    interval = 1
+    while True:
+        time.sleep(interval)
+        job = jobStatus(credential, job_id)
+        if job['status'] == 'error':
+            raise QueryError, 'query error: ' + job['error']
+        if job['status'] == 'done':
+            break
+        interval *= 2
+        if interval > max_interval:
+            interval = max_interval
+
+
+def download(credential, job_id, out):
+    url = args.api_url + 'download'
+    postData = {'credential': credential, 'id': job_id}
+    res = httpJsonPost(url, postData)
+    bufSize = 64 * 1<<10 # 64k
+    while True:
+        buf = res.read(bufSize)
+        out.write(buf)
+        if len(buf) < bufSize:
+            break
+
+
+def deleteJob(credential, job_id):
+    url = args.api_url + 'delete'
+    postData = {'credential': credential, 'id': job_id}
+    httpJsonPost(url, postData)
+
+
+def getPassword():
+    password_from_envvar = os.environ.get(args.password_env, '')
+    if password_from_envvar != '':
+        return password_from_envvar
+    else:
+        return getpass.getpass('password? ')
+
+
+if __name__ == '__main__':
+    main()

la2016/colorPostage/.gitignore

+/coords.txt
+/png2
+/png3
\ No newline at end of file

la2016/colorPostage/README.md

+# Postage Stamps in Color
+If you want postage-stamps of your objects in color, you can upload an object list with this tool.
+
+## Basic Usage
+```sh
+cat > coords.txt <<EOT
+# ra         dec             outfile(optional)
+4.62630      16.44671        a.png
+4.63742      16.44747        b.png
+4.62537      16.44345        c.png
+4.65146      16.44694        d.png
+4.64854      16.44695        e.png
+4.65057      16.44706        f.png
+4.62701      16.44535        g.png
+4.65151      16.44745        h.png
+4.63186      16.44911        i.png
+4.63830      16.44271        j.png
+EOT
+
+python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt
+```
+
+## Advanced Usage
+```
+usage: colorPostage.py [-h] --outDir OUTDIR --user USER [--filters FILTERS FILTERS FILTERS] [--fov FOV] [--rerun {la2014}] [--color {hsc,sdss}] input
+
+positional arguments:
+  input
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --outDir OUTDIR, -o OUTDIR
+  --user USER, -u USER
+  --filters FILTERS FILTERS FILTERS, -f FILTERS FILTERS FILTERS
+  --fov FOV
+  --rerun {la2014}
+  --color {hsc,sdss}
+```
\ No newline at end of file

la2016/colorPostage/colorPostage.py

+#!/bin/env python3
+'''
+> head coords.txt
+# ra         dec             outfile(optional)
+4.62630      16.44671        a.png
+4.63742      16.44747        b.png
+4.62537      16.44345        c.png
+4.65146      16.44694        d.png
+4.64854      16.44695        e.png
+4.65057      16.44706        f.png
+4.62701      16.44535        g.png
+4.65151      16.44745        h.png
+4.63186      16.44911        i.png
+4.63830      16.44271        j.png
+> python colorPostage.py --user YOUR_ACCOUNT --outDir pngs coords.txt  
+
+'''
+
+import PIL.Image
+import numpy
+import argparse
+import tarfile
+import subprocess
+import tempfile
+import getpass
+import os
+import os.path
+import contextlib
+import logging
+logging.basicConfig(level=logging.INFO)
+try:
+    import pyfits
+except:
+    import astropy.io.fits as pyfits
+
+FLUXMAG0 = 10 ** (27 / 2.5)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--outDir', '-o', required=True)
+    parser.add_argument('--user', '-u', required=True)
+    parser.add_argument('--filters', '-f',
+                        nargs=3, default=['HSC-I', 'HSC-R', 'HSC-G'])
+    parser.add_argument('--fov', default='30asec')
+    parser.add_argument('--rerun', default='any',
+                        choices='la2016 la2014'.split())
+    parser.add_argument('--color', choices='hsc sdss'.split(), default='hsc')
+    parser.add_argument('input', type=argparse.FileType('r'))
+    args = parser.parse_args()
+
+    password = getpass.getpass('Password: ')
+    checkPassword(args.user, password)
+
+    coords, outs = loadCoords(args.input)
+
+    mkdir_p(args.outDir)
+
+    batchSize = 300
+    for batchI, batchCoords in enumerate(batch(coords, batchSize)):
+        with requestFileFor(batchCoords, args.filters, args.fov, args.rerun) as requestFile:
+            tarMembers = queryTar(args.user, password, requestFile)
+            for i, rgb in rgbBundle(tarMembers):
+                j = batchI * batchSize + i
+                outFile = os.path.join(args.outDir, outs[j])
+                logging.info('-> {}'.format(outFile))
+                makeColorPng(rgb, outFile, args.color)
+
+
+TOP_PAGE = 'https://hscla.mtk.nao.ac.jp/das_cutout/la2014/'
+API = 'https://hscla.mtk.nao.ac.jp/das_cutout/la2014/cgi-bin/cutout'
+
+
+def loadCoords(input):
+    import re
+    comment = re.compile('\s*(?:$|#)')
+    num = 1