HATLAS-SGP master catalogue

Preparation of VIKING data

VISTA telescope/VIKING catalogue: the catalogue comes from dmu0_VIKING.

In the catalogue, we keep:

  • The identifier (it's unique in the catalogue);
  • The position;
  • The stellarity;
  • The magnitude for each band.
  • The kron magnitude to be used as total magnitude (no “auto” magnitude is provided).

We don't know when the maps have been observed. We will use the year of the reference paper.

In [1]:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
This notebook was run with herschelhelp_internal version: 
44f1ae0 (Thu Nov 30 18:27:54 2017 +0000)
In [2]:
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))

from collections import OrderedDict
import os

from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table
import numpy as np

from herschelhelp_internal.flagging import  gaia_flag_column
from herschelhelp_internal.masterlist import nb_astcor_diag_plot, remove_duplicates
from herschelhelp_internal.utils import astrometric_correction, mag_to_flux
In [3]:
OUT_DIR =  os.environ.get('TMP_DIR', "./data_tmp")
try:
    os.makedirs(OUT_DIR)
except FileExistsError:
    pass

RA_COL = "viking_ra"
DEC_COL = "viking_dec"

I - Column selection

In [4]:
# Bands: Z,Y,J,H,K
imported_columns = OrderedDict({
        'SOURCEID': "viking_id",
        'ra': "viking_ra",
        'dec': "viking_dec",
        'PSTAR':  "viking_stellarity",
        'ZPETROMAG': "m_vista_z", 
        'ZPETROMAGERR': "merr_vista_z", 
        'ZAPERMAG3': "m_ap_vista_z", 
        'ZAPERMAG3ERR': "merr_ap_vista_z",
        'YPETROMAG': "m_vista_y", 
        'YPETROMAGERR': "merr_vista_y", 
        'YAPERMAG3': "m_ap_vista_y", 
        'YAPERMAG3ERR': "merr_ap_vista_y",
        'JPETROMAG': "m_vista_j", 
        'JPETROMAGERR': "merr_vista_j", 
        'JAPERMAG3': "m_ap_vista_j", 
        'JAPERMAG3ERR': "merr_ap_vista_j",        
        'HPETROMAG': "m_vista_h", 
        'HPETROMAGERR': "merr_vista_h", 
        'HAPERMAG3': "m_ap_vista_h", 
        'HAPERMAG3ERR': "merr_ap_vista_h",        
        'KSPETROMAG': "m_vista_k", 
        'KSPETROMAGERR': "merr_vista_k", 
        'KSAPERMAG3': "m_ap_vista_k", 
        'KSAPERMAG3ERR': "merr_ap_vista_k",
    })


catalogue = Table.read("../../dmu0/dmu0_VISTA-VIKING/data/VIKING_SGP.fits")[list(imported_columns)]
for column in imported_columns:
    catalogue[column].name = imported_columns[column]

epoch = 2011

# Clean table metadata
catalogue.meta = None
In [5]:
# Conversion from Vega magnitudes to AB is done using values from 
# http://casu.ast.cam.ac.uk/surveys-projects/vista/technical/filter-set
vega_to_ab = {
    "z": 0.521,
    "y": 0.618,
    "j": 0.937,
    "h": 1.384,
    "k": 1.839
}
In [6]:
# Adding flux and band-flag columns
for col in catalogue.colnames:
    if col.startswith('m_'):
        
        errcol = "merr{}".format(col[1:])
        
        # Some object have a magnitude to 0, we suppose this means missing value
        catalogue[col][catalogue[col] <= 0] = np.nan
        catalogue[errcol][catalogue[errcol] <= 0] = np.nan  
   
        # Convert magnitude from Vega to AB
        catalogue[col] += vega_to_ab[col[-1]]

        flux, error = mag_to_flux(np.array(catalogue[col]), np.array(catalogue[errcol]))
        
        # Fluxes are added in µJy
        catalogue.add_column(Column(flux * 1.e6, name="f{}".format(col[1:])))
        catalogue.add_column(Column(error * 1.e6, name="f{}".format(errcol[1:])))
        
        # Band-flag column
        if "ap" not in col:
            catalogue.add_column(Column(np.zeros(len(catalogue), dtype=bool), name="flag{}".format(col[1:])))
        
# TODO: Set to True the flag columns for fluxes that should not be used for SED fitting.
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.
Check the NumPy 1.11 release notes for more information.
  ma.MaskedArray.__setitem__(self, index, value)
In [7]:
catalogue[:10].show_in_notebook()
Out[7]:
<Table masked=True length=10>
idxviking_idviking_raviking_decviking_stellaritym_vista_zmerr_vista_zm_ap_vista_zmerr_ap_vista_zm_vista_ymerr_vista_ym_ap_vista_ymerr_ap_vista_ym_vista_jmerr_vista_jm_ap_vista_jmerr_ap_vista_jm_vista_hmerr_vista_hm_ap_vista_hmerr_ap_vista_hm_vista_kmerr_vista_km_ap_vista_kmerr_ap_vista_kf_vista_zferr_vista_zflag_vista_zf_ap_vista_zferr_ap_vista_zf_vista_yferr_vista_yflag_vista_yf_ap_vista_yferr_ap_vista_yf_vista_jferr_vista_jflag_vista_jf_ap_vista_jferr_ap_vista_jf_vista_hferr_vista_hflag_vista_hf_ap_vista_hferr_ap_vista_hf_vista_kferr_vista_kflag_vista_kf_ap_vista_kferr_ap_vista_k
degdeg
0601324958322337.037589563-29.95628189020.923.0910.30908623.30290.284804nannannannannannannannannannannannannannannannan2.106750.599749False1.733150.45463nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
1601324958338337.038307107-29.92748553630.0521.85680.12239622.41790.128321nannannannannannannannannannannannannannannannan6.565650.740153False3.915950.46282nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
2601324958351337.037720203-30.12967976090.0522.80540.29073322.93990.203524nannannannannannannannannannannannannannannannan2.740450.733825False2.421220.453864nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
3601324958353337.037916617-30.07798642770.0522.3690.22475422.91320.198416nannannannannannannannannannannannannannannannan4.096420.847984False2.481620.453511nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
4601324958359337.038755364-29.90671280210.0521.34870.098952522.46580.133998nannannannannannannannannannannannannannannannan10.48350.955455False3.746860.462427nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
5601324958366337.037578834-30.20498469810.99981920.68370.056175520.62480.027535620.46360.085060720.34580.043373320.26970.078032720.33870.046599720.27270.22084220.21730.08866420.1520.19748820.40080.10726519.34351.00082False20.42110.51790423.69061.85601False26.40341.0547728.32092.03545False26.57861.1407528.24325.74475False29.7222.4271831.56435.74134False25.09922.47967
6601324958377337.037671251-30.27975156750.0521.80020.14166722.88590.198471nannannannannannannannannannannannannannannannan6.917280.902565False2.544830.46519nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
7601324958383337.039225078-29.97691391350.0523.17350.37215323.26210.274852nannannannannannannannannannannannannannannannan1.952620.669291False1.799610.455569nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
8601324958388337.03778046-30.35372406280.0522.46430.17450223.2730.278117nannannannannannannannannannannannannannannannan3.752220.603066False1.781620.45637nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan
9601324958391337.038175768-30.23294658760.0522.83380.32203523.04620.224061nannannannannannannannannannannannannannannannan2.669730.791856False2.195380.453056nannanFalsenannannannanFalsenannannannanFalsenannannannanFalsenannan

II - Removal of duplicated sources

We remove duplicated objects from the input catalogues.

In [8]:
SORT_COLS = ['merr_ap_vista_y', 'merr_ap_vista_h', 'merr_ap_vista_j', 'merr_ap_vista_k']
FLAG_NAME = 'viking_flag_cleaned'

nb_orig_sources = len(catalogue)

catalogue = remove_duplicates(catalogue, RA_COL, DEC_COL, sort_col=SORT_COLS,flag_name=FLAG_NAME)

nb_sources = len(catalogue)

print("The initial catalogue had {} sources.".format(nb_orig_sources))
print("The cleaned catalogue has {} sources ({} removed).".format(nb_sources, nb_orig_sources - nb_sources))
print("The cleaned catalogue has {} sources flagged as having been cleaned".format(np.sum(catalogue[FLAG_NAME])))
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/astropy/table/column.py:1096: MaskedArrayFutureWarning: setting an item on a masked array which has a shared mask will not copy the mask and also change the original mask array in the future.
Check the NumPy 1.11 release notes for more information.
  ma.MaskedArray.__setitem__(self, index, value)
The initial catalogue had 12380676 sources.
The cleaned catalogue has 12350404 sources (30272 removed).
The cleaned catalogue has 30050 sources flagged as having been cleaned

III - Astrometry correction

We match the astrometry to the Gaia one. We limit the Gaia catalogue to sources with a g band flux between the 30th and the 70th percentile. Some quick tests show that this give the lower dispersion in the results.

In [9]:
gaia = Table.read("../../dmu0/dmu0_GAIA/data/GAIA_SGP.fits")
gaia_coords = SkyCoord(gaia['ra'], gaia['dec'])
In [10]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec, near_ra0=True)
In [11]:
delta_ra, delta_dec =  astrometric_correction(
    SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]),
    gaia_coords, near_ra0=True
)

print("RA correction: {}".format(delta_ra))
print("Dec correction: {}".format(delta_dec))
RA correction: 0.12487526277027428 arcsec
Dec correction: -0.11282892556891966 arcsec
In [12]:
catalogue[RA_COL] +=  delta_ra.to(u.deg)
catalogue[DEC_COL] += delta_dec.to(u.deg)
In [13]:
nb_astcor_diag_plot(catalogue[RA_COL], catalogue[DEC_COL], 
                    gaia_coords.ra, gaia_coords.dec, near_ra0=True)

IV - Flagging Gaia objects

In [14]:
catalogue.add_column(
    gaia_flag_column(SkyCoord(catalogue[RA_COL], catalogue[DEC_COL]), epoch, gaia)
)
In [15]:
GAIA_FLAG_NAME = "viking_flag_gaia"

catalogue['flag_gaia'].name = GAIA_FLAG_NAME
print("{} sources flagged.".format(np.sum(catalogue[GAIA_FLAG_NAME] > 0)))
468817 sources flagged.

V - Flagging objects near bright stars

VI - Saving to disk

In [16]:
catalogue.write("{}/VISTA-VIKING.fits".format(OUT_DIR), overwrite=True)