Herschel Stripe 82 master catalogue

Checks and diagnostics

from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))

This notebook was run with herschelhelp_internal version: 
33f5ec7 (Wed Dec 6 16:56:17 2017 +0000)

%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))
plt.style.use('ggplot')

import locale
locale.setlocale(locale.LC_ALL, 'en_GB')

import os
import time
import itertools


from astropy.coordinates import SkyCoord
from astropy.table import Table
from astropy import units as u
import numpy as np
from matplotlib_venn import venn3

from herschelhelp_internal.masterlist import nb_compare_mags, nb_ccplots, find_last_ml_suffix, quick_checks

OUT_DIR = os.environ.get('OUT_DIR', "./data")
SUFFIX = find_last_ml_suffix()
#SUFFIX = "20170627"

master_catalogue_filename = "master_catalogue_herschel-stripe-82_{}_{}.fits".format('RANDOM10PCSAMPLE', SUFFIX)
master_catalogue = Table.read("{}/{}".format(OUT_DIR, master_catalogue_filename))

print("Diagnostics done using: {}".format(master_catalogue_filename))

Diagnostics done using: master_catalogue_herschel-stripe-82_RANDOM10PCSAMPLE_20180113.fits

0 - Quick checks

quick_checks(master_catalogue).show_in_notebook()

/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(296, False) will return an array of dtype('bool')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(296, 0) will return an array of dtype('int64')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)

Table shows only problematic columns.

<Table length=78>

idx	Column	All nan	#Measurements	#Zeros	#Negative	Minimum value
0	f_suprime_g	False	4881980	18	0	0.0
1	f_suprime_r	False	4885212	1	0	0.0
2	f_suprime_i	False	4883943	8	0	0.0
3	f_suprime_z	False	4875494	10	0	0.0
4	f_suprime_y	False	4858808	3	0	0.0
5	f_ap_suprime_n816	False	244106	56	0	0.0
6	f_suprime_n816	False	4862290	235	0	0.0
7	f_ap_suprime_n921	False	247726	39	0	0.0
8	f_suprime_n921	False	4865964	121	0	0.0
9	merr_vics82_k	True	0	0	0	0.0
10	ferr_vics82_k	True	0	0	0	0.0
11	merr_ap_vics82_k	True	0	0	0	0.0
12	ferr_ap_vics82_k	True	0	0	0	0.0
13	merr_vics82_j	True	0	0	0	0.0
14	ferr_vics82_j	True	0	0	0	0.0
15	merr_ap_vics82_j	True	0	0	0	0.0
16	ferr_ap_vics82_j	True	0	0	0	0.0
17	merr_ap_gpc1_g	False	636518	535	0	0.0
18	m_gpc1_g	False	638145	0	1	-0.013756999746
19	merr_gpc1_g	False	638145	704	0	0.0
20	merr_ap_gpc1_r	False	666433	145	0	0.0
21	merr_gpc1_r	False	666783	107	0	0.0
22	merr_ap_gpc1_i	False	669872	220	0	0.0
23	merr_gpc1_i	False	669924	192	0	0.0
24	m_ap_gpc1_z	False	664458	0	1	-0.442647993565
25	merr_ap_gpc1_z	False	664458	402	0	0.0
26	merr_gpc1_z	False	664716	210	0	0.0
27	m_ap_gpc1_y	False	655993	0	16	-2.51508998871
28	merr_ap_gpc1_y	False	655993	655	0	0.0
29	m_gpc1_y	False	647856	0	18	-3.26448988914
30	merr_gpc1_y	False	647856	302	0	0.0
31	ferr_ap_gpc1_g	False	636518	535	0	0.0
32	ferr_gpc1_g	False	638145	704	0	0.0
33	ferr_ap_gpc1_r	False	666433	145	0	0.0
34	ferr_gpc1_r	False	666783	107	0	0.0
35	ferr_ap_gpc1_i	False	669872	220	0	0.0
36	ferr_gpc1_i	False	669924	192	0	0.0
37	ferr_ap_gpc1_z	False	664458	402	0	0.0
38	ferr_gpc1_z	False	664716	210	0	0.0
39	ferr_ap_gpc1_y	False	655993	655	0	0.0
40	ferr_gpc1_y	False	647856	302	0	0.0
41	f_ap_irac_i1	False	553489	0	578	-18.0009937286
42	merr_ap_irac_i1	False	553489	0	578	-489.406951904
43	f_irac_i1	False	4919163	0	1221	-421.66459907
44	merr_irac_i1	False	552789	0	521	-683.07462602
45	f_ap_irac_i2	False	553757	0	607	-6.89538049698
46	merr_ap_irac_i2	False	553757	0	607	-17975.5429688
47	f_irac_i2	False	4919431	0	1066	-158.127334595
48	merr_irac_i2	False	553756	0	1065	-836.396972656
49	f_decam_g	False	4897371	0	174313	-4723124.5
50	f_decam_r	False	4899740	0	49336	-4488797.5
51	f_decam_z	False	4919073	0	67273	-19518390272.0
52	merr_decam_g	False	2380846	0	174313	-55569.4882812
53	merr_decam_r	False	2383215	0	49336	-16852.3769531
54	merr_decam_z	False	2402548	0	67273	-96064.6015625
55	merr_ap_decam_g	False	2380194	0	160451	-117573.453125
56	merr_ap_decam_r	False	2382105	0	85424	-314441.5
57	merr_ap_decam_z	False	2401503	0	201314	-1560339.0
58	f_ap_rcs_g	True	0	0	0	0.0
59	ferr_ap_rcs_g	True	0	0	0	0.0
60	m_ap_rcs_g	True	0	0	0	0.0
61	merr_ap_rcs_g	True	0	0	0	0.0
62	f_ap_rcs_r	True	0	0	0	0.0
63	ferr_ap_rcs_r	True	0	0	0	0.0
64	m_ap_rcs_r	True	0	0	0	0.0
65	merr_ap_rcs_r	True	0	0	0	0.0
66	f_ap_rcs_i	True	0	0	0	0.0
67	ferr_ap_rcs_i	True	0	0	0	0.0
68	m_ap_rcs_i	True	0	0	0	0.0
69	merr_ap_rcs_i	True	0	0	0	0.0
70	f_ap_rcs_z	True	0	0	0	0.0
71	ferr_ap_rcs_z	True	0	0	0	0.0
72	m_ap_rcs_z	True	0	0	0	0.0
73	merr_ap_rcs_z	True	0	0	0	0.0
74	f_ap_rcs_y	True	0	0	0	0.0
75	ferr_ap_rcs_y	True	0	0	0	0.0
76	m_ap_rcs_y	True	0	0	0	0.0
77	merr_ap_rcs_y	True	0	0	0	0.0

I - Summary of wavelength domains

flag_obs = master_catalogue['flag_optnir_obs']
flag_det = master_catalogue['flag_optnir_det']

venn3(
    [
        np.sum(flag_obs == 4),
        np.sum(flag_obs == 2),
        np.sum(flag_obs == 6),
        np.sum(flag_obs == 1),
        np.sum(flag_obs == 5),
        np.sum(flag_obs == 3),
        np.sum(flag_obs == 7)
    ],
    set_labels=('mid-IR', 'near-IR', 'Optical'),
    subset_label_formatter=lambda x: "{}%".format(int(100*x/len(flag_obs)))
)
plt.title("Wavelength domain observations");

venn3(
    [
        np.sum(flag_det[flag_obs == 7] == 4),
        np.sum(flag_det[flag_obs == 7] == 2),
        np.sum(flag_det[flag_obs == 7] == 6),
        np.sum(flag_det[flag_obs == 7] == 1),
        np.sum(flag_det[flag_obs == 7] == 5),
        np.sum(flag_det[flag_obs == 7] == 3),
        np.sum(flag_det[flag_obs == 7] == 7)
    ],
    set_labels=('mid-IR', 'near-IR', 'Optical'),
    subset_label_formatter=lambda x: "{}%".format(int(100*x/np.sum(flag_det != 0)))
)
plt.title("Detection of the {} sources detected\n in any wavelength domains "
          "(among {} sources)".format(
              locale.format('%d', np.sum(flag_det != 0), grouping=True),
              locale.format('%d', len(flag_det), grouping=True)));

/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/matplotlib_venn/_venn3.py:53: UserWarning: Circle A has zero area
  warnings.warn("Circle A has zero area")
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/matplotlib_venn/_venn3.py:57: UserWarning: Circle B has zero area
  warnings.warn("Circle B has zero area")

II - Comparing magnitudes in similar filters

The master list if composed of several catalogues containing magnitudes in similar filters on different instruments. We are comparing the magnitudes in these corresponding filters.

u_bands = [ "SDSS u"]
g_bands = [ "SDSS g", "DECam g" ,"GPC1 g", "Suprime g", "RCS g"]
r_bands = [ "SDSS r", "DECam r" ,"GPC1 r", "Suprime r", "RCS r"]
i_bands = [ "SDSS i"            ,"GPC1 i", "Suprime i", "RCS i"]
z_bands = [ "SDSS z", "DECam z" ,"GPC1 z", "Suprime z", "RCS z"]
y_bands = [                      "GPC1 y", "Suprime y", "RCS y", "VHS Y", "UKIDSS Y"] #This will compare lower case y to upper case Y
J_bands = [                                                      "VHS J", "UKIDSS J", "VICS82 J"]
H_bands = [                                                      "VHS H", "UKIDSS H"]
K_bands = [                                                      "VHS K", "UKIDSS K", "VICS82 K"]

for band_of_a_kind in [u_bands, g_bands, r_bands, i_bands, z_bands, y_bands, J_bands, H_bands, K_bands]:
    for band1, band2 in itertools.combinations(band_of_a_kind, 2):
        
        basecol1, basecol2 = band1.replace(" ", "_").lower(), band2.replace(" ", "_").lower()
        
        col1, col2 = "m_ap_{}".format(basecol1), "m_ap_{}".format(basecol2)
        nb_compare_mags(master_catalogue[col1], master_catalogue[col2], 
                        labels=("{} (aperture)".format(band1), "{} (aperture)".format(band2)))
        
        col1, col2 = "m_{}".format(basecol1), "m_{}".format(basecol2)
        nb_compare_mags(master_catalogue[col1], master_catalogue[col2], 
                        labels=("{} (total)".format(band1), "{} (total)".format(band2)))

DECam g (aperture) - SDSS g (aperture):
- Median: -0.65
- Median Absolute Deviation: 0.21
- 1% percentile: -1.369743415771485
- 99% percentile: 1.1470082854003918

DECam g (total) - SDSS g (total):
- Median: 0.03
- Median Absolute Deviation: 0.18
- 1% percentile: -0.8870277509765617
- 99% percentile: 1.6710442353515624

GPC1 g (aperture) - SDSS g (aperture):
- Median: -0.91
- Median Absolute Deviation: 0.37
- 1% percentile: -3.206002183227539
- 99% percentile: 1.5728337866821276

GPC1 g (total) - SDSS g (total):
- Median: -0.07
- Median Absolute Deviation: 0.26
- 1% percentile: -2.8801525574951183
- 99% percentile: 1.6551279635009788

No sources have both SDSS g (aperture) and Suprime g (aperture) values.
No sources have both SDSS g (total) and Suprime g (total) values.
No sources have both SDSS g (aperture) and RCS g (aperture) values.
RCS g (total) - SDSS g (total):
- Median: -0.15
- Median Absolute Deviation: 0.20
- 1% percentile: -1.1508145864868167
- 99% percentile: 1.2041443676757795

GPC1 g (aperture) - DECam g (aperture):
- Median: -0.12
- Median Absolute Deviation: 0.28
- 1% percentile: -3.1324508666992186
- 99% percentile: 2.209981250762938

GPC1 g (total) - DECam g (total):
- Median: -0.02
- Median Absolute Deviation: 0.28
- 1% percentile: -3.503301296234131
- 99% percentile: 1.7441903114318813

Suprime g (aperture) - DECam g (aperture):
- Median: 0.02
- Median Absolute Deviation: 0.23
- 1% percentile: -2.4182591247558594
- 99% percentile: 1.8705764389038144

Suprime g (total) - DECam g (total):
- Median: -0.15
- Median Absolute Deviation: 0.24
- 1% percentile: -2.5489704895019534
- 99% percentile: 1.5885691070556645

No sources have both DECam g (aperture) and RCS g (aperture) values.
RCS g (total) - DECam g (total):
- Median: -0.22
- Median Absolute Deviation: 0.21
- 1% percentile: -2.1725115966796875
- 99% percentile: 0.9619197845458984

Suprime g (aperture) - GPC1 g (aperture):
- Median: 0.25
- Median Absolute Deviation: 0.30
- 1% percentile: -1.9736169052124022
- 99% percentile: 2.9869889640808087

Suprime g (total) - GPC1 g (total):
- Median: 0.12
- Median Absolute Deviation: 0.29
- 1% percentile: -1.4026131439208984
- 99% percentile: 3.274006023406982

No sources have both GPC1 g (aperture) and RCS g (aperture) values.
RCS g (total) - GPC1 g (total):
- Median: -0.13
- Median Absolute Deviation: 0.32
- 1% percentile: -1.882863235473633
- 99% percentile: 2.9682432556152345

No sources have both Suprime g (aperture) and RCS g (aperture) values.
RCS g (total) - Suprime g (total):
- Median: -0.15
- Median Absolute Deviation: 0.21
- 1% percentile: -2.40746826171875
- 99% percentile: 1.0876546478271485

DECam r (aperture) - SDSS r (aperture):
- Median: -0.68
- Median Absolute Deviation: 0.21
- 1% percentile: -1.3852449727783214
- 99% percentile: 1.1101013143310563

DECam r (total) - SDSS r (total):
- Median: -0.13
- Median Absolute Deviation: 0.15
- 1% percentile: -1.0457528029785144
- 99% percentile: 0.9778500347900343

GPC1 r (aperture) - SDSS r (aperture):
- Median: -0.65
- Median Absolute Deviation: 0.24
- 1% percentile: -1.6492644418945306
- 99% percentile: 0.7977198767089918

GPC1 r (total) - SDSS r (total):
- Median: 0.09
- Median Absolute Deviation: 0.12
- 1% percentile: -1.150610781066894
- 99% percentile: 1.0046281422119154

No sources have both SDSS r (aperture) and Suprime r (aperture) values.
No sources have both SDSS r (total) and Suprime r (total) values.
No sources have both SDSS r (aperture) and RCS r (aperture) values.
RCS r (total) - SDSS r (total):
- Median: -0.12
- Median Absolute Deviation: 0.15
- 1% percentile: -1.019848545166015
- 99% percentile: 1.0965001182861318

GPC1 r (aperture) - DECam r (aperture):
- Median: 0.12
- Median Absolute Deviation: 0.15
- 1% percentile: -1.838051061630249
- 99% percentile: 1.5183926773071295

GPC1 r (total) - DECam r (total):
- Median: 0.23
- Median Absolute Deviation: 0.15
- 1% percentile: -1.316351318359375
- 99% percentile: 1.4407266616821286

Suprime r (aperture) - DECam r (aperture):
- Median: 0.11
- Median Absolute Deviation: 0.21
- 1% percentile: -2.228728008270264
- 99% percentile: 1.4809819793701173

Suprime r (total) - DECam r (total):
- Median: 0.01
- Median Absolute Deviation: 0.18
- 1% percentile: -1.1619326782226564
- 99% percentile: 1.5908895874023425

No sources have both DECam r (aperture) and RCS r (aperture) values.
RCS r (total) - DECam r (total):
- Median: -0.01
- Median Absolute Deviation: 0.17
- 1% percentile: -1.0698253631591799
- 99% percentile: 1.1014166641235348

Suprime r (aperture) - GPC1 r (aperture):
- Median: 0.07
- Median Absolute Deviation: 0.15
- 1% percentile: -1.3094873237609863
- 99% percentile: 1.4882897758483766

Suprime r (total) - GPC1 r (total):
- Median: -0.11
- Median Absolute Deviation: 0.14
- 1% percentile: -0.9736609268188476
- 99% percentile: 1.6416873550415039

No sources have both GPC1 r (aperture) and RCS r (aperture) values.
RCS r (total) - GPC1 r (total):
- Median: -0.23
- Median Absolute Deviation: 0.14
- 1% percentile: -1.1326136779785156
- 99% percentile: 1.4416136932373051

No sources have both Suprime r (aperture) and RCS r (aperture) values.
RCS r (total) - Suprime r (total):
- Median: -0.07
- Median Absolute Deviation: 0.18
- 1% percentile: -2.3652862548828124
- 99% percentile: 1.0521179199218729

GPC1 i (aperture) - SDSS i (aperture):
- Median: -0.57
- Median Absolute Deviation: 0.18
- 1% percentile: -1.290400276611329
- 99% percentile: 0.20386654101562612

GPC1 i (total) - SDSS i (total):
- Median: 0.11
- Median Absolute Deviation: 0.09
- 1% percentile: -0.5160627268371595
- 99% percentile: 0.7460765848388664

No sources have both SDSS i (aperture) and Suprime i (aperture) values.
No sources have both SDSS i (total) and Suprime i (total) values.
No sources have both SDSS i (aperture) and RCS i (aperture) values.
RCS i (total) - SDSS i (total):
- Median: -0.09
- Median Absolute Deviation: 0.16
- 1% percentile: -0.9390832550659163
- 99% percentile: 1.4268233815002451

Suprime i (aperture) - GPC1 i (aperture):
- Median: 0.01
- Median Absolute Deviation: 0.12
- 1% percentile: -0.7142147827148437
- 99% percentile: 1.2686882781982451

Suprime i (total) - GPC1 i (total):
- Median: -0.19
- Median Absolute Deviation: 0.11
- 1% percentile: -0.7223954772949219
- 99% percentile: 1.327408523559564

No sources have both GPC1 i (aperture) and RCS i (aperture) values.
RCS i (total) - GPC1 i (total):
- Median: -0.26
- Median Absolute Deviation: 0.10
- 1% percentile: -0.825885009765625
- 99% percentile: 0.6832805633544881

No sources have both Suprime i (aperture) and RCS i (aperture) values.
RCS i (total) - Suprime i (total):
- Median: 0.01
- Median Absolute Deviation: 0.19
- 1% percentile: -2.2906301116943357
- 99% percentile: 1.1509046173095727

DECam z (aperture) - SDSS z (aperture):
- Median: -0.59
- Median Absolute Deviation: 0.26
- 1% percentile: -2.4457099262695317
- 99% percentile: 2.0554444185790963

DECam z (total) - SDSS z (total):
- Median: -0.09
- Median Absolute Deviation: 0.24
- 1% percentile: -2.6325343649902337
- 99% percentile: 1.8004996293945332

GPC1 z (aperture) - SDSS z (aperture):
- Median: -0.54
- Median Absolute Deviation: 0.22
- 1% percentile: -1.564947363952636
- 99% percentile: 0.9453999288330075

GPC1 z (total) - SDSS z (total):
- Median: 0.17
- Median Absolute Deviation: 0.13
- 1% percentile: -1.1928884832763664
- 99% percentile: 1.1529006408691411

No sources have both SDSS z (aperture) and Suprime z (aperture) values.
No sources have both SDSS z (total) and Suprime z (total) values.
No sources have both SDSS z (aperture) and RCS z (aperture) values.
RCS z (total) - SDSS z (total):
- Median: -0.05
- Median Absolute Deviation: 0.27
- 1% percentile: -2.5915972451171867
- 99% percentile: 1.4944283287963875

GPC1 z (aperture) - DECam z (aperture):
- Median: 0.09
- Median Absolute Deviation: 0.14
- 1% percentile: -2.1351356887817383
- 99% percentile: 1.4540224456787108

GPC1 z (total) - DECam z (total):
- Median: 0.27
- Median Absolute Deviation: 0.15
- 1% percentile: -1.1103314208984376
- 99% percentile: 1.454379558563233

Suprime z (aperture) - DECam z (aperture):
- Median: 0.10
- Median Absolute Deviation: 0.24
- 1% percentile: -2.8512701416015624
- 99% percentile: 1.6853060150146484

Suprime z (total) - DECam z (total):
- Median: 0.02
- Median Absolute Deviation: 0.19
- 1% percentile: -1.7147566223144532
- 99% percentile: 1.6015958404540975

No sources have both DECam z (aperture) and RCS z (aperture) values.
RCS z (total) - DECam z (total):
- Median: 0.03
- Median Absolute Deviation: 0.23
- 1% percentile: -1.6561821365356446
- 99% percentile: 1.3318465232849168

Suprime z (aperture) - GPC1 z (aperture):
- Median: -0.00
- Median Absolute Deviation: 0.15
- 1% percentile: -1.2945173072814942
- 99% percentile: 1.120867881774903

Suprime z (total) - GPC1 z (total):
- Median: -0.17
- Median Absolute Deviation: 0.13
- 1% percentile: -1.0127337074279785
- 99% percentile: 1.1070509338378889

No sources have both GPC1 z (aperture) and RCS z (aperture) values.
RCS z (total) - GPC1 z (total):
- Median: -0.24
- Median Absolute Deviation: 0.15
- 1% percentile: -1.1760531616210939
- 99% percentile: 1.1817067146301268

No sources have both Suprime z (aperture) and RCS z (aperture) values.
RCS z (total) - Suprime z (total):
- Median: -0.04
- Median Absolute Deviation: 0.21
- 1% percentile: -1.9193576049804686
- 99% percentile: 1.0926615142822262

Suprime y (aperture) - GPC1 y (aperture):
- Median: 0.11
- Median Absolute Deviation: 0.23
- 1% percentile: -1.7991735839843748
- 99% percentile: 1.8983014678955072

Suprime y (total) - GPC1 y (total):
- Median: -0.15
- Median Absolute Deviation: 0.25
- 1% percentile: -1.6497690963745117
- 99% percentile: 2.1316806030273425

No sources have both GPC1 y (aperture) and RCS y (aperture) values.
RCS y (total) - GPC1 y (total):
- Median: 0.47
- Median Absolute Deviation: 0.39
- 1% percentile: -1.4632400512695314
- 99% percentile: 2.727959060668946

VHS Y (aperture) - GPC1 y (aperture):
- Median: 0.05
- Median Absolute Deviation: 0.20
- 1% percentile: -1.4580602073669433
- 99% percentile: 1.079011898040772

VHS Y (total) - GPC1 y (total):
- Median: -0.15
- Median Absolute Deviation: 0.18
- 1% percentile: -1.2063194274902345
- 99% percentile: 1.3054595184326165

UKIDSS Y (aperture) - GPC1 y (aperture):
- Median: 0.15
- Median Absolute Deviation: 0.22
- 1% percentile: -1.1051537322998046
- 99% percentile: 1.1605104827880872

UKIDSS Y (total) - GPC1 y (total):
- Median: -0.08
- Median Absolute Deviation: 0.16
- 1% percentile: -1.1228889083862303
- 99% percentile: 1.4015650177001957

No sources have both Suprime y (aperture) and RCS y (aperture) values.
RCS y (total) - Suprime y (total):
- Median: 0.53
- Median Absolute Deviation: 0.32
- 1% percentile: -1.3908543395996094
- 99% percentile: 2.1912711334228554

VHS Y (aperture) - Suprime y (aperture):
- Median: -0.04
- Median Absolute Deviation: 0.13
- 1% percentile: -5.783206024169922
- 99% percentile: 0.5436531829833986

VHS Y (total) - Suprime y (total):
- Median: -0.01
- Median Absolute Deviation: 0.18
- 1% percentile: -4.1186545753479
- 99% percentile: 1.0063447952270506

UKIDSS Y (aperture) - Suprime y (aperture):
- Median: 0.08
- Median Absolute Deviation: 0.13
- 1% percentile: -1.0824210357666015
- 99% percentile: 0.6076422119140625

UKIDSS Y (total) - Suprime y (total):
- Median: 0.10
- Median Absolute Deviation: 0.15
- 1% percentile: -1.2867385101318358
- 99% percentile: 1.1423934173583983

No sources have both RCS y (aperture) and VHS Y (aperture) values.
VHS Y (total) - RCS y (total):
- Median: -0.47
- Median Absolute Deviation: 0.31
- 1% percentile: -3.0394248962402344
- 99% percentile: 0.746319580078125

No sources have both RCS y (aperture) and UKIDSS Y (aperture) values.
UKIDSS Y (total) - RCS y (total):
- Median: -0.39
- Median Absolute Deviation: 0.29
- 1% percentile: -2.155640106201172
- 99% percentile: 0.8123551559448239

No sources have both VHS Y (aperture) and UKIDSS Y (aperture) values.
No sources have both VHS Y (total) and UKIDSS Y (total) values.
UKIDSS J (aperture) - VHS J (aperture):
- Median: 0.03
- Median Absolute Deviation: 0.07
- 1% percentile: -0.5630474090576172
- 99% percentile: 0.544174575805664

UKIDSS J (total) - VHS J (total):
- Median: 0.09
- Median Absolute Deviation: 0.15
- 1% percentile: -0.8497722434997559
- 99% percentile: 1.2258838081359853

VICS82 J (aperture) - VHS J (aperture):
- Median: 0.11
- Median Absolute Deviation: 0.14
- 1% percentile: -0.514088249206543
- 99% percentile: 1.2630663871765173

VICS82 J (total) - VHS J (total):
- Median: -0.07
- Median Absolute Deviation: 0.19
- 1% percentile: -1.1885685729980469
- 99% percentile: 1.122412071228025

VICS82 J (aperture) - UKIDSS J (aperture):
- Median: 0.10
- Median Absolute Deviation: 0.13
- 1% percentile: -0.5394633865356445
- 99% percentile: 1.7428174400329572

VICS82 J (total) - UKIDSS J (total):
- Median: -0.18
- Median Absolute Deviation: 0.17
- 1% percentile: -1.2610341644287109
- 99% percentile: 1.0374440765380846

UKIDSS H (aperture) - VHS H (aperture):
- Median: -0.02
- Median Absolute Deviation: 0.07
- 1% percentile: -0.7017165756225586
- 99% percentile: 0.3929036521911624

UKIDSS H (total) - VHS H (total):
- Median: 0.01
- Median Absolute Deviation: 0.16
- 1% percentile: -1.1395844078063966
- 99% percentile: 1.1315725898742681

UKIDSS K (aperture) - VHS K (aperture):
- Median: 0.01
- Median Absolute Deviation: 0.09
- 1% percentile: -0.6475127029418946
- 99% percentile: 0.4219000053405762

UKIDSS K (total) - VHS K (total):
- Median: 0.06
- Median Absolute Deviation: 0.18
- 1% percentile: -1.1438078880310059
- 99% percentile: 1.1813421249389648

VICS82 K (aperture) - VHS K (aperture):
- Median: -0.00
- Median Absolute Deviation: 0.13
- 1% percentile: -0.6133130645751953
- 99% percentile: 0.9901255798339845

VICS82 K (total) - VHS K (total):
- Median: -0.12
- Median Absolute Deviation: 0.18
- 1% percentile: -1.1994578552246093
- 99% percentile: 1.008749771118168

VICS82 K (aperture) - UKIDSS K (aperture):
- Median: 0.02
- Median Absolute Deviation: 0.13
- 1% percentile: -0.5552535057067871
- 99% percentile: 1.2199306488037163

VICS82 K (total) - UKIDSS K (total):
- Median: -0.16
- Median Absolute Deviation: 0.19
- 1% percentile: -1.3577183532714843
- 99% percentile: 1.186549072265621

III - Comparing magnitudes to reference bands

Cross-match the master list to SDSS and 2MASS to compare its magnitudes to SDSS and 2MASS ones.

master_catalogue_coords = SkyCoord(master_catalogue['ra'], master_catalogue['dec'])

III.a - Comparing u, g, r, i, and z bands to SDSS

The catalogue is cross-matched to SDSS-DR13 withing 0.2 arcsecond.

We compare the u, g, r, i, and z magnitudes to those from SDSS using fiberMag for the aperture magnitude and petroMag for the total magnitude.

sdss = Table.read("../../dmu0/dmu0_SDSS-DR13/data/SDSS-DR13_Herschel-Stripe-82.fits")
sdss_coords = SkyCoord(sdss['ra'] * u.deg, sdss['dec'] * u.deg)

idx, d2d, _ = sdss_coords.match_to_catalog_sky(master_catalogue_coords)
mask = (d2d < 0.2 * u.arcsec)

sdss = sdss[mask]
ml_sdss_idx = idx[mask]

for band_of_a_kind in [u_bands, g_bands, r_bands, i_bands, z_bands]:
    for band in band_of_a_kind:
        
        sdss_mag_ap_colname = "fiberMag_{}".format(band[-1])
        sdss_mag_tot_colname = "petroMag_{}".format(band[-1])
        master_cat_mag_ap_colname = "m_ap_{}".format(band.replace(" ", "_").lower())
        master_cat_mag_tot_colname = "m_{}".format(band.replace(" ", "_").lower())
        
        # Set SDSS magnitudes to NaN when the magnitude  is negative as SDSS uses large
        # negative numbers for missing magnitudes
        sdss[sdss_mag_ap_colname][sdss[sdss_mag_ap_colname] < 0.] = np.nan
        sdss[sdss_mag_tot_colname][sdss[sdss_mag_tot_colname] < 0.] = np.nan
  

        sdss_mag_ap = sdss[sdss_mag_ap_colname]
        master_cat_mag_ap = master_catalogue[master_cat_mag_ap_colname][ml_sdss_idx]
    
        nb_compare_mags(sdss_mag_ap, master_cat_mag_ap,
                            labels=("SDSS (Main survey DR13) {} (fiberMag)".format(band[-1]), "{} (aperture)".format(band)))
    
        sdss_mag_tot = sdss[sdss_mag_tot_colname]
        master_cat_mag_tot = master_catalogue[master_cat_mag_tot_colname][ml_sdss_idx]
        
        nb_compare_mags(sdss_mag_tot, master_cat_mag_tot,
                        labels=("SDSS (Main survey DR13) {} (petroMag)".format(band[-1]), "{} (total)".format(band)))

SDSS u (aperture) - SDSS (Main survey DR13) u (fiberMag):
- Median: 0.69
- Median Absolute Deviation: 0.46
- 1% percentile: -0.9915702382812488
- 99% percentile: 4.199792940368658

SDSS u (total) - SDSS (Main survey DR13) u (petroMag):
- Median: 0.21
- Median Absolute Deviation: 0.74
- 1% percentile: -3.8369227404785153
- 99% percentile: 4.479896454772954

SDSS g (aperture) - SDSS (Main survey DR13) g (fiberMag):
- Median: 0.31
- Median Absolute Deviation: 0.15
- 1% percentile: -0.47871348526000956
- 99% percentile: 1.2509398686218285

SDSS g (total) - SDSS (Main survey DR13) g (petroMag):
- Median: -0.04
- Median Absolute Deviation: 0.22
- 1% percentile: -3.2007564193725573
- 99% percentile: 1.5934776985168446

/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:12: RuntimeWarning: invalid value encountered in less
/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/ipykernel/__main__.py:13: RuntimeWarning: invalid value encountered in less

DECam g (aperture) - SDSS (Main survey DR13) g (fiberMag):
- Median: -0.45
- Median Absolute Deviation: 0.16
- 1% percentile: -1.2813874244689942
- 99% percentile: 1.168358058929444

DECam g (total) - SDSS (Main survey DR13) g (petroMag):
- Median: -0.10
- Median Absolute Deviation: 0.24
- 1% percentile: -3.305940704345703
- 99% percentile: 2.042019958496093

GPC1 g (aperture) - SDSS (Main survey DR13) g (fiberMag):
- Median: -0.56
- Median Absolute Deviation: 0.26
- 1% percentile: -2.765476360321045
- 99% percentile: 1.6859664916992188

GPC1 g (total) - SDSS (Main survey DR13) g (petroMag):
- Median: -0.11
- Median Absolute Deviation: 0.27
- 1% percentile: -4.2331067657470705
- 99% percentile: 1.6772725105285626

Suprime g (aperture) - SDSS (Main survey DR13) g (fiberMag):
- Median: -0.33
- Median Absolute Deviation: 0.15
- 1% percentile: -1.1898948669433593
- 99% percentile: 1.161732482910156

Suprime g (total) - SDSS (Main survey DR13) g (petroMag):
- Median: -0.03
- Median Absolute Deviation: 0.27
- 1% percentile: -3.135676040649414
- 99% percentile: 1.8215815734863274

No sources have both SDSS (Main survey DR13) g (fiberMag) and RCS g (aperture) values.
RCS g (total) - SDSS (Main survey DR13) g (petroMag):
- Median: -0.29
- Median Absolute Deviation: 0.28
- 1% percentile: -3.6734525299072267
- 99% percentile: 1.660163574218752

SDSS r (aperture) - SDSS (Main survey DR13) r (fiberMag):
- Median: 0.22
- Median Absolute Deviation: 0.12
- 1% percentile: -0.4703202220153814
- 99% percentile: 0.7856137927246094

SDSS r (total) - SDSS (Main survey DR13) r (petroMag):
- Median: -0.04
- Median Absolute Deviation: 0.14
- 1% percentile: -2.531239594879151
- 99% percentile: 1.0697426069641118

DECam r (aperture) - SDSS (Main survey DR13) r (fiberMag):
- Median: -0.55
- Median Absolute Deviation: 0.13
- 1% percentile: -1.2494139862060547
- 99% percentile: 0.7610930252075252

DECam r (total) - SDSS (Main survey DR13) r (petroMag):
- Median: -0.19
- Median Absolute Deviation: 0.17
- 1% percentile: -2.980473175048828
- 99% percentile: 1.074421691894531

GPC1 r (aperture) - SDSS (Main survey DR13) r (fiberMag):
- Median: -0.40
- Median Absolute Deviation: 0.15
- 1% percentile: -1.664783935546875
- 99% percentile: 0.7864620971679673

GPC1 r (total) - SDSS (Main survey DR13) r (petroMag):
- Median: 0.05
- Median Absolute Deviation: 0.15
- 1% percentile: -2.5348509216308592
- 99% percentile: 1.213526039123535

Suprime r (aperture) - SDSS (Main survey DR13) r (fiberMag):
- Median: -0.34
- Median Absolute Deviation: 0.10
- 1% percentile: -1.040871810913086
- 99% percentile: 0.6163643646240211

Suprime r (total) - SDSS (Main survey DR13) r (petroMag):
- Median: -0.06
- Median Absolute Deviation: 0.19
- 1% percentile: -2.7978500366210937
- 99% percentile: 1.2967376708984382

No sources have both SDSS (Main survey DR13) r (fiberMag) and RCS r (aperture) values.
RCS r (total) - SDSS (Main survey DR13) r (petroMag):
- Median: -0.18
- Median Absolute Deviation: 0.16
- 1% percentile: -2.9737467193603515
- 99% percentile: 1.1879370880126947

SDSS i (aperture) - SDSS (Main survey DR13) i (fiberMag):
- Median: 0.18
- Median Absolute Deviation: 0.12
- 1% percentile: -0.5666089828491205
- 99% percentile: 0.7385578533935562

SDSS i (total) - SDSS (Main survey DR13) i (petroMag):
- Median: -0.04
- Median Absolute Deviation: 0.13
- 1% percentile: -2.995337109832765
- 99% percentile: 1.0213562768554678

GPC1 i (aperture) - SDSS (Main survey DR13) i (fiberMag):
- Median: -0.39
- Median Absolute Deviation: 0.11
- 1% percentile: -1.1544958877563476
- 99% percentile: 0.2978730392456064

GPC1 i (total) - SDSS (Main survey DR13) i (petroMag):
- Median: 0.05
- Median Absolute Deviation: 0.12
- 1% percentile: -2.4359721183776855
- 99% percentile: 0.9707852172851563

Suprime i (aperture) - SDSS (Main survey DR13) i (fiberMag):
- Median: -0.38
- Median Absolute Deviation: 0.10
- 1% percentile: -1.0213645935058593
- 99% percentile: 0.7098453521728619

Suprime i (total) - SDSS (Main survey DR13) i (petroMag):
- Median: -0.12
- Median Absolute Deviation: 0.18
- 1% percentile: -3.1270497322082518
- 99% percentile: 1.3416518783569336

No sources have both SDSS (Main survey DR13) i (fiberMag) and RCS i (aperture) values.
RCS i (total) - SDSS (Main survey DR13) i (petroMag):
- Median: -0.20
- Median Absolute Deviation: 0.17
- 1% percentile: -3.412097816467285
- 99% percentile: 1.129906044006348

SDSS z (aperture) - SDSS (Main survey DR13) z (fiberMag):
- Median: 0.25
- Median Absolute Deviation: 0.19
- 1% percentile: -0.7316756486816389
- 99% percentile: 1.5328369016113332

SDSS z (total) - SDSS (Main survey DR13) z (petroMag):
- Median: -0.03
- Median Absolute Deviation: 0.31
- 1% percentile: -3.515508124420166
- 99% percentile: 2.0787191370544447

DECam z (aperture) - SDSS (Main survey DR13) z (fiberMag):
- Median: -0.38
- Median Absolute Deviation: 0.19
- 1% percentile: -1.392816925048828
- 99% percentile: 1.9866805076599126

DECam z (total) - SDSS (Main survey DR13) z (petroMag):
- Median: -0.13
- Median Absolute Deviation: 0.32
- 1% percentile: -3.8523220825195312
- 99% percentile: 1.8408506774902273

GPC1 z (aperture) - SDSS (Main survey DR13) z (fiberMag):
- Median: -0.30
- Median Absolute Deviation: 0.18
- 1% percentile: -1.4519528579711913
- 99% percentile: 1.1004284858703666

GPC1 z (total) - SDSS (Main survey DR13) z (petroMag):
- Median: 0.12
- Median Absolute Deviation: 0.26
- 1% percentile: -3.6426555824279783
- 99% percentile: 1.6995499610900824

Suprime z (aperture) - SDSS (Main survey DR13) z (fiberMag):
- Median: -0.26
- Median Absolute Deviation: 0.18
- 1% percentile: -1.3362243652343748
- 99% percentile: 1.096382350921631

Suprime z (total) - SDSS (Main survey DR13) z (petroMag):
- Median: -0.01
- Median Absolute Deviation: 0.37
- 1% percentile: -3.7131628036499023
- 99% percentile: 1.8880386352539036

No sources have both SDSS (Main survey DR13) z (fiberMag) and RCS z (aperture) values.
RCS z (total) - SDSS (Main survey DR13) z (petroMag):
- Median: -0.10
- Median Absolute Deviation: 0.37
- 1% percentile: -3.960337562561035
- 99% percentile: 2.035121383666991

III.b - Comparing J and K bands to 2MASS

The catalogue is cross-matched to 2MASS-PSC withing 0.2 arcsecond.

We compare the UKIDSS total J and K magnitudes to those from 2MASS.

twomass = Table.read("../../dmu0/dmu0_2MASS-point-sources/data/2MASS-PSC_Herschel-Stripe-82.fits")
twomass_coords = SkyCoord(twomass['raj2000'], twomass['dej2000'])

idx, d2d, _ = twomass_coords.match_to_catalog_sky(master_catalogue_coords)
mask = (d2d < 0.2 * u.arcsec)

twomass = twomass[mask]
ml_twomass_idx = idx[mask]

nb_compare_mags(twomass['jmag'], master_catalogue['m_ukidss_j'][ml_twomass_idx],
                labels=("2MASS J", "UKIDSS LAS J (total)"))

UKIDSS LAS J (total) - 2MASS J:
- Median: 0.94
- Median Absolute Deviation: 0.05
- 1% percentile: 0.11236026573181146
- 99% percentile: 1.4694202251434318

nb_compare_mags(twomass['kmag'], master_catalogue['m_ukidss_k'][ml_twomass_idx],
                labels=("2MASS K", "UKIDSS LAS K (total)"))

UKIDSS LAS K (total) - 2MASS K:
- Median: 1.95
- Median Absolute Deviation: 0.08
- 1% percentile: 1.0191856248474123
- 99% percentile: 2.7435803141784705

IV - Comparing aperture magnitudes to total ones.

nb_ccplots(
    master_catalogue['m_rcs_r'],
    master_catalogue['m_ap_rcs_r'] - master_catalogue['m_rcs_r'],
    "r total magnitude (CFHT)", "r aperture mag - total mag (CFHT)",
    master_catalogue["stellarity"],
    invert_x=True
)

Number of source used: 0 / 4924329 (0.00%)
HELP warning: no sources with observations in both bands

V - Color-color and magnitude-color plots

nb_ccplots(
    master_catalogue['m_gpc1_g'] - master_catalogue['m_gpc1_i'],
    master_catalogue['m_ukidss_j'] - master_catalogue['m_ukidss_k'],
    "g - i (PanSTARRS)", "J - K (UKIDSS LAS)",
    master_catalogue["stellarity"]
)

Number of source used: 112069 / 4924329 (2.28%)

nb_ccplots(
    master_catalogue['m_gpc1_i'] - master_catalogue['m_irac_i1'],
    master_catalogue['m_gpc1_g'] - master_catalogue['m_gpc1_i'],
    "PanSTARRS i - IRAC1", "g - i (PanSTARRS)",
    master_catalogue["stellarity"]
)

Number of source used: 132885 / 4924329 (2.70%)

nb_ccplots(
    master_catalogue['m_rcs_g'] - master_catalogue['m_rcs_i'],
    master_catalogue['m_rcs_r'] - master_catalogue['m_rcs_y'],
    "g - i (CFHT)", "r - y (CFHT)",
    master_catalogue["stellarity"]
)

Number of source used: 0 / 4924329 (0.00%)
HELP warning: no sources with observations in both bands

nb_ccplots(
    master_catalogue['m_ukidss_j'] - master_catalogue['m_ukidss_k'],
    master_catalogue['m_gpc1_g'] - master_catalogue['m_ukidss_j'],
    "J - K (UKIDSS LAS)", "g - J (PanSTARRS, UKIDSS)",
    master_catalogue["stellarity"]
)

Number of source used: 112253 / 4924329 (2.28%)

nb_ccplots(
    master_catalogue['m_gpc1_i'] - master_catalogue['m_gpc1_z'],
    master_catalogue['m_gpc1_z'] - master_catalogue['m_ukidss_j'],
    "i - z (PanSTARRS)", "z - J (PanSTARRS, UKIDSS LAS)",
    master_catalogue["stellarity"]
)

Number of source used: 133599 / 4924329 (2.71%)

nb_ccplots(
    master_catalogue['m_gpc1_i'] - master_catalogue['m_gpc1_z'],
    master_catalogue['m_irac_i1'] - master_catalogue['m_irac_i2'],
    "i - z (PanSTARRS)", "IRAC1 - IRAC2",
    master_catalogue["stellarity"]
)

Number of source used: 138070 / 4924329 (2.80%)