This notebook collates the final output files ready for writing to csv for ingestion to a VO server. At the bottom of the notebook we also summarise the pipeline products which are processed on a given field. This are generated using the dmu32 meta_main.yml files which contain links to the XID+, CIGALE and photo-z catalogues which feed in to the final catalogues for publishing.
Summary of notebook:
from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
import datetime
print("This notebook was executed on: \n{}".format(datetime.datetime.now()))
from astropy.table import Table, Column
from astropy import units as u
import numpy as np
from pymoc import MOC
from herschelhelp_internal.masterlist import find_last_ml_suffix
import yaml
import os
import time
import humanfriendly
TODAY = os.environ.get('SUFFIX', time.strftime("_%Y%m%d"))
Here we take the DR1 definition from dmu1. We then propagate those suffixes through to dmu32. Note that this does not include a specification of how the masterlist made its way through the whole HELP process.
dr1 = Table.read("../dmu1/dr1_overview.fits")
dr1.show_in_notebook()
GAVO_FOLDER = '/mnt/hedam/data_vo/'
stilts_command = 'stilts tpipe {in_file} omode=out ofmt=csv out={GAVO_FOLDER}{out_file}'
final_data = open('help_to_vo.sh', 'w+')
for field in dr1:
final_help_product = './dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1])
cigale_input = './dmu32_{}/data/{}_{}_cigale.fits'.format(field[0], field[0], field[1])
if os.path.exists(final_help_product):
print(final_help_product)
#Test with Cigale input files
final_data.write(stilts_command.format(in_file=final_help_product,
GAVO_FOLDER=GAVO_FOLDER,
out_file='herschelhelp/main/{}_{}_all.csv \n'.format(field[0],
field[1])))
elif os.path.exists(cigale_input):
print(cigale_input)
final_data.write(stilts_command.format(in_file=cigale_input,
GAVO_FOLDER=GAVO_FOLDER,
out_file='herschelhelp/main/{}_{}_incomplete.csv \n'.format(field[0],
field[1])))
else:
final_data.write('# No data for {} \n'.format(field[0]))
#final_data.write('./dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1]))
final_data.close()
The out put of this notebook is a shell script which will write all the fits files to csv files in the vo folder
depths_to_vo = open('depths_to_vo.sh', 'w+')
for field in dr1:
final_depth_product = '../dmu1/dmu1_ml_{}/data/depths_{}_{}.fits'.format(field[0], field[0].lower(), field[1])
if os.path.exists(final_depth_product):
print(final_depth_product)
#Test with Cigale input files
depths_to_vo.write(stilts_command.format(in_file=final_depth_product,
GAVO_FOLDER=GAVO_FOLDER,
out_file='depth/{}_{}.csv \n'.format(field[0],
field[1])))
else:
depths_to_vo.write('# No depths for {} \n'.format(field[0]))
#final_data.write('./dmu32_{}/data/{}_{}.fits'.format(field[0], field[0], field[1]))
depths_to_vo.close()
Here we get information about what is available on each field to summarise the data products available per field. We take the cigale, xid+ and photo-z filenames from the per field meta_main.yml files here and check they are there and how large they are. This then given a summary of all the data present.
dr1_data_products = dr1.copy()
fields_info = yaml.load(open("../dmu2/meta_main.yml", 'r'))
dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'xid_objects'))
dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'photoz_objects'))
dr1_data_products.add_column(Column(data =np.full(len(dr1_data_products), 0, dtype=int), name = 'cigale_objects'))
for field in fields_info['fields']:
print(field['name'] + ':')
xid_objects = 0
photoz_objects = 0
cigale_objects = 0
dmu32_info = yaml.load(open('./dmu32_{}/meta_main.yml'.format(field['name'].replace("HATLAS-", "")), 'r'))
try:
for n, xid_file in enumerate(dmu32_info['xid']):
print(n,xid_file,xid_file.replace('dmu_products', '..'))
xid_objects = len(Table.read(xid_file.replace('dmu_products', '..')))
print(" - xid: {}".format(dmu32_info['xid'][n]))
except FileNotFoundError:
print(" - xid not done.")
try:
#print(dmu32_info['photoz'].replace('dmu_products', '..'))
photoz_objects = len(Table.read(dmu32_info['photoz'].replace('dmu_products', '..')))
print(" - photoz: {}".format(dmu32_info['photoz']))
except FileNotFoundError:
print(" - photoz not done.")
try:
cigale_objects = len(Table.read(dmu32_info['cigale'].replace('dmu_products', '..')))
print(" - cigale: {}".format(dmu32_info['cigale']))
except FileNotFoundError:
print(" - cigale not done.")
dr1_data_products['xid_objects'][dr1_data_products['field'] == field['name'].replace("HATLAS-", "")] = xid_objects
dr1_data_products['photoz_objects'][dr1_data_products['field'] == field['name'].replace("HATLAS-", "")] = photoz_objects
dr1_data_products['cigale_objects'][dr1_data_products['field'] == field['name'].replace("HATLAS-", "")] = cigale_objects
dr1_data_products.show_in_notebook()
has_xid = dr1_data_products['xid_objects'] != 0
has_photoz = dr1_data_products['photoz_objects'] != 0
has_cigale = dr1_data_products['cigale_objects'] != 0
print("XID+: There are {} fields with XID+ fluxes computed totalling {area:.2f} square degrees.".format(
np.sum(has_xid),
area=np.sum(dr1_data_products['area_sq_degrees'][has_xid])))
print(" This is a total of {} objects with an average of {perc:.0f}% of objects on processed areas.".format(
np.sum(dr1_data_products['xid_objects'][has_xid]),
perc=100 * np.sum(dr1_data_products['xid_objects'][has_xid])/np.sum(dr1_data_products['objects'][has_xid])))
print("Photo-zs: There are {} fields with photozs computed totalling {area:.2f} square degrees.".format(
np.sum(has_photoz),
area=np.sum(dr1_data_products['area_sq_degrees'][has_photoz])))
print(" This is a total of {} objects with an average of {perc:.0f}% of objects on processed areas.".format(
np.sum(dr1_data_products['photoz_objects'][has_photoz]),
perc=100 * np.sum(dr1_data_products['photoz_objects'][has_photoz])/np.sum(dr1_data_products['objects'][has_photoz])))
print("CIGALE SEDs: There are {} fields with SEDs computed totalling {area:.2f} square degrees.".format(
np.sum(has_cigale),
area=np.sum(dr1_data_products['area_sq_degrees'][has_cigale])))
print(" This is a total of {} objects with an average of {perc:.0f}% of objects on processed areas.".format(
np.sum(dr1_data_products['cigale_objects'][has_cigale]),
perc=100 * np.sum(dr1_data_products['cigale_objects'][has_cigale])/np.sum(dr1_data_products['objects'][has_cigale])))
dr1_data_products.write('dr1_data_products_overview{}.csv'.format(TODAY), overwrite=True)