{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ingesting Private Datasets (v2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# imports\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import h5py\n",
    "import specdb\n",
    "import glob\n",
    "\n",
    "from astropy.table import Table\n",
    "from linetools import utils as ltu\n",
    "\n",
    "from specdb.build import privatedb as pbuild\n",
    "from specdb.build import utils as spbu\n",
    "from specdb.specdb import IgmSpec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test on Single Folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tree = specdb.__path__[0]+'/build/tests/files'\n",
    "#os.getenv('DROPBOX_DIR')+'/QSOPairs/data/MMT_redux/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(pbuild)\n",
    "flux_files = pbuild.grab_files(tree)\n",
    "len(flux_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([u'/Users/xavier/local/Python/specdb/specdb/build/tests/files//./SDSSJ001605.89+005654.3_b800_F.fits.gz',\n",
       "  u'/Users/xavier/local/Python/specdb/specdb/build/tests/files//./SDSSJ001607.27+005653.1_b800_F.fits.gz'],\n",
       " None,\n",
       " None)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flux_files[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Directory Tree -- Step by Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "tree2 = specdb.__path__[0]+'/data/test_privateDB'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "branches = glob.glob(tree2+'/*')\n",
    "branches[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get started"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "id_key = 'TEST_ID'\n",
    "maindb, tkeys = spbu.start_maindb(id_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reload(pbuild)\n",
    "mflux_files, meta_file, _ = pbuild.grab_files(branches[0])\n",
    "len(mflux_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS//./J095240.17+515250.03.fits.gz',\n",
       " u'/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS//./J095243.05+515121.15.fits.gz']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mflux_files[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS/COS_meta.json'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'maxpix': 60000,\n",
       " u'meta_dict': {u'TELESCOPE': u'HST'},\n",
       " u'parse_head': {u'DATE-OBS': u'DATE',\n",
       "  u'GRATING': u'OPT_ELEM',\n",
       "  u'INSTR': u'INSTRUME',\n",
       "  u'R': True}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_dict = ltu.loadjson(meta_file)\n",
    "meta_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ztbl  (read from file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<Table length=6>\n",
       "<table id=\"table4652212112\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>RA</th><th>DEC</th><th>ZEM</th><th>ZEM_SOURCE</th><th>SPEC_FILE</th></tr></thead>\n",
       "<thead><tr><th>float64</th><th>float64</th><th>float64</th><th>str5</th><th>str35</th></tr></thead>\n",
       "<tr><td>331.992916667</td><td>12.9956388889</td><td>1.0</td><td>UNKNW</td><td>SDSSJ220758.30+125944.3_F.fits</td></tr>\n",
       "<tr><td>261.35275</td><td>30.6344166667</td><td>1.1</td><td>UNKNW</td><td>SDSSJ172524.66+303803.9_F.fits</td></tr>\n",
       "<tr><td>345.184833333</td><td>1.92825</td><td>1.2</td><td>UNKNW</td><td>SDSSJ230044.36+015541.7_r600_F.fits</td></tr>\n",
       "<tr><td>345.184833333</td><td>1.92825</td><td>1.2</td><td>UNKNW</td><td>SDSSJ230044.36+015541.7_b400_F.fits</td></tr>\n",
       "<tr><td>148.167375</td><td>51.8805638889</td><td>1.3</td><td>UNKNW</td><td>J095240.17+515250.03.fits.gz</td></tr>\n",
       "<tr><td>148.179375</td><td>51.855875</td><td>1.4</td><td>UNKNW</td><td>J095243.05+515121.15.fits.gz</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=6>\n",
       "      RA           DEC      ... ZEM_SOURCE              SPEC_FILE             \n",
       "   float64       float64    ...    str5                   str35               \n",
       "------------- ------------- ... ---------- -----------------------------------\n",
       "331.992916667 12.9956388889 ...      UNKNW      SDSSJ220758.30+125944.3_F.fits\n",
       "    261.35275 30.6344166667 ...      UNKNW      SDSSJ172524.66+303803.9_F.fits\n",
       "345.184833333       1.92825 ...      UNKNW SDSSJ230044.36+015541.7_r600_F.fits\n",
       "345.184833333       1.92825 ...      UNKNW SDSSJ230044.36+015541.7_b400_F.fits\n",
       "   148.167375 51.8805638889 ...      UNKNW        J095240.17+515250.03.fits.gz\n",
       "   148.179375     51.855875 ...      UNKNW        J095243.05+515121.15.fits.gz"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ztbl = Table.read(specdb.__path__[0]+'/data/test_privateDB/testDB_ztbl.fits')\n",
    "ztbl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reload(pbuild)\n",
    "meta = pbuild.mk_meta(mflux_files, ztbl, fname=True, mdict=meta_dict['meta_dict'], parse_head=meta_dict['parse_head'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table length=2&gt;\n",
       "<table id=\"table4652211664\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>RA_GROUP</th><th>DEC_GROUP</th><th>STYPE</th><th>zem_GROUP</th><th>sig_zem</th><th>flag_zem</th><th>SPEC_FILE</th><th>DATE-OBS</th><th>GRATING</th><th>R</th><th>INSTR</th><th>TELESCOPE</th><th>EPOCH</th><th>GROUP_ID</th></tr></thead>\n",
       "<thead><tr><th>float64</th><th>float64</th><th>str3</th><th>float64</th><th>float64</th><th>str8</th><th>unicode96</th><th>str10</th><th>str5</th><th>float64</th><th>str3</th><th>unicode3</th><th>float64</th><th>int64</th></tr></thead>\n",
       "<tr><td>148.167375</td><td>51.8805638889</td><td>QSO</td><td>1.3</td><td>0.0</td><td>UNKNW</td><td>/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS//./J095240.17+515250.03.fits.gz</td><td>2015-05-31</td><td>G130M</td><td>17000.0</td><td>COS</td><td>HST</td><td>2000.0</td><td>0</td></tr>\n",
       "<tr><td>148.179375</td><td>51.855875</td><td>QSO</td><td>1.4</td><td>0.0</td><td>UNKNW</td><td>/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS//./J095243.05+515121.15.fits.gz</td><td>2015-12-08</td><td>G130M</td><td>17000.0</td><td>COS</td><td>HST</td><td>2000.0</td><td>1</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=2>\n",
       " RA_GROUP    DEC_GROUP   STYPE zem_GROUP ... INSTR TELESCOPE  EPOCH  GROUP_ID\n",
       " float64      float64     str3  float64  ...  str3  unicode3 float64  int64  \n",
       "---------- ------------- ----- --------- ... ----- --------- ------- --------\n",
       "148.167375 51.8805638889   QSO       1.3 ...   COS       HST  2000.0        0\n",
       "148.179375     51.855875   QSO       1.4 ...   COS       HST  2000.0        1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta[0:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without fname=True\n",
    "    Requires SPEC_FILE in ztbl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "meta2 = pbuild.mk_meta(mflux_files, ztbl, fname=False, mdict=meta_dict['meta_dict'], parse_head=meta_dict['parse_head'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table length=2&gt;\n",
       "<table id=\"table4653773904\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>RA_GROUP</th><th>DEC_GROUP</th><th>STYPE</th><th>zem_GROUP</th><th>sig_zem</th><th>flag_zem</th><th>SPEC_FILE</th><th>DATE-OBS</th><th>GRATING</th><th>R</th><th>INSTR</th><th>TELESCOPE</th><th>EPOCH</th><th>GROUP_ID</th></tr></thead>\n",
       "<thead><tr><th>float64</th><th>float64</th><th>str3</th><th>float64</th><th>float64</th><th>str8</th><th>unicode96</th><th>str10</th><th>str5</th><th>float64</th><th>str3</th><th>unicode3</th><th>float64</th><th>int64</th></tr></thead>\n",
       "<tr><td>148.167375</td><td>51.8805638889</td><td>QSO</td><td>1.3</td><td>0.0</td><td>UNKNW</td><td>/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS//./J095240.17+515250.03.fits.gz</td><td>2015-05-31</td><td>G130M</td><td>17000.0</td><td>COS</td><td>HST</td><td>2000.0</td><td>0</td></tr>\n",
       "<tr><td>148.179375</td><td>51.855875</td><td>QSO</td><td>1.4</td><td>0.0</td><td>UNKNW</td><td>/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS//./J095243.05+515121.15.fits.gz</td><td>2015-12-08</td><td>G130M</td><td>17000.0</td><td>COS</td><td>HST</td><td>2000.0</td><td>1</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=2>\n",
       " RA_GROUP    DEC_GROUP   STYPE zem_GROUP ... INSTR TELESCOPE  EPOCH  GROUP_ID\n",
       " float64      float64     str3  float64  ...  str3  unicode3 float64  int64  \n",
       "---------- ------------- ----- --------- ... ----- --------- ------- --------\n",
       "148.167375 51.8805638889   QSO       1.3 ...   COS       HST  2000.0        0\n",
       "148.179375     51.855875   QSO       1.4 ...   COS       HST  2000.0        1"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add Group and IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... INSTR TELESCOPE EPOCH GROUP_ID\n",
      "-------- --------- ----- --------- ------- ... ----- --------- ----- --------\n"
     ]
    }
   ],
   "source": [
    "gdict = {}\n",
    "flag_g = spbu.add_to_group_dict('COS', gdict)\n",
    "maindb = pbuild.add_ids(maindb, meta, flag_g, tkeys, id_key, first=(flag_g==1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table length=2&gt;\n",
       "<table id=\"table4585864144\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>flag_group</th><th>sig_zem</th><th>flag_zem</th><th>RA</th><th>DEC</th><th>STYPE</th><th>zem</th><th>TEST_ID</th></tr></thead>\n",
       "<thead><tr><th>int64</th><th>float64</th><th>str8</th><th>float64</th><th>float64</th><th>str3</th><th>float64</th><th>int64</th></tr></thead>\n",
       "<tr><td>1</td><td>0.0</td><td>UNKNW</td><td>148.167375</td><td>51.8805638889</td><td>QSO</td><td>1.3</td><td>0</td></tr>\n",
       "<tr><td>1</td><td>0.0</td><td>UNKNW</td><td>148.179375</td><td>51.855875</td><td>QSO</td><td>1.4</td><td>1</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=2>\n",
       "flag_group sig_zem flag_zem     RA          DEC      STYPE   zem   TEST_ID\n",
       "  int64    float64   str8    float64      float64     str3 float64  int64 \n",
       "---------- ------- -------- ---------- ------------- ----- ------- -------\n",
       "         1     0.0    UNKNW 148.167375 51.8805638889   QSO     1.3       0\n",
       "         1     0.0    UNKNW 148.179375     51.855875   QSO     1.4       1"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "maindb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'COS': 1}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spectra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "hdf = h5py.File('tmp.hdf5','w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adding test group to DB\n"
     ]
    }
   ],
   "source": [
    "reload(pbuild)\n",
    "pbuild.ingest_spectra(hdf, 'test', meta, max_npix=meta_dict['maxpix'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Finish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pbuild.write_hdf(hdf, 'TEST_DB', maindb, [str('SDSS')], gdict, 'v01')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Directory Tree -- All in One"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ztbl = Table.read(specdb.__path__[0]+'/data/test_privateDB/testDB_ztbl.fits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on branch: /Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS\n",
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... INSTR TELESCOPE EPOCH GROUP_ID\n",
      "-------- --------- ----- --------- ------- ... ----- --------- ----- --------\n",
      "Adding COS group to DB\n",
      "Working on branch: /Users/xavier/local/Python/specdb/specdb/data/test_privateDB/ESI\n",
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... GRATING EPOCH GROUP_ID tGRB\n",
      "-------- --------- ----- --------- ------- ... ------- ----- -------- ----\n",
      "Adding ESI group to DB\n",
      "Working on branch: /Users/xavier/local/Python/specdb/specdb/data/test_privateDB/LRIS\n",
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... GRATING TELESCOPE EPOCH GROUP_ID\n",
      "-------- --------- ----- --------- ------- ... ------- --------- ----- --------\n",
      "Adding LRIS group to DB\n",
      "Wrote tmp.hdf5 DB file\n"
     ]
    }
   ],
   "source": [
    "reload(pbuild)\n",
    "pbuild.mk_db('TEST_DB', tree2, 'tmp.hdf5', ztbl, fname=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Working on branch: /Users/xavier/local/Python/specdb/specdb/data/test_privateDB/COS\n",
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... INSTR TELESCOPE EPOCH GROUP_ID\n",
      "-------- --------- ----- --------- ------- ... ----- --------- ----- --------\n",
      "Adding COS group to DB\n",
      "Working on branch: /Users/xavier/local/Python/specdb/specdb/data/test_privateDB/ESI\n",
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... GRATING EPOCH GROUP_ID tGRB\n",
      "-------- --------- ----- --------- ------- ... ------- ----- -------- ----\n",
      "Adding ESI group to DB\n",
      "Working on branch: /Users/xavier/local/Python/specdb/specdb/data/test_privateDB/LRIS\n",
      "The following sources were previously in the DB\n",
      "RA_GROUP DEC_GROUP STYPE zem_GROUP sig_zem ... GRATING TELESCOPE EPOCH GROUP_ID\n",
      "-------- --------- ----- --------- ------- ... ------- --------- ----- --------\n",
      "Adding LRIS group to DB\n",
      "Wrote tmp2.hdf5 DB file\n"
     ]
    }
   ],
   "source": [
    "# Without fname\n",
    "pbuild.mk_db('TEST_DB', tree2, 'tmp2.hdf5', ztbl, fname=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### By script\n",
    "    specdb_privatedb testDB ../../specdb/data/test_privateDB tst3_DB.hdf5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check ESI meta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using tmp2.hdf5 for the DB file\n",
      "Available groups: [u'COS', u'ESI', u'LRIS']\n"
     ]
    }
   ],
   "source": [
    "igmsp = IgmSpec(db_file='tmp2.hdf5', verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "&lt;Table length=2&gt;\n",
       "<table id=\"table4666987408\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>RA_GROUP</th><th>DEC_GROUP</th><th>STYPE</th><th>zem_GROUP</th><th>sig_zem</th><th>flag_zem</th><th>DATE-OBS</th><th>R</th><th>EPOCH</th><th>GROUP_ID</th><th>tGRB</th><th>PRIV_ID</th><th>NPIX</th><th>WV_MIN</th><th>WV_MAX</th><th>SPEC_FILE</th><th>INSTR</th><th>TELESCOPE</th><th>GRATING</th></tr></thead>\n",
       "<thead><tr><th>float64</th><th>float64</th><th>str3</th><th>float64</th><th>float64</th><th>str8</th><th>str10</th><th>float64</th><th>float64</th><th>int64</th><th>str21</th><th>int64</th><th>int64</th><th>float64</th><th>float64</th><th>str98</th><th>str3</th><th>str7</th><th>str3</th></tr></thead>\n",
       "<tr><td>261.3528</td><td>30.6344</td><td>QSO</td><td>1.100</td><td>0.0</td><td>UNKNW</td><td>2015-05-19</td><td>4545.0</td><td>2000.0</td><td>0</td><td>2009-11-23:10:12:13.2</td><td>2</td><td>27931</td><td>3993.5</td><td>10131.6</td><td>/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/ESI//./SDSSJ172524.66+303803.9_F.fits</td><td>ESI</td><td>Keck-II</td><td>ECH</td></tr>\n",
       "<tr><td>331.9929</td><td>12.9956</td><td>QSO</td><td>1.000</td><td>0.0</td><td>UNKNW</td><td>2008-06-04</td><td>4545.0</td><td>2000.0</td><td>1</td><td>2007-08-13:10:22:23.3</td><td>3</td><td>27926</td><td>3993.5</td><td>10129.9</td><td>/Users/xavier/local/Python/specdb/specdb/data/test_privateDB/ESI//./SDSSJ220758.30+125944.3_F.fits</td><td>ESI</td><td>Keck-II</td><td>ECH</td></tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Table length=2>\n",
       "RA_GROUP DEC_GROUP STYPE zem_GROUP ... INSTR TELESCOPE GRATING\n",
       "float64   float64   str3  float64  ...  str3    str7     str3 \n",
       "-------- --------- ----- --------- ... ----- --------- -------\n",
       "261.3528   30.6344   QSO     1.100 ...   ESI   Keck-II     ECH\n",
       "331.9929   12.9956   QSO     1.000 ...   ESI   Keck-II     ECH"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "igmsp['ESI'].meta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## JSON files for meta table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "parse_head = {'DATE-OBS':'DATE', 'TELESCOPE':'TELESCOP','INSTR':'INSTRUME', 'R': True}\n",
    "mdict = dict(GRATING='ALL', R=8000.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "db_dict = dict(parse_head=parse_head, meta_dict=mdict, maxpix=60000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jdict = ltu.jsonify(db_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ltu.savejson('tst.json', jdict, easy_to_read=True, overwrite=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}