documentations/jupyter.notebooks/cdms.demo.2021.12.09.ipynb (1,089 lines of code) (raw):
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### In-situ Parquet\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"![In-situ architecture](./in-situ-architecture.png \"in-situ-architecture\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Today's demo:\n",
"![In-situ architecture Demo](./in-situ-architecture-demo.png \"in-situ-architecture-demo\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import necessary libraries and check if the ingester pod is up"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<Response [200]>"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"a = requests.get(url='http://localhost:9801/1.0/doc')\n",
"a"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Ingesting a new file from S3"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"##### Input Data Format\n",
"```\n",
"{\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"ATOMIC EUREC4A 2020\",\n",
" \"observations\": [\n",
" {\n",
" \"time\": \"2020-01-18T00:00:00Z\",\n",
" \"latitude\": 10.5648704,\n",
" \"longitude\": -55.9281152,\n",
" \"depth\": -5.2,\n",
" \"platform\": {\n",
" \"code\": \"3B\",\n",
" \"id\": \"1026\",\n",
" \"type\": \"saildrone\"\n",
" },\n",
" \"device\": \"101\",\n",
" \"meta\": \"report_payload/payload/1026/2020/01/report_payload-payload-1578754800000-1580515140000.snappy.parquet\",\n",
" \"wind_speed\": 6.84,\n",
" \"wind_speed_quality\": 2,\n",
" \"eastward_wind\": -6.64,\n",
" \"northward_wind\": -2.49,\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 69.4\n",
" },\n",
" {\n",
" \"time\": \"2020-01-18T00:00:00Z\",\n",
" \"latitude\": 10.5648704,\n",
" \"longitude\": -55.9281152,\n",
" \"depth\": 0.5,\n",
" \"platform\": {\n",
" \"code\": \"3B\",\n",
" \"id\": \"1026\",\n",
" \"type\": \"saildrone\"\n",
" },\n",
" \"device\": \"130\",\n",
" \"meta\": \"report_payload/payload/1026/2020/01/report_payload-payload-1578754800000-1580515140000.snappy.parquet\",\n",
" \"sea_water_temperature\": 27.4648,\n",
" \"sea_water_temperature_quality\": 2,\n",
" \"sea_water_salinity\": 35.238,\n",
" \"sea_water_salinity_quality\": 2\n",
" }\n",
" ]\n",
"}\n",
"```"
]
},
{
"cell_type": "markdown",
"source": [
"##### In-situ incoming data S3 file Structure\n",
"\n",
"![In-situ incoming data S3 file Structure](./in-situ-s3-input-data.png \"in-situ-s3-input-data\")"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"s3_url = 's3://cdms-dev-fsu-in-situ-stage/KTDQ_20180730v20001_str.json.gz'\n",
"a = requests.put(url='http://localhost:9801/1.0/ingest_json_s3', headers={'Content-Type':'application/json'}, data=json.dumps({\"s3_url\": s3_url}))\n",
"a.content"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"b'{\"message\": \"Internal Server Error\"}\\n'"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s3_url = 's3://cdms-dev-fsu-in-situ-stage/KTDQ_20180730v20001_str.json.gz'\n",
"a = requests.put(url='http://localhost:9801/1.0/ingest_json_s3', headers={'Content-Type':'application/json'}, data=json.dumps({\"s3_url\": s3_url}))\n",
"a.content"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Sample Response:\n",
"`{\"message\": \"ingested, different sha512\", \"cause\": \"missing S3 sha512\", \"job_id\": \"6e0f3669-4bef-4917-9d40-03e342233ed4\"}`"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Replacing exisiting file from S3"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Using query endpoint similar to DOMS "
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"b'{\"message\": \"Internal Server Error\"}\\n'"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"s3_url = 's3://cdms-dev-fsu-in-situ-stage/KTDQ_20180730v20001_str.json.gz'\n",
"job_id = '8fd1f707-c540-4152-88b8-ea5bc10d738c'\n",
"a = requests.put(url='http://localhost:9801/1.0/replace_json_s3', headers={'Content-Type':'application/json'}, data=json.dumps({\"s3_url\": s3_url, \"job_id\": job_id}))\n",
"a.content"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### Parquet Partitions\n",
"![In-situ Parquet Partition](./in-situ-parquet-partition.png \"in-situ-parquet-partition.png\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### data subsetting API imitating DOMS API"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"http://localhost:9801/1.0/query_data_doms?startIndex=0&itemsPerPage=20&provider=Saildrone&project=atlantic_to_med_2019_to_2020&platform=3B&minDepth=-5.2&maxDepth=-5.1&variable=relative_humidity&startTime=2020-06-01T00:00:00Z&endTime=2020-06-03T00:00:00Z&platform=3B&bbox=14,38.03801,14.04,38.03802\n",
"{\n",
" \"total\": 451,\n",
" \"results\": [\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.96,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336048,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.55,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:02:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 60.2,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.11,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.99,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0336064,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.78,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:11:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 51.8,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.26,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.87,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0336032,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.05,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:12:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 39.6,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.36,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.83,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336064,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.39,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:13:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 30.8,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.62,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -1.08,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336048,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.52,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:14:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 35.4,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.86,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.8,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.32,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:15:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 31.2,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.54,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.39,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.02,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:16:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 20.9,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.09,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.73,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0336,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.88,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:18:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 39.7,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.14,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.69,\n",
" \"latitude\": 38.0380128,\n",
" \"longitude\": 14.0335984,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.83,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:19:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 39.7,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.08,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.76,\n",
" \"latitude\": 38.0380128,\n",
" \"longitude\": 14.0336,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.4,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:20:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 28.5,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.59,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.6,\n",
" \"latitude\": 38.0380128,\n",
" \"longitude\": 14.0335952,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.05,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:24:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 29.7,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.21,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.8,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.8,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:25:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 45.0,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.13,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.58,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336032,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.88,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:26:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 33.4,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.05,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.47,\n",
" \"latitude\": 38.0380192,\n",
" \"longitude\": 14.0336016,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.78,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:27:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 31.1,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 0.91,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.28,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0335984,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.49,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:28:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 29.7,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 0.56,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.38,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0335968,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.52,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:29:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 36.2,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 0.64,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.34,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0335968,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.75,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:30:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 24.4,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 0.82,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.22,\n",
" \"latitude\": 38.0380128,\n",
" \"longitude\": 14.0335984,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.87,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:31:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 14.2,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 0.9,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.62,\n",
" \"latitude\": 38.0380128,\n",
" \"longitude\": 14.0335968,\n",
" \"meta\": null,\n",
" \"northward_wind\": -0.81,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:32:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 37.4,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.02,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" },\n",
" {\n",
" \"air_pressure\": null,\n",
" \"air_pressure_quality\": null,\n",
" \"air_temperature\": null,\n",
" \"air_temperature_quality\": null,\n",
" \"depth\": -5.2,\n",
" \"eastward_wind\": -0.24,\n",
" \"latitude\": 38.038016,\n",
" \"longitude\": 14.0336,\n",
" \"meta\": null,\n",
" \"northward_wind\": -1.47,\n",
" \"platform\": {\n",
" \"type\": \"saildrone\",\n",
" \"code\": \"3B\",\n",
" \"id\": \"1053\"\n",
" },\n",
" \"relative_humidity\": null,\n",
" \"relative_humidity_quality\": null,\n",
" \"time\": \"2020-06-01T17:33:00Z\",\n",
" \"wind_component_quality\": 2,\n",
" \"wind_from_direction\": 9.3,\n",
" \"wind_from_direction_quality\": null,\n",
" \"wind_speed\": 1.49,\n",
" \"wind_speed_quality\": 2,\n",
" \"downwelling_longwave_flux_in_air\": null,\n",
" \"downwelling_longwave_flux_in_air_quality\": null,\n",
" \"downwelling_shortwave_flux_in_air\": null,\n",
" \"downwelling_shortwave_flux_in_air_quality\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air\": null,\n",
" \"surface_downwelling_photosynthetic_photon_flux_in_air_quality\": null,\n",
" \"sea_water_salinity\": null,\n",
" \"sea_water_salinity_quality\": null,\n",
" \"sea_water_temperature\": null,\n",
" \"sea_water_temperature_quality\": null,\n",
" \"provider\": \"Saildrone\",\n",
" \"project\": \"atlantic_to_med_2019_to_2020\",\n",
" \"platform_code\": \"3B\",\n",
" \"job_id\": \"5c60306a-52a6-4f50-b745-22baa13d6363\"\n",
" }\n",
" ],\n",
" \"last\": \"http://localhost:9801/1.0/query_data_doms?startIndex=22&itemsPerPage=20&provider=Saildrone&project=atlantic_to_med_2019_to_2020&platform=3B&minDepth=-5.2&maxDepth=-5.1&variable=relative_humidity&startTime=2020-06-01T00:00:00Z&endTime=2020-06-03T00:00:00Z&bbox=14,38.03801,14.04,38.03802\",\n",
" \"first\": \"http://localhost:9801/1.0/query_data_doms?startIndex=0&itemsPerPage=20&provider=Saildrone&project=atlantic_to_med_2019_to_2020&platform=3B&minDepth=-5.2&maxDepth=-5.1&variable=relative_humidity&startTime=2020-06-01T00:00:00Z&endTime=2020-06-03T00:00:00Z&bbox=14,38.03801,14.04,38.03802\",\n",
" \"next\": \"http://localhost:9801/1.0/query_data_doms?startIndex=1&itemsPerPage=20&provider=Saildrone&project=atlantic_to_med_2019_to_2020&platform=3B&minDepth=-5.2&maxDepth=-5.1&variable=relative_humidity&startTime=2020-06-01T00:00:00Z&endTime=2020-06-03T00:00:00Z&bbox=14,38.03801,14.04,38.03802\",\n",
" \"prev\": \"http://localhost:9801/1.0/query_data_doms?startIndex=0&itemsPerPage=20&provider=Saildrone&project=atlantic_to_med_2019_to_2020&platform=3B&minDepth=-5.2&maxDepth=-5.1&variable=relative_humidity&startTime=2020-06-01T00:00:00Z&endTime=2020-06-03T00:00:00Z&bbox=14,38.03801,14.04,38.03802\"\n",
"}\n"
]
}
],
"source": [
"start_index = 0\n",
"items_per_page = 20\n",
"provider = 'Saildrone'\n",
"project = 'atlantic_to_med_2019_to_2020'\n",
"platform = '3B'\n",
"min_depth = -5.2\n",
"max_depth = -5.1\n",
"variable = 'relative_humidity'\n",
"start_time = '2020-06-01T00:00:00Z'\n",
"end_time = '2020-06-03T00:00:00Z'\n",
"west_aka_min_lon = 14\n",
"south_aka_min_lat = 38.03801\n",
"east_aka_max_long = 14.04\n",
"north_aka_max_lat = 38.03802\n",
"\n",
"\n",
"requesting_url = f'http://localhost:9801/1.0/query_data_doms?startIndex={start_index}&itemsPerPage={items_per_page}&'\\\n",
" f'provider={provider}&project={project}&platform={platform}&'\\\n",
" f'minDepth={min_depth}&maxDepth={max_depth}&variable={variable}&'\\\n",
" f'startTime={start_time}&endTime={end_time}&'\\\n",
" f'platform=3B&bbox={west_aka_min_lon},{south_aka_min_lat},{east_aka_max_long},{north_aka_max_lat}'\n",
"print(requesting_url)\n",
"a = requests.get(url=requesting_url)\n",
"result = json.loads(a.content.decode())\n",
"print(json.dumps(result, indent=2))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 4
}